diff --git a/RELEASES.json b/RELEASES.json index 0359dce6f23efefe8e3b19ad8359a5f6598c1324..9aac7bc7d95cb4b47d10bfda884e9371dbd5830a 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,4 +1,4 @@ { - "current": "84d02c9227", + "current": "8962a7d2b8", "releases": [] } \ No newline at end of file diff --git a/VERSION b/VERSION index 0315f57b4ac157a04872a2446d0ee5aba482755e..733c6c5eed6d0caad6eaa8dc658373825d895258 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -84d02c9227 +8962a7d2b8 diff --git a/api/amazon_quotes_loop.py b/api/amazon_quotes_loop.py index bc940e798b934cdd34296beeff9146ad21e7b0c9..631e8d3a1a28d0b83356308f7111ec5343e5eefa 100644 --- a/api/amazon_quotes_loop.py +++ b/api/amazon_quotes_loop.py @@ -9,12 +9,10 @@ import logging from pathlib import Path import threading import time -import uuid JOB = "amazon-rival-quotes" #: The loop asks each tenant whether its daily run is due this often. POLL_SECONDS = 600 -SNAPSHOT_TABLES = ("competitor_snapshots", "competitor_images") #: Called as `fn(tenant, changed_asins)` after a run finishes, where `changed_asins` is a tuple of the #: ASINs whose newest quote changed. Each import is lazy and a failure is one log line, never a failed run. @@ -28,29 +26,17 @@ _thread = None _stop = None -class SnapshotBlocked(RuntimeError): - """A safe admission cause that can be recorded without exposing provider details.""" - - def _say(message): print(f"[{JOB}] {message}", flush=True) def readers(fields): - """Read prices and complete facts with the tenant credential and the shared Amazon pacers.""" + """The three SP-API reads of one tenant's credential. Each takes a list of at most 20 ASINs.""" from harness import amazon_client as cx - from harness import amazon_competitor_facts as facts - - def complete(asins): - catalog, offers = facts.read_facts(fields, asins), facts.read_offers(fields, asins) - return {asin: {**catalog[asin], **{key: offers[asin][key] for key in ("prime", "offers")}, - "cause": " ".join(c for c in (catalog[asin]["cause"], offers[asin]["cause"]) if c) or None} - for asin in asins} - return {"offers": lambda asins: cx.item_offers(fields, asins), # The pricing sync draws this same `competitive_summary` pacer in this worker process. "summaries": lambda asins: cx.competitive_summaries(fields, asins), - "facts": complete} + "ranks": lambda asins: cx.catalog_sales_ranks(fields, asins)} @contextmanager @@ -78,69 +64,9 @@ def _after_quotes(tenant, changed): "the run result is unchanged.") -def _snapshot_cause(runtime, *, check_lease=True): - from harness import amazon_store - - enabled = amazon_store.read_database_enablement(runtime) - if not enabled["available"]: - return "Competitor database settings could not be read. Restore settings access and retry the run." - if not all(enabled["databases"].get("ut_amzn_" + table) for table in SNAPSHOT_TABLES): - return "A competitor database is disabled. Enable competitor snapshots and images in Settings, Connectors and retry." - if check_lease and amazon_store.first_load_job_live(runtime): - return "The Amazon mirror writer lease is held. Let the current writer finish and retry the run." - return None - - -def _publish_snapshots(tenant, runtime, result): - from harness import amazon_competitor_facts as facts, amazon_store, mirror_store - - lock = mirror_store.for_connector("amazon").lock - if not lock.acquire(blocking=False): - raise SnapshotBlocked("The Amazon mirror writer is busy.") - holder = f"{JOB}:{result['id']}" - leased = False - try: - leased = amazon_store.acquire_amazon_lease(runtime, holder) - if not leased: - raise SnapshotBlocked("The Amazon mirror writer lease is held.") - cause = _snapshot_cause(runtime, check_lease=False) - if cause: - raise SnapshotBlocked(cause) - report = {"tables": {}, "problems": []} - with mirror_store.sync_mirror("amazon", tenant, runtime, report=report, expected_tables=SNAPSHOT_TABLES): - con, _path = amazon_store._connect_tenant_mirror(tenant, runtime) - try: - stored = facts.import_snapshots(con, result["snapshots"]) - for table in SNAPSHOT_TABLES: - total = con.execute(f"SELECT count(*) FROM amzn_{table}").fetchone()[0] - report["tables"][table] = {"state": "done", "rows": total} - finally: - con.close() - if report["problems"]: - raise RuntimeError("Competitor mirror publication failed") - result["counts"].update(snapshots=stored["snapshots"], images=stored["images"]) - finally: - try: - if leased: - amazon_store.release_amazon_lease(runtime, holder) - finally: - lock.release() - - -def _run_blocked(runtime, cause): - from core import amazon_rivals as rivals - - run_id = uuid.uuid4().hex - rivals.start_run(runtime, run_id, asin_total=0) - return {"id": run_id, "progress": 0, "cause": cause, - "counts": {"facts": dict.fromkeys(rivals.FACT_COUNT_COLUMNS, 0)}, "changed": (), "snapshots": []} - - def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep): """Run today's quotes for one tenant when due. None when it is not due or has no Amazon store.""" from core import amazon_rivals as rivals - if str(tenant) != str(runtime.key) or str(runtime.key) != str(runtime.tenant.key): - raise ValueError("Competitor quotes require the matching tenant runtime") try: if not rivals.due(runtime, now=now): return None @@ -154,23 +80,9 @@ def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep): fields = amazon_store.credential(tenant, runtime) if not fields: return None - started = time.monotonic() - cause = _snapshot_cause(runtime) - result = (_run_blocked(runtime, cause) if cause else - rivals.run_quotes(runtime, lambda: _mirror(runtime), **readers(fields), - seller_id=str(fields.get("seller_id") or "").strip() or None, now=now, - log=lambda message: _say(f"tenant={tenant} {message}"), sleep=sleep)) - if result["snapshots"]: - try: - _publish_snapshots(tenant, runtime, result) - except Exception as exc: - cause = (str(exc) if isinstance(exc, SnapshotBlocked) else - f"Competitor snapshots could not be published ({type(exc).__name__}).") - result["cause"] = (f"{cause} Check database settings and mirror access, let any writer finish, " - f"and retry within {rivals.RETRY_RUN_HOURS} hours.") - result["counts"]["elapsed_seconds"] = round(time.monotonic() - started, 3) - rivals.mark_run(runtime, result["id"], progress=result["progress"], finished=True, - counts=result["counts"], cause=result["cause"]) + result = rivals.run_quotes(runtime, lambda: _mirror(runtime), **readers(fields), + seller_id=str(fields.get("seller_id") or "").strip() or None, now=now, + log=lambda message: _say(f"tenant={tenant} {message}"), sleep=sleep) if result["cause"] is None: _after_quotes(tenant, result["changed"]) return result diff --git a/api/amazon_relational.py b/api/amazon_relational.py index 53d40081bd1c3945907d39b4822779f86a1f3b96..5dfa1369ea500ef3a22bbdf97259d115c9a409b2 100644 --- a/api/amazon_relational.py +++ b/api/amazon_relational.py @@ -650,8 +650,6 @@ def _fields_for(cur, key, *, read_through=False): "pinned": name == pinned, **({"description": amazon_schema.REVIEW_UNAVAILABLE_NOTE} if name == "asin" and key in (PREFIX + "review_topics", PREFIX + "review_trends") else {}), - **({"description": amazon_schema.SQP_LIMIT_NOTE} - if name == "asin" and key == PREFIX + "search_query_performance" else {}), **({"format": fmt} if fmt else {}), }, flow="amazon_relational")) for lookup in derived.get(name, ()): @@ -725,8 +723,7 @@ READ_THROUGH = frozenset( and (spec.get("report") or spec.get("api") or spec.get("monthly")))) | frozenset( PREFIX + table for table in amazon_store.DERIVED_TABLES) | frozenset( # Change history grows without a cap (standing rule 1), so it is never stored. - PREFIX + table for table in amazon_store.HISTORY_TABLES) | frozenset(( - PREFIX + "competitor_snapshots", PREFIX + "competitor_images")) + PREFIX + table for table in amazon_store.HISTORY_TABLES) def _quoted(name): @@ -1601,8 +1598,6 @@ def apply_plan(rt, built, username="amazon", today=None): doc[key]["description"] = ads_descriptions[key] if key in (PREFIX + "review_topics", PREFIX + "review_trends"): doc[key]["description"] = amazon_schema.REVIEW_UNAVAILABLE_NOTE - if key == PREFIX + "search_query_performance": - doc[key]["description"] = amazon_schema.SQP_LIMIT_NOTE if key == PREFIX + "listing_items": doc[key]["amazonListingIdentity"] = listing_version return doc diff --git a/api/app_templates.py b/api/app_templates.py index 09211ad2b0ea6323891e476567e8d8f62b617d2e..53f494823e188440830ec798e8dfc0afa49837f2 100644 --- a/api/app_templates.py +++ b/api/app_templates.py @@ -127,22 +127,12 @@ TEMPLATES = ( # Project management reads its own module tables (ut_amazon_*), so it declares no # source: a cold mirror must not refuse an App whose rows live in the store. _amazon("amazon_profit", "Project management", "See profit, ads and stock for every SKU.", - "Create value", (), - optional_sources=("ut_amzn_search_query_performance", "ut_amzn_order_items", - "ut_amzn_brand_analytics", "ut_amzn_listings", "ut_amzn_catalog_items", - "ut_amzn_listing_items", "ut_amzn_competitor_snapshots", - "ut_amzn_competitor_images"), - optional_connectors=("odoo",), proposes=False), + "Create value", (), optional_connectors=("odoo",), proposes=False), {"kind": "alerts", "label": "Alerts", "purpose": "Act on changes in your databases.", "connector": None, "connectors": (), "permission": "automation", "tags": ["Automation"], "sources": (), "optional_sources": (), "optional_connectors": (), "process": "None", "mark": "alerts", "module": None, "router": "routes_automation", "view": "alerts", "tools": (), "proposes": False}, - {"kind": "reports", "label": "Reports", "purpose": "Download reports shared with you.", - "connector": "odoo", "connectors": ("odoo",), "tags": ["Odoo"], "sources": (), - "optional_sources": (), "optional_connectors": (), "process": "None", "mark": "reports", - "module": "modules.reports", "router": "routes_reports", "view": "reports", - "tools": (), "proposes": False}, ) if not all(validate_connectors(row) for row in TEMPLATES): raise ValueError("An App template omits a source connector.") diff --git a/api/application_registry.py b/api/application_registry.py index 4f177256f6a0a5c66396d9c29a3a63a7b7176900..bc5c7f81a18f77ee9b642ce384013d5e649c21a9 100644 --- a/api/application_registry.py +++ b/api/application_registry.py @@ -624,16 +624,6 @@ def _source_rows_exist(session, sources): return False -def _reports_readable(runtime, user, is_admin): - """Report grants admit the Reports App independently of its App sharing record.""" - try: - from modules import reports - except Exception as exc: # A failed Reports import withholds its App. - raise_if_resource_error(exc) - return False - return bool(is_admin or reports.granted_keys(runtime, user)) - - def _readable_template_app(session, app_id): """Resolve App identity before reading any source or exposing a refusal.""" from deps import err @@ -645,10 +635,7 @@ def _readable_template_app(session, app_id): except Exception: raise err(503, "app_unavailable", "This App cannot be read right now.") from None if (not isinstance(app, dict) or app.get("id") != app_id or app.get("tenant") != session.tenant - or app.get("retired") is True - or not (_reports_readable(session.runtime, session.uname, session.admin) - if app.get("templateKind") == "reports" - else may_read(app, session.uname, session.admin))): + or app.get("retired") is True or not may_read(app, session.uname, session.admin)): raise err(404, "app_not_found", "This App is unavailable or is not shared with you.") kind = app.get("templateKind") catalogue = _app_templates.BY_KIND.get(str(kind or "")) @@ -1318,7 +1305,7 @@ def validate_application(payload, tenant, *, semantic_records=None, field_contra def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None, - is_default=False, readable=None): + is_default=False): row=copy.deepcopy(_app_projection(app)) row.setdefault("origin", None) if row.get("type") == "code": @@ -1327,10 +1314,9 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas compatible = [database for database in row.get("compatibleDatabases") or () if database in _allowed(row.get("compatibleDatabases"), allowed_databases)] row["compatibleDatabases"] = compatible - app_readable = may_read(app, user, is_admin) - row["capabilities"]={"read":app_readable if readable is None else readable,"edit":may_edit(app,user,is_admin), + row["capabilities"]={"read":may_read(app,user,is_admin),"edit":may_edit(app,user,is_admin), "share":may_edit(app,user,is_admin),"delete":may_edit(app,user,is_admin), - "duplicate":app_readable,"star":app_readable, + "duplicate":may_read(app,user,is_admin),"star":may_read(app,user,is_admin), "default":bool(is_admin)} row["starred"] = user in (app.get("starredBy") or ()) row["isDefault"] = bool(is_default) @@ -1391,7 +1377,6 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre """List Applications. Unfiltered is the App home; a database filter is how a View resolves.""" doc = load(runtime) rows=[] - reports_reader = None for app in doc["applications"].values(): if (isinstance(app, dict) and getattr(runtime, "key", None) and app.get("tenant") != runtime.key): @@ -1401,19 +1386,13 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre if (isinstance(app, dict) and app.get("templateKind") == "alerts" and not (may_open or _user_wall(runtime, user))("automation")): continue - report_app = isinstance(app, dict) and app.get("templateKind") == "reports" - if report_app: - if reports_reader is None: - reports_reader = _reports_readable(runtime, user, is_admin) - if not reports_reader or app.get("retired") is True: - continue if isinstance(app, dict) and app.get("templateKind") in ("customer_card", FINANCE_TEMPLATE_KIND): continue if database: # A View resolves its App only through this read, so a retired seed stays listed here. - if not report_app and not may_read(app,user,is_admin): continue + if not may_read(app,user,is_admin): continue if database not in (app.get("compatibleDatabases") or ()): continue - elif not report_app and not _listed(app, user): continue + elif not _listed(app, user): continue if (not _database_free(app) and not _allowed(app.get("compatibleDatabases"), allowed_databases)): catalogue = _app_templates.BY_KIND.get(str(app.get("templateKind") or "")) @@ -1425,7 +1404,6 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre default_key = f"{database}::{app.get('type')}" if database else "" rows.append(_public(app,user,is_admin,allowed_databases=allowed_databases, target_database=database, - readable=True if report_app else None, is_default=bool(default_key and doc["defaults"].get(default_key)==app.get("id")))) # S67-T39: the dependent-View count rides a DATABASE-filtered read only, and only for rows # the caller may delete, so the unfiltered nav read (App home tiles) pays nothing for it. diff --git a/api/connectors_amazon.py b/api/connectors_amazon.py index 5357479870133c043fdc7dca3acb206884e669ce..db1a5915be16a299703bfa2f4158270fb67aea72 100644 --- a/api/connectors_amazon.py +++ b/api/connectors_amazon.py @@ -611,11 +611,16 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None): answer[table]["warning"] = warning if table == az.SQP_TABLE or table in az.GROUP_A_TABLES or table in ("transactions", "brand_analytics"): answer[table]["limit"] = limit - for table in (*az.DERIVED_TABLES, "competitor_snapshots", "competitor_images"): - answer[table] = { - "rowsLoaded": mirrored.get(table), "lastSuccess": None, "inProgress": False, - "unknownCause": az.amazon_schema.PROVENANCE_COPY[az.amazon_schema.PROVENANCE[table]], - } + answer["sku_keywords"] = { + "rowsLoaded": mirrored.get("sku_keywords"), + "lastSuccess": None, + "inProgress": False, + "unknownCause": az.amazon_schema.PROVENANCE_COPY["derived"], + } + answer["keyword_ranks"] = { + "rowsLoaded": mirrored.get("keyword_ranks"), "lastSuccess": None, "inProgress": False, + "unknownCause": az.amazon_schema.PROVENANCE_COPY["apify_dataset"], + } # A history table is written in its source's transaction, so it shares the source's last success. for table in az.HISTORY_TABLES: rows = mirrored.get(table) diff --git a/api/external_freshness.py b/api/external_freshness.py index 7eb771cbeae127c1689c33fb3b9d26cec5eb53c7..09e06f19dc381e7504e81e57f5863fd582461af0 100644 --- a/api/external_freshness.py +++ b/api/external_freshness.py @@ -82,24 +82,18 @@ def _amazon_raw(table_key, source=None): return None -def amazon_sync_state(cur, rt=None, *, entry=None): +def amazon_sync_state(cur, rt=None): """`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read.""" - if rt is not None or entry is not None: + if rt is not None: from core import connector_sync, connector_spec - if entry is None: - entry = connector_sync.status(rt, "amazon_sp", fresh=True) + entry = connector_sync.status(rt, "amazon_sp", fresh=True) if entry is not None: last_pass = entry.get("pass") or {} held = last_pass.get("held") or {} stamps = last_pass.get("tableSync") or {} tables = {row.source.removeprefix("amzn_") for row in connector_spec.AMAZON_TABLES} - state = {"amzn_sync_mark_" + name: stamp for name in tables - if (stamp := stamps.get(name) or (entry.get("lastGoodSyncAt") if name not in held else None))} - if cur is not None and any(name not in stamps for name in held): - previous = amazon_sync_state(cur) - state.update({"amzn_sync_mark_" + name: stamp for name in held if name not in stamps - and (stamp := _amazon_table_stamp(previous, name)) is not None}) - return state + return {"amzn_sync_mark_" + name: stamp for name in tables + if (stamp := stamps.get(name) or (entry.get("lastGoodSyncAt") if name not in held else None))} return {str(entity): at for entity, at in cur.execute( "SELECT entity, updated_at FROM _sync_state " "WHERE phase = 'done' AND starts_with(entity, 'amzn_')").fetchall()} @@ -141,8 +135,8 @@ def amazon_sync_stamp(cur, table_key, state=None): return _amazon_table_stamp(state, table) -def _amazon_refreshes(session, keys, *, entry, served=False): - """Use the supplied canonical entry or served receipts for authorized Amazon databases.""" +def _amazon_refreshes(session, keys): + """Read only sync metadata for the authorized Amazon databases on this tenant's cursor.""" import assistant_mirror cur, state, refreshed = None, None, {} @@ -154,7 +148,7 @@ def _amazon_refreshes(session, keys, *, entry, served=False): cur = None continue if state is None: - state = amazon_sync_state(cur, entry=None if served else entry) + state = amazon_sync_state(cur, session.runtime) refreshed[key] = amazon_sync_stamp(cur, key, state=state) except Exception: continue @@ -244,7 +238,7 @@ def _mirror_refreshes(session, definitions, keys): return refreshed -def _served_behind(session, key, *, worker=None): +def _served_behind(session, key): """process_role's install lag for the connector behind one mirror database, or None.""" key = str(key or "") connector = ("amazon" if key.startswith("ut_amzn_") else @@ -253,82 +247,49 @@ def _served_behind(session, key, *, worker=None): return None try: import process_role - if worker is None: - worker = process_role.stored_worker(session.runtime) - return process_role.served_behind(getattr(session.runtime, "key", None) or session.tenant, connector, - worker=worker) + return process_role.served_behind(getattr(session.runtime, "key", None) or session.tenant, connector) except Exception: return None -def _retained_fact(served, state, reason): - """Keep the successful source receipt while naming why its rows have not advanced.""" - fact = {"source_synced_at": utc_stamp(served), "source_sync": state, "source_sync_reason": reason} - if (day := _sync_date(served)) is not None: - fact["source_synced_on"] = day - return fact - - -def _stale_fact(served, lag=None): +def _stale_fact(served): """A fact for a mirror whose newest published sync this deployment could not install.""" day = _sync_date(served) or (utc_stamp(served) or "")[:10] or None reason = (f"This deployment still serves the data synced on {day}. The newest sync could not be installed." if day else "This deployment serves an older copy of the data. The newest sync could not be installed.") - if lag and lag.get("cause"): - reason += " " + lag["cause"] - if lag and (retry := utc_stamp(lag.get("nextRetryAt"))): - reason += f" Next retry at {retry}." - return _retained_fact(served, "stale", reason) - - -def _amazon_holds(entry, keys): - """Map each authorized table to its own canonical hold or its held input tables.""" - held = (entry.get("pass") or {}).get("held") or {} - result = {} - for key in keys: - raw = _amazon_raw(key) - table = raw.removeprefix("amzn_") if raw else "" - dependencies = (table, *_AMAZON_DERIVED_INPUTS.get(table, ())) - rows = [held[name] for name in dependencies if isinstance(held.get(name), dict)] - if rows: - result[key] = rows - return result - - -def _held_fact(served, entries): - """Name the kept rows and each stored hold cause without changing their source date.""" - import dash_norm + return {"source_synced_at": None, "source_sync": "stale", "source_sync_reason": reason} - day = _sync_date(served) or (utc_stamp(served) or "")[:10] - reason = f"Held at the rows synced on {day}." if day else "Held at the previously synced rows." - causes = dict.fromkeys(dash_norm.plain(entry.get("cause") or "The last sync did not finish this table.") - for entry in entries) - reason += " " + " ".join(causes) - retries = sorted({stamp for entry in entries if (stamp := utc_stamp(entry.get("nextTryAt")))}) - if retries: - reason += f" Next retry at {retries[0]}." - return _retained_fact(served, "stale", reason) +def snapshot(session, databases): + """Index source receipts and edit times once for this authorized discovery call.""" + from core import connector_sync, user_tables -def _edit_receipts(rt, edited): - """Read the latest edit for all personal databases in one history aggregate.""" + wanted = set(databases) + rt = session.runtime + try: + definitions = rt.get_projection(user_tables.STORE_KEY, drop=("rows",)) + if not isinstance(definitions, dict): + raise ValueError("source definitions unavailable") + except Exception: + return {key: {"source_synced_at": None, "source_sync": "never_synced", + "source_sync_reason": "The source's connector status could not be verified."} + for key in wanted} + lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions}) + connected = {key for key in wanted if key in definitions and user_tables.is_connected(key, st=lent)} + edited = {key for key in wanted if key in definitions and key not in connected and key != "ut_activity"} edits = {} if edited: + # D-11777: edits append only to the history rows, so one SQL aggregate gives each table's + # newest edit. A store with no history backend (TypeError) or an unreachable one keeps the + # definition's `updated` stamp below and never fails discovery. try: from core import history_rows latest = history_rows.backend(rt).latest(sorted(edited)) - except Exception: # Unavailable history keeps the definition's recorded edit time. + except Exception: latest = {} for key, at in latest.items() if isinstance(latest, dict) else (): if key in edited: edits[key] = _latest(edits.get(key), at) - return edits - - -def _connector_receipts(rt, connected, definitions): - """Read matching jobs without borrowing another connection's successful sync.""" - from core import connector_sync - receipts = {} if connected: try: @@ -344,84 +305,27 @@ def _connector_receipts(rt, connected, definitions): or declared.get("connectionId") and job.get("connectionId") != declared["connectionId"]): continue receipts[key] = job - return receipts - - -class _StatusSnapshot: - """Keep the request's fresh canonical status available to readers that request freshness again.""" - - def __init__(self, runtime, statuses): - self._runtime, self._statuses = runtime, statuses - - def get(self, name, fresh=False): - from core import connector_sync - - if name == connector_sync.STATUS_KEY: - return self._statuses - return self._runtime.get(name, fresh=True) if fresh else self._runtime.get(name) - - def __getattr__(self, name): - return getattr(self._runtime, name) - - -def _odoo_freshness(rt): - """Read Odoo's metadata without restoring or querying its mirror.""" - import odoo_relational - from core import user_tables - - docs = {} - for name in (odoo_relational.CONFIG_KEY, user_tables.ODOO_SYNC_STATUS_KEY): - try: - docs[name] = rt.get(name) or {} - except Exception: # A missing status must not turn a discovery read into an error. - docs[name] = {} - return odoo_relational.sync_freshness(user_tables.lend(rt, **docs)) - - -def _synced_fact(values): - """Use a successful source timestamp or preserve an explicitly date-only receipt.""" - stamp = next((stamp for value in values if (stamp := utc_stamp(value)) is not None), None) - if stamp is not None: - return {"source_synced_at": stamp, "source_sync": "synced"} - day = next((day for value in values if (day := _sync_date(value)) is not None), None) - if day is not None: - return {"source_synced_on": day, "source_sync": "synced_date_only", - "source_sync_reason": "The source reports only a sync date, without a time."} - return None - - -def snapshot(session, databases): - """Index source receipts and edit times once for this authorized discovery call.""" - from core import connector_sync, user_tables - - wanted, rt = set(databases), session.runtime - try: - definitions = rt.get_projection(user_tables.STORE_KEY, drop=("rows",)) - if not isinstance(definitions, dict): - raise ValueError("source definitions unavailable") - except Exception: - return {key: {"source_synced_at": None, "source_sync": "never_synced", - "source_sync_reason": "The source's connector status could not be verified."} - for key in wanted} - lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions}) - connected = {key for key in wanted if key in definitions and user_tables.is_connected(key, st=lent)} - edited = {key for key in wanted if key in definitions and key not in connected and key != "ut_activity"} - edits, receipts = _edit_receipts(rt, edited), _connector_receipts(rt, connected, definitions) odoo_keys = {key for key in connected if key.startswith("ut_odoo_")} | (wanted & {"customer_data", "product_data"}) + odoo = {} + if odoo_keys: + import odoo_relational + + docs = {} + for name in (odoo_relational.CONFIG_KEY, user_tables.ODOO_SYNC_STATUS_KEY): + try: + docs[name] = rt.get(name) or {} + except Exception: + docs[name] = {} + odoo = odoo_relational.sync_freshness(user_tables.lend(rt, **docs)) amazon_keys = {key for key in connected if key.startswith("ut_amzn_")} - statuses = connector_sync.status_all(rt, fresh=True) if amazon_keys or odoo_keys else {} - odoo = _odoo_freshness(_StatusSnapshot(rt, statuses)) if odoo_keys else {} - worker = (statuses.get("sync_worker") or {}).get("default") or {} - amazon_entry = (statuses.get("amazon_sp") or {}).get("default") - behind = {group: _served_behind(session, next(iter(keys)), worker=worker) - for group, keys in (("odoo", odoo_keys), ("amazon", amazon_keys)) if keys} - amazon = _amazon_refreshes(session, amazon_keys, entry=amazon_entry, - served=behind.get("amazon") is not None) if amazon_keys else {} - held = _amazon_holds(amazon_entry or {}, amazon_keys) + amazon = _amazon_refreshes(session, amazon_keys) if amazon_keys else {} mirror_keys = {key for key in connected - amazon_keys if not user_tables.materialises(key, st=lent, defn=definitions[key])} mirror = _mirror_refreshes(session, definitions, mirror_keys) if mirror_keys else {} facts = {} + # One install-lag read per connector: a secondary web worker reads it from the primary's file. + behind = {group: _served_behind(session, next(iter(keys))) + for group, keys in (("odoo", odoo_keys), ("amazon", amazon_keys)) if keys} odoo_failed = {str(item.get("key")) for item in (odoo.get("failedDatabases") or ()) if isinstance(item, dict) and item.get("key")} for key in wanted: @@ -432,10 +336,7 @@ def snapshot(session, databases): elif key == "ut_activity": fact["source_sync"] = "not_applicable" elif behind.get("amazon" if key in amazon_keys else "odoo" if key in odoo_keys else None) is not None: - fact = _stale_fact(amazon.get(key) if key in amazon_keys else mirror.get(key), - behind["amazon" if key in amazon_keys else "odoo"]) - elif key in held: - fact = _held_fact(amazon.get(key), held[key]) + fact = _stale_fact(amazon.get(key) if key in amazon_keys else mirror.get(key)) else: health = definition.get(user_tables.CONNECTOR_HEALTH_FIELD) or {} job = receipts.get(key) or {} @@ -447,9 +348,13 @@ def snapshot(session, databases): if job.get("state") == "ready": successes.append(job.get("at")) values = [amazon.get(key)] if key.startswith("ut_amzn_") else [mirror.get(key), *successes] - synced = _synced_fact(values) - if synced is not None: - fact.update(synced) + stamp = next((stamp for value in values if (stamp := utc_stamp(value)) is not None), None) + synced_on = next((day for value in values if (day := _sync_date(value)) is not None), None) + if stamp is not None: + fact.update(source_synced_at=stamp, source_sync="synced") + elif synced_on is not None: + fact.update(source_synced_on=synced_on, source_sync="synced_date_only", + source_sync_reason="The source reports only a sync date, without a time.") elif (own_failed or job.get("state") == "failed" or key in odoo_keys and odoo.get("phase") == "error"): fact["source_sync_reason"] = "The connector's latest sync failed before a successful sync was recorded." diff --git a/api/main.py b/api/main.py index ac90d6b651748ec5d11ef566e51b81cd641998f9..030a81143f307ca0df6acf21a906b7bd8483ff7d 100644 --- a/api/main.py +++ b/api/main.py @@ -758,13 +758,6 @@ import app_routes _mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount. _mount_router("routes_amazon_profit") # S78-T03: the Command Center refresh and settings doors. _mount_router("routes_amazon_projects") -_mount_router("routes_amazon_pdp") -_mount_router("routes_pdp_text") -_mount_router("routes_pdp_images") -if _mount_router("routes_pdp_publish"): - # Amazon fetches published image URLs without a session, so the media router carries no auth. - from routes_pdp_publish import media_router as _pdp_media_router - app.include_router(_pdp_media_router) _mount_router("routes_competitors") _mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount. _mount_router("routes_sku_actions") @@ -1350,8 +1343,6 @@ def _pull_amazon(why, vendor=True): # D-13614: before the lease check, since the chain that holds the lease is the writer that publishes. _install_amazon_publication(tenant_key, rt, why) if _amazon.first_load_job_live(rt): - from harness import mirror_store - mirror_store.record_sync_attempt("amazon", rt) print("[aios-api] the first-load job holds the Amazon writer lease", flush=True) return False # Hotfix 2026-09-24: an unfinished first load goes through the Sync now desk, whose @@ -1387,7 +1378,6 @@ def _pull_amazon(why, vendor=True): except Exception as exc: # A writer that took the lease after the first-load check owns Amazon: skip, record no pass. if isinstance(exc, RuntimeError) and "holds the amazon writer lease" in str(exc): - mirror_store.record_sync_attempt("amazon", rt) print(f"[aios-api] amazon sync skipped ({why}): another job holds the Amazon writer lease", flush=True) return False @@ -1759,16 +1749,8 @@ def _amazon_sync_receipt(report, rt=None): def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None): - """Name a classified failure or a completed publication's typed entity hold.""" + """Record one classified failure; leave successful receipt text untouched.""" if ok: - if provider == "odoo": - from harness import mirror_store - held = mirror_store.odoo_moving_entities(cause) - if held: - entity = sorted(held)[0] - others = f" {len(held) - 1} other entities were held." if len(held) > 1 else "" - return (f"Odoo {entity} changed during all 3 full reads. Its previous rows were kept." - + others + " Other completed entities published.") return detail classifier, sentence = _sync_now_cause(cause) if classifier == "nothing_due": @@ -1969,35 +1951,9 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None, **({"after_entity": _after_entity} if _after_entity else {})) if not report and selected_entities != []: raise _SyncNowRefused("Odoo did not run a sync. Check whether the connection is paused.") - held = mirror_store.odoo_moving_entities(report) - if any(row.get("phase") != "live" and name not in held for name, row in report.items()): + if any(row.get("phase") != "live" for row in report.values()): raise _SyncNowRefused("Odoo backfill is incomplete. Run Sync now again to continue it.") - waiting = mirror_store.pending_sync_entities("odoo", tenant, rt) - if held and waiting == []: - import json - store = mirror_store.for_connector("odoo") - with store.lock: - published, _ = store._connect(tenant, rt) - candidate = None - try: - path = rt.datastore_path.with_suffix(".odoo.candidate.duckdb") - candidate = datastore.apply_limits(datastore.duckdb.connect(str(path), read_only=True)) - retained = {row[0] for row in candidate.execute("SELECT name FROM _candidate_held").fetchall()} - publication = json.loads(store._download(rt, mirror_store._PASS_KEY) or b"{}") - installed = datastore._publication(published, "odoo") - if (retained == set(held) and publication.get("state") == "published" - and publication.get("tenant") == tenant and publication.get("connector") == "odoo" - and publication.get("completed_at") and installed.get("state") == "complete" - and publication.get("snapshot_id") - and publication.get("snapshot_id") == installed.get("snapshot_id")): - waiting = retained - finally: - if candidate is not None: - candidate.close() - published.close() - if ((held and (set(waiting or ()) != set(held) - or not any(row.get("phase") == "live" for row in report.values()))) - or (not held and waiting is not None)): + if mirror_store.pending_sync_entities("odoo", tenant, rt) is not None: raise _SyncNowRefused("Odoo publication is held. The previous snapshot remains available.") problems = [] elif provider == "amazon_sp": @@ -2090,15 +2046,14 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None, detail = _sync_now_detail(provider, tenant, ok=False, detail=amazon_detail, cause=amazon_failure or problems[0]) else: - ok, detail = True, _sync_now_detail(provider, tenant, ok=True, detail="Sync finished.", - cause=report if provider == "odoo" else None) + ok, detail = True, "Sync finished." if amazon_detail and not problems: detail += " " + amazon_detail except _SyncNowRefused as exc: detail = str(exc) except Exception as exc: # noqa: BLE001 detail = _sync_now_detail(provider, tenant, ok=False, cause=exc) - if progress and not amazon_detail and not (provider == "odoo" and ok and held): + if progress and not amazon_detail: import re suffix = re.search(r" \(cause code: [a-z_]+\)$", detail) cause_code = suffix[0] if suffix else "" @@ -3565,8 +3520,6 @@ def _amazon_desk_pass(tenant, *, project=True): job.join() state["job"] = None _project_desk(state["outcome"], tenant) - import sync_runtime - sync_runtime.relay_at_boundary() receipt = connector_sync.sync_request("amazon_sp", tenant) if (not receipt or receipt["state"] != "queued" or receipt.get("requestedBy") == "first load" or max(receipt.get("notBefore") or 0, diff --git a/api/process_role.py b/api/process_role.py index 91f636f690fbb2d90d7c0f7d170b95f0e7a04d69..06c98d7a035d27d82ae2c29f4bd2bbdc2f39fce7 100644 --- a/api/process_role.py +++ b/api/process_role.py @@ -56,20 +56,6 @@ READY_FILE = "READY.json" PRIMARY_LOCK = "web-primary.lock" PRIMARY_FILE = "web-primary.json" INSTALL_ATTEMPTS = 3 -INSTALL_RETRY_SECONDS = 3600 -_INSTALL_CAUSES = ( - (TimeoutError, "The install timed out."), - (ConnectionError, "The saved sync could not be downloaded."), - (PermissionError, "The install could not access the saved sync."), - (FileNotFoundError, "The saved sync is missing a required file."), - (OSError, "The install could not read or write its files."), - (MemoryError, "The install ran out of memory."), - (ValueError, "The saved sync failed validation."), - (TypeError, "The saved sync has an invalid structure."), - (LookupError, "The saved sync is missing required data."), - (RuntimeError, "The install stopped with an internal error."), - (Exception, "The install failed unexpectedly."), -) INCOMPLETE_GRACE_SECONDS = 3600 #: The C-status provider key the sync worker writes its heartbeat, catch-up and mirror ids under. WORKER_STATUS = "sync_worker" @@ -550,34 +536,37 @@ def pending_handoffs(root=None): def _failure_map(root=None): - """Return each published snapshot's own failed install receipt.""" + """{(tenant, connector): {snapshotId, attempts, type, at}} for each published snapshot that failed to install.""" if not is_primary(): stored = primary_state(root).get("installFailures") or {} return {tuple(key.split("/", 1)): row for key, row in stored.items() if "/" in key and isinstance(row, dict)} - return {key: dict(row) for key, snapshot in dict(_STORED["mirrors"]).items() - if (row := _INSTALL_FAILURES.get((key, snapshot)))} - - -def served_behind(tenant, connector, root=None, *, worker=None): + last, failures = dict(LAST_INSTALL_ERROR), {} + for key, snapshot in dict(_STORED["mirrors"]).items(): + attempts = _INSTALL_FAILURES.get((key, snapshot), 0) + if attempts: + # LAST_INSTALL_ERROR is one slot: its type and time belong only to the snapshot it names. + same = (last.get("tenant"), last.get("connector"), last.get("snapshotId")) == (*key, snapshot) + failures[key] = {"snapshotId": snapshot, "attempts": attempts, + "type": last.get("type") if same else None, "at": last.get("at") if same else None} + return failures + + +def served_behind(tenant, connector, root=None): """`{published, served, attempts, error}` when the store published a snapshot this container failed to install. None when nothing is published, the published snapshot is served, or no install of it failed yet. """ key = (str(tenant or "").strip().lower(), str(connector)) - worker = worker or {} - published = ((worker.get("mirrors") or {}).get(key[1]) or {}).get("snapshotId") or _known_mirrors(root).get(key) + published = _known_mirrors(root).get(key) if not published: return None served = _installed_map(root).get(key) - failure = _failure_map(root).get(key) or (worker.get("installs") or {}).get(_install_key(key, published)) or {} - if (failure.get("tenant"), failure.get("connector"), failure.get("snapshotId")) != (*key, published): - return None + failure = _failure_map(root).get(key) or {} attempts = int(failure.get("attempts") or 0) if failure.get("snapshotId") == published else 0 if served == published or attempts < 1: return None - return {"published": published, "served": served or failure.get("served"), "attempts": attempts, - "error": failure.get("type"), "cause": failure.get("cause"), "nextRetryAt": failure.get("nextRetryAt")} + return {"published": published, "served": served, "attempts": attempts, "error": failure.get("type")} # ----------------------------------------------------------------------------- several web workers @@ -785,44 +774,6 @@ def _remove_dir(path): shutil.rmtree(path, ignore_errors=True) -def _install_key(key, snapshot): - return "/".join((*key, snapshot)) - - -def _install_due(key, snapshot, stamp): - failure = _INSTALL_FAILURES.get((key, snapshot)) or {} - return failure.get("attempts", 0) < INSTALL_ATTEMPTS or stamp >= failure["nextRetryAt"] - - -def _record_install(key, snapshot, stamp, *, exc=None, rt=None, changed=True): - """Keep one scoped install receipt and persist its safe cause and retry deadline.""" - from core import connector_sync - - identity = (key, snapshot) - was_failed = identity in _INSTALL_FAILURES - row = {"tenant": key[0], "connector": key[1], "snapshotId": snapshot, "at": stamp, - "state": "failed" if exc is not None else "installed"} - if exc is not None: - kind, cause = next((kind, cause) for kind, cause in _INSTALL_CAUSES if isinstance(exc, kind)) - attempts = (_INSTALL_FAILURES.get(identity) or {}).get("attempts", 0) + 1 - row.update(attempts=attempts, type=kind.__name__, cause=cause, served=_INSTALLED.get(key), - nextRetryAt=stamp + (INSTALL_RETRY_SECONDS if attempts >= INSTALL_ATTEMPTS else POLL_SECONDS)) - _INSTALL_FAILURES[identity] = row - LAST_INSTALL_ERROR.clear() - LAST_INSTALL_ERROR.update(row) - else: - row["changed"] = changed is not False - _INSTALL_FAILURES.pop(identity, None) - if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector"), - LAST_INSTALL_ERROR.get("snapshotId")) == (*key, snapshot): - LAST_INSTALL_ERROR.clear() - if rt is not None and (exc is not None or was_failed): - written = connector_sync.update_status(rt, WORKER_STATUS, "installs", {_install_key(key, snapshot): row}) - if not written: - print(f"[process-role] install status not stored for {key[0]}/{key[1]}", flush=True) - return row - - def install_ready(root=None, installer=None, *, now=None): """Install the newest READY snapshot per tenant and connector, then delete older hand-offs. @@ -835,22 +786,25 @@ def install_ready(root=None, installer=None, *, now=None): for key, rows in ready_handoffs(root).items(): completed_at, newest_dir, ready = rows[-1] snapshot = ready.get("snapshotId") or newest_dir.name - if _INSTALLED.get(key) != snapshot and _install_due(key, snapshot, stamp): + if _INSTALLED.get(key) != snapshot and _INSTALL_FAILURES.get((key, snapshot), 0) < INSTALL_ATTEMPTS: try: changed = installer(key[1], key[0], ready) - except Exception as exc: # Retry failed installs without exposing their diagnostic text. - failure = _record_install(key, snapshot, stamp, exc=exc) + except Exception as exc: # noqa: BLE001 + _INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1 + LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp, + attempts=_INSTALL_FAILURES[(key, snapshot)], type=type(exc).__name__) print(f"[process-role] hand-off install {key[0]}/{key[1]} {snapshot} failed " - f"({failure['type']})", flush=True) + f"({type(exc).__name__}: {exc})", flush=True) continue _INSTALLED[key] = snapshot - _record_install(key, snapshot, stamp, changed=changed) if changed is False: # Already resident: nothing was rebuilt, so this is not an install. for _at, older, _ready in rows[:-1]: _remove_dir(older) continue LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp) + if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key: + LAST_INSTALL_ERROR.clear() installed.append(key) print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]}", flush=True) for _at, older, _ready in rows[:-1]: @@ -908,31 +862,29 @@ def install_stored(tenant, rt, mirrors, installer=None, *, now=None): installer = installer or _install_from_store stamp = time.time() if now is None else now installed = [] - stored = (stored_worker(rt).get("installs") or {}) if rt is not None else {} for connector, row in sorted((mirrors or {}).items()): snapshot = row.get("snapshotId") if isinstance(row, dict) else None if not snapshot: continue - key = (str(tenant).strip().lower(), str(connector)) + key = (str(tenant), str(connector)) _STORED["mirrors"][key] = snapshot - previous = stored.get(_install_key(key, snapshot)) or {} - if (previous.get("state") == "failed" and (key, snapshot) not in _INSTALL_FAILURES - and (previous.get("tenant"), previous.get("connector"), previous.get("snapshotId")) == (*key, snapshot)): - _INSTALL_FAILURES[(key, snapshot)] = previous - if _INSTALLED.get(key) == snapshot or not _install_due(key, snapshot, stamp): + if _INSTALLED.get(key) == snapshot or _INSTALL_FAILURES.get((key, snapshot), 0) >= INSTALL_ATTEMPTS: continue try: changed = installer(key[1], key[0], rt, snapshot) - except Exception as exc: # Preserve the serving copy and retry this snapshot on its own deadline. - failure = _record_install(key, snapshot, stamp, exc=exc, rt=rt) + except Exception as exc: # noqa: BLE001 + _INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1 + LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp, + attempts=_INSTALL_FAILURES[(key, snapshot)], type=type(exc).__name__) print(f"[process-role] store install {key[0]}/{key[1]} {snapshot} failed " - f"({failure['type']})", flush=True) + f"({type(exc).__name__}: {exc})", flush=True) continue _INSTALLED[key] = snapshot - _record_install(key, snapshot, stamp, rt=rt, changed=changed) if changed is False: continue LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp) + if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key: + LAST_INSTALL_ERROR.clear() installed.append(key) print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]} from the store", flush=True) return installed @@ -1283,23 +1235,6 @@ def start_web_services(): return True -def _install_diagnostics(stored): - """Read this tenant's current failed snapshot and last changed install across web processes.""" - tenant = writer_tenant() - rows = {**(stored.get("installs") or {}), **{ - _install_key(key, snapshot): row for (key, snapshot), row in dict(_INSTALL_FAILURES).items()}} - rows = sorted((row for row in rows.values() if isinstance(row, dict) and row.get("tenant") == tenant), - key=lambda row: row.get("at", 0), reverse=True) - mirrors = {key[1]: snapshot for key, snapshot in _known_mirrors().items() if key[0] == tenant} - mirrors.update({connector: row.get("snapshotId") for connector, row in (stored.get("mirrors") or {}).items()}) - failure = next((row for row in rows if row.get("state") == "failed" - and mirrors.get(row["connector"]) == row["snapshotId"] - and _INSTALLED.get((tenant, row["connector"])) != row["snapshotId"]), None) - installed = next((row for row in rows if row.get("state") == "installed" and row.get("changed")), None) - local = dict(LAST_INSTALL) if LAST_INSTALL.get("tenant") == tenant else None - return local or installed, failure - - def status(): """{role, worker: {pid, alive, heartbeatAgeS, restarts, lastInstall}} for the settings response. @@ -1324,13 +1259,12 @@ def status(): fleet[slug] = {"pid": None, "alive": bool(row.get("alive")), "restarts": row.get("jobs", 0), "lastExit": None, "jobId": row.get("jobId")} import sync_runtime - last_install, last_error = _install_diagnostics(stored) return {"role": ROLE, "roleSource": ROLE_SOURCE, "webWorkers": web_workers(), "primary": is_primary(), "syncRuntime": sync_runtime.runtime_name()[0], "tenant": writer_tenant(), "scope": worker_scope(), "tenantWorkers": fleet, "tenantWorkersOverCap": list(_FLEET_STATE["overCap"]), "syncWorkersMax": sync_worker_cap(), "worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age, - "restarts": _WORKER["restarts"], "lastInstall": last_install, - "lastInstallError": last_error, + "restarts": _WORKER["restarts"], "lastInstall": dict(LAST_INSTALL) or None, + "lastInstallError": dict(LAST_INSTALL_ERROR) or None, "attestation": worker_provenance.visible(stored) if stored.get("heartbeat") == beat else None}, "writer": {"armed": _WRITER["armed"], "held": _WRITER_HELD.is_set(), "holder": _WRITER["holder"]}} diff --git a/api/routes_amazon_projects.py b/api/routes_amazon_projects.py index f9d5f5709efad1bbc4f33bf163bd85d2280616db..7e3fe423357e2cce5e6bfdb37e2bac79b9e70f34 100644 --- a/api/routes_amazon_projects.py +++ b/api/routes_amazon_projects.py @@ -38,7 +38,7 @@ FIELDS = {"name": "name", "status": "status", "priority": "priority", "owner": " "completedOn": "completed_on", "resultConfigs": "result_configs", "contributors": "contributors"} #: Project JSON keys a client may echo back. They are ignored, never written. READ_ONLY = frozenset(("id", "health", "targetCount", "createdAt", "updatedAt", "createdBy", - "rollup", "impact", "via", "figures", "type", "listed")) + "rollup", "impact", "via", "figures")) def _call(fn, *args, **kwargs): @@ -116,7 +116,6 @@ def _project(record, today): impact = dict(record.get("impact") or {}) impact.pop(projects.RESULT_CONFIGS_KEY, None) return {"id": record["id"], "name": record["name"], "status": record["status"], - "type": record.get("type", "project"), "listed": record.get("listed", True), "priority": record["priority"], "owner": identity["name"] if identity else record["owner"], "ownerUsername": identity["username"] if identity else None, "startOn": record["start_on"], "targetOn": record["target_on"], "progress": record["progress"], "notes": record["notes"], @@ -268,7 +267,7 @@ def _row_ids(cells): return sorted(cells) -def _write_cells(st, table_key, cells, *, by=amazon_profit.WRITER, via=amazon_profit.VIA): +def _write_cells(st, table_key, cells): import core.user_tables as ut def _set(cur): @@ -284,7 +283,7 @@ def _write_cells(st, table_key, cells, *, by=amazon_profit.WRITER, via=amazon_pr row[key] = value return cur - ut.update_with_activity(table_key, _set, st=st, by=by, via=via, + ut.update_with_activity(table_key, _set, st=st, by=amazon_profit.WRITER, via=amazon_profit.VIA, row_ids=_row_ids(cells)) @@ -875,11 +874,6 @@ def edit_project(project_id: str, body: Any = Body(default=None), session: Sessi def delete_project(project_id: str, session: Session = Depends(require_session)): """Delete one project and its targets.""" runtime = _writer(session) - record = _call(projects.get, runtime, project_id) - if record.get("type") == "pdp": - from routes_amazon_pdp import delete_workflow - delete_workflow(project_id, session=session) - return {"ok": True, "id": project_id} pairs = _target_pairs(runtime, project_id) _call(projects.delete, runtime, project_id, by=_by(session)) _repaint(runtime, pairs) diff --git a/api/routes_brand.py b/api/routes_brand.py index ead1b3b667e1ba96d49c21b4cf1f8f02f25fbd48..88c9b7dfa3c0598ea9ca1e303949b99638ceed8a 100644 --- a/api/routes_brand.py +++ b/api/routes_brand.py @@ -35,16 +35,13 @@ from __future__ import annotations import os import re -from hashlib import sha256 from xml.etree import ElementTree from pathlib import Path -from fastapi import APIRouter, Depends, Request -from fastapi.concurrency import run_in_threadpool -from fastapi.responses import FileResponse, Response +from fastapi import APIRouter, Depends +from fastapi.responses import FileResponse from deps import Session, err, require_session -from core import tenant_brand router = APIRouter(prefix="/api/v1") @@ -80,53 +77,6 @@ _MEDIA = {".png": "image/png", ".svg": "image/svg+xml"} _HERE = Path(__file__).resolve().parent -def _logo_call(fn, *args): - try: - return fn(*args) - except tenant_brand.LogoStorageError as error: - raise err(503, "logo_storage_unavailable", str(error)) from None - except ValueError as error: - raise err(422, "invalid_logo", str(error)) from None - - -def _logo_admin(session): - if not session.admin: - raise err(403, "admin_required", "Only an administrator can change the tenant logo.") - - -@router.get("/tenant-brand/logo") -def tenant_logo(session: Session = Depends(require_session)): - """Serve this tenant's logo to any signed-in user without changing state.""" - result = _logo_call(tenant_brand.logo, session.runtime) - if result is None: - raise err(404, "no_logo", "This workspace has no logo.") - data, mime = result - return Response(data, media_type=mime, headers={ - "ETag": f'"{sha256(data).hexdigest()}"', "Cache-Control": "private, no-cache", - "X-Content-Type-Options": "nosniff"}) - - -@router.put("/tenant-brand/logo") -async def put_tenant_logo(request: Request, session: Session = Depends(require_session)): - """Accept at most 2 MiB of raw PNG or JPEG bytes from an administrator.""" - _logo_admin(session) - data = bytearray() - async for chunk in request.stream(): - if len(data) + len(chunk) > tenant_brand.MAX_BYTES: - raise err(413, "logo_too_large", tenant_brand.SIZE_LIMIT) - data.extend(chunk) - return await run_in_threadpool( - _logo_call, tenant_brand.put_logo, session.runtime, bytes(data), session.uname) - - -@router.delete("/tenant-brand/logo", status_code=204) -def delete_tenant_logo(session: Session = Depends(require_session)): - """Remove this tenant's logo for an administrator.""" - _logo_admin(session) - _logo_call(tenant_brand.delete_logo, session.runtime) - return Response(status_code=204) - - def _roots(): env = (os.environ.get("AIOS_BRAND_DIR") or "").strip() out = [Path(env)] if env else [] diff --git a/api/routes_changes.py b/api/routes_changes.py index 05d0c274e2f7e52a5e2cd127e7440837fd768143..c790086a5f275d39f389d72bd294c2c1bba1d08b 100644 --- a/api/routes_changes.py +++ b/api/routes_changes.py @@ -62,7 +62,6 @@ import core.store as store import core.user_tables as user_tables import modules.customer_data as customer_data import modules.product_data as product_data -import virtual_sources from deps import Session, err, require_changes_session, require_session @@ -201,8 +200,6 @@ def _may_watch(session, scope): @router.get("/changes") def changes(scope: str = "", session: Session | None = Depends(require_changes_session)): """Serve legacy change checks without making Pg a resident client.""" - if virtual_sources.get(_clean_scope(scope)) is not None: - return {"scope": _clean_scope(scope), "tokens": {}, "backend": store.backend()} names = buckets_for(scope) if names is None: raise err(400, "bad_scope", "name the table you are watching, e.g. ?scope=customer") diff --git a/api/routes_competitors.py b/api/routes_competitors.py index 436cd676061454d0a78332a8515123b3a82cdc27..31f61ba1d56024cf954fae709a2fa7199f49b04a 100644 --- a/api/routes_competitors.py +++ b/api/routes_competitors.py @@ -39,19 +39,6 @@ def _open_mirror(runtime): cur.close() -@contextmanager -def _boundary_errors(): - try: - yield - except rivals.MirrorUnreadable as exc: - raise err(503, "amazon_mirror_unreadable", - f"The Amazon data could not be read ({exc}). Try again shortly.") from None - except rivals.BackendUnavailable: - raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None - except RuntimeError: - raise err(503, "store_not_ready", "Amazon data is still loading. Try again shortly.") from None - - def _rt(session): session.require(routes_amazon_profit.amazon_profit.TABLE_KEY) try: @@ -164,23 +151,6 @@ def competitors_list(grain: str, key: str, session: Session = Depends(require_se return _payload(runtime, grain, key, target, parent, routes_amazon_profit._may_edit(session), problem) -@router.get("/competitors/pool") -def competitors_pool(grain: str, key: str, session: Session = Depends(require_session)): - """Read the eligible Setup pool and its stored daily facts through the caller's tenant session.""" - runtime = _rt(session) - key = _grain(grain, key) - with _boundary_errors(), _open_mirror(runtime) as cur: - target, parent = _target(cur, grain, key) - if parent is not None: - raise err(400, "parent_has_no_rivals", rivals.PARENT_ROLLS_UP) - pool = rivals.pool(runtime, cur, target) - payload = {field: pool[field] for field in ('asin', 'productType', 'dataAsOf', 'week', 'month', - 'keywordRanksOn', 'cause', 'held', 'removed', 'competitors')} - if not payload['competitors']: - payload['cause'] = rivals.NO_POOL - return {**payload, 'canEdit': routes_amazon_profit._may_edit(session)} - - def _writable(session, body): runtime = _rt(session) if not routes_amazon_profit._may_edit(session): @@ -188,8 +158,16 @@ def _writable(session, body): body = body if isinstance(body, dict) else {} grain = body.get("grain") key = _grain(grain, body.get("key")) - with _boundary_errors(), _open_mirror(runtime) as cur: - target, parent = _target(cur, grain, key) + try: + with _open_mirror(runtime) as cur: + target, parent = _target(cur, grain, key) + except rivals.MirrorUnreadable as exc: + raise err(503, "amazon_mirror_unreadable", + f"The Amazon data could not be read ({exc}). Try again shortly.") from None + except rivals.BackendUnavailable: + raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None + except RuntimeError: + raise err(503, "store_not_ready", "Amazon data is still loading. Try again shortly.") from None if parent is not None: raise err(400, "parent_has_no_rivals", rivals.PARENT_ROLLS_UP) try: diff --git a/api/routes_grid.py b/api/routes_grid.py index f519d9e8cb47f59af8f132ff2425575dfabed319..d942214b31b8cc0f76e609b9d8507f1cfb6f1041 100644 --- a/api/routes_grid.py +++ b/api/routes_grid.py @@ -20,7 +20,6 @@ from inspect import signature from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query from deps import Session, err, module_gate, perms, require_session -import virtual_sources router = APIRouter(prefix="/api/v1") @@ -50,7 +49,6 @@ def row_patch_receipt(scope): raise err(400, 'bad_op_id', str(exc)) from exc session = bound.arguments['session'] scope_key = scope if isinstance(scope, str) else scope(bound.arguments) - virtual_sources.refuse_write(scope_key) bound.arguments['body'] = {key: value for key, value in body.items() if key != 'opId'} def apply(): @@ -235,7 +233,6 @@ def grid_choice_remove(scope: str = "customer", body: dict = Body(default=None), body = body if isinstance(body, dict) else {} scope = _scope_or_400(scope) - virtual_sources.refuse_write(scope) key = str(body.get("key") or "").strip()[:80] options = _cr.normalise(body.get("options") if isinstance(body.get("options"), list) else []) confirm = body.get("confirm") is True @@ -478,8 +475,6 @@ def workspace(scope: str = "customer", from routes_customers import grid_assembly scope = _scope_or_400(scope) - if virtual_sources.get(scope) is not None: - return {"workspace": virtual_sources.assembly(session, scope, with_rows=False)["workspace"]} # Wave 21 C4 — the per-scope gate the dependency no longer asserts: customer/cohort need # the customer module; product asserts PRODUCT_MODULE in its branch; ut_* needs only a # session (its wall is the table's own, inside the assembly). @@ -1529,7 +1524,6 @@ def bulk_cells(body: dict = Body(default=None), from routes_products import MODULE as PRODUCT_MODULE, scoped_pool scope = _scope_or_400((body or {}).get("scopeKey")) - virtual_sources.refuse_write(scope) if scope != "product": # Fail-closed with the reason: the shared stratum is a PRODUCT-topic mechanism today. raise err(400, "scope_not_bulk_writable", @@ -2132,7 +2126,6 @@ def grid_events_route(body: dict = Body(default=None), # typo degraded silently to customer-scope behaviour on a WRITE. Read and write must agree on # what a scope is, or the surface you read is not the surface you wrote. scope = _scope_or_400((body or {}).get("scopeKey")) - virtual_sources.refuse_write(scope) if scope.startswith("ut_") and any( isinstance(event, dict) and event.get("type") == "overlay_patch" for event in events): # Managed records require Route/Zone tools, including keys without stored definitions. diff --git a/api/routes_odoo_tables.py b/api/routes_odoo_tables.py index 3160d8b2afe92dc02eeff0bd8cfaa6cc8f3bc588..2967033ad9b0b55f0107550f789b4475a66d94fa 100644 --- a/api/routes_odoo_tables.py +++ b/api/routes_odoo_tables.py @@ -5096,12 +5096,10 @@ def _materialised_window(table_key, session, offset, limit, filters, filterConj, def _materialised_window_read(table_key, session, offset, limit, filters, filterConj, sorts, search, related, groupBy=None, groupKey=None, ids=None, cursor=None): from harness import datastore - import virtual_sources query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj, "sorts": _json_arg(sorts, "sorts"), "search": search}) - virtual = virtual_sources.get(table_key) is not None - payload = virtual_sources.rows_payload(session, table_key, related=related) if virtual else None + payload = None window_offset = offset if table_key == 'ut_activity': if cursor is not None: @@ -5140,9 +5138,7 @@ def _materialised_window_read(table_key, session, offset, limit, filters, filter options = dict(select=spec['id'] if read.get('selection') else '_row_index', where=where, params=tuple(params), order_by=order or spec['id'], offset=window_offset, limit=limit or None, cur=read['cur'], **read['frm']) - if table_key == 'ut_activity' or virtual: - if virtual and cursor is not None: - raise err(400, 'window_cursor_unsupported', 'Continue this source with an offset.') + if table_key == 'ut_activity': win = datastore.window(**options) win['next_cursor'] = (read['payload'] or {}).get('next_cursor') else: @@ -5289,10 +5285,6 @@ def odoo_table_rows(table_key: str, "limit, filters, sorts and search in the query string.") ids = _record_ids(body) - import virtual_sources - if virtual_sources.get(table_key) is not None: - return _materialised_window_read(table_key, session, offset, limit, filters, filterConj, - sorts, search, related, groupBy, groupKey, ids=ids, cursor=cursor) # Admission resolves this Amazon database without publishing a process-wide binding. page = _page_defs(session) if str(table_key).startswith(("ut_amzn_", "ut_odoo_model_")) else None admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None) diff --git a/api/routes_shares.py b/api/routes_shares.py index 22cee0bc521db77ac76e5c74ebc5627691f82283..a76f43418359cab15f7be4106d5b3833fc2be1d7 100644 --- a/api/routes_shares.py +++ b/api/routes_shares.py @@ -1221,8 +1221,6 @@ def get_cohort(scope: str, cohort_id: str, session: Session = Depends(require_se @router.get("/share/{kind}/{oid}") def get_share(kind: str, oid: str, session: Session = Depends(require_session)): kind = _kind_or_400(kind) - if kind == "report" and not session.admin: - raise err(404, "no_object", "no such item, or it is not shared with this account") if kind == "field": table_key, field_key = shares.split_field_oid(oid) if table_key and field_key: @@ -1281,9 +1279,8 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)): **rec, "role": role, "mayAdminister": may_admin, - "maxGrantableRole": ( - "view" if kind == "report" else "edit" if claiming else shares.max_grantable_role( - kind, oid, session.uname, is_admin=_admin_share, st=session.runtime)), + "maxGrantableRole": "edit" if claiming else shares.max_grantable_role( + kind, oid, session.uname, is_admin=_admin_share, st=session.runtime), "ownerLabel": owner_label, # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry # them. `assignable_people` serves bare display names because `user`-kind CELLS store @@ -1305,19 +1302,35 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)): def _people(tenant): - """Active tenant identities, excluding test accounts, ordered by display name.""" - return sorted(users.assignable_identities(tenant), key=lambda p: p["name"].lower()) - - -def _report_entries_or_400(entries): - """A report grants download to named people alone.""" - for entry in entries: - if (not isinstance(entry, dict) or not str(entry.get("user") or "").strip() - or str(entry.get("user")).strip() == shares.EVERYONE): - raise err(400, "bad_entries", - "Add people by name. A report is shared one person at a time.") - if str(entry.get("role") or "view").strip().lower() != "view": - raise err(400, "bad_role", "A report grant can only allow download.") + """[{username, name}] for this tenant — same population as `assignable_people`, with the + BINDING identity alongside the display one. + + ⛔⛔ W49-T23 / D-235 — A BLANK TENANT RETURNS NOBODY, NOT EVERYBODY. This read + `if want and str(u.get('tenant') …) != want`, so a blank or `None` slug skipped the filter + entirely and handed one workspace every tenant's roster: the exact `if want and …` fail-OPEN + that W33-T37 closed in `core.users.assignable_people` / `avatar_map` one layer down, left + behind in the copy that lives here. The three callers all pass `session.tenant`, which is + `claims["t"]` and is already proven non-blank by `require_session` (an unresolvable slug is a + 401 before any route runs), so this changes no answer any live caller gets today — it removes + the shape, before a fourth caller arrives that does not carry that proof. Fail-closed, because + from this function a blank slug and "every tenant at once" are the same observation, and one + of the two callers below turns this roster into who may be GRANTED access. + """ + try: + reg = users.registry() or {} + except Exception: + return [] + want = str(tenant or '').strip().lower() + if not want: + return [] + out = [] + for uname, u in reg.items(): + if not isinstance(u, dict) or u.get('active') is False: + continue + if str(u.get('tenant') or 'royal-imports').strip().lower() != want: + continue + out.append({"username": str(uname), "name": str(u.get('name') or uname)}) + return sorted(out, key=lambda p: p["name"].lower()) def _stranger(session, kind, oid): @@ -1342,11 +1355,6 @@ def _put_share(kind, oid, body, session, *, _view_snapshot=None, _source_owner=N _notify=True): """Apply the shared share admission, ceiling and atomic writer for both doors.""" kind = _kind_or_400(kind) - if kind == "report" and not session.admin: - if _stranger(session, kind, oid): - raise err(404, "no_object", "no such item, or it is not shared with this account") - raise err(403, "not_owner", - "Only an administrator can change who can download this report.") if kind == "field": table_key, field_key = shares.split_field_oid(oid) if table_key and field_key: @@ -1420,8 +1428,6 @@ def _put_share(kind, oid, body, session, *, _view_snapshot=None, _source_owner=N raise err(400, "bad_entries", "entries must be a list of {user, role}. Send [] to un-share, which is how " "revoking is expressed") - if kind == "report": - _report_entries_or_400(entries) _entries_or_400(session, entries) # ⭐⭐ R4 / W40-T02 — THE CEILING. `may_administer` above now opens this door to an `edit` # grantee on a VIEW, so R4's other half ("a re-share may never exceed the role the re-sharer @@ -1617,8 +1623,7 @@ def _put_share(kind, oid, body, session, *, _view_snapshot=None, _source_owner=N try: out = shares.set_grants(kind, oid, entries, - owner=(None if kind == "report" else - session.uname if claiming else rec["owner"]), + owner=session.uname if claiming else rec["owner"], granter=session.uname, st=session.runtime, expect=rec["entries"]) diff --git a/api/routes_starred.py b/api/routes_starred.py index 93c34a74a2c936a4c1e468999425b0282d2690ad..c4a589d8da4228ddfb99b1cc3a09174fe45876c2 100644 --- a/api/routes_starred.py +++ b/api/routes_starred.py @@ -1231,9 +1231,6 @@ def starred_records(database: str = "", session: Session = Depends(require_sessi key = _clean_id(database) if not key: raise err(400, "bad_request", "no database was named") - import virtual_sources - if virtual_sources.get(key) is not None: - return {"database": key, "ids": [], "count": 0, "view": None} return _records_payload(session, key, _database_or_refuse(session, key)) @@ -1252,8 +1249,6 @@ def set_starred_record(body: dict = Body(default=None), raw_id = _clean_id(body.get("id")) if not key: raise err(400, "bad_request", "no database was named") - import virtual_sources - virtual_sources.refuse_write(key) if not raw_id.isdigit(): # Named rather than coerced: see `_record_ids` on why a non-numeric id cannot be stored. raise err(400, "bad_request", "a record id is a number") diff --git a/api/routes_tables.py b/api/routes_tables.py index bbf775f2cc661332ac92786b1a5763693aa958bd..3e6392232dcf77aec71b972287ea2a7b8fafab33 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -56,7 +56,6 @@ from fastapi import HTTPException, Request, Response from deps import Session, err, require_session from harness.runtime import DatastoreMismatch from routes_grid import row_patch_receipt -import virtual_sources router = APIRouter(prefix="/api/v1") @@ -286,8 +285,6 @@ def related_metrics(table_key: str, body: dict = Body(default=None), if (not isinstance(pin, dict) or not isinstance(pids, list) or any(type(pid) is not int or pid <= 0 for pid in pids)): raise err(400, "bad_metric_scope", "Name the origin record, relation and linked records.") - if virtual_sources.get(table_key) is not None: - return virtual_sources.related_metrics(session, table_key, pin, pids) session.require(table_key) viewer = copy.copy(session) viewer.runtime = _MetricReadRuntime(session.runtime) @@ -393,8 +390,6 @@ def _table_lend(runtime, table_key, defs_only=False): def _managed_or_refuse(session, table_key, *, mutation=False): """C1 generic-door policy. Dedicated Route/Zone handlers use their own capability.""" from core import route_system - if mutation: - virtual_sources.refuse_write(table_key) definition = _ut().get_one(table_key, st=session.runtime) # Route/Zone are virtual system descriptors until their first dedicated write, so looking # only in `user_tables` would leave a generic mutation path open during that empty state. @@ -589,7 +584,6 @@ def _defn_or_refuse(session, table_key, st=None, defs_only=False, scope_applied= proves nothing about which read produced it. `core.store.is_projected(defn)` is how a gate asserts the fast path was TAKEN. """ - virtual_sources.refuse_write(table_key) ut = _ut() # ⭐⭐ W31 QA — THE SWEEP, AND IT IS ONE LINE BECAUSE IT IS DONE HERE RATHER THAN PER ROUTE. # Owner: *"this is just one case, I need you to check and apply the fix everywhere too."* @@ -672,8 +666,8 @@ def _records_lock_message(table_key, definition, st): return f"This {name} is locked, so only user-created field values can be edited." -def _editor_or_refuse(session, table_key, st=None): - """Require editor authority on a database this session may see. `(defn, st)`.""" +def _records_or_refuse(session, table_key, st=None, *, cell_updates=None): + """Require editor authority and reserve locked database edits for authored field values.""" # Share one snapshot across the read, editor and record-mode checks. st = st if st is not None else _ut().lend(session.runtime) # ⭐ W36-T21 — `scope_applied=True` because every row write behind this wall is bounded by a @@ -683,12 +677,6 @@ def _editor_or_refuse(session, table_key, st=None): defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True) if not _ut().may_edit_database(table_key, session.uname, is_admin=session.admin, st=st): raise err(403, "database_read_only", "Can edit access is required to change this database.") - return defn, st - - -def _records_or_refuse(session, table_key, st=None, *, cell_updates=None): - """Require editor authority and reserve locked database edits for authored field values.""" - defn, st = _editor_or_refuse(session, table_key, st) if not _ut().records_mutable(table_key, st=st): if cell_updates and table_key != "ut_activity": from core import shared_overlay @@ -1063,9 +1051,6 @@ def _activity_grid_rows(session): @router.get('/tables/{table_key}/activity') def table_activity(table_key: str, cursor: str = None, limit: int = 50, session: Session = Depends(require_session)): - if virtual_sources.get(table_key) is not None: - from core import activity_log - return {'records': [], 'next_cursor': None, 'fields': activity_log.definition()['fields']} if table_key.startswith('ut_'): _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True) elif table_key not in ('customer_data', 'product_data'): @@ -2137,9 +2122,6 @@ def _scoped_table_rows(session, table_key, st, *, cursor=None, limit=None): def ut_assembly(session: Session, table_key: str, storage_key: str = "", consume_corrections: bool = False, with_rows: bool = True, st=None, *, fields_only: bool = False, needed=None, cursor=None, limit=None): - if virtual_sources.get(table_key) is not None: - return virtual_sources.assembly(session, table_key, with_rows=with_rows and not fields_only, - storage_key=storage_key) from core import grid_events actor = grid_events.request_admission(session, table_key + '_table_workspace') result = _ut_assembly(actor, table_key, storage_key, consume_corrections, with_rows, st, @@ -3365,8 +3347,6 @@ def table_rows(table_key: str, session: Session = Depends(require_session), (found by this route's own gate check, not by luck).""" import aios_grid - if virtual_sources.get(table_key) is not None: - return virtual_sources.rows_payload(session, table_key, related=related) if table_key == 'ut_activity': if related: raise err(409, 'activity_related_unavailable', 'Open the Activity database to read events.') @@ -4541,8 +4521,7 @@ def restore_history(table_key: str, event_id: str, body: dict = Body(default=Non from core import grid_events _managed_or_refuse(session, table_key, mutation=True) - # A locked database admits authored values only, so its record wall waits for the retained values. - _editor_or_refuse(session, table_key) + _records_or_refuse(session, table_key) body = body if isinstance(body, dict) else {} row_id = str(body.get("rowId") or "").strip() if not row_id: @@ -4572,7 +4551,6 @@ def restore_history(table_key: str, event_id: str, body: dict = Body(default=Non raise err(404, "history_event_not_found", "that retained value version is unavailable") if any(key in (g.get("hidden") or ()) for key in updates): raise err(403, "field_hidden", "that retained version includes a hidden field") - _records_or_refuse(session, table_key, cell_updates=updates) prior_history_ids = { str(item.get("id")) for item in _ut().value_history( table_key, row_id, st=session.runtime, retention_days=days, diff --git a/api/sync_runtime.py b/api/sync_runtime.py index db7d5927f779bc768646f10fefb52a81b74c7f1a..f6e99682593cd01dfd19dd1561f592a6e5838df3 100644 --- a/api/sync_runtime.py +++ b/api/sync_runtime.py @@ -6,7 +6,7 @@ publish is 878.8 s inside the 1,200 s limit, so a pass fits one job. Decision (lead ruling, 2026-11-03): the default job is RESIDENT (`sync_worker.py --job`). It runs every worker loop as the old child did, because about ten heavy loops (automation, Amazon ads, inbox, payouts, aged stock, Sheet refresh, restock, returns, materialization, first-load floor) -live only in the worker. A resident job hands over to a new job only at a pass boundary of the +live only in the worker. A resident job hands over to a new job only at an idle point of the resync loop, when the web's deployed version differs from its own or under 3 h of its 24 h timeout remain. AIOS_SYNC_JOB_MODE=per-pass keeps one job per pass (`sync_worker.py --once`). The job runs the Space's own image (hf.co/spaces/), proved by HF job @@ -17,7 +17,7 @@ AIOS_SYNC_RUNTIME picks the runtime: `hf-job`, `external` (a separately run resi HF sets SPACE_HOST in a Space. The DigitalOcean staging config sets SPACE_ID only. Web side: `tick` submits a job for each tenant with no live job: always when resident, only when -its work is due when per-pass. Worker side: `relay_at_boundary` hands queued work to the store. +its work is due when per-pass. Worker side: `idle_wait` is the resync loop's only exit point. The tenant's C-status `sync_worker` entry carries `job` {id, submittedAt, image, tenant}, `schedule` {nextPassAt, at, reason} and `web` {version, at}. Stamps are ISO-8601 UTC, the format of RESYNC_STATE's. @@ -29,7 +29,6 @@ import math import os import threading import time -from copy import deepcopy RUNTIMES = ("hf-job", "external", "off") RUNTIME_ENV = "AIOS_SYNC_RUNTIME" @@ -234,51 +233,8 @@ def relay_due(now=None): return "" -def _handoff_queued_requests(): - """Persist ordinary queued receipts without changing durable events or interrupting a claim.""" - from core import connector_sync - import process_role - with connector_sync._SYNC_REQUEST_LOCK: - receipts = deepcopy(connector_sync._SYNC_REQUESTS) - runtimes = dict(connector_sync._SYNC_REQUEST_RUNTIMES) - if any(row.get("state") == "running" for row in receipts.values()): - return None - try: - for row in receipts.values(): - if row.get("state") != "queued" or row.get("tables") is not None: - continue - tenant = row["tenant"] - rt = runtimes.get(tenant) or process_role._status_runtime(tenant) - if rt.key != tenant: - raise ValueError("The relay desk does not match the receipt tenant.") - connector_sync.queue_remote_request(rt, row, row.get("connectionId")) - except Exception as exc: - print(f"[sync-worker] job: relay handoff failed ({type(exc).__name__}); keeping this job running", flush=True) - return None - return receipts - - -def relay_at_boundary(next_at=None, *, exit_fn=None): - """Relay a resident job after its current write and projection finish, before another claim.""" - from core import connector_sync - if not ONCE["on"] or ONCE["mode"] != "resident": - return False - reason = relay_due() - if not reason: - return False - receipts = _handoff_queued_requests() - if receipts is None: - return False - with connector_sync._SYNC_REQUEST_LOCK: - # No receipt can be admitted or claimed between the final handoff check and exit. - if connector_sync._SYNC_REQUESTS != receipts: - return False - finish(time.time() if next_at is None else next_at, reason, exit_fn=exit_fn) - return True - - def idle_wait(event, seconds, *, exit_fn=None): - """Wait between passes, relaying a resident job before a queued wake can start another pass.""" + """The resync loop's wait, and a job's only exit point. Outside a job it is `event.wait(seconds)`.""" if not ONCE["on"]: return event.wait(seconds) if seconds is None or not math.isfinite(float(seconds)): @@ -286,21 +242,18 @@ def idle_wait(event, seconds, *, exit_fn=None): deadline = time.time() + max(0.0, min(float(seconds), MAX_IDLE_SECONDS)) resident = ONCE["mode"] == "resident" while True: - if resident and relay_at_boundary(deadline, exit_fn=exit_fn): - return False remaining = deadline - time.time() if remaining <= IDLE_GRACE_SECONDS: return event.wait(max(0.0, remaining)) # Per-pass: the grace lets the desk poll admit a request queued while the pass ran. if event.wait(min(RELAY_CHECK_SECONDS if resident else IDLE_GRACE_SECONDS, remaining)): - if resident and relay_at_boundary(deadline, exit_fn=exit_fn): - return False return True - if resident: + reason = relay_due() if resident else "idle" + if not reason: continue busy, wake_at = _desk_state(time.time()) if not busy: - finish(deadline if wake_at is None else min(deadline, wake_at), "idle", exit_fn=exit_fn) + finish(deadline if wake_at is None else min(deadline, wake_at), reason, exit_fn=exit_fn) return False @@ -345,7 +298,7 @@ def no_lease_line(tenant, seconds=None): def _pass_in_flight(): - """True while the store resync loop runs and owns its pass boundaries.""" + """True while the store resync loop runs: it relays at its own idle point.""" return any(t.name == "store-resync" and t.is_alive() for t in threading.enumerate()) @@ -356,7 +309,9 @@ def run_once(*, exit_fn=None, alive=None): if alive(): continue if ONCE["mode"] == "resident": - if relay_at_boundary(exit_fn=exit_fn): + reason = relay_due() + if reason and not _desk_state(time.time())[0]: + finish(time.time(), reason, exit_fn=exit_fn) return False continue finish(time.time() + MAX_IDLE_SECONDS, diff --git a/api/usage_ledger.py b/api/usage_ledger.py index 15edbb026b4df0e778fc8ebf2b88017aad48dc30..4d42c06bc5bf7e6f06bddf11fbc908bf05a8be34 100644 --- a/api/usage_ledger.py +++ b/api/usage_ledger.py @@ -3,13 +3,16 @@ R9: *"ONE meter for every AI surface. The dashboard REPORTS weekly usage against an allowance and does NOT cut anyone off this wave; over the allowance it still works, shown red."* -⛔ THIS FILE MEASURES AND NEVER ENFORCES. Token allowances remain report-only. -Image-making limits live in `image_budget`, which reads measured image dollars and its own holds. +⛔ SO THIS FILE MEASURES AND NEVER ENFORCES. There is no ceiling here, no refusal, and no caller of +this module may branch on its answer. The one limit the product does enforce is per COLUMN and lives +where it was built (`ai_enrich.ceiling_report`); this is the tenant-wide REPORT, and mixing the two +would make an over-allowance week silently stop somebody's enrichment run. -**THE SURFACES.** `assistant` (a person asking the Assistant a question) · `field_agent` (an AI +**THE FOUR SURFACES.** `assistant` (a person asking the Assistant a question) · `field_agent` (an AI enrichment column filling cells) · `ai_review` (a model deciding a review stage) · `automation_draft` -(a sentence becoming a draft flow) · `image_gen` (image making, edits and supporting reads or briefs). -The gate in `verify_api.py` derives LLM entry points from the code rather than this list. +(a sentence becoming a draft flow). They are the four LLM entry points the product has, and the gate +in `verify_api.py` DERIVES them from the code rather than reading this list, so a fifth entry point +is caught by the gate and not by somebody remembering to edit a tuple. ⛔⛔ **`record()` MUST BE HANDED ITS TARGET, AND THAT IS A MEASUREMENT, NOT A STYLE CHOICE.** The obvious design is to bind `(runtime, username)` into a `contextvars.ContextVar` inside @@ -29,17 +32,17 @@ that hides the gap is worse than one that admits it. """ from __future__ import annotations -from decimal import Decimal, InvalidOperation, ROUND_HALF_UP import os import time -#: Weekly counters and image-settlement retry receipts contain no prompts or generated content. +#: The store bucket. Small by construction — counters only, never a per-call log: a log would grow +#: without bound on a route that runs per row, and nothing in R9 asks a question a log answers. STORE_KEY = "ai_usage" -#: The AI surfaces (C7). ⚠ A surface this list does not name is REFUSED rather than stored under +#: The four AI surfaces (C7). ⚠ A surface this list does not name is REFUSED rather than stored under #: whatever string arrived, because a typo'd surface is a meter that reads low forever with a second #: row nobody looks at. -SURFACES = ("assistant", "field_agent", "ai_review", "automation_draft", "image_gen", "pdp_text") +SURFACES = ("assistant", "field_agent", "ai_review", "automation_draft") #: The weekly allowance the dashboard measures against (R9). ⛔ IT IS A REPORTING LINE, NOT A GATE. #: Env-tunable so a deployment can set a real number without a release. @@ -129,39 +132,26 @@ def _int_or_none(value): return None -def usd_micros(value): - """Return nonnegative dollars as whole micro-dollars, rounded half up, or None if unknown.""" - if isinstance(value, bool) or not isinstance(value, (int, float, str, Decimal)): - return None - try: - amount = Decimal(str(value)) - if amount.is_finite() and amount >= 0: - return int((amount * 1_000_000).to_integral_value(rounding=ROUND_HALF_UP)) - except (InvalidOperation, ValueError, OverflowError): - return None - return None - - -def _book_dollars(row, micros, calls, entry_id): - """Book measured dollars or a separate unknown count, resolving a single-call receipt once.""" - receipts = row.setdefault("_cost_entries", {}) if entry_id else {} - repeated = bool(entry_id and entry_id in receipts) - if repeated and (receipts[entry_id] is not None or micros is None): - return False - if repeated: - row["usd_unmeasured"] = max(0, int(row.get("usd_unmeasured") or 0) - 1) - if micros is None: - row["usd_unmeasured"] = int(row.get("usd_unmeasured") or 0) + calls - else: - row["usd_micros"] = int(row.get("usd_micros") or 0) + micros - if entry_id: - receipts[entry_id] = micros - return not repeated - - def record(surface, provider, model, tokens_in=None, tokens_out=None, *, - total=None, calls=1, cost_usd=None, st=None, user="", week=None, entry_id=""): - """Report a call, with optional held week and single-call retry id, without enforcing a limit.""" + total=None, calls=1, st=None, user=""): + """Book ONE AI call. Returns True when the line landed durably, False when it could not. + + C7's signature is the five positional parameters; everything after `*` is this module's own + extension and each one has a reason: + + · `total` — for a caller that knows the total and not the split (`ai_enrich.run_field` + accumulates a per-run total through an `_ask` contract a gate injects against, and changing + that contract to carry a split would break the injected shape for no gain here). + · `calls` — so a RUN over many rows books one line with its real call count instead of one + store write per row. + · `st` / `user` — the target. Required in practice; see this module's header for the measured + reason a contextvar cannot supply them. + + ⛔ IT NEVER RAISES. A meter that can break the feature it measures is worse than no meter: a + store blip during an enrichment run would abort the run and lose the cells. + ⛔ AND IT NEVER SILENTLY SUCCEEDS. A call it cannot attribute increments `UNATTRIBUTED`, which + `GET /usage` shows an admin. + """ key = str(surface or "").strip() if key not in SURFACES: # Refused rather than stored: a surface nobody named is a row nobody reads, and the meter @@ -172,16 +162,15 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *, if tot is None and (ins is not None or outs is not None): tot = (ins or 0) + (outs or 0) n = max(0, int(calls) if isinstance(calls, int) and not isinstance(calls, bool) else 1) - if not n or (entry_id and n != 1): + if not n: return False uname = str(user or "").strip().lower() if st is None: UNATTRIBUTED["calls"] += n UNATTRIBUTED["surfaces"][key] = UNATTRIBUTED["surfaces"].get(key, 0) + n return False - week = week or week_of() + week = week_of() prov, mdl = str(provider or "")[:40], str(model or "")[:80] - micros = usd_micros(cost_usd) def _up(data): data = data if isinstance(data, dict) else {} @@ -189,16 +178,18 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *, row = ((weeks.setdefault(week, {}) .setdefault(uname or "-", {})) .setdefault(key, {})) - if _book_dollars(row, micros, n, entry_id): - row["calls"] = int(row.get("calls") or 0) + n - if tot is None: - row["unmeasured"] = int(row.get("unmeasured") or 0) + n - else: - row["tokens"] = int(row.get("tokens") or 0) + tot - if ins is not None: - row["tokens_in"] = int(row.get("tokens_in") or 0) + ins - if outs is not None: - row["tokens_out"] = int(row.get("tokens_out") or 0) + outs + row["calls"] = int(row.get("calls") or 0) + n + if tot is None: + # ⛔ "the call happened, the tokens are unknown" IS A STATE, and it has to be + # representable or the meter quietly under-reports. The dashboard shows it as a count of + # unmeasured calls beside the token total. + row["unmeasured"] = int(row.get("unmeasured") or 0) + n + else: + row["tokens"] = int(row.get("tokens") or 0) + tot + if ins is not None: + row["tokens_in"] = int(row.get("tokens_in") or 0) + ins + if outs is not None: + row["tokens_out"] = int(row.get("tokens_out") or 0) + outs if prov: row["provider"] = prov if mdl: @@ -210,8 +201,12 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *, return weeks try: - # Image dollars must persist before their budget holds can disappear. - st.update(STORE_KEY, _up, flush="sync" if key == "image_gen" else "async") + # ⚠ `flush='async'` — the coalescing mode ([[store-async-flush]]). This is the hot path of + # every AI surface in the product (an enrichment run books one line per RUN, but the + # assistant books one per question), and a blocking hub upload per call would put a network + # round trip inside the answer a person is waiting for. The mutation applies to the + # in-process cache immediately, so `GET /usage` reads its own writes. + st.update(STORE_KEY, _up, flush="async") return True except Exception: # noqa: BLE001 UNATTRIBUTED["calls"] += n @@ -220,8 +215,7 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *, def _blank_row(): - return {"calls": 0, "tokens": 0, "tokens_in": 0, "tokens_out": 0, "unmeasured": 0, - "usd_micros": 0, "usd_unmeasured": 0} + return {"calls": 0, "tokens": 0, "tokens_in": 0, "tokens_out": 0, "unmeasured": 0} def _fold(rows): @@ -229,7 +223,7 @@ def _fold(rows): for row in rows: if not isinstance(row, dict): continue - for field in out: + for field in ("calls", "tokens", "tokens_in", "tokens_out", "unmeasured"): got = row.get(field) if isinstance(got, int) and not isinstance(got, bool): out[field] += max(0, got) diff --git a/platform/core/amazon_projects.py b/platform/core/amazon_projects.py index ac296c76ca935eac1d25e60a4a7e2f6b27ea3673..3fddd9b8abcff538be82cb16772ae3f129a7eaac 100644 --- a/platform/core/amazon_projects.py +++ b/platform/core/amazon_projects.py @@ -26,7 +26,6 @@ STATUSES = ('Backlog', 'Planned', 'In progress', 'Completed', 'Canceled') COMPLETED = 'Completed' CANCELED = 'Canceled' PRIORITIES = ('No priority', 'Urgent', 'High', 'Medium', 'Low') -TYPES = ('project', 'pdp') #: Health turns At risk this many days before the target date. AT_RISK_DAYS = 7 CHUNK = 200 @@ -244,8 +243,6 @@ def _record(values, columns=PROJECT_COLUMNS): value = values[len(columns) + 3] if len(values) > len(columns) + 3 else None members = json.loads(value) if isinstance(value, str) else value result['contributors'] = sorted(members or [], key=lambda member: member['username']) - result['type'] = values[len(columns) + 4] if len(values) > len(columns) + 4 else 'project' - result['listed'] = bool(values[len(columns) + 5]) if len(values) > len(columns) + 5 else True return result @@ -751,53 +748,11 @@ def _activity_target_preview(value, truncated): return f'{shown}. Target list incomplete' if shown else 'Target list incomplete' -def _pdp_item_stages(value, pdp): - """Accept complete typed workflow item identities before comparing historical stages.""" - if not isinstance(value, dict) or value.get('type') != 'pdp' or not isinstance(value.get('items'), list): - return None - stages = {} - for item in value['items']: - if (not isinstance(item, dict) or not isinstance(item.get('asin'), str) - or not isinstance(item.get('stage'), str) or item['stage'] not in pdp.STAGE_PROGRESS - or item['asin'] in stages): - return None - stages[item['asin']] = item['stage'] - return stages - - -def _pdp_activity_facts(event, field, before, after): - """Project only C4 action facts retained in the bounded stored event.""" - from core import amazon_pdp as pdp - if event.get('via') != pdp.VIA: - return None - unavailable = {'action': 'unavailable'} - if event.get('changes_truncated'): - return unavailable - old, new = (_pdp_item_stages(value, pdp) for value in (before, after)) - if field == 'setup' and before is None and new is not None: - scope = after.get('scope') - scope = scope if isinstance(scope, list) and scope and all(key in pdp.SCOPE_PARTS for key in scope) else None - return {'action': 'created', 'scope': scope} - if old is not None and new is not None: - changed = {stage for asin, stage in new.items() if asin in old and old[asin] != stage} - if len(changed) == 1: - stage = next(iter(changed)) - if field == ('publish' if stage == 'published' else stage): - return {'action': 'moved', 'stage': stage} if stage != 'published' else {'action': 'published'} - return {'action': 'setup'} if field in ('setup', 'competitors') and not changed else unavailable - if isinstance(after, dict) and isinstance(after.get('slot'), str) and after['slot'] in pdp.SLOTS: - part = pdp.SLOTS[after['slot']]['key'] - if after.get('part') == part and field in ('draft', 'approve', 'request'): - return {'action': {'draft': 'saved', 'approve': 'approved', 'request': 'requested'}[field]} - return {'action': 'ai'} if field == 'ai' else unavailable - - def _activity_entries(event, actor_names, task_names): """Project one log event into named activity entries.""" out = [] for index, (field, change) in enumerate((event.get('changes') or {}).items()): before, after = change.get('before'), change.get('after') - pdp_facts = _pdp_activity_facts(event, field, before, after) if field == 'project': action, field = ('created' if before is None else 'deleted'), None elif field == 'targets': @@ -820,8 +775,6 @@ def _activity_entries(event, actor_names, task_names): 'by': event['by'], 'byUsername': event['by'], 'byName': actor_names.get(event['by'], event['by']), 'action': action, 'field': field, 'from': shown[0], 'to': shown[1]}) - if pdp_facts is not None: - out[-1]['pdp'] = pdp_facts return out @@ -1142,8 +1095,6 @@ class PgProjectStore: notes TEXT, completed_on DATE, source_id TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, created_by TEXT, rollup JSONB, impact JSONB, - type TEXT NOT NULL DEFAULT 'project' CHECK (type IN ({_in_sql(TYPES)})), - listed BOOLEAN NOT NULL DEFAULT TRUE, PRIMARY KEY (tenant, id), UNIQUE (tenant, source_id), CHECK ((status = '{COMPLETED}' AND completed_on IS NOT NULL) OR (status <> '{COMPLETED}' AND completed_on IS NULL)))''') @@ -1161,11 +1112,8 @@ class PgProjectStore: # One row's lookup is one indexed read on this index. con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_project_targets_entity" ' f'ON {self.t(TARGETS)} (tenant, grain, key, project_id)') - additions = {'rollup': 'JSONB', 'impact': 'JSONB', - 'type': f"TEXT NOT NULL DEFAULT 'project' CHECK (type IN ({_in_sql(TYPES)}))", - 'listed': 'BOOLEAN NOT NULL DEFAULT TRUE'} - for field, declaration in additions.items(): - con.execute(f'ALTER TABLE {self.t(PROJECTS)} ADD COLUMN IF NOT EXISTS {field} {declaration}') + for field in ('rollup', 'impact'): + con.execute(f'ALTER TABLE {self.t(PROJECTS)} ADD COLUMN IF NOT EXISTS {field} JSONB') @contextmanager def _write(self): @@ -1178,20 +1126,13 @@ class PgProjectStore: cols = ', '.join(f'p.{c}' for c in PROJECT_COLUMNS) count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c ' 'WHERE c.tenant = p.tenant AND c.project_id = p.id)') - return (f'SELECT {prefix}{cols}, {count}, {self._rollup_projection()}, {members}, ' - f'{self._type_projection()}, {self._listed_projection()} ' + return (f'SELECT {prefix}{cols}, {count}, {self._rollup_projection()}, {members} ' f'FROM {self.t(PROJECTS)} AS p WHERE p.tenant = %s{where}') def _rollup_projection(self): # Old schemas read blank figures until a guarded write adds the columns. return "to_jsonb(p)->'rollup', to_jsonb(p)->'impact'" - def _type_projection(self): - return "COALESCE(to_jsonb(p)->>'type', 'project')" - - def _listed_projection(self): - return "COALESCE((to_jsonb(p)->>'listed')::boolean, TRUE)" - def _members_projection(self, con): if con.execute('SELECT to_regclass(%s)', (self.t(MEMBERS),)).fetchone()[0] is None: return 'NULL' @@ -1237,19 +1178,15 @@ class PgProjectStore: 'WHERE tenant = %s AND id = %s', (stamp, json.dumps(impact), self.tenant_slug, project_id)) - def _insert(self, con, record, pairs, stamp): - columns = (*PROJECT_COLUMNS, 'impact', 'type', 'listed') - values = (*[record[c] for c in PROJECT_COLUMNS], - json.dumps(_with_result_configs(None, record.get('result_configs', []))), - record.get('type', 'project'), record.get('listed', True)) - con.execute(f'INSERT INTO {self.t(PROJECTS)} ({", ".join(columns)}) ' - f'VALUES ({_marks(len(PROJECT_COLUMNS))}, %s::jsonb, %s, %s)', values) - self._insert_targets(con, record['id'], pairs, stamp) - self._replace_members(con, record['id'], record.get('contributors', []), [], stamp) - def insert(self, record, pairs, stamp): with self._write() as con: - self._insert(con, record, pairs, stamp) + columns = (*PROJECT_COLUMNS, 'impact') + values = (*[record[c] for c in PROJECT_COLUMNS], + json.dumps(_with_result_configs(None, record.get('result_configs', [])))) + con.execute(f'INSERT INTO {self.t(PROJECTS)} ({", ".join(columns)}) ' + f'VALUES ({_marks(len(columns) - 1)}, %s::jsonb)', values) + self._insert_targets(con, record['id'], pairs, stamp) + self._replace_members(con, record['id'], record['contributors'], [], stamp) return self._one(con, record['id']) def get(self, project_id): @@ -1261,8 +1198,6 @@ class PgProjectStore: def update(self, project_id, changes, stamp): with self._write() as con: current = self._one(con, project_id, lock=True) - if current['type'] == 'pdp' and set(changes) & {'progress', 'status', 'completed_on'}: - raise ValueError('PDP item stages control project progress and completion.') candidate = _fields(changes, previous=current) if 'contributors' in changes: candidate['contributors'] = _member_values(changes['contributors'], self.tenant_slug) @@ -1278,25 +1213,23 @@ class PgProjectStore: self._replace_members(con, project_id, candidate['contributors'], current['contributors'], stamp) return self._one(con, project_id) - def _delete(self, con, project_id): - for table in (MEMBERS, TARGETS): - con.execute(f'DELETE FROM {self.t(table)} WHERE tenant = %s AND project_id = %s', - (self.tenant_slug, project_id)) - row = con.execute(f'DELETE FROM {self.t(PROJECTS)} WHERE tenant = %s AND id = %s RETURNING id', - (self.tenant_slug, project_id)).fetchone() - if row is None: - raise KeyError('Project not found.') - def delete(self, project_id): with self._write() as con: - self._delete(con, project_id) + con.execute(f'DELETE FROM {self.t(MEMBERS)} WHERE tenant = %s AND project_id = %s', + (self.tenant_slug, project_id)) + con.execute(f'DELETE FROM {self.t(TARGETS)} WHERE tenant = %s AND project_id = %s', + (self.tenant_slug, project_id)) + row = con.execute(f'DELETE FROM {self.t(PROJECTS)} WHERE tenant = %s AND id = %s RETURNING id', + (self.tenant_slug, project_id)).fetchone() + if row is None: + raise KeyError('Project not found.') return {'deleted': project_id} def list_all(self): with self._connection() as con: if not self._exists(con): return [] - rows = con.execute(self._select_sql(f' AND {self._listed_projection()} ORDER BY p.created_at, p.id', + rows = con.execute(self._select_sql(' ORDER BY p.created_at, p.id', members=self._members_projection(con)), (self.tenant_slug,)).fetchall() return [_record(row) for row in rows] @@ -1320,12 +1253,10 @@ class PgProjectStore: count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c ' 'WHERE c.tenant = p.tenant AND c.project_id = p.id)') rows = con.execute( - f'SELECT t.grain, t.key, {cols}, {count}, {self._rollup_projection()}, {members}, ' - f'{self._type_projection()}, {self._listed_projection()} ' + f'SELECT t.grain, t.key, {cols}, {count}, {self._rollup_projection()}, {members} ' f'FROM {self.t(TARGETS)} AS t ' f'JOIN {self.t(PROJECTS)} AS p ON p.tenant = t.tenant AND p.id = t.project_id ' - f'WHERE t.tenant = %s AND {self._listed_projection()} AND ({match}) ' - 'ORDER BY p.created_at, p.id', + f'WHERE t.tenant = %s AND ({match}) ORDER BY p.created_at, p.id', (self.tenant_slug, *(v for pair in part for v in pair))).fetchall() for row in rows: out[(row[0], row[1])].append(_record(row[2:])) diff --git a/platform/core/amazon_rivals.py b/platform/core/amazon_rivals.py index a42821c5f1c0d7d29b3c76f1f3ff2ec2a92783b6..b426ba1486fd647d318a1f0826428465fd75d08e 100644 --- a/platform/core/amazon_rivals.py +++ b/platform/core/amazon_rivals.py @@ -1,11 +1,10 @@ -"""Per-tenant PostgreSQL money-search rival pools and daily quotes. +"""Per-tenant PostgreSQL rivals of our ASINs, found from Brand Analytics, plus their daily quotes. Discovery reads the Amazon mirror through a cursor the caller opens: core imports no harness module (verify_layering) and no App module (rule 13). Tables are created on the first write. """ from contextlib import contextmanager -from dataclasses import dataclass from datetime import date, datetime, timedelta, timezone import json import re @@ -20,6 +19,8 @@ RUNS = 'amazon_rival_runs' WANTED = 'amazon_rival_wanted' TABLES = (RIVALS, DISCOVERY, QUOTES, RUNS, WANTED) +#: R23: at most this many auto-found rivals per ASIN, by click share. +AUTO_LIMIT = 10 #: Brand Analytics keeps the top 3 clicked ASINs of each search term. TOP_CLICKED = 3 SOURCES = frozenset(('auto', 'added')) @@ -28,35 +29,16 @@ CHUNK = 200 BA_TABLE = 'amzn_brand_analytics' LISTINGS_TABLE = 'amzn_listings' PRICING_TABLE = 'amzn_pricing' -SQP_TABLE = 'amzn_search_query_performance' -RANKS_TABLE = 'amzn_apify_rank_observations' -SNAPSHOTS_TABLE = 'amzn_competitor_snapshots' -IMAGES_TABLE = 'amzn_competitor_images' -POOL_SNAPSHOT_FIELDS = ('snapshot_date', 'brand', 'title', 'bullets', 'price', 'currency', 'prime', 'offers') -POOL_IMAGE_FIELDS = ('variant', 'link', 'width', 'height') NO_TOP3 = "Not in any search term's top 3 clicked" NO_WEEK = 'Brand Analytics has no dated weekly reports' -NO_MONTH = 'No completed monthly Search Query Performance report for this product yet.' -NO_MONEY = 'No searches with market money for this product yet.' -NO_KEYWORD_RANKS = "No keyword ranks for this product's searches yet." -NO_PRODUCT_TYPE = 'Product type not read yet. The next daily run reads it.' -WAITING_PRODUCT_TYPE = '{count} found, waiting for the daily read of their product type.' NOT_OURS = 'Not an ASIN in your Amazon listings' PARENT_ROLLS_UP = 'A Parent ASIN has no rival list. Its fields roll up its child ASINs.' -NO_POOL = 'No search report or keyword ranks for this product yet.' ASIN_RE = re.compile(r'[A-Z0-9]{10}') -POOL_FIELDS = { - RIVALS: {'origin': ('TEXT', "'click'"), 'money_won': ('DOUBLE PRECISION', '0'), - 'terms_won': ('INTEGER', '0'), 'term_rank': ('INTEGER', 'NULL'), - 'eligible': ('BOOLEAN', 'TRUE')}, - DISCOVERY: {'month': ('DATE', 'NULL'), 'keyword_ranks_on': ('DATE', 'NULL'), - 'product_type': ('TEXT', 'NULL'), 'held': ('INTEGER', '0')}, -} RIVAL_COLUMNS = ('asin', 'rival_asin', 'source', 'removed', 'click_share', 'term', 'title', 'week', - 'added_by', 'updated_at', *POOL_FIELDS[RIVALS]) -DISCOVERY_COLUMNS = ('asin', 'week', 'discovered_at', 'cause', 'auto_count', *POOL_FIELDS[DISCOVERY]) + 'added_by', 'updated_at') +DISCOVERY_COLUMNS = ('asin', 'week', 'discovered_at', 'cause', 'auto_count') QUOTE_COLUMNS = ('asin', 'quote_on', 'offer_price', 'buy_box_price', 'sales_rank', 'display_group', 'offer_cause', 'buy_box_cause', 'rank_cause', 'offers', 'currency', 'run_id', 'quoted_at') RUN_COLUMNS = ('id', 'started_at', 'progress', 'progress_at', 'finished_at', 'asin_total', 'priced', @@ -193,355 +175,96 @@ def children(cur, parent): return [r[0] for r in rows if r[0]] -def _money_sql(cur): - """Newest completed monthly searches per ASIN, including zero-money searches for freshness.""" - required = {'asin', 'search_query_data_search_query', 'start_date', 'end_date', - 'purchase_data_total_purchase_count', 'purchase_data_total_median_purchase_price_amount'} - if not required <= _columns(cur, SQP_TABLE): - return ('SELECT NULL::VARCHAR AS asin, NULL::VARCHAR AS term, NULL::DATE AS month, ' - 'NULL::DOUBLE AS market_money WHERE FALSE') - return f''' - WITH reports AS ( - SELECT upper(trim(CAST(asin AS VARCHAR))) AS asin, - lower(trim(CAST(search_query_data_search_query AS VARCHAR))) AS term, - TRY_CAST(start_date AS DATE) AS start_on, TRY_CAST(end_date AS DATE) AS end_on, - TRY_CAST(purchase_data_total_purchase_count AS DOUBLE) * - TRY_CAST(purchase_data_total_median_purchase_price_amount AS DOUBLE) AS market_money - FROM {SQP_TABLE} - WHERE date_diff('day', TRY_CAST(start_date AS DATE), TRY_CAST(end_date AS DATE)) > 7 - AND TRY_CAST(end_date AS DATE) < CURRENT_DATE), - latest AS ( - SELECT * FROM reports QUALIFY end_on = max(end_on) OVER (PARTITION BY asin)) - SELECT asin, term, CAST(date_trunc('month', start_on) AS DATE) AS month, - max(market_money) AS market_money - FROM latest GROUP BY asin, term, month''' - - -def _click_sql(cur): - """Normalized dated top-three clicks, with percentage shares kept in their source units.""" +def _candidates(cur, targets, week, per_asin): + """Ranked co-clicked ASINs per target, bounded in SQL to `per_asin` rows each.""" cols = _columns(cur, BA_TABLE) - if not {'search_term', 'clicked_asin', 'click_share_rank', 'click_share', - '_period_start', '_period_end'} <= cols: - return ('SELECT NULL::VARCHAR AS term, NULL::VARCHAR AS rival, NULL::DOUBLE AS share, ' - 'NULL::VARCHAR AS title, NULL::INTEGER AS term_rank, NULL::DATE AS week WHERE FALSE') - title = 'CAST(clicked_item_name AS VARCHAR)' if 'clicked_item_name' in cols else 'NULL::VARCHAR' - return f'''SELECT lower(trim(CAST(search_term AS VARCHAR))) AS term, - upper(trim(CAST(clicked_asin AS VARCHAR))) AS rival, - TRY_CAST(click_share AS DOUBLE) AS share, {title} AS title, - TRY_CAST(click_share_rank AS INTEGER) AS term_rank, {_week_sql()} AS week - FROM {BA_TABLE} WHERE {weekly_period_sql()} - AND TRY_CAST(click_share_rank AS INTEGER) BETWEEN 1 AND {TOP_CLICKED}''' - - -def _rank_rows_sql(cur): - """Raw Keyword ranks observations include rivals absent from the own-placement view.""" - if not {'keyword', 'asin', 'date', 'organic_rank', 'is_sponsored'} <= _columns(cur, RANKS_TABLE): - return ('SELECT NULL::VARCHAR AS term, NULL::VARCHAR AS rival, NULL::DATE AS observed_on, ' - 'NULL::INTEGER AS term_rank, NULL::BOOLEAN AS sponsored WHERE FALSE') - return f'''SELECT lower(trim(CAST(keyword AS VARCHAR))) AS term, - upper(trim(CAST(asin AS VARCHAR))) AS rival, TRY_CAST(date AS DATE) AS observed_on, - TRY_CAST(organic_rank AS INTEGER) AS term_rank, - TRY_CAST(is_sponsored AS BOOLEAN) AS sponsored FROM {RANKS_TABLE}''' - - -def _search_dates(cur, targets): - rows = _read(cur, f''' - WITH money AS ({_money_sql(cur)}), observations AS ({_rank_rows_sql(cur)}) - SELECT m.asin, max(m.month), max(o.observed_on), count(DISTINCT m.term) - FILTER (WHERE m.market_money > 0) - FROM money m LEFT JOIN observations o ON o.term = m.term AND m.market_money > 0 - WHERE m.asin IN {_in(targets)} GROUP BY m.asin''', targets) - return {a: dict(month=_day(m), keyword_ranks_on=_day(k), money_terms=n) for a, m, k, n in rows} - - -def _newest_snapshots_sql(cur, fields=('product_type', 'brand')): - if not {'asin', 'snapshot_date', 'product_type', 'brand'} <= _columns(cur, SNAPSHOTS_TABLE): - return 'SELECT NULL::VARCHAR AS asin, ' + ', '.join(f'NULL::VARCHAR AS {f}' for f in fields) + ' WHERE FALSE' - return (f'SELECT upper(trim(asin)) AS asin, {", ".join(fields)} FROM {SNAPSHOTS_TABLE} ' - 'QUALIFY row_number() OVER (PARTITION BY upper(trim(asin)) ORDER BY snapshot_date DESC) = 1') - - -def _brand(value): - return ''.join((value or '').casefold().split()) - - -def _same_type(target_type, rival_type): - return bool(target_type and rival_type and target_type == rival_type) - - -def _other_brand(brand, own_brands): - return not _brand(brand) or _brand(brand) not in own_brands - - -@dataclass -class PoolFacts: - """Newest ASIN facts and all historical own brands determine pool eligibility.""" - - latest: dict - own_brands: set - own_asins: set - - def product_type(self, target): - return self.latest.get(target, {}).get('product_type') - - def eligible(self, target, rival): - fact = self.latest.get(rival, {}) - return (rival not in self.own_asins and _same_type(self.product_type(target), fact.get('product_type')) - and _other_brand(fact.get('brand'), self.own_brands)) - - -def _pool_facts(cur, values, own, prefetched=None, candidate_rows=()): - latest = {r[1]: dict(product_type=r[-2], brand=r[-1]) for r in candidate_rows} - for part in _chunks(sorted(values - latest.keys())): - rows = _read(cur, f'SELECT asin, product_type, brand FROM ({_newest_snapshots_sql(cur)}) ' - f'WHERE asin IN {_in(part)}', part) - latest.update({a: dict(product_type=t, brand=b) for a, t, b in rows}) - brands = set() - if {'asin', 'brand'} <= _columns(cur, SNAPSHOTS_TABLE): - rows = _read(cur, f'SELECT DISTINCT s.brand FROM {SNAPSHOTS_TABLE} s ' - f'JOIN {LISTINGS_TABLE} l ON upper(trim(s.asin)) = upper(trim(l.asin1))') - brands.update(_brand(row[0]) for row in rows if _brand(row[0])) - latest.update(prefetched or {}) - brands.update(_brand(latest[a].get('brand')) for a in own if a in latest and _brand(latest[a].get('brand'))) - return PoolFacts(latest, brands, own) - - -def _candidates(cur, targets, week): - """Every non-own clicked or organic page-one ASIN on each target's monthly money searches.""" + share = 'TRY_CAST("click_share" AS DOUBLE)' if 'click_share' in cols else 'CAST(NULL AS DOUBLE)' + title = 'CAST("clicked_item_name" AS VARCHAR)' if 'clicked_item_name' in cols else 'CAST(NULL AS VARCHAR)' sql = f''' - WITH money AS ({_money_sql(cur)}), clicks AS ({_click_sql(cur)}), - observations AS ({_rank_rows_sql(cur)}), - newest_observations AS ( - SELECT * FROM observations - QUALIFY observed_on = max(observed_on) OVER (PARTITION BY term)), - click AS ( - SELECT term, rival, max(share) AS share, min(term_rank) AS term_rank, max(title) AS title - FROM clicks WHERE week = CAST(? AS DATE) GROUP BY term, rival), - page1 AS ( - SELECT term, rival, min(term_rank) AS term_rank FROM newest_observations - WHERE sponsored = FALSE GROUP BY term, rival), - placements AS ( - SELECT coalesce(c.term, p.term) AS term, coalesce(c.rival, p.rival) AS rival, - c.share, c.title, coalesce(c.term_rank, p.term_rank) AS term_rank, - c.term IS NOT NULL AS clicked, p.term IS NOT NULL AS page1 - FROM click c FULL OUTER JOIN page1 p ON p.term = c.term AND p.rival = c.rival), + WITH ours AS ( + SELECT DISTINCT upper(trim(CAST("asin1" AS VARCHAR))) AS asin FROM {LISTINGS_TABLE} + WHERE "asin1" IS NOT NULL), + week AS ( + SELECT "search_term" AS term, upper(trim(CAST("clicked_asin" AS VARCHAR))) AS clicked, + {share} AS share, {title} AS title + FROM {BA_TABLE} + WHERE TRY_CAST("click_share_rank" AS INTEGER) BETWEEN 1 AND {TOP_CLICKED} + AND ({weekly_period_sql()}) AND {_week_sql()} = CAST(? AS DATE)), hits AS ( - SELECT m.asin, p.*, m.market_money * coalesce(p.share, 0) / 100 AS money_won - FROM money m JOIN placements p ON p.term = m.term - WHERE m.asin IN {_in(targets)} AND m.market_money > 0 AND p.rival <> '' - AND p.rival NOT IN (SELECT upper(trim(CAST(asin1 AS VARCHAR))) - FROM {LISTINGS_TABLE} WHERE asin1 IS NOT NULL)), - totals AS ( - SELECT asin, rival, max(share) AS share, sum(money_won) AS money_won, - count(*) FILTER (WHERE clicked AND term_rank = 1) AS terms_won, - CASE WHEN bool_or(clicked) AND bool_or(page1) THEN 'both' - WHEN bool_or(clicked) THEN 'click' ELSE 'page1' END AS origin - FROM hits GROUP BY asin, rival), - representative AS ( - SELECT asin, rival, term, title, term_rank FROM hits - QUALIFY row_number() OVER (PARTITION BY asin, rival - ORDER BY money_won DESC, term_rank, term) = 1) - SELECT t.asin, t.rival, t.share, r.term, r.title, t.origin, t.money_won, t.terms_won, r.term_rank, - s.product_type, s.brand - FROM totals t JOIN representative r USING (asin, rival) - LEFT JOIN ({_newest_snapshots_sql(cur)}) s ON s.asin = t.rival - ORDER BY t.asin, t.money_won DESC, t.rival''' - return _read(cur, sql, (week, *targets)) + SELECT DISTINCT term, clicked AS asin FROM week WHERE clicked IN {_in(targets)}), + cands AS ( + SELECT h.asin, w.clicked AS rival, w.share, w.term, w.title + FROM hits h JOIN week w ON w.term = h.term + WHERE w.clicked <> '' AND w.clicked NOT IN (SELECT asin FROM ours)), + best AS ( + SELECT asin, rival, max(share) AS share, + arg_max(term, coalesce(share, -1)) AS term, arg_max(title, coalesce(share, -1)) AS title + FROM cands GROUP BY asin, rival), + ranked AS ( + SELECT *, row_number() OVER (PARTITION BY asin ORDER BY share DESC NULLS LAST, rival) AS rn + FROM best) + SELECT asin, rival, share, term, title FROM ranked WHERE rn <= ? ORDER BY asin, rn''' + return _read(cur, sql, (week, *targets, per_asin)) # --------------------------------------------------------------------------- discovery -def discover(rt, cur, asins=None, *, candidate_rows=None, prefetched=None): - """Refresh uncapped money-search pools while preserving removed and hand-added rivals.""" +def discover(rt, cur, asins=None): + """Rewrite the auto rivals of our ASINs (or of `asins`) from the newest dated week. + + A removed rival stays removed and a hand-added one stays added. Returns counts and `rowsRead`.""" backend = _backend(rt) requested = None if asins is None else _asins(asins) targets = sorted(our_asins(cur, requested)) stamp = _now() discovery = {} for missing in sorted(set(requested or ()) - set(targets)): - discovery[missing] = dict(asin=missing, week=None, discovered_at=stamp, cause=NOT_OURS, - auto_count=0, month=None, keyword_ranks_on=None, product_type=None, held=0) + discovery[missing] = dict(asin=missing, week=None, discovered_at=stamp, cause=NOT_OURS, auto_count=0) week = newest_week(cur) if targets else None - dates = _search_dates(cur, targets) if targets else {} - rows = (_candidates(cur, targets, week) if candidate_rows is None else candidate_rows) if targets else [] - stored = backend.pool_rows(targets) - before = backend.rivals_of(targets) - values = set(targets) | {r[1] for r in rows} | {r['rival_asin'] for r in stored} - facts = _pool_facts(cur, values, our_asins(cur), prefetched, rows) - found, held = _eligible_candidates(rows, stored, facts, week, stamp) + found = {} + rows_read = 0 + if targets and week is not None: + kept = backend.kept(targets) + per_asin = AUTO_LIMIT + max((len(v) for v in kept.values()), default=0) + rows = _candidates(cur, targets, week, per_asin) + rows_read = len(rows) + for target, rival, share, term, title in rows: + if rival in kept.get(target, ()) or len(found.get(target, ())) >= AUTO_LIMIT: + continue + found.setdefault(target, []).append(dict( + asin=target, rival_asin=rival, source='auto', removed=False, + click_share=None if share is None else float(share), term=term, title=title, + week=week, added_by=None, updated_at=stamp)) for target in targets: - count = (len(found.get(target, ())) if facts.product_type(target) else - sum(r['source'] == 'auto' for r in stored if r['asin'] == target and not r['removed'])) - meta = dates.get(target) or dict(month=None, keyword_ranks_on=None, money_terms=0) - cause = _pool_cause(facts.product_type(target), held.get(target, 0), week, meta, count) - discovery[target] = dict(asin=target, week=week, discovered_at=stamp, cause=cause, auto_count=count, - month=meta['month'], keyword_ranks_on=meta['keyword_ranks_on'], - product_type=facts.product_type(target), held=held.get(target, 0)) - ready = [t for t in targets if facts.product_type(t)] - backend.replace_auto(ready, [row for t in ready for row in found.get(t, ())], list(discovery.values())) - backend.set_eligibility(stored, facts) - members = backend.rivals_of(targets) - pools = {target: len(members.get(target, ())) for target in targets} + count = len(found.get(target, ())) + cause = NO_WEEK if week is None else (None if count else NO_TOP3) + discovery[target] = dict(asin=target, week=week, discovered_at=stamp, cause=cause, auto_count=count) + backend.replace_auto(targets, [row for t in targets for row in found.get(t, ())], list(discovery.values())) return {'asins': len(targets), 'rivals': sum(len(v) for v in found.values()), 'week': week, - 'pools': pools, 'rowsRead': len(rows), 'held': held, - 'changed': tuple(t for t in targets if set(before.get(t, ())) != set(members.get(t, ()))), - 'causes': {k: v['cause'] for k, v in discovery.items() if v['cause']}} - - -def _eligible_candidates(rows, stored, facts, week, stamp): - kept = {(r['asin'], r['rival_asin']) for r in stored if r['removed'] or r['source'] == 'added'} - candidates = {(r[0], r[1]) for r in rows} | { - (r['asin'], r['rival_asin']) for r in stored if r['source'] == 'added' and not r['removed']} - removed = {(r['asin'], r['rival_asin']) for r in stored if r['removed']} - held = {} - for target, rival in candidates - removed: - if (rival not in facts.own_asins and not facts.product_type(rival) - and _other_brand(facts.latest.get(rival, {}).get('brand'), facts.own_brands)): - held[target] = held.get(target, 0) + 1 - found = {} - for target, rival, share, term, title, origin, money, terms, term_rank, _type, _name in rows: - if (target, rival) in kept or not facts.eligible(target, rival): - continue - found.setdefault(target, []).append(dict( - asin=target, rival_asin=rival, source='auto', removed=False, eligible=True, - click_share=None if share is None else float(share), term=term, title=title, - week=week, added_by=None, updated_at=stamp, origin=origin, - money_won=float(money), terms_won=int(terms), term_rank=term_rank)) - return found, held - - -def _discovery_cause(week, meta, count): - if meta['month'] is None: - return NO_MONTH - if not meta['money_terms']: - return NO_MONEY - if meta['keyword_ranks_on'] is None: - return NO_KEYWORD_RANKS - if week is None: - return NO_WEEK - return None if count else NO_TOP3 - - -def _pool_cause(product_type, held, week, meta, count): - if not product_type: - return NO_PRODUCT_TYPE - if held: - return WAITING_PRODUCT_TYPE.format(count=held) - return _discovery_cause(week, meta, count) + 'rowsRead': rows_read, 'causes': {k: v['cause'] for k, v in discovery.items() if v['cause']}} def stale(rt, cur, target): - """True when the ASIN's discovery predates a weekly, monthly or Keyword ranks source.""" + """True when this ASIN's discovery is missing or predates the newest dated week.""" row = _backend(rt).discovery([target]).get(target) if row is None: return True - types = _read(cur, f'SELECT product_type FROM ({_newest_snapshots_sql(cur)}) WHERE asin = ?', [target]) - product_type = types[0][0] if types else None - if row['product_type'] != product_type or (not product_type and row['cause'] != NO_PRODUCT_TYPE): - return True - newest = {'week': newest_week(cur), **(_search_dates(cur, [target]).get(target) or {})} - return any(newest.get(key) is not None and - (row.get(key) is None or row[key] < newest[key]) - for key in ('week', 'month', 'keyword_ranks_on')) + newest = newest_week(cur) + return newest is not None and (row['week'] is None or row['week'] < newest) # --------------------------------------------------------------------------- doors def rivals(rt, target): - """One ASIN's rivals: hand-added first, then money won and rank average, with discovery dates.""" + """One ASIN's rivals: hand-added first, then auto by click share, with its discovery row.""" target = asin(target) backend = _backend(rt) rows = backend.rivals(target) found = backend.discovery([target]).get(target) return {'asin': target, 'rivals': rows, 'removed': backend.removed_count(target), 'week': found['week'] if found else None, - 'month': found['month'] if found else None, - 'keywordRanksOn': found['keyword_ranks_on'] if found else None, 'discoveredAt': found['discovered_at'] if found else None, - 'cause': found['cause'] if found else None, - 'productType': found['product_type'] if found else None, - 'held': found['held'] if found else 0} - - -def pool(rt, cur, target): - """Read eligible pool rows through the caller's mirror cursor without writes or provider reads.""" - target = asin(target) - backend = _backend(rt) - if cur is None: - return {**rivals(rt, target), 'rivals': [], 'competitors': [], 'dataAsOf': None, 'cause': NO_POOL} - own = our_asins(cur) - week = newest_week(cur) - sources = _candidates(cur, [target], week) if target in own else [] - stored = backend.pool_rows([target]) - facts = _pool_facts(cur, {target} | {r[1] for r in sources} | {r['rival_asin'] for r in stored}, - own, candidate_rows=sources) - found, held = _eligible_candidates(sources, stored, facts, week, _now()) - if facts.product_type(target): - rows = found.get(target, []) + [dict(r, eligible=True) for r in stored - if r['source'] == 'added' and not r['removed'] - and facts.eligible(target, r['rival_asin'])] - else: - rows = [r for r in stored if r['eligible'] and not r['removed'] and r['rival_asin'] not in own - and _other_brand(facts.latest.get(r['rival_asin'], {}).get('brand'), facts.own_brands)] - if target not in own: - rows = [] - averages = backend.rank_averages([r['rival_asin'] for r in rows]) - rows = [{**r, **averages.get(r['rival_asin'], {'rank_average': None, 'rank_days': 0})} for r in rows] - meta = _search_dates(cur, [target]).get(target) or dict(month=None, keyword_ranks_on=None, money_terms=0) - cause = _pool_cause(facts.product_type(target), held.get(target, 0), week, meta, len(rows)) - rows = sorted(rows, key=_pool_order) - snapshots = _pool_snapshots(cur, [target, *(r['rival_asin'] for r in rows)]) - images = _pool_images(cur, [r['rival_asin'] for r in rows]) - dates = [s['snapshot_date'] for s in snapshots.values() if s['snapshot_date'] is not None] - return {**rivals(rt, target), 'rivals': rows, 'week': week, - 'month': meta['month'], 'keywordRanksOn': meta['keyword_ranks_on'], - 'productType': facts.product_type(target), 'held': held.get(target, 0), - 'cause': cause if target in own else NOT_OURS, 'dataAsOf': max(dates, default=None), - 'competitors': [_pool_competitor(r, snapshots.get(r['rival_asin'], {}), - images.get(r['rival_asin'], [])) for r in rows]} - - -def _pool_snapshots(cur, values): - snapshots = {} - for part in _chunks(sorted(set(values))): - rows = _read(cur, f'SELECT * FROM ({_newest_snapshots_sql(cur, POOL_SNAPSHOT_FIELDS)}) ' - f'WHERE asin IN {_in(part)}', part) - for target, *fields in rows: - snapshot = dict(zip(POOL_SNAPSHOT_FIELDS, fields)) - snapshot['snapshot_date'] = _day(snapshot['snapshot_date']) - snapshot['bullets'] = json.loads(snapshot['bullets'] or '[]') - snapshots[target] = snapshot - return snapshots - - -def _pool_images(cur, values): - """Resolve only the newest snapshot's variants against changed-link history.""" - images = {} - if not {'asin', *POOL_IMAGE_FIELDS} <= _columns(cur, IMAGES_TABLE): - return images - for part in _chunks(values): - rows = _read(cur, f''' - SELECT s.asin, i.variant, i.link, i.width, i.height - FROM ({_newest_snapshots_sql(cur, ('image_variants',))}) s, - json_each(s.image_variants) v, {IMAGES_TABLE} i - WHERE s.asin IN {_in(part)} AND i.asin = s.asin - AND i.variant = json_extract_string(v.value, '$.variant') - AND i.link = json_extract_string(v.value, '$.link') - ORDER BY s.asin, CAST(v.key AS INTEGER)''', part) - for target, *fields in rows: - images.setdefault(target, []).append(dict(zip(POOL_IMAGE_FIELDS, fields))) - return images - - -def _pool_competitor(row, snapshot, images): - days = row['rank_days'] - return {'asin': row['rival_asin'], 'brand': snapshot.get('brand'), 'title': snapshot.get('title'), - 'origin': row['origin'], 'moneyWon': row['money_won'], 'termsWon': row['terms_won'], - 'term': row['term'], 'termRank': row['term_rank'], 'bestSellerRank': row['rank_average'], - 'rankDays': days, 'rankCause': f'Rank average needs 7 days of ranks. {days} so far.' if days < 7 else None, - 'price': snapshot.get('price'), 'currency': snapshot.get('currency'), 'prime': snapshot.get('prime'), - 'offers': snapshot.get('offers'), 'imageCount': len(images), 'images': images, - 'bullets': snapshot.get('bullets', []), 'snapshotOn': snapshot.get('snapshot_date')} + 'cause': found['cause'] if found else None} def add(rt, target, rival, *, user, title=None): @@ -549,10 +272,9 @@ def add(rt, target, rival, *, user, title=None): target, rival = asin(target), asin(rival, 'rival') if rival == target: raise ValueError('An ASIN cannot be its own rival.') - return _backend(rt).add(dict(asin=target, rival_asin=rival, source='added', removed=False, eligible=False, + return _backend(rt).add(dict(asin=target, rival_asin=rival, source='added', removed=False, click_share=None, term=None, title=title, week=None, - added_by=str(user or '') or None, updated_at=_now(), - origin='added', money_won=0, terms_won=0, term_rank=None)) + added_by=str(user or '') or None, updated_at=_now())) def remove(rt, target, rival, *, user): @@ -616,19 +338,6 @@ def runs(rt, limit=10): return _backend(rt).runs(int(limit)) -def rank_average(rt, asins): - """Mean daily rank in each ASIN's newest display group, requiring seven of its newest 30 days.""" - values = _asins(asins) - found = _backend(rt).rank_averages(values) if values else {} - return {a: (found.get(a) or {}).get('rank_average') for a in values} - - -def _pool_order(row): - average = row.get('rank_average') - return (-float(row.get('money_won') or 0), - average is None, average if average is not None else 0, row['rival_asin']) - - # --------------------------------------------------------------------------- the daily quotes run #: The run row gets a progress marker after each block of this many ASINs. @@ -642,8 +351,7 @@ MAX_BACKOFF_SECONDS = 300.0 #: An unfinished run, or a finished run with a cause, is retried after this many hours. RETRY_RUN_HOURS = 3 -READS = {'offers': 'offers', 'summaries': 'Buy Box price', 'facts': 'catalog facts'} -FACT_COUNT_COLUMNS = {'title': 'title', 'product_type': 'product_type', 'image': 'image', 'rank': 'sales_rank'} +READS = {'offers': 'offers', 'summaries': 'Buy Box price', 'ranks': 'sales rank'} NO_MIRROR = 'The Amazon data is not on this worker yet, so no ASIN was quoted.' NO_OWN_OFFER = 'Amazon returned no offer of yours on this ASIN.' NO_BUY_BOX = 'Amazon returned no Buy Box offer for this ASIN.' @@ -741,7 +449,7 @@ def _summary_quote(response): def _keyed(kind, got): - if kind == 'facts': + if kind == 'ranks': return {str(k).upper(): v for k, v in (got or {}).items()} if kind == 'offers': return {str((((r or {}).get('body') or {}).get('payload') or {}).get('ASIN') or '').upper(): r @@ -781,16 +489,7 @@ def _read_batches(kind, fn, asins, *, sleep, log, counts): attempt = 0 while True: try: - got = _keyed(kind, fn(part)) - answers.update(got) - if kind == 'facts': - for target, fact in got.items(): - if fact.get('cause'): - causes[target] = fact['cause'] - if '(429)' in fact['cause']: - counts['throttled'] += 1 - elif 'Amazon refused' in fact['cause']: - counts['refused'] += 1 + answers.update(_keyed(kind, fn(part))) break except Exception as exc: # noqa: BLE001 if _throttled(exc) and attempt < THROTTLE_RETRIES: @@ -806,19 +505,14 @@ def _read_batches(kind, fn, asins, *, sleep, log, counts): return answers, causes -def _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts, - prefetched=None, fact_causes=None): +def _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts): """One quote row per ASIN of `block`, each with a value or a cause for every field it owns.""" mine = [a for a in block if a in own] others = [a for a in block if a not in own] offered, offer_causes = _read_batches('offers', readers['offers'], mine, sleep=sleep, log=log, counts=counts) summed, sum_causes = _read_batches('summaries', readers['summaries'], others, sleep=sleep, log=log, counts=counts) - cached = prefetched or {} - facts, rank_causes = _read_batches('facts', readers['facts'], [a for a in block if a not in cached], - sleep=sleep, log=log, counts=counts) - facts.update({a: cached[a] for a in block if a in cached}) - rank_causes.update(fact_causes or {}) + ranked, rank_causes = _read_batches('ranks', readers['ranks'], block, sleep=sleep, log=log, counts=counts) stamp = _now() rows = [] for target in block: @@ -837,31 +531,20 @@ def _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, c row.update(_summary_quote(summed[target])) else: row['buy_box_cause'] = NO_PRICE - rank = facts.get(target) or {} + rank = ranked.get(target) or {} value = rank.get('rank') if isinstance(rank.get('rank'), int) and not isinstance(rank.get('rank'), bool) else None row.update(sales_rank=value, display_group=rank.get('group') if value is not None else None, - rank_cause=None if value is not None else rank_causes.get(target) or NO_RANK) + rank_cause=rank_causes.get(target) or (None if value is not None else NO_RANK)) rows.append(row) - return rows, [_snapshot(row, facts.get(row['asin']) or {}, row['asin'] in own) for row in rows] - - -def _snapshot(quote, facts, own): - """C3 facts with price and rank taken from the exact persisted quote row.""" - return {**{key: facts.get(key) for key in ('product_type', 'brand', 'title', 'bullets', - 'image_variants', 'prime', 'offers', 'rank_cat', 'image')}, - 'asin': quote['asin'], 'snapshot_date': quote['quote_on'], - 'price': quote['offer_price' if own else 'buy_box_price'], 'currency': quote['currency'], - 'sales_rank': quote['sales_rank'], 'display_group': quote['display_group']} + return rows -def _tally(counts, rows, own, snapshots): +def _tally(counts, rows, own): for row in rows: counts['priced'] += int((row['offer_price'] if row['asin'] in own else None) is not None or row['buy_box_price'] is not None) counts['ranked'] += int(row['sales_rank'] is not None) counts['caused'] += int(bool(row['offer_cause'] or row['buy_box_cause'] or row['rank_cause'])) - for key, field in FACT_COUNT_COLUMNS.items(): - counts['facts'][key] += sum(row.get(field) is not None and row[field] != '' for row in snapshots) def _changed(before, row): @@ -869,47 +552,35 @@ def _changed(before, row): return old is None or any(old.get(k) != row.get(k) for k in QUOTED_FIELDS) -def _prepare_pools(rt, open_mirror, facts_reader, sleep, log, counts): - with open_mirror() as cur: - if cur is None: - return None - own = our_asins(cur) - sources = _candidates(cur, sorted(own), newest_week(cur)) if own else [] - stored = _backend(rt).pool_rows(sorted(own)) - values = own | {r[1] for r in sources} | {r['rival_asin'] for r in stored if not r['removed']} - known = _pool_facts(cur, values, own, candidate_rows=sources) - missing = sorted(a for a in values if not known.product_type(a)) - prefetched, causes = _read_batches('facts', facts_reader, missing, sleep=sleep, log=log, counts=counts) - prefetched = {a: prefetched.get(a, {}) for a in missing} - with open_mirror() as cur: - if cur is None: - return None - found = discover(rt, cur, candidate_rows=sources, prefetched=prefetched) - return own, found, prefetched, causes - - -def run_quotes(rt, open_mirror, *, offers, summaries, facts, seller_id=None, run_id=None, now=None, +def run_quotes(rt, open_mirror, *, offers, summaries, ranks, seller_id=None, run_id=None, now=None, log=None, sleep=time.sleep): - """Discover, read unknown types outside the mirror, filter pools, then quote with cached facts.""" + """Rediscover rivals, then quote our ASINs, every rival and every wanted ASIN for today. + + `open_mirror()` is a context manager yielding a mirror cursor or None. It closes before the first + SP-API read. `offers`, `summaries` and `ranks` each take a list of at most 20 ASINs and return + `item_offers` responses, `competitive_summaries` responses and `{asin: {rank, group}}`. A raised + read is a cause on each ASIN of its batch, and a 429 backs off first. Returns the run's counts + and `changed`, the ASINs whose quote differs from their previous newest quote.""" _backend(rt) log = log or (lambda _message: None) now = now or datetime.now(timezone.utc) quote_on = now.date().isoformat() run_id = str(run_id or f'{quote_on}-{uuid.uuid4().hex[:8]}') counts = dict(ours=0, rivals=0, wanted=0, asins=0, priced=0, ranked=0, caused=0, refused=0, throttled=0, - backoffs=0, discovered=0, week=None, pools={}, facts=dict.fromkeys(FACT_COUNT_COLUMNS, 0)) + backoffs=0, discovered=0, week=None) start_run(rt, run_id, asin_total=0) log(f'run {run_id} started') done = 0 try: try: - prepared = _prepare_pools(rt, open_mirror, facts, sleep, log, counts) - if prepared is None: - return _finish(rt, run_id, done, counts, NO_MIRROR, log) - own, found, prefetched, fact_causes = prepared + with open_mirror() as cur: + if cur is None: + return _finish(rt, run_id, done, counts, NO_MIRROR, log) + found = discover(rt, cur) + own = our_asins(cur) except MirrorUnreadable as exc: return _finish(rt, run_id, done, counts, f'The Amazon data could not be read ({exc}).', log) - counts.update(discovered=found['rivals'], week=found['week'], pools=found['pools']) + counts.update(discovered=found['rivals'], week=found['week']) others = sorted(set(rival_asins(rt)) - own) extra = sorted(set(wanted(rt)) - own - set(others)) targets = sorted(own) + others + extra @@ -918,23 +589,18 @@ def run_quotes(rt, open_mirror, *, offers, summaries, facts, seller_id=None, run log(f'run {run_id} quotes {len(targets)} ASINs: {len(own)} ours, {len(others)} rivals, ' f'{len(extra)} wanted') before = newest_quotes(rt, targets) if targets else {} - readers = {'offers': offers, 'summaries': summaries, 'facts': facts} - changed, snapshots = list(found['changed']), [] + readers = {'offers': offers, 'summaries': summaries, 'ranks': ranks} + changed = [] for block in _chunks(targets, PROGRESS_EVERY): - rows, captured = _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts, - prefetched, fact_causes) + rows = _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts) put_quotes(rt, rows) - snapshots.extend(captured) - _tally(counts, rows, own, captured) + _tally(counts, rows, own) changed += [row['asin'] for row in rows if _changed(before, row)] done += len(block) mark_run(rt, run_id, progress=done, counts=counts) log(f'run {run_id} progress {done}/{len(targets)}') - cause = (f'Amazon could not finish every read at its rate or refused a read. ' - f'Retry the run within {RETRY_RUN_HOURS} hours after the rate limit delay.' - if counts['throttled'] or counts['refused'] else None) - result = _finish(rt, run_id, done, counts, cause, log) - result.update(changed=tuple(dict.fromkeys(changed)), snapshots=snapshots) + result = _finish(rt, run_id, done, counts, None, log) + result['changed'] = tuple(changed) return result except Exception as exc: mark_run(rt, run_id, progress=done, finished=True, counts=counts, @@ -947,7 +613,7 @@ def _finish(rt, run_id, done, counts, cause, log): mark_run(rt, run_id, progress=done, finished=True, counts=counts, cause=cause) log(f'run {run_id} finished: {counts["asins"]} ASINs, {counts["priced"]} priced, {counts["ranked"]} ranked, ' f'{counts["caused"]} with a cause' + (f'. {cause}' if cause else '')) - return {'id': run_id, 'progress': done, 'cause': cause, 'counts': dict(counts), 'changed': (), 'snapshots': []} + return {'id': run_id, 'progress': done, 'cause': cause, 'counts': dict(counts), 'changed': ()} # --------------------------------------------------------------------------- competition status and rollups @@ -1217,70 +883,6 @@ def validate(rt, tables, plan): 'rows': len(parents), 'mismatches': bad['parent']}] -# --------------------------------------------------------------------------- pool validation - -def _pool_expected(cur, targets): - """Recompute raw click money independently of discovery's candidate and money queries.""" - required = {'asin', 'search_query_data_search_query', 'start_date', 'end_date', - 'purchase_data_total_purchase_count', 'purchase_data_total_median_purchase_price_amount'} - clicks = {'search_term', 'clicked_asin', 'click_share', 'click_share_rank', '_period_start', '_period_end'} - if not required <= _columns(cur, SQP_TABLE) or not clicks <= _columns(cur, BA_TABLE): - return {} - rows = _read(cur, f''' - WITH monthly AS ( - SELECT upper(trim(CAST(asin AS VARCHAR))) AS owner, - lower(trim(CAST(search_query_data_search_query AS VARCHAR))) AS query, - TRY_CAST(end_date AS DATE) AS ends, - TRY_CAST(purchase_data_total_purchase_count AS DOUBLE) * - TRY_CAST(purchase_data_total_median_purchase_price_amount AS DOUBLE) AS money - FROM {SQP_TABLE} - WHERE TRY_CAST(end_date AS DATE) > TRY_CAST(start_date AS DATE) + INTERVAL '7 days' - AND TRY_CAST(end_date AS DATE) < CURRENT_DATE), - newest_month AS (SELECT owner, max(ends) AS ends FROM monthly GROUP BY owner), - searches AS ( - SELECT m.owner, m.query, max(m.money) AS money FROM monthly m - JOIN newest_month n ON n.owner = m.owner AND n.ends = m.ends - GROUP BY m.owner, m.query HAVING max(m.money) > 0), - report AS ( - SELECT lower(trim(CAST(search_term AS VARCHAR))) AS query, - upper(trim(CAST(clicked_asin AS VARCHAR))) AS rival, - TRY_CAST(click_share AS DOUBLE) AS share, - TRY_CAST(click_share_rank AS INTEGER) AS position, {_week_sql()} AS starts - FROM {BA_TABLE} WHERE {weekly_period_sql()}), - term_clicks AS ( - SELECT query, rival, max(share) AS share, min(position) AS position FROM report - WHERE starts = (SELECT max(starts) FROM report) AND position BETWEEN 1 AND 3 - GROUP BY query, rival) - SELECT s.owner, c.rival, sum(s.money * coalesce(c.share, 0) * 0.01), - sum(CASE WHEN c.position = 1 THEN 1 ELSE 0 END) - FROM searches s JOIN term_clicks c ON c.query = s.query - WHERE s.owner IN {_in(targets)} GROUP BY s.owner, c.rival''', targets) - return {(a, r): (float(m), int(n)) for a, r, m, n in rows} - - -def validate_pool(cur, rt, targets): - """Tie stored auto-pool money and term wins to an independent raw-source aggregation.""" - targets = _asins(targets) - expected = _pool_expected(cur, targets) if targets else {} - bad_money, bad_terms, count = [], [], 0 - backend = _backend(rt) - for target in targets: - for row in backend.rivals(target): - if row['source'] != 'auto': - continue - count += 1 - money, terms = expected.get((target, row['rival_asin']), (0, 0)) - label = f'{target} {row["rival_asin"]}' - if _differs(row['money_won'], money, 0.000001): - bad_money.append(label) - if row['terms_won'] != terms: - bad_terms.append(label) - return [{'check': 'pool money won ties to Brand Analytics and search report rows', - 'ok': not bad_money, 'rows': count, 'mismatches': bad_money}, - {'check': 'pool terms won ties to Brand Analytics and search report rows', - 'ok': not bad_terms, 'rows': count, 'mismatches': bad_terms}] - - # --------------------------------------------------------------------------- storage def _ddl(table): @@ -1351,10 +953,6 @@ class PgRivalStore: """Create every rival table and index, only on a write.""" for name in TABLES: con.execute(_ddl(self.t(name))[name]) - for name, fields in POOL_FIELDS.items(): - for column, (kind, default) in fields.items(): - con.execute(f'ALTER TABLE {self.t(name)} ADD COLUMN IF NOT EXISTS {column} {kind} ' - f'DEFAULT {default}') con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_rivals_rival" ON {self.t(RIVALS)} (tenant, rival_asin)') @contextmanager @@ -1364,49 +962,29 @@ class PgRivalStore: self.ensure_schema(con) yield con - def _select(self, con, name, columns, where, args, order='', *, include_ineligible=False): + def _select(self, con, name, columns, where, args, order=''): if not self._exists(con, name): return [] - projection = columns - fields = POOL_FIELDS.get(name, {}) - if fields.keys() & set(columns) or name == RIVALS: - existing = {r[0] for r in con.execute( - 'SELECT column_name FROM information_schema.columns WHERE table_schema = %s AND table_name = %s', - (self.schema, name)).fetchall()} - projection = [f'{fields[c][1]} AS {c}' if c in fields and c not in existing else c for c in columns] - if name == RIVALS and 'eligible' in existing and not include_ineligible: - where = f'({where}) AND eligible = TRUE' if where else 'eligible = TRUE' - rows = con.execute(f'SELECT {", ".join(projection)} FROM {self.t(name)} WHERE tenant = %s' + rows = con.execute(f'SELECT {", ".join(columns)} FROM {self.t(name)} WHERE tenant = %s' f'{" AND " + where if where else ""}{order}', (self.tenant_slug, *args)).fetchall() return [dict(zip(columns, row)) for row in rows] @staticmethod def _rival(row): - return {**row, 'removed': bool(row['removed']), 'eligible': bool(row['eligible']), 'week': _day(row['week']), - 'origin': 'added' if row['source'] == 'added' else row['origin'], + return {**row, 'removed': bool(row['removed']), 'week': _day(row['week']), 'click_share': None if row['click_share'] is None else float(row['click_share'])} - def pool_rows(self, targets): - """Stored choices include hidden additions and removal tombstones.""" + def kept(self, targets): + """`{asin: {rival}}` of removed and hand-added rivals, which discovery must not touch.""" + out = {} with self._connection() as con: - return [self._rival(row) for part in _chunks(list(targets)) for row in self._select( - con, RIVALS, RIVAL_COLUMNS, f'asin IN ({_marks(len(part))})', part, - include_ineligible=True)] - - def set_eligibility(self, rows, facts): - with self._write() as con: - for row in rows: - if row['removed']: - continue - target, rival = row['asin'], row['rival_asin'] - allowed = (facts.eligible(target, rival) if facts.product_type(target) else - row['eligible'] and rival not in facts.own_asins - and _other_brand(facts.latest.get(rival, {}).get('brand'), facts.own_brands)) - if allowed != row['eligible']: - con.execute(f'UPDATE {self.t(RIVALS)} SET eligible = %s ' - 'WHERE tenant = %s AND asin = %s AND rival_asin = %s', - (allowed, self.tenant_slug, target, rival)) + for part in _chunks(targets): + for row in self._select(con, RIVALS, ('asin', 'rival_asin'), + f"asin IN ({_marks(len(part))}) AND (removed = %s OR source = 'added')", + (*part, True)): + out.setdefault(row['asin'], set()).add(row['rival_asin']) + return out def replace_auto(self, targets, rows, discovery): with self._write() as con: @@ -1425,9 +1003,7 @@ class PgRivalStore: + ', '.join(f'({_marks(len(cols))})' for _ in part) + ' ON CONFLICT (tenant, asin) DO UPDATE SET week = excluded.week, ' 'discovered_at = excluded.discovered_at, cause = excluded.cause, ' - 'auto_count = excluded.auto_count, month = excluded.month, ' - 'keyword_ranks_on = excluded.keyword_ranks_on, ' - 'product_type = excluded.product_type, held = excluded.held', + 'auto_count = excluded.auto_count', tuple(v for row in part for v in (self.tenant_slug, *(row[c] for c in DISCOVERY_COLUMNS)))) def discovery(self, targets): @@ -1435,41 +1011,14 @@ class PgRivalStore: rows = [] for part in _chunks(list(targets)): rows += self._select(con, DISCOVERY, DISCOVERY_COLUMNS, f'asin IN ({_marks(len(part))})', part) - return {row['asin']: {**row, **{key: _day(row[key]) for key in - ('week', 'month', 'keyword_ranks_on')}} for row in rows} + return {row['asin']: {**row, 'week': _day(row['week'])} for row in rows} def rivals(self, target): with self._connection() as con: - rows = self._select(con, RIVALS, RIVAL_COLUMNS, 'asin = %s AND removed = %s', (target, False)) - averages = self.rank_averages([row['rival_asin'] for row in rows]) - enriched = [{**self._rival(row), **averages.get(row['rival_asin'], - {'rank_average': None, 'rank_days': 0})} for row in rows] - return sorted(enriched, key=lambda row: (row['source'] != 'added', *_pool_order(row))) - - def rank_averages(self, values): - out = {} - with self._connection() as con: - if not self._exists(con, QUOTES): - return out - for part in _chunks(values): - rows = con.execute(f''' - WITH newest AS ( - SELECT asin, max(quote_on) AS newest FROM {self.t(QUOTES)} - WHERE tenant = %s AND asin IN ({_marks(len(part))}) GROUP BY asin), - groups AS ( - SELECT n.asin, n.newest, q.display_group FROM newest n - JOIN {self.t(QUOTES)} q ON q.asin = n.asin AND q.quote_on = n.newest - WHERE q.tenant = %s) - SELECT q.asin, CASE WHEN count(q.sales_rank) >= 7 THEN avg(q.sales_rank) END, - count(q.sales_rank) - FROM groups m JOIN {self.t(QUOTES)} q ON q.asin = m.asin - AND q.display_group = m.display_group AND q.quote_on >= m.newest - 29 - AND q.quote_on <= m.newest - WHERE q.tenant = %s GROUP BY q.asin''', - (self.tenant_slug, *part, self.tenant_slug, self.tenant_slug)).fetchall() - out.update({a: dict(rank_average=None if avg is None else float(avg), rank_days=n) - for a, avg, n in rows}) - return out + rows = self._select(con, RIVALS, RIVAL_COLUMNS, 'asin = %s AND removed = %s', (target, False), + " ORDER BY CASE WHEN source = 'added' THEN 0 ELSE 1 END, " + 'CASE WHEN click_share IS NULL THEN 1 ELSE 0 END, click_share DESC, rival_asin') + return [self._rival(row) for row in rows] def removed_count(self, target): with self._connection() as con: @@ -1483,15 +1032,11 @@ class PgRivalStore: with self._write() as con: got = con.execute(f'INSERT INTO {self.t(RIVALS)} ({", ".join(cols)}) VALUES ({_marks(len(cols))}) ' "ON CONFLICT (tenant, asin, rival_asin) DO UPDATE SET source = 'added', " - "origin = 'added', money_won = 0, terms_won = 0, term_rank = NULL, " 'removed = excluded.removed, added_by = excluded.added_by, ' 'updated_at = excluded.updated_at, ' 'title = coalesce(excluded.title, title) ' f'RETURNING {", ".join(RIVAL_COLUMNS)}', (self.tenant_slug, *(row[c] for c in RIVAL_COLUMNS))).fetchone() - if not dict(zip(RIVAL_COLUMNS, got))['eligible']: - con.execute(f'UPDATE {self.t(DISCOVERY)} SET week = NULL, month = NULL, keyword_ranks_on = NULL ' - 'WHERE tenant = %s AND asin = %s', (self.tenant_slug, row['asin'])) return self._rival(dict(zip(RIVAL_COLUMNS, got))) def remove(self, target, rival, user, stamp): @@ -1505,13 +1050,15 @@ class PgRivalStore: return self._rival(dict(zip(RIVAL_COLUMNS, got))) def rivals_of(self, targets): - """`{asin: [rival ASIN]}` of the rivals not removed.""" + """`{asin: [rival ASIN]}` of the rivals not removed, hand-added first then by click share.""" out = {} with self._connection() as con: for part in _chunks(list(targets)): for row in self._select(con, RIVALS, ('asin', 'rival_asin'), f'asin IN ({_marks(len(part))}) AND removed = %s', (*part, False), - ' ORDER BY asin, rival_asin'): + " ORDER BY asin, CASE WHEN source = 'added' THEN 0 ELSE 1 END, " + 'CASE WHEN click_share IS NULL THEN 1 ELSE 0 END, click_share DESC, ' + 'rival_asin'): out.setdefault(row['asin'], []).append(row['rival_asin']) return out @@ -1532,10 +1079,6 @@ class PgRivalStore: with self._connection() as con: if not (self._exists(con, QUOTES) and self._exists(con, RIVALS)): return own, rows - columns = {r[0] for r in con.execute( - 'SELECT column_name FROM information_schema.columns WHERE table_schema = %s AND table_name = %s', - (self.schema, RIVALS)).fetchall()} - eligibility = 'AND r.eligible = TRUE ' if 'eligible' in columns else '' for part in _chunks(list(targets)): marks = _marks(len(part)) for target, rank, group in con.execute( @@ -1546,8 +1089,7 @@ class PgRivalStore: rows += [tuple(r) for r in con.execute( f'SELECT r.asin, q.{RIVAL_PRICE}, q.sales_rank, q.display_group FROM {self.t(RIVALS)} AS r ' f'JOIN {self.t(QUOTES)} AS q ON q.tenant = r.tenant AND q.asin = r.rival_asin ' - f'WHERE r.tenant = %s AND r.removed = %s {eligibility}' - f'AND r.asin IN ({marks}) AND {newest}', + f'WHERE r.tenant = %s AND r.removed = %s AND r.asin IN ({marks}) AND {newest}', (self.tenant_slug, False, *part)).fetchall()] return own, rows diff --git a/platform/core/connector_spec.py b/platform/core/connector_spec.py index ff83ec8f0a986dd97a4693b679fab3312a3eaecf..00226057ac3e82af1ee4ca221358510a5d4ef7fc 100644 --- a/platform/core/connector_spec.py +++ b/platform/core/connector_spec.py @@ -843,16 +843,6 @@ AMAZON_TABLES = ( identity=("rank_id",), links=(("keyword_id", _AMZN + "sku_keywords", "keyword_id", "many_to_one"),), ), - ConnectorTableSpec( - key=_AMZN + "competitor_snapshots", native="competitor_run:amazon_quotes_loop.run_tenant", - source="amzn_competitor_snapshots", label="Amazon Marketing: Competitor snapshots", pinned="asin", - identity=("asin", "snapshot_date"), - ), - ConnectorTableSpec( - key=_AMZN + "competitor_images", native="competitor_run:amazon_quotes_loop.run_tenant", - source="amzn_competitor_images", label="Amazon Marketing: Competitor images", pinned="asin", - identity=("asin", "variant", "link"), - ), ) #: Snapshot reports that keep change history in `amzn__history`. Equals the REPORT_TYPES keys @@ -883,38 +873,6 @@ AMAZON_TABLES = AMAZON_TABLES + tuple( # Local projections supplement measured vendor columns without changing the table contract. AMAZON_FIELDS = { - _AMZN + "competitor_snapshots": tuple( - {"name": name, "type": kind, "label": label, "source": "competitor_run." + name} - for name, kind, label in ( - ("asin", "VARCHAR", "ASIN"), - ("snapshot_date", "DATE", "Snapshot date"), - ("product_type", "VARCHAR", "Product type"), - ("brand", "VARCHAR", "Brand"), - ("title", "VARCHAR", "Title"), - ("bullets", "VARCHAR", "Bullet points"), - ("image_variants", "VARCHAR", "Image variants"), - ("price", "DOUBLE", "Price"), - ("currency", "VARCHAR", "Currency"), - ("prime", "BOOLEAN", "Prime"), - ("offers", "BIGINT", "Offers"), - ("sales_rank", "BIGINT", "Sales rank"), - ("display_group", "VARCHAR", "Display group"), - ("rank_cat", "VARCHAR", "Rank category"), - ("image", "VARCHAR", "Image"), - ) - ), - _AMZN + "competitor_images": tuple( - {"name": name, "type": kind, "label": label, "source": "competitor_run." + name} - for name, kind, label in ( - ("asin", "VARCHAR", "ASIN"), - ("variant", "VARCHAR", "Variant"), - ("link", "VARCHAR", "Link"), - ("sha256", "VARCHAR", "SHA256"), - ("width", "BIGINT", "Width"), - ("height", "BIGINT", "Height"), - ("snapshot_date", "DATE", "Snapshot date"), - ) - ), _AMZN + "keyword_ranks": tuple( {"name": name, "type": kind, "label": label, "source": source} for name, kind, label, source in ( @@ -3637,9 +3595,6 @@ AMAZON_DEFAULT_COLUMNS = { "cart_adds", "purchases", "backend_keywords", "source_basis"), _AMZN + "keyword_ranks": ("keyword", "date", "organic_rank", "sponsored_rank", "sponsored_rank_basis", "sku", "asin"), - _AMZN + "competitor_snapshots": ("asin", "snapshot_date", "brand", "title", "product_type", - "price", "currency", "prime", "offers", "sales_rank", "rank_cat"), - _AMZN + "competitor_images": ("asin", "variant", "link", "sha256", "width", "height", "snapshot_date"), } # A history table opens on its source's columns plus the validity dates. An undeclared source stays undeclared. AMAZON_DEFAULT_COLUMNS.update({_AMZN + _t + "_history": (*AMAZON_DEFAULT_COLUMNS[_AMZN + _t], "valid_from", "valid_to") @@ -3767,7 +3722,7 @@ AMAZON_GROUPS = { **{_AMZN + t: "Advertising" for t in ( "brand_analytics", "search_query_performance", "search_catalog_performance", "market_basket", "repeat_purchase", "promotion_performance", "coupon_performance", - "sku_keywords", "keyword_ranks", "competitor_snapshots", "competitor_images")}, + "sku_keywords", "keyword_ranks")}, } AMAZON_GROUPS.update({_AMZN + _t + "_history": AMAZON_GROUPS[_AMZN + _t] for _t in AMAZON_HISTORY_SOURCES}) @@ -3785,15 +3740,15 @@ AMAZON_ADS_REPORTS = ("campaigns", "targeting", "advertised_products", "search_t _ADS_PRODUCT = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"} _ADS_REPORT_LABEL = {"campaigns": "campaigns", "targeting": "targeting", "advertised_products": "advertised products", "search_terms": "search terms"} -#: `native` is the Ads report type id. Unknown: the exact v1 spelling; the first credentialed -#: report settles it (PRD P17), and each id is one literal here. +#: `native` is the Reporting v1 report group that fills the table (`amazon_ads_schema.REPORT_GROUPS`). +#: v1 has no report types. SD search terms has no group, because v1 does not offer it. _ADS_NATIVE = { - "sp": {"campaigns": "spCampaigns", "targeting": "spTargeting", - "advertised_products": "spAdvertisedProduct", "search_terms": "spSearchTerm"}, - "sb": {"campaigns": "sbCampaigns", "targeting": "sbTargeting", - "advertised_products": "sbAds", "search_terms": "sbSearchTerm"}, - "sd": {"campaigns": "sdCampaigns", "targeting": "sdTargeting", - "advertised_products": "sdAdvertisedProduct", "search_terms": "sdSearchTerm"}, + "sp": {"campaigns": "campaigns", "targeting": "targeting", + "advertised_products": "advertised_products", "search_terms": "search_terms"}, + "sb": {"campaigns": "campaigns", "targeting": "targeting", + "advertised_products": "advertised_products_sb", "search_terms": "search_terms"}, + "sd": {"campaigns": "campaigns", "targeting": "targeting", + "advertised_products": "advertised_products", "search_terms": "not_offered"}, } _ADS_PINNED = {"campaigns": "campaign_name", "targeting": "targeting", "advertised_products": "asin", "search_terms": "search_term"} @@ -3822,21 +3777,22 @@ AMAZON_ADS_TABLES = tuple( #: which verify_connectors holds equal to the `"amazon"` registration. AMAZON_ADS_GROUPS = {spec.key: "Advertising" for spec in AMAZON_ADS_TABLES} -#: Days the first sync reaches back, per table: the longest window Amazon documents per ad type -#: (SP 95, SB 60, SD 65, search terms 60). Unknown: v1 retention (primer F12). The first -#: credentialed sync measures the oldest day Amazon returns. -AMAZON_ADS_BACKFILL_DAYS = { - spec.key: (60 if spec.key.endswith("_search_terms") - else {"sp": 95, "sb": 60, "sd": 65}[spec.key[len(_ADS):len(_ADS) + 2]]) - for spec in AMAZON_ADS_TABLES} +#: Days the first sync reaches back: Reporting v1 keeps 24 months of daily data, and standing +#: rule 1 takes all of it. +AMAZON_ADS_BACKFILL_DAYS = {spec.key: 730 for spec in AMAZON_ADS_TABLES} #: Standing rule 1: the throughput limit that cannot be removed, with cause and fix. On screen. AMAZON_ADS_THROUGHPUT = ( - "Amazon Reporting v1 runs one report at a time per advertiser and about 100 reports a day. " - "Sponsored Products, Brands and Display need 11 report types, and one report can take up to " - "3 hours, so one day of refresh may not finish in a day. Loopable runs the reports one at a " - "time and keeps every day it already has. If the refresh falls behind, ask Amazon Ads support " - "to raise the report limit for this advertiser.") + "Amazon Ads runs one report at a time for each Loopable app login (its client ID) and about " + "100 reports a day, shared by every advertiser on that login. Sponsored Products, Brands and " + "Display need 5 reports for each refresh, and one report can take up to 3 hours, so one day of " + "refresh may not finish in a day. Loopable runs the reports one at a time and keeps every day " + "it already has. If the refresh falls behind, ask Amazon Ads support to raise the report limit " + "for this client ID.") + +#: On screen: the attribution window Amazon applies to Sponsored Products seller metrics. +AMAZON_ADS_SP_ATTRIBUTION = ("Sponsored Products sales, orders and units count purchases within 7 " + "days of a click, Amazon's default for sellers.") #: Standing rule 1: the report v1 does not offer, with cause and fix. On screen. AMAZON_ADS_NOT_OFFERED = { @@ -3859,9 +3815,10 @@ def amazon_ads_description(key): if unoffered: return f"{unoffered} {AMAZON_ADS_THROUGHPUT}" days = AMAZON_ADS_BACKFILL_DAYS[spec.key] - return (f"{spec.label} by day, from Amazon Ads Reporting v1. History is kept from the day you " - f"connect, and the first sync reaches back {days} days, the longest Amazon keeps. " - f"{AMAZON_ADS_THROUGHPUT}") + attribution = AMAZON_ADS_SP_ATTRIBUTION + " " if spec.key.startswith(_ADS + "sp_") else "" + return (f"{spec.label} by day, from Amazon Ads Reporting v1. {attribution}History is kept from " + f"the day you connect, and the first sync reaches back {days} days, the longest Amazon " + f"keeps. {AMAZON_ADS_THROUGHPUT}") def table_group(key): diff --git a/platform/core/external_tools.py b/platform/core/external_tools.py index 991e7aa5c67831c7fb3a9a0f2d3589be43d22ec3..947192850271c25735a301380855545465237c70 100644 --- a/platform/core/external_tools.py +++ b/platform/core/external_tools.py @@ -267,7 +267,7 @@ _APPLICATION = { "templateKind": {"enum": ["route", "catalog", "competitor_watch", "ranking_diagnostic", "amazon_operator", "restock_planner", "reimbursements", "aged_stock", "opportunity_finder", "returns_refunds", "repeat_buyers", "payouts", "account_health", - "amazon_dashboard", "amazon_profit", "alerts", "reports"]}, + "amazon_dashboard", "amazon_profit", "alerts"]}, "fieldMap": {"type": "object", "additionalProperties": _STRING}, "navPlacement": _object({"section": {"const": "app"}, "order": {"type": "integer", "minimum": 0, "maximum": 999}}, diff --git a/platform/core/shares.py b/platform/core/shares.py index 014886f99889d043f1e8541870d2c0b017fac5f1..c2cf3b0d4e20f03213787f4f9014d2cebc571ab8 100644 --- a/platform/core/shares.py +++ b/platform/core/shares.py @@ -80,7 +80,7 @@ SHARES_KEY = 'object_shares' #: here is this tuple member. That is the point of one registry: the new object's WALL is written #: once in `core.perm_scope`, and its DOORS once in `routes_shares.py` — never a fourth #: permission decision in a fourth file ([[one-evaluator-per-question]]). -KINDS = ('view', 'folder', 'database', 'field', 'cohort', 'workbook', 'report') +KINDS = ('view', 'folder', 'database', 'field', 'cohort', 'workbook') #: `*` is "everyone who can already open the surface". It is NOT "every account on the platform". #: Spelled as a single character so it can never collide with a username (usernames are lower-case @@ -692,32 +692,13 @@ def reconcile_database_grants(st=None, *, tenant=None, table_keys=None): return repaired[0] -def _synthetic_qa_publication(target, table_keys): - """Identify private scratch publications outside the legacy account plane.""" - from core import user_tables, users - - qa = users.qa_identity(users._QA_USERNAME) - if (not qa or not table_keys or getattr(target, 'key', None) != qa['tenant'] - or getattr(getattr(target, 'tenant', None), 'key', None) != qa['tenant']): - return False - for key in table_keys: - if not isinstance(key, str) or not key.startswith(user_tables.KEY_PREFIX): - return False - definition = user_tables.get_definition(key, st=target) - if (not isinstance(definition, dict) or definition.get('key') != key - or definition.get('createdBy') != qa['username'] - or definition.get('source') != user_tables.BLANK_SOURCE): - return False - return True - - def _strict_accounts(): """Every account record, read strictly: the account rows on PostgreSQL, else the users document.""" from core import account_rows, users # On PostgreSQL the users document is a layout marker; reading it repairs no account at all. if account_rows._pg_source(users.store): return dict(account_rows._all(users.store)) - registry = users.registry() + registry = users.store._read_strict('users') if account_rows.is_marker(registry): raise RuntimeError('the account registry is unavailable') return registry @@ -727,9 +708,6 @@ def repair_published_database_grants(st=None, *, table_keys): """Attempt one repair after publication, deferring failures to boot/editor except freeze refusal.""" from core import freeze_notice try: - table_keys = tuple(table_keys) - if _synthetic_qa_publication(_st(st), table_keys): - return 0 return reconcile_database_grants(st, table_keys=table_keys) except freeze_notice.DataInputFrozen: raise diff --git a/platform/harness/amazon_ads_client.py b/platform/harness/amazon_ads_client.py index 817839bd349de4f40ce9c5f8df185b9384128a55..f241c4af3be42c10925183cc6802886b8b491ab0 100644 --- a/platform/harness/amazon_ads_client.py +++ b/platform/harness/amazon_ads_client.py @@ -4,7 +4,7 @@ read-only scope: `advertising::campaign_management` can change any campaign (primer F10). So the refusal lives here, in the request builder, before any credential read, token exchange or socket: - reports create, status and row pages on Reporting v1 (R8) + reports Reporting v1: create, retrieve, the advertiser account query, and the report files streams create and list subscriptions for the dedicated V1 destination named writes `send_named_write(tenant_key, kind, payload, *, approval_id)`, kind in `bid`, `budget`, `negative_keyword`, `scheduled_change`, each operation one path constant @@ -17,8 +17,11 @@ harness module never imports `modules` at load). A missing approval source refus made until Royal's Direct Advertiser credential arrives (R7, R17). NA host `advertising-api.amazon.com`, one profile id per marketplace (F13). The profile id is -stored with the credential, so the client needs no profile listing call. +stored with the credential. v1 names the account by `advertiserAccountId`, which the client looks +up once per connection from `query/advertiserAccounts`. Every path, header and field comes from +Amazon's saved contracts in `AutoWiki/research/amazon-ads-v1-schema/`. """ +import gzip import importlib import json import logging @@ -31,6 +34,7 @@ import urllib.parse import urllib.request from contextlib import contextmanager from datetime import date, datetime, timedelta, timezone +from typing import NamedTuple from zoneinfo import ZoneInfo, ZoneInfoNotFoundError KEY = "amazon_ads" @@ -46,22 +50,28 @@ DEFAULT_REGION = "na" LWA_TOKEN_URL = "https://api.amazon.com/auth/o2/token" CREDENTIAL_FIELDS = ("client_id", "client_secret", "refresh_token", "profile_id") -#: Reporting v1 limits (primer F11, F12): one report in flight per advertiser, about 100 a day, -#: up to 3 hours per report, at most 31 days per request. +#: Reporting v1 limits (limits.md, dates.md): one report in flight per client ID, about 100 a day, +#: up to 3 hours per report, at most 120 days per request. REPORT_CONCURRENCY = 1 REPORTS_PER_DAY = 100 REPORT_MAX_WAIT_SECONDS = 3 * 3600 -MAX_DAYS_PER_REQUEST = 31 -REPORT_POLL_SECONDS = 30.0 +MAX_DAYS_PER_REQUEST = 120 +REPORT_POLL_SECONDS = 60.0 +#: Amazon's error text reaches the sync report verbatim, up to this many characters. +ERROR_TEXT_LIMIT = 600 _TIMEOUT = 60 -#: The only non-write calls: Reporting v1 create, status and row pages. Unknown: the exact v1 -#: paths; the first credentialed call settles them, and each is one literal here. +#: The only non-write calls: the Reporting v1 operations, each a POST to one literal path from the +#: Reports and AdvertiserAccounts contracts. Every other `adsApi` path refuses, a GET included. REPORT_OPERATIONS = { - "createReport": ("POST", "reporting/v1/reports"), - "getReport": ("GET", "reporting/v1/reports/{reportId}"), - "getReportRows": ("GET", "reporting/v1/reports/{reportId}/rows"), + "createReport": ("POST", "adsApi/v1/create/reports"), + "retrieveReport": ("POST", "adsApi/v1/retrieve/reports"), + "queryAdvertiserAccounts": ("POST", "adsApi/v1/query/advertiserAccounts"), } +#: The contract names `Amazon-Ads-ClientId`. The limits page names the older header. v1 sends both. +V1_CLIENT_HEADERS = ("Amazon-Ads-ClientId", "Amazon-Advertising-API-ClientId") +#: A report file is a presigned S3 URL: https, on this host suffix, GET with no Ads header. +REPORT_FILE_HOST_SUFFIX = ".amazonaws.com" # Subscription management never admits an advertising write. STREAM_OPERATIONS = { @@ -208,7 +218,15 @@ class AdsWriteRefused(AdsError): f"the named writes: bid, budget, negative keyword and scheduled change.") -class AdsReportUnavailable(AdsError): +class AdsRejected(AdsError): + """Amazon answered an error. `errors` keeps each entry's code, message and fieldLocation.""" + + def __init__(self, message, errors=()): + super().__init__(message) + self.errors = list(errors) + + +class AdsReportUnavailable(AdsRejected): """Amazon does not offer this report to this advertiser.""" @@ -225,25 +243,10 @@ def fixture_mode(): return os.environ.get(FIXTURE_ENV) == "1" -def _template(path): - return tuple(path.strip("/").split("/")) - - -def _matches(template, path): - parts = str(path or "").strip("/").split("/") - if len(parts) != len(template): - return False - return all((bool(re.fullmatch(r"[A-Za-z0-9_-]{1,128}", part))) - if seg.startswith("{") and seg.endswith("}") else part == seg - for part, seg in zip(parts, template)) - - def report_operation(method, path): - """The Reporting v1 operation name for one call, or None.""" - for name, (verb, tmpl) in REPORT_OPERATIONS.items(): - if str(method).upper() == verb and _matches(_template(tmpl), path): - return name - return None + """The Reporting v1 operation name for one call, or None. A path matches only its literal.""" + return next((name for name, (verb, endpoint) in REPORT_OPERATIONS.items() + if str(method).upper() == verb and str(path or "").strip("/") == endpoint), None) class _Prepared: @@ -503,12 +506,17 @@ def _send(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None) if not _admitted(prepared): raise AdsWriteRefused("the request did not come from the request builder") if fixture_mode(): - return FIXTURE.answer(prepared, tenant_key, profile_id) + headers = _headers(prepared, FIXTURE.credential(tenant_key), FIXTURE_TOKEN, profile_id) + return FIXTURE.answer(prepared, tenant_key, profile_id, headers) return _live(prepared, tenant_key, rt=rt, profile_id=profile_id, connection_id=connection_id) # --------------------------------------------------------------------------- live transport +_NO_CREDENTIAL = ("No Amazon Ads credential is stored for this tenant. " + "Connect Amazon Ads under Amazon Ads Setup.") + + def credential(tenant_key, rt=None, connection_id=None): """The tenant's stored Amazon Ads credential fields, or None. The keychain is the only source.""" try: @@ -538,11 +546,50 @@ def _token(fields): raise AdsError(f"Amazon Ads sign in failed ({type(exc).__name__})") from None +def _headers(prepared, fields, token, profile_id=None): + """The headers one admitted call sends. A v1 call names its account in the body, not a scope.""" + if prepared.operation in REPORT_OPERATIONS: + return {"Authorization": f"Bearer {token}", **dict.fromkeys(V1_CLIENT_HEADERS, fields["client_id"]), + "Content-Type": "application/json", "Accept": "application/json"} + headers = {"Authorization": f"Bearer {token}", + "Amazon-Advertising-API-ClientId": fields["client_id"], + "Amazon-Advertising-API-Scope": str(profile_id or fields["profile_id"]), + "Content-Type": "application/json", "Accept": "application/json"} + if prepared.operation in STREAM_OPERATIONS: + headers["Content-Type"] = STREAM_CONTENT_TYPE + headers["Accept"] = STREAM_CONTENT_TYPE + return headers + + +def _errors(detail): + """Amazon's error entries from an error body, each with its code, message and fieldLocation.""" + if not isinstance(detail, dict): + return [] + entries = detail["errors"] if isinstance(detail.get("errors"), list) else [detail] + return [{"code": str(e.get("code") or ""), "message": str(e.get("details") or e.get("message") or ""), + "fieldLocation": str(e.get("fieldLocation") or "")} + for e in entries if isinstance(e, dict) and (e.get("code") or e.get("details") or e.get("message"))] + + +def _error_text(errors): + """Amazon's errors as one line a person reads, verbatim, cut to `ERROR_TEXT_LIMIT`.""" + return " ".join(" ".join(filter(None, (e["code"] + ":" if e["code"] else "", e["message"], + f"(field {e['fieldLocation']})" if e["fieldLocation"] else ""))) + for e in errors)[:ERROR_TEXT_LIMIT] + + +def _rejection(prefix, errors, *, may_be_unoffered=True): + """The exception for Amazon's errors: not offered when Amazon says so, else rejected.""" + text = _error_text(errors) + if may_be_unoffered and re.search(r"not (?:supported|offered|available)", text, re.I): + return AdsReportUnavailable(f"Amazon does not offer this report: {text}", errors) + return AdsRejected(f"{prefix}: {text}" if text else prefix, errors) + + def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None): fields = credential(tenant_key, rt, connection_id=connection_id) if fields is None: - raise AdsError("No Amazon Ads credential is stored for this tenant. " - "Connect Amazon Ads under Amazon Ads Setup.") + raise AdsError(_NO_CREDENTIAL) if prepared.operation in STREAM_OPERATIONS and ( str(profile_id or fields["profile_id"]) != str(fields["profile_id"]) or str(fields.get("region") or DEFAULT_REGION).lower() != "na"): @@ -551,13 +598,7 @@ def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None) url = f"{host}/{prepared.path}" if prepared.query: url += "?" + urllib.parse.urlencode(prepared.query) - headers = {"Authorization": f"Bearer {_token(fields)}", - "Amazon-Advertising-API-ClientId": fields["client_id"], - "Amazon-Advertising-API-Scope": str(profile_id or fields["profile_id"]), - "Content-Type": "application/json", "Accept": "application/json"} - if prepared.operation in STREAM_OPERATIONS: - headers["Content-Type"] = STREAM_CONTENT_TYPE - headers["Accept"] = STREAM_CONTENT_TYPE + headers = _headers(prepared, fields, _token(fields), profile_id) data = json.dumps(prepared.body).encode() if prepared.body is not None else None req = urllib.request.Request(url, data=data, method=prepared.method, headers=headers) try: @@ -566,12 +607,10 @@ def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None) except urllib.error.HTTPError as exc: try: detail = json.loads(exc.read().decode("utf-8", "replace") or "{}") - message = str(detail.get("details") or detail.get("message") or "")[:200] - except Exception: # noqa: BLE001 - message = "" - if exc.code == 400 and re.search(r"not (?:supported|offered|available)", message, re.I): - raise AdsReportUnavailable(f"Amazon does not offer this report: {message}") from None - raise AdsError(f"Amazon Ads answered HTTP {exc.code} on {prepared.operation}: {message}") from None + except Exception: # noqa: BLE001 an unreadable body still raises its status + detail = {} + raise _rejection(f"Amazon Ads answered HTTP {exc.code} on {prepared.operation}", _errors(detail), + may_be_unoffered=exc.code == 400) from None except Exception as exc: # noqa: BLE001 raise AdsError(f"Amazon Ads could not be reached ({type(exc).__name__})") from None @@ -580,89 +619,276 @@ def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None) _REPORT_LOCKS = {} _REPORT_LOCKS_GUARD = threading.Lock() +#: Process caches: the advertiser account per connection, the optional fields Amazon refused per +#: (client id, group) with Amazon's text, and the report file URLs a retrieve returned. +_ACCOUNTS = {} +_REFUSED = {} +_ISSUED_FILES = set() + + +class GroupReport(NamedTuple): + """One finished group report: its id, flat rows per member table, and notes a person reads.""" + report_id: str + rows: dict + notes: tuple -def _report_lock(tenant_key, profile_id): +def _report_lock(client_id): + """One lock per client ID: Amazon runs one report at a time for each app login.""" with _REPORT_LOCKS_GUARD: - return _REPORT_LOCKS.setdefault((tenant_key, str(profile_id or "")), threading.Lock()) + return _REPORT_LOCKS.setdefault(str(client_id), threading.Lock()) + + +def _report_credential(tenant_key, rt, connection_id): + """The credential a report runs under: its client id keys the lock, its profile names the account.""" + if fixture_mode(): + return FIXTURE.credential(tenant_key) + fields = credential(tenant_key, rt, connection_id=connection_id) + if fields is None: + raise AdsError(_NO_CREDENTIAL) + return fields + + +def _lists_profile(account, profile_id): + return (isinstance(account, dict) and bool(account.get("advertiserAccountId")) + and any(isinstance(alt, dict) and str(alt.get("profileId")) == str(profile_id) + for alt in account.get("alternateIds") or ())) + +def _advertiser_account(tenant_key, *, rt, profile_id, connection_id): + """The v1 `advertiserAccountId` whose alternate ids carry the stored profile id, cached per connection.""" + key, token = (tenant_key, connection_id or "", str(profile_id)), None + while key not in _ACCOUNTS: + page = request(tenant_key, "POST", REPORT_OPERATIONS["queryAdvertiserAccounts"][1], + body={"maxResults": 100, **({"nextToken": token} if token else {})}, + rt=rt, profile_id=profile_id, connection_id=connection_id) + match = next((a for a in page.get("advertiserAccounts") or () if _lists_profile(a, profile_id)), None) + token = page.get("nextToken") + if match is not None: + _ACCOUNTS[key] = str(match["advertiserAccountId"]) + elif not token: + raise AdsError(f"No Amazon Ads advertiser account lists profile {profile_id}. " + f"Check the profile ID on this Amazon Ads connection.") + return _ACCOUNTS[key] -def run_report(tenant_key, table, start, end, *, rt=None, profile_id=None, connection_id=None, sleep=time.sleep, - now=time.monotonic, max_wait=None, report_id=None): - """Create one Reporting v1 report, wait for it, and read every row page. -> (report_id, rows). - `max_wait` caps this call's poll loop in seconds. `report_id` resumes that report without createReport. +def _create_body(group, start, end, account, fields): + """A CreateReportRequest for one group and window, filtered to the group's ad products.""" + from harness import amazon_ads_schema as schema + return {"accessRequestedAccounts": [{"advertiserAccountId": account}], + "reports": [{"format": "GZIP_JSON", + "periods": [{"datePeriod": {"startDate": start.isoformat(), "endDate": end.isoformat()}}], + "query": {"fields": list(fields), + "filter": {"on": {"field": schema.AD_PRODUCT_FIELD, "comparisonOperator": "IN", + "not": False, "values": list(group.tables)}}}}]} + + +def _first_report(reply, operation): + """The one report a 207 multi-status answer carries, or its errors raised verbatim.""" + errors = [e for entry in reply.get("error") or () if isinstance(entry, dict) + for e in _errors({"errors": entry.get("errors") or []})] + if errors or reply.get("error"): + raise _rejection(f"Amazon Ads refused {operation}", errors) + for entry in reply.get("success") or (): + report = entry.get("report") if isinstance(entry, dict) else None + if isinstance(report, dict) and re.fullmatch(r"[A-Za-z0-9_.:-]{1,128}", str(report.get("reportId") or "")): + return report + raise AdsError(f"Amazon Ads returned no report on {operation}") + + +def _named_fields(exc, fields): + """The requested fields an error names, by id in its text or by index in its field location.""" + text = " ".join([str(exc)] + [e["fieldLocation"] for e in exc.errors]) + named = {f for f in fields if re.search(rf"(? its report file URLs.""" + started = now() + while True: + report = _first_report(request(tenant_key, "POST", REPORT_OPERATIONS["retrieveReport"][1], + body={"reportIds": [report_id]}, **opts), "retrieveReport") + state = str(report.get("status") or "").upper() + if state == "COMPLETED": + urls = [str(part.get("url") or "") for part in report.get("completedReportParts") or () + if isinstance(part, dict)] + _ISSUED_FILES.update(urls) + return urls + if state == "FAILED": + reason = " ".join(filter(None, (str(report.get("failureCode") or ""), + str(report.get("failureReason") or ""))))[:ERROR_TEXT_LIMIT] or state + if re.search(r"not (?:supported|offered|available)", reason, re.I): + raise AdsReportUnavailable(f"Amazon does not offer this report: {reason}") + raise AdsError(f"Amazon Ads report {report_id} ended FAILED: {reason}") + elapsed = now() - started + if elapsed > REPORT_MAX_WAIT_SECONDS: + raise AdsReportPending(f"Amazon Ads report {report_id} did not finish in 3 hours", report_id) + if limit is not None and elapsed >= limit: + raise AdsReportPending(f"Amazon Ads report {report_id} is still being prepared", report_id) + pause = 0 if fixture_mode() else REPORT_POLL_SECONDS + sleep(pause if limit is None else min(pause, limit - elapsed)) + + +def _flat(row, prefix=""): + """A nested v1 row -> flat dotted field ids. A flat row stays as it is.""" + out = {} + for key, value in row.items(): + if isinstance(value, dict): + out.update(_flat(value, f"{prefix}{key}.")) + else: + out[f"{prefix}{key}"] = value + return out + + +def _part_rows(raw): + """One report file -> flat rows. GZIP or plain, a JSON array or JSON lines.""" + try: + text = (gzip.decompress(raw) if raw[:2] == b"\x1f\x8b" else raw).decode("utf-8", "replace").strip() + if not text: + return [] + try: + data = json.loads(text) + except ValueError: + data = [json.loads(line) for line in text.splitlines() if line.strip()] + except (OSError, EOFError, ValueError) as exc: + raise AdsError(f"Amazon Ads sent a report file Loopable cannot read ({type(exc).__name__})") from None + return [_flat(row) for row in (data if isinstance(data, list) else [data]) if isinstance(row, dict)] + + +def _read_file(url): + """GET one report file with no Ads header. Only an https S3 URL a retrieve returned is admitted.""" + parts = urllib.parse.urlsplit(str(url or "")) + host = parts.hostname or "" + if parts.scheme != "https" or not host.endswith(REPORT_FILE_HOST_SUFFIX) or url not in _ISSUED_FILES: + raise AdsWriteRefused(f"{parts.scheme or 'no scheme'} {_safe_path(host)} is not a report file Amazon returned") + _ISSUED_FILES.discard(url) + headers = {} + if fixture_mode(): + return _part_rows(FIXTURE.download(url, headers)) + try: + with urllib.request.urlopen(urllib.request.Request(url, method="GET", headers=headers), + timeout=_TIMEOUT) as r: + raw = r.read() + except Exception as exc: # noqa: BLE001 the URL carries a signature: never echo it + raise AdsError(f"Amazon Ads report file could not be read ({type(exc).__name__})") from None + return _part_rows(raw) + + +def _split(group, rows): + """Flat rows -> {member table: rows} by `adProduct.value`, and a note for rows no member takes.""" + from harness import amazon_ads_schema as schema + out, tables, stray = {table: [] for table in group.members}, group.tables, 0 + for row in rows: + table = tables.get(str(row.get(schema.AD_PRODUCT_FIELD) or "").upper()) + if table is None or not row.get(schema.vendor_name("date")): + stray += 1 + else: + out[table].append(row) + note = (f"Amazon Ads sent {stray} rows with no date or with another ad product. Loopable left them out.",) + return out, note if stray else () + + +def run_group_report(tenant_key, group, start, end, *, rt=None, profile_id=None, connection_id=None, + sleep=time.sleep, now=time.monotonic, max_wait=None, report_id=None): + """Create one Reporting v1 group report, wait for it, and read every file. -> GroupReport. + + `max_wait` caps this call's poll loop in seconds. `report_id` resumes that report without a create. """ from harness import amazon_ads_schema as schema - spec = schema.TABLE_SPECS[schema.short_name(table)] + group = schema.GROUPS[group] if isinstance(group, str) else group if (end - start).days + 1 > MAX_DAYS_PER_REQUEST: - raise ValueError("a report window is at most 31 days") - body = {"reportTypeId": spec["report_type"], "adProduct": spec["ad_type"].upper(), - "startDate": start.isoformat(), "endDate": end.isoformat(), - "timeUnit": "DAILY", "columns": list(schema.vendor_columns(table)), "format": "JSON"} + raise ValueError(f"a report window is at most {MAX_DAYS_PER_REQUEST} days") + fields = _report_credential(tenant_key, rt, connection_id) + opts = {"rt": rt, "profile_id": str(profile_id or fields["profile_id"]), "connection_id": connection_id} + refused = _REFUSED.setdefault((str(fields["client_id"]), group.name), {}) limit = None if max_wait is None else max(0.0, min(float(max_wait), REPORT_MAX_WAIT_SECONDS)) - # One report in flight per advertiser (F11): the lock serialises this process's reports. - with _report_lock(tenant_key, profile_id): + with _report_lock(fields["client_id"]): if report_id is None: - created = request(tenant_key, "POST", REPORT_OPERATIONS["createReport"][1], body=body, - rt=rt, profile_id=profile_id, connection_id=connection_id) - report_id = str(created.get("reportId") or "") - else: - report_id = str(report_id) - if not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", report_id): - raise AdsError("Amazon Ads returned no report id") - started = now() - status_path = REPORT_OPERATIONS["getReport"][1].replace("{reportId}", report_id) - while True: - status = request(tenant_key, "GET", status_path, rt=rt, profile_id=profile_id, connection_id=connection_id) - state = str(status.get("status") or "").upper() - if state == "COMPLETED": - break - if state in ("FAILED", "FAILURE", "CANCELLED"): - reason = str(status.get("failureReason") or state)[:200] - if re.search(r"not (?:supported|offered|available)", reason, re.I): - raise AdsReportUnavailable(f"Amazon does not offer this report: {reason}") - raise AdsError(f"Amazon Ads report {report_id} ended {state}: {reason}") - elapsed = now() - started - if elapsed > REPORT_MAX_WAIT_SECONDS: - raise AdsReportPending(f"Amazon Ads report {report_id} did not finish in 3 hours", report_id) - if limit is not None and elapsed >= limit: - raise AdsReportPending(f"Amazon Ads report {report_id} is still being prepared", report_id) - pause = 0 if fixture_mode() else REPORT_POLL_SECONDS - sleep(pause if limit is None else min(pause, limit - elapsed)) - rows, token = [], None - rows_path = REPORT_OPERATIONS["getReportRows"][1].replace("{reportId}", report_id) - while True: - page = request(tenant_key, "GET", rows_path, query={"nextToken": token} if token else None, - rt=rt, profile_id=profile_id, connection_id=connection_id) - rows.extend(page.get("rows") or []) - token = page.get("nextToken") - if not token: - return report_id, rows + account = _advertiser_account(tenant_key, **opts) + report_id = _create(tenant_key, group, start, end, account, refused, opts)["reportId"] + urls = _await(tenant_key, str(report_id), opts, sleep=sleep, now=now, limit=limit) + rows = [row for url in urls for row in _read_file(url)] + split, stray = _split(group, rows) + notes = tuple(f"Amazon Ads refused the field {field} on this report, so Loopable reads it without that " + f"field. Amazon said: {text}" for field, text in refused.items()) + return GroupReport(str(report_id), split, notes + stray) + + +def run_report(tenant_key, table, start, end, **options): + """One table's rows from its group report. -> (report_id, rows). The store reads `run_group_report`.""" + from harness import amazon_ads_schema as schema + group = schema.group_of(table) + if group is None: + raise AdsReportUnavailable(f"Amazon Reporting v1 offers no report for {schema.short_name(table)}") + result = run_group_report(tenant_key, group, start, end, **options) + return result.report_id, result.rows[schema.short_name(table)] # --------------------------------------------------------------------------- recording fixture +FIXTURE_TOKEN = "fixture-access-token" +FIXTURE_CREDENTIAL = {"client_id": "amzn1.application-oa2-client.fixture", "client_secret": "fixture", + "refresh_token": "fixture", "profile_id": "1000000001", "region": "na"} +FIXTURE_FILE_HOST = "https://fixture-ads-reports.s3.amazonaws.com" +FIXTURE_CREATED = "2026-10-20T09:00:00Z" +_FIXTURE_PREFIX = {"SPONSORED_PRODUCTS": ("71", "SP"), "SPONSORED_BRANDS": ("72", "SB"), + "SPONSORED_DISPLAY": ("73", "SD")} +#: Per campaign and day: the metric totals every finer row splits evenly. +_FIXTURE_TOTALS = {"metric.impressions": 400, "metric.clicks": 12, "metric.totalCost": 5.0, + "metric.sales": 38.0, "metric.purchases": 4, "metric.unitsSold": 4} + + +def _multi_status(report=None, errors=None): + """A ReportMultiStatusResponse with one success or one error entry at index 0.""" + return {"success": [{"index": 0, "report": report}] if report else [], + "error": [{"index": 0, "errors": errors}] if errors else []} + + class _Fixture: - """The recording fixture `AMAZON_ADS_FIXTURE=1` routes to. Records each call; answers reports - with deterministic rows, two report polls, and paged rows.""" + """The recording fixture `AMAZON_ADS_FIXTURE=1` routes to, in the saved contracts' v1 shapes. - PAGE_SIZE = 25 + It records method, path, operation, headers and body of every call, report file downloads + included. Toggles: `drop_days`, `unsupported` (group names that end FAILED), `refuse_fields` + (a create naming one answers 400), `nested` (files carry nested JSON lines), `responder`. + """ + + FILES = 2 def __init__(self): self.reset() def reset(self): - self.calls, self.reports = [], {} - self.subscriptions = {} + self.calls, self.reports, self.subscriptions, self.credentials = [], {}, {}, {} self.in_flight = self.max_in_flight = 0 - self.drop_days, self.unsupported = set(), set() + self.drop_days, self.unsupported, self.refuse_fields = set(), set(), set() + self.nested = False self.responder = None - def answer(self, prepared, tenant_key, profile_id): + def credential(self, tenant_key): + """The fixture credential for one tenant. `credentials[tenant]` overrides its fields.""" + return {**FIXTURE_CREDENTIAL, **self.credentials.get(tenant_key, {})} + + def answer(self, prepared, tenant_key, profile_id, headers=None): self.calls.append({"method": prepared.method, "path": prepared.path, "operation": prepared.operation, "body": prepared.body, - "query": dict(prepared.query), "tenant": tenant_key, + "query": dict(prepared.query), "tenant": tenant_key, "headers": dict(headers or {}), "named": dict(prepared.named) if prepared.named else None}) if self.responder is not None: return self.responder(prepared) @@ -678,88 +904,174 @@ class _Fixture: if prepared.named is not None: items = next(iter(prepared.body.values())) return {"success": [{"index": i} for i in range(len(items))], "error": []} + if prepared.operation == "queryAdvertiserAccounts": + return self._accounts(prepared.body, profile_id) if prepared.operation == "createReport": - report_id = f"fx{len(self.reports) + 1:05d}" - self.in_flight += 1 - self.max_in_flight = max(self.max_in_flight, self.in_flight) - self.reports[report_id] = {"body": prepared.body, "polls": 0, "rows": None} - return {"reportId": report_id, "status": "PENDING"} - report_id = prepared.path.split("/")[3] + return self._create(prepared.body) + return self._retrieve(prepared.body["reportIds"][0]) + + def _accounts(self, body, profile_id): + """Two pages: an unrelated account, then the account that lists the caller's profile.""" + if not body.get("nextToken"): + return {"advertiserAccounts": [{"advertiserAccountId": "amzn1.ads-account.g.fxother", + "alternateIds": [{"profileId": "999", "countryCode": "CA"}]}], + "nextToken": "fixture-page-2"} + return {"advertiserAccounts": [{"advertiserAccountId": f"amzn1.ads-account.g.fx{profile_id}", + "alternateIds": [{"profileId": str(profile_id), "countryCode": "US", + "entityId": "ENTITYFX"}]}]} + + def _create(self, body): + if not (body.get("accessRequestedAccounts") or [{}])[0].get("advertiserAccountId"): + return _multi_status(errors=[{"code": "400001", "message": "An advertiser account is required.", + "fieldLocation": "accessRequestedAccounts"}]) + refused = [f for f in body["reports"][0]["query"]["fields"] if f in self.refuse_fields] + if refused: + raise _rejection("Amazon Ads answered HTTP 400 on createReport", + [{"code": "400004", "message": f"Field {refused[0]} is invalid for this request.", + "fieldLocation": ""}], may_be_unoffered=False) + report_id = f"fx{len(self.reports) + 1:05d}" + self.in_flight += 1 + self.max_in_flight = max(self.max_in_flight, self.in_flight) + self.reports[report_id] = {"body": body, "polls": 0, "files": None, "done": False} + return _multi_status(self._record(report_id, "PENDING")) + + def _record(self, report_id, status, **extra): + """A contract `Report`: every required key, the request's format, periods and query echoed.""" + request = self.reports[report_id]["body"]["reports"][0] + return {"reportId": report_id, "status": status, "format": request["format"], + "periods": request["periods"], "query": request["query"], + "creationDateTime": FIXTURE_CREATED, "lastUpdatedDateTime": FIXTURE_CREATED, **extra} + + def _retrieve(self, report_id): + from harness import amazon_ads_schema as schema report = self.reports[report_id] - if prepared.operation == "getReport": - report["polls"] += 1 - if report["body"]["reportTypeId"] in self.unsupported: - self.in_flight -= 1 - return {"status": "FAILED", "failureReason": "Report type is not supported for this advertiser"} - if report["polls"] < 2: - return {"status": "PROCESSING"} - return {"status": "COMPLETED"} - if report["rows"] is None: - report["rows"] = fixture_rows(report["body"], drop_days=self.drop_days) - start = int(prepared.query.get("nextToken") or 0) - page = report["rows"][start:start + self.PAGE_SIZE] - end = start + len(page) - if end >= len(report["rows"]): + report["polls"] += 1 + query = report["body"]["reports"][0]["query"] + group = schema.group_for(query["fields"], query["filter"]["on"]["values"]) + if group is not None and group.name in self.unsupported: + self._finish(report) + return _multi_status(self._record(report_id, "FAILED", failureCode="REPORT_NOT_SUPPORTED", + failureReason="This report is not supported for this advertiser.")) + if report["polls"] < 2: + return _multi_status(self._record(report_id, "PROCESSING")) + if report["files"] is None: + rows = fixture_rows(report["body"], drop_days=self.drop_days) + report["files"] = {f"{FIXTURE_FILE_HOST}/{report_id}/part-{i}.json.gz?X-Amz-Signature=fx{report_id}{i}": + _fixture_file(rows[i::self.FILES], self.nested) for i in range(self.FILES)} + return _multi_status(self._record(report_id, "COMPLETED", completedDateTime=FIXTURE_CREATED, completedReportParts=[ + {"url": url, "sizeInBytes": len(raw), "urlExpirationDateTime": "2026-12-31T00:00:00Z"} + for url, raw in report["files"].items()])) + + def _finish(self, report): + if not report["done"]: + report["done"] = True self.in_flight -= 1 - return {"rows": page} - return {"rows": page, "nextToken": str(end)} + + def download(self, url, headers): + """Serve one report file and record the GET with the headers the client sent.""" + self.calls.append({"method": "GET", "path": url, "operation": "downloadReportPart", "body": None, + "query": {}, "tenant": None, "headers": dict(headers), "named": None}) + report = self.reports[urllib.parse.urlsplit(url).path.split("/")[1]] + raw = report["files"][url] + report.setdefault("read", set()).add(url) + if report["read"] == set(report["files"]): + self._finish(report) + return raw + + +def _fixture_file(rows, nested): + """GZIP JSON: an array of flat rows, or nested objects one per line.""" + def nest(row): + out = {} + for field, value in row.items(): + head, _, tail = field.partition(".") + out.setdefault(head, {})[tail] = value + return out + text = "\n".join(json.dumps(nest(r)) for r in rows) if nested else json.dumps(rows) + return gzip.compress(text.encode("utf-8")) + + +def _fixture_query(body): + report = body["reports"][0] + period = report["periods"][0]["datePeriod"] + return (report["query"]["fields"], report["query"]["filter"]["on"]["values"], + date.fromisoformat(period["startDate"]), date.fromisoformat(period["endDate"])) + + +def _fixture_days(start, end, drop_days): + return [start + timedelta(days=i) for i in range((end - start).days + 1) + if (start + timedelta(days=i)).isoformat() not in drop_days] + + +def fixture_totals(body, *, drop_days=()): + """The fixture's own totals for one create body. -> {(ad product, date, campaign id): {field: total}}.""" + _fields, products, start, end = _fixture_query(body) + out = {} + for day in _fixture_days(start, end, drop_days): + for index, product in enumerate(products): + for c in (1, 2): + n = day.toordinal() % 7 + c + index + out[(product, day.isoformat(), f"{_FIXTURE_PREFIX[product][0]}10{c}")] = { + field: total * n for field, total in _FIXTURE_TOTALS.items()} + return out def fixture_rows(body, *, drop_days=()): - """Deterministic Ads report rows for one request body, every day in its window.""" - report_type, columns = body["reportTypeId"], body["columns"] - start, end = date.fromisoformat(body["startDate"]), date.fromisoformat(body["endDate"]) - ad = report_type[:2] - out, day = [], start - while day <= end: - if day.isoformat() not in drop_days: - for c in (1, 2): - for g in ((0,) if report_type.endswith("Campaigns") else (1, 2)): - n = day.toordinal() % 7 + c + g - row = { - "date": day.isoformat(), "campaignId": f"{ad.encode().hex()}10{c}", - "campaignName": f"{ad.upper()} campaign {c}", "campaignStatus": "ENABLED", - "campaignBudgetAmount": 25.0 * c, "campaignBudgetCurrencyCode": "USD", - "adGroupId": f"20{c}{g}", "adGroupName": f"Ad group {c}.{g}", - "keywordId": f"30{c}{g}", "targetingId": f"30{c}{g}", - "targeting": f"term {c}{g}", "targetingExpression": f"asin is B00FX{c}{g}", - "matchType": "exact", "keywordBid": 0.5 + g / 10, - "adId": f"40{c}{g}", "advertisedSku": f"SKU-{c}{g}", - "advertisedAsin": f"B00FX0{c}{g}0", "promotedSku": f"SKU-{c}{g}", - "promotedAsin": f"B00FX0{c}{g}0", "searchTerm": f"search {c}{g}", - "impressions": 100 * n, "clicks": 3 * n, "cost": round(1.25 * n, 2), - "sales": round(9.5 * n, 2), "sales14d": round(9.5 * n, 2), - "purchases": n, "purchases14d": n, "unitsSold": n, "unitsSoldClicks14d": n, - } - out.append({k: row.get(k) for k in columns}) - day += timedelta(days=1) + """Deterministic v1 rows for one create body: one row per distinct value of every requested + dimension and day, only the requested fields and ad products. Two ad groups split a campaign.""" + fields, _products, start, end = _fixture_query(body) + groups = (1, 2) if "adGroup.id" in fields else (0,) + out = [] + for (product, day, campaign), totals in fixture_totals(body, drop_days=drop_days).items(): + prefix, short = _FIXTURE_PREFIX[product] + c = campaign[-1] + for g in groups: + row = {"date.value": day, "adProduct.value": product, "campaign.id": campaign, + "campaign.name": f"{short} campaign {c}", "campaign.deliveryStatus": "DELIVERING", + "campaign.budgetAmount": str(25.0 * int(c)), "budgetCurrency.value": "USD", + "adGroup.id": f"{prefix}20{c}{g}", "adGroup.name": f"Ad group {c}.{g}", + "adGroupTarget.id": f"{prefix}30{c}{g}", "targeting.value": f"term {c}{g}", + "targetingMatchType.value": "EXACT", "adGroupTarget.bid": str(0.5 + g / 10), + "ad.id": f"{prefix}40{c}{g}", "advertisedProduct.id": f"B0FX{prefix}{c}{g}00", + "advertisedProductMarketplace.value": "ATVPDKIKX0DER", "advertisedProductSku.value": f"SKU-{c}{g}", + "searchTerm.value": f"search {c}{g}", + **{f: (t // len(groups) if isinstance(t, int) else t / len(groups)) for f, t in totals.items()}} + out.append({f: row[f] for f in fields}) return out FIXTURE = _Fixture() +def _reset_caches(): + _ACCOUNTS.clear() + _REFUSED.clear() + _ISSUED_FILES.clear() + + @contextmanager def recording_fixture(): - """Route every call to a fresh recording fixture for the block. -> the fixture.""" + """Route every call to a fresh recording fixture and fresh report caches for the block. -> the fixture.""" previous = os.environ.get(FIXTURE_ENV) os.environ[FIXTURE_ENV] = "1" FIXTURE.reset() + _reset_caches() try: yield FIXTURE finally: FIXTURE.reset() + _reset_caches() if previous is None: os.environ.pop(FIXTURE_ENV, None) else: os.environ[FIXTURE_ENV] = previous -__all__ = ["KEY", "LABEL", "ENTRY_TYPE", "FIXTURE_ENV", "HOSTS", "REPORT_OPERATIONS", "NAMED_WRITES", - "STREAM_OPERATIONS", "STREAM_CONTENT_TYPE", "stream_operation", +__all__ = ["KEY", "LABEL", "ENTRY_TYPE", "FIXTURE_ENV", "HOSTS", "REPORT_OPERATIONS", "V1_CLIENT_HEADERS", + "NAMED_WRITES", "STREAM_OPERATIONS", "STREAM_CONTENT_TYPE", "stream_operation", "FIELD_RULES", "APPROVAL_SOURCE", "SCHEDULE_APPROVAL_SOURCE", "SENT_WRITE_SOURCE", "SCHEDULE_WEEKDAY_CONVENTION", "REPORT_CONCURRENCY", "REPORTS_PER_DAY", - "MAX_DAYS_PER_REQUEST", "AdsError", "AdsWriteRefused", "AdsReportUnavailable", - "AdsReportPending", "fixture_mode", "build_request", "report_operation", - "send_named_write", "request", "credential", "run_report", "FIXTURE", - "recording_fixture", "fixture_rows"] + "MAX_DAYS_PER_REQUEST", "AdsError", "AdsWriteRefused", "AdsRejected", "AdsReportUnavailable", + "AdsReportPending", "GroupReport", "fixture_mode", "build_request", "report_operation", + "send_named_write", "request", "credential", "run_group_report", "run_report", "FIXTURE", + "recording_fixture", "fixture_rows", "fixture_totals"] diff --git a/platform/harness/amazon_ads_schema.py b/platform/harness/amazon_ads_schema.py index 63db63a14f3a11e50d56558c051f7d8beae54395..9f2bf47485653a36a74328f661172d6ad5cf171c 100644 --- a/platform/harness/amazon_ads_schema.py +++ b/platform/harness/amazon_ads_schema.py @@ -1,16 +1,17 @@ """amazon_ads_schema.py: the vendor schema artifact for the Amazon Ads tables (S79-T12, contract C6). -One entry per ad type and report: the report type, the columns we store, the Ads column each one -is read from, the row identity and the backfill window. The table keys, labels and backfill days -come from `core.connector_spec`, so the two files cannot disagree on a name. +One entry per ad type and report: the columns we store, the Reporting v1 field each one is read +from, the row identity and the backfill window. The table keys, labels and backfill days come from +`core.connector_spec`, so the two files cannot disagree on a name. -Unknown: the exact Reporting v1 column spellings. The vendor names below are the Ads reporting -vocabulary the primer records (F11, F12). The first credentialed report settles them (PRD P17), -and a rename is one literal in `_VENDOR`. +Field ids come from Amazon's v1 field library, saved at +`AutoWiki/research/amazon-ads-v1-schema/reporting_v1_fields.json`. v1 has no report types: one +report per `REPORT_GROUPS` row covers several ad types, and its rows split by `adProduct.value`. """ import hashlib import sys from pathlib import Path +from typing import NamedTuple _HERE = Path(__file__).resolve().parent if str(_HERE.parent) not in sys.path: @@ -22,8 +23,6 @@ AD_TYPES = _cs.AMAZON_ADS_AD_TYPES REPORTS = _cs.AMAZON_ADS_REPORTS DATABASE_PREFIX = "ut_amzn_" MIRROR_PREFIX = "amzn_ads_" -#: Amazon accepts at most 31 days per report request. -MAX_DAYS_PER_REQUEST = 31 #: The C6 columns every Ads table carries, in this order. C6_COLUMNS = ("campaign_id", "ad_group_id", "target_id", "advertised_sku", "asin", "date", @@ -54,22 +53,71 @@ _EXTRA = { } _TAIL = ("profile_id", "currency") -#: Our column -> the Ads column, shared by all three ad types unless `_VENDOR_BY_TYPE` says not. +#: Our column -> its Reporting v1 field id, one map for every ad type. The store stamps profile_id. _VENDOR = { - "campaign_id": "campaignId", "campaign_name": "campaignName", - "campaign_status": "campaignStatus", "campaign_budget_amount": "campaignBudgetAmount", - "ad_group_id": "adGroupId", "ad_group_name": "adGroupName", "target_id": "keywordId", - "targeting": "targeting", "match_type": "matchType", "bid": "keywordBid", - "ad_id": "adId", "advertised_sku": "advertisedSku", "asin": "advertisedAsin", - "search_term": "searchTerm", "date": "date", "impressions": "impressions", - "clicks": "clicks", "cost": "cost", "sales": "sales", "orders": "purchases", - "units": "unitsSold", "currency": "campaignBudgetCurrencyCode", -} -_VENDOR_BY_TYPE = { - "sp": {"sales": "sales14d", "orders": "purchases14d", "units": "unitsSoldClicks14d"}, - "sd": {"target_id": "targetingId", "targeting": "targetingExpression", - "advertised_sku": "promotedSku", "asin": "promotedAsin"}, + "date": "date.value", "campaign_id": "campaign.id", "campaign_name": "campaign.name", + "campaign_status": "campaign.deliveryStatus", "campaign_budget_amount": "campaign.budgetAmount", + "ad_group_id": "adGroup.id", "ad_group_name": "adGroup.name", "target_id": "adGroupTarget.id", + "targeting": "targeting.value", "match_type": "targetingMatchType.value", "bid": "adGroupTarget.bid", + "ad_id": "ad.id", "advertised_sku": "advertisedProductSku.value", "asin": "advertisedProduct.id", + "search_term": "searchTerm.value", "impressions": "metric.impressions", "clicks": "metric.clicks", + "cost": "metric.totalCost", "sales": "metric.sales", "orders": "metric.purchases", + "units": "metric.unitsSold", "currency": "budgetCurrency.value", } +#: The v1 field that names each row's ad type, and its value per ad type. +AD_PRODUCT_FIELD = "adProduct.value" +AD_PRODUCTS = {"sp": "SPONSORED_PRODUCTS", "sb": "SPONSORED_BRANDS", "sd": "SPONSORED_DISPLAY"} +#: The name the v1 field library lists each ad type under. +LIBRARY_AD_PRODUCTS = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"} + +METRICS = ("metric.impressions", "metric.clicks", "metric.totalCost", "metric.sales", + "metric.purchases", "metric.unitsSold") +_HEAD = ("date.value", AD_PRODUCT_FIELD, "campaign.id", "campaign.name") +_AD_GROUP = ("adGroup.id", "adGroup.name") +_TAIL_FIELDS = ("budgetCurrency.value",) + METRICS +_TARGET_TEXT = ("targeting.value", "targetingMatchType.value") + + +class ReportGroup(NamedTuple): + """One Reporting v1 report and the member tables its rows split into by `adProduct.value`. + + `optional` fields list no ad product in Amazon's library: a create that refuses one drops it. + """ + name: str + members: tuple + core: tuple + optional: tuple = () + + @property + def tables(self): + """`adProduct.value` -> the member table its rows land in.""" + return {AD_PRODUCTS[table.split("_", 1)[0]]: table for table in self.members} + + def fields(self, drop=()): + """The fields one create asks for, without the optional fields Amazon refused.""" + return tuple(f for f in self.core + self.optional if f not in drop) + + +#: ⛔ Every requested dimension is a group-by and sets the row grain. A group asks only for its +#: tables' identity dimensions, attributes of those, metrics, the ad product and required partners. +#: Every core field lists every ad product of its group in the library: SB lacks the SKU field and +#: the marketplace field `advertisedProduct.id` requires, so SB advertised products is its own report. +REPORT_GROUPS = ( + ReportGroup("campaigns", ("sp_campaigns", "sb_campaigns", "sd_campaigns"), + _HEAD + ("campaign.deliveryStatus", "campaign.budgetAmount") + _TAIL_FIELDS), + ReportGroup("targeting", ("sp_targeting", "sb_targeting", "sd_targeting"), + _HEAD + _AD_GROUP + ("adGroupTarget.id",) + _TAIL_FIELDS, + _TARGET_TEXT + ("adGroupTarget.bid",)), + ReportGroup("advertised_products", ("sp_advertised_products", "sd_advertised_products"), + _HEAD + _AD_GROUP + ("ad.id", "advertisedProduct.id", "advertisedProductMarketplace.value", + "advertisedProductSku.value") + _TAIL_FIELDS), + ReportGroup("advertised_products_sb", ("sb_advertised_products",), + _HEAD + _AD_GROUP + ("ad.id",) + _TAIL_FIELDS), + ReportGroup("search_terms", ("sp_search_terms", "sb_search_terms"), + _HEAD + _AD_GROUP + ("adGroupTarget.id", "searchTerm.value") + _TAIL_FIELDS, _TARGET_TEXT), +) +GROUPS = {group.name: group for group in REPORT_GROUPS} +_GROUP_OF = {table: group for group in REPORT_GROUPS for table in group.members} def _short(ad_type, report): @@ -89,25 +137,24 @@ def coltype(name): return "BIGINT" if name in _INT else "DOUBLE" if name in _DBL else "VARCHAR" -def vendor_name(ad_type, column): - """The Ads column one of our columns is read from, or None when the vendor has none.""" - if column == "profile_id": - return None - return _VENDOR_BY_TYPE.get(ad_type, {}).get(column) or _VENDOR.get(column) +def vendor_name(column): + """The v1 field one of our columns is read from, or None when the vendor has none.""" + return _VENDOR.get(column) TABLE_SPECS = {} for _spec in _cs.AMAZON_ADS_TABLES: _ad_type, _report = _spec.key[len(DATABASE_PREFIX) + len("ads_"):].split("_", 1) + _group = _GROUP_OF.get(_short(_ad_type, _report)) TABLE_SPECS[_short(_ad_type, _report)] = { "key": _spec.key, "source": _spec.source, "label": _spec.label, - "ad_type": _ad_type, "report": _report, "report_type": _spec.native, + "ad_type": _ad_type, "report": _report, "group": _group.name if _group else None, "columns": _columns(_report), "row_identity": tuple(_spec.identity), "backfill_days": _cs.AMAZON_ADS_BACKFILL_DAYS[_spec.key], "offered": _spec.key not in _cs.AMAZON_ADS_NOT_OFFERED, "not_offered_reason": _cs.AMAZON_ADS_NOT_OFFERED.get(_spec.key), } -del _spec, _ad_type, _report +del _spec, _ad_type, _report, _group def short_name(table): @@ -127,11 +174,16 @@ def columns_for(table): return TABLE_SPECS[short_name(table)]["columns"] -def vendor_columns(table): - """The Ads columns one report asks for, in our column order, each once.""" - spec = TABLE_SPECS[short_name(table)] - names = (vendor_name(spec["ad_type"], c) for c in spec["columns"]) - return tuple(dict.fromkeys(n for n in names if n)) +def group_of(table): + """The report group that fills one Ads table, or None for a table v1 does not offer.""" + return _GROUP_OF.get(short_name(table)) + + +def group_for(fields, ad_products): + """The report group a create request asks for, by its fields and ad product filter, or None.""" + fields, ad_products = set(fields), set(ad_products) + return next((g for g in REPORT_GROUPS if set(g.tables) == ad_products + and set(g.core) <= fields <= set(g.core + g.optional)), None) def _cell(name, value): @@ -149,11 +201,11 @@ def _cell(name, value): def row_from_vendor(table, vendor_row, *, profile_id=None, report_id=None): - """One Ads report row -> one stored row, with its stable `_row_key` and report origin.""" + """One flat v1 report row (dotted field ids) -> one stored row, with its `_row_key` and report origin.""" spec = TABLE_SPECS[short_name(table)] out = {} for column in spec["columns"]: - source = vendor_name(spec["ad_type"], column) + source = vendor_name(column) value = vendor_row.get(source) if source else None if column == "profile_id": value = profile_id @@ -213,7 +265,8 @@ def stream_coltype(name): return "DOUBLE" return "VARCHAR" -__all__ = ["AD_TYPES", "REPORTS", "C6_COLUMNS", "LABELS", "TABLE_SPECS", "MAX_DAYS_PER_REQUEST", +__all__ = ["AD_TYPES", "REPORTS", "C6_COLUMNS", "LABELS", "TABLE_SPECS", "AD_PRODUCT_FIELD", "AD_PRODUCTS", + "LIBRARY_AD_PRODUCTS", "METRICS", "ReportGroup", "REPORT_GROUPS", "GROUPS", "ORIGIN_COLUMNS", "coltype", "vendor_name", "short_name", "columns_for", - "vendor_columns", "row_from_vendor", "row_key", "STREAM_WINDOWS", "STREAM_METRICS", "STREAM_GRAIN", + "group_of", "group_for", "row_from_vendor", "row_key", "STREAM_WINDOWS", "STREAM_METRICS", "STREAM_GRAIN", "STREAM_COLUMNS", "STREAM_TABLE_SPECS", "stream_spec", "stream_coltype"] diff --git a/platform/harness/amazon_ads_store.py b/platform/harness/amazon_ads_store.py index c7a11c0375fef456a22e65181063ef78c292d039..e470ff2df3ad201d5092a607fc581dc10bae89e8 100644 --- a/platform/harness/amazon_ads_store.py +++ b/platform/harness/amazon_ads_store.py @@ -10,8 +10,8 @@ back the longest window Amazon documents per table (`connector_spec.AMAZON_ADS_B A later sync re-reads the table's daily or intraday cadence window and upserts by row identity. No sync deletes a day an earlier sync stored, so a day Amazon has aged out stays here. -⛔ STANDING RULE 1. Every report is read whole and paged. The limits that cannot be removed (one -report at a time, about 100 a day, a report v1 does not offer) are written to `_sync_state` as +⛔ STANDING RULE 1. Every report is read whole, every file of it. The limits that cannot be removed +(one report at a time per client ID, about 100 a day, a report v1 does not offer) are written to `_sync_state` as `amzn_limit_ads_` with the reason a person reads, and stated on the table description. Receipts use the Amazon mirror's markers: `amzn_sync_mark_ads_
` is the newest landed day, @@ -178,7 +178,7 @@ def _day_marker(con, key): def windows(con, table, connect_day, today, connection_id=None): - """The report windows one sync asks for, oldest first, each at most 31 days.""" + """The report windows one sync asks for, oldest first, each at most 120 days (`client.MAX_DAYS_PER_REQUEST`).""" table = schema.short_name(table) spec = schema.TABLE_SPECS[table] cadence = amazon_client.CADENCE["ads_" + table] @@ -237,13 +237,13 @@ def _upsert(con, table, rows): cols = _columns_for(table) by_key = {r["_row_key"]: r for r in rows} keys = list(by_key) + one = f'({", ".join("?" for _ in cols)})' for i in range(0, len(keys), INSERT_BATCH_ROWS): part = keys[i:i + INSERT_BATCH_ROWS] con.execute(f'DELETE FROM "{name}" WHERE "_row_key" IN ({",".join("?" for _ in part)})', part) - con.executemany( - f'INSERT INTO "{name}" ({", ".join(chr(34) + c + chr(34) for c in cols)}) ' - f'VALUES ({", ".join("?" for _ in cols)})', - [[by_key[k].get(c) for c in cols] for k in part]) + # One multi-row INSERT per batch: DuckDB's executemany is about 5 times slower at 24 months a pass. + con.execute(f'INSERT INTO "{name}" ({", ".join(chr(34) + c + chr(34) for c in cols)}) ' + f'VALUES {", ".join(one for _ in part)}', [by_key[k].get(c) for k in part for c in cols]) return len(by_key) @@ -389,16 +389,158 @@ def _land(con, table, rows, report_id, start, end, profile, connection_id, label def _run_report(key, table, start, end, rt, profile_id, connection_id, **extra): - """Require named transport support instead of sending a sibling's credential.""" + """Run the group report that fills `table` under this connection's own credential. -> GroupReport.""" options = {"rt": rt, "profile_id": profile_id, **extra} if connection_id: options["connection_id"] = connection_id - try: - return client.run_report(key, table, start, end, **options) - except TypeError as exc: - if connection_id and "unexpected keyword argument 'connection_id'" in str(exc): - raise client.AdsError("The Amazon Ads client needs named connection support before this connection can sync.") from None - raise + return client.run_group_report(key, schema.group_of(table), start, end, **options) + + +class _Pass: + """One connection's sync pass: its budget, clock, and what its group reports already covered. + + `eligible` holds the tables a group report may land into this pass: requested, offered, due and + not yet finished. `covered` holds each table's spans landed this pass, `pulled` the rows a + sibling's report landed, `outcome` a group's failure, `notes` Amazon's text per table. + """ + + def __init__(self, con, key, rt, connection, report, *, today, used, deadline, stamp, checkpointed): + self.con, self.key, self.rt, self.report = con, key, rt, report + self.cid, self.label, self.profile = connection[:3] + self.today, self.used, self.deadline, self.stamp = today, used, deadline, stamp + self.checkpointed = checkpointed + self.eligible, self.covered, self.pulled, self.outcome, self.notes = set(), {}, {}, {}, {} + self.blocked = False + + def wait(self, spec, reason): + self.report["waits"].append(f"{self.label}: {spec['label']}: {reason}") + + def checkpoint(self, table): + if table_name(table) not in self.checkpointed: + self.checkpointed.append(table_name(table)) + + def covers(self, table, start, end): + """True when reports this pass already read every day from `start` to `end` for `table`.""" + spans = self.covered.get(table, ()) + return all(any(lo <= start + timedelta(days=i) <= hi for lo, hi in spans) + for i in range((end - start).days + 1)) + + def bounds(self): + """The poll limit for one new report: none, or the time left in this bounded pass.""" + return {} if self.deadline is None else {"max_wait": max(0.0, self.deadline - _clock()), "now": _clock} + + def fail(self, table, exc): + """Record a group report's failure for every member. -> (state, reason) for `table`.""" + self.outcome[schema.group_of(table).name] = exc + return self.failure(table, exc) + + def failure(self, table, exc): + spec = schema.TABLE_SPECS[table] + if isinstance(exc, client.AdsReportUnavailable): + return "not_offered", _not_offered_reason(spec, exc) + reason = str(exc)[:client.ERROR_TEXT_LIMIT] + self.report["problems"].append(f"{self.label}: {spec['label']}: {reason}") + return "failed", reason + + def land(self, table, result, start, end): + """Land one group report into `table` and each eligible sibling, covering the span. -> rows into `table`.""" + own = 0 + for member in schema.group_of(table).members: + if member != table and member not in self.eligible: + continue + _ensure_table(self.con, member) + rows = _land(self.con, member, result.rows.get(member, []), result.report_id, start, end, + self.profile, self.cid, self.label) + self.covered.setdefault(member, []).append((start, end)) + self.notes[member] = list(dict.fromkeys(self.notes.get(member, []) + list(result.notes))) + self.checkpoint(member) + if member == table: + own = rows + else: + self.pulled[member] = self.pulled.get(member, 0) + rows + for note in result.notes: + if f"{self.label}: {note}" not in self.report["problems"]: + self.report["problems"].append(f"{self.label}: {note}") + return own + + +def _resume(p): + """Read every report this connection left preparing before it creates another. -> {table: outcome}.""" + resumed = {} + for table in TABLES: + pending = _read_pending(p.con, table, p.cid) + if pending is None: + continue + spec = schema.TABLE_SPECS[table] + _ensure_table(p.con, table) + p.checkpoint(table) + expired = f"Amazon Ads report {pending['report_id']} did not finish in 3 hours" + age = (p.stamp() - pending["created"]).total_seconds() + if age >= client.REPORT_MAX_WAIT_SECONDS: + _clear_pending(p.con, table, p.cid) + resumed[table] = (*p.failure(table, client.AdsError(expired)), 0) + continue + limit = client.REPORT_MAX_WAIT_SECONDS - age + if p.deadline is not None: + limit = min(limit, max(0.0, p.deadline - _clock())) + try: + result = _run_report(p.key, table, pending["start"], pending["end"], p.rt, p.profile, p.cid, + max_wait=limit, report_id=pending["report_id"], now=_clock) + except client.AdsReportPending: + if (p.stamp() - pending["created"]).total_seconds() < client.REPORT_MAX_WAIT_SECONDS: + p.blocked = True + p.wait(spec, PREPARING_REASON) + resumed[table] = ("deferred", PREPARING_REASON, 0) + continue + _clear_pending(p.con, table, p.cid) + resumed[table] = (*p.failure(table, client.AdsError(expired)), 0) + continue + except client.AdsError as exc: + _clear_pending(p.con, table, p.cid) + resumed[table] = (*p.fail(table, exc), 0) + continue + _clear_pending(p.con, table, p.cid) + resumed[table] = ("done", "", p.land(table, result, pending["start"], pending["end"])) + return resumed + + +def _run_windows(p, table, connect_day): + """Run the group report for each window of `table` no report covered yet. -> (state, reason, rows).""" + spec = schema.TABLE_SPECS[table] + failed = p.outcome.get(schema.group_of(table).name) + if failed is not None: + return (*p.failure(table, failed), 0) + pulled = 0 + for start, end in windows(p.con, table, connect_day, p.today, p.cid): + if p.covers(table, start, end): + continue + if p.used >= client.REPORTS_PER_DAY: + reason = (f"{connector_spec.AMAZON_ADS_THROUGHPUT} Today's {client.REPORTS_PER_DAY} " + f"reports are used, so days from {start.isoformat()} load on the next sync.") + p.report["problems"].append(f"{p.label}: {spec['label']}: waiting for tomorrow's report limit") + p.report["waits"].append(p.report["problems"][-1]) + return "deferred", reason, pulled + if p.deadline is not None and _clock() >= p.deadline: + p.wait(spec, TIME_LIMIT_REASON) + return "deferred", TIME_LIMIT_REASON, pulled + p.used += 1 + _spend_budget(p.con, p.today, p.used, p.cid) + created = p.stamp() + try: + result = _run_report(p.key, table, start, end, p.rt, p.profile, p.cid, **p.bounds()) + except client.AdsReportPending as exc: + if (p.deadline is None or not exc.report_id + or (p.stamp() - created).total_seconds() >= client.REPORT_MAX_WAIT_SECONDS): + return (*p.failure(table, exc), pulled) + _write_pending(p.con, table, p.cid, exc.report_id, start, end, created) + p.checkpoint(table) + p.blocked = True + p.wait(spec, PREPARING_REASON) + return "deferred", PREPARING_REASON, pulled + except client.AdsError as exc: + return (*p.fail(table, exc), pulled) + pulled += p.land(table, result, start, end) + return "done", "", pulled def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None, log=print, @@ -409,6 +551,7 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None through `amazon_store.persist_mirror`. A caller that lends `con` persists itself. `max_seconds` bounds the pass's wait on Amazon. A report still preparing then waits in the pending ledger, and the next pass reads it before it creates any other report. + One group report fills every requested member table, so a sibling's covered days spend no report. """ key = amazon_store._tenant_key(tenant_key) report = {"tenant": key, "tables": {}, "problems": [], "limits": {}, "connections": [], @@ -425,10 +568,9 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None own = con is None if own: con, _path = amazon_store._connect_tenant_mirror(key, rt) - bounded = max_seconds is not None started = _clock() - deadline = started + float(max_seconds) if bounded else None - waits = report["waits"] + deadline = None if max_seconds is None else started + float(max_seconds) + synced, checkpointed = [], [] try: datastore._ensure_sync_state(con) now = _utc_now(now) @@ -437,164 +579,24 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None def stamp(): return now + timedelta(seconds=max(0.0, _clock() - started)) - def wait(label, spec, reason): - waits.append(f"{label}: {spec['label']}: {reason}") - - synced = [] - checkpointed = [] - for cid, label, profile, legacy in connections: + requested = [schema.short_name(t) for t in (tables or TABLES)] + for connection in connections: + cid, label, _profile, legacy = connection connect_day = _connect_day(con, today, cid, legacy=legacy) report.setdefault("connectDay", connect_day.isoformat()) entry = {"id": cid, "label": label, "connectDay": connect_day.isoformat(), "tables": {}} report["connections"].append(entry) - used = _budget(con, today, cid, legacy=legacy) - # Resume every report this connection left preparing before it creates another one. - resumed, blocked = {}, False - for table in TABLES: - pending = _read_pending(con, table, cid) - if pending is None: - continue - spec = schema.TABLE_SPECS[table] - _ensure_table(con, table) - if table_name(table) not in checkpointed: - checkpointed.append(table_name(table)) - age = (stamp() - pending["created"]).total_seconds() - expired = (f"Amazon Ads report {pending['report_id']} did not finish in 3 hours") - if age >= client.REPORT_MAX_WAIT_SECONDS: - _clear_pending(con, table, cid) - report["problems"].append(f"{label}: {spec['label']}: {expired}") - resumed[table] = ("failed", expired, 0, None) - continue - limit = client.REPORT_MAX_WAIT_SECONDS - age - if bounded: - limit = min(limit, max(0.0, deadline - _clock())) - try: - report_id, rows = _run_report(key, table, pending["start"], pending["end"], rt, profile, cid, - max_wait=limit, report_id=pending["report_id"], now=_clock) - except client.AdsReportUnavailable as exc: - _clear_pending(con, table, cid) - resumed[table] = ("not_offered", _not_offered_reason(spec, exc), 0, None) - continue - except client.AdsReportPending: - if (stamp() - pending["created"]).total_seconds() >= client.REPORT_MAX_WAIT_SECONDS: - _clear_pending(con, table, cid) - report["problems"].append(f"{label}: {spec['label']}: {expired}") - resumed[table] = ("failed", expired, 0, None) - else: - blocked = True - wait(label, spec, PREPARING_REASON) - resumed[table] = ("deferred", PREPARING_REASON, 0, None) - continue - except client.AdsError as exc: - _clear_pending(con, table, cid) - reason = str(exc)[:200] - report["problems"].append(f"{label}: {spec['label']}: {reason}") - resumed[table] = ("failed", reason, 0, None) - continue - _clear_pending(con, table, cid) - pulled = _land(con, table, rows, report_id, pending["start"], pending["end"], profile, cid, label) - resumed[table] = ("done", "", pulled, (pending["start"], pending["end"])) - requested = [schema.short_name(t) for t in (tables or TABLES)] + p = _Pass(con, key, rt, connection, report, today=today, used=_budget(con, today, cid, legacy=legacy), + deadline=deadline, stamp=stamp, checkpointed=checkpointed) + p.eligible = {t for t in requested if schema.TABLE_SPECS[t]["offered"] and _cadence_due(con, t, now, cid)} + resumed = _resume(p) for table in list(dict.fromkeys(requested + list(resumed))): - spec = schema.TABLE_SPECS[table] - _ensure_table(con, table) - origin_changed = _refresh_origin(con, table, cid, label, legacy) - pulled, state, mark, reason = 0, "done", _marker_table(table), "" - prior = resumed.get(table) - landed = None - if prior is not None: - state, reason, pulled, landed = prior - if state != "done" or table not in requested: - pass - elif not spec["offered"]: - state, reason = "not_offered", spec["not_offered_reason"] - elif not _cadence_due(con, table, now, cid): - state = "skipped" - elif blocked: - state, reason = "deferred", PREPARING_REASON - wait(label, spec, reason) - else: - for start, end in windows(con, table, connect_day, today, cid): - if landed and landed[0] <= start and end <= landed[1]: - continue # The resumed report just read these days. - if used >= client.REPORTS_PER_DAY: - state = "deferred" - reason = (f"{connector_spec.AMAZON_ADS_THROUGHPUT} Today's {client.REPORTS_PER_DAY} " - f"reports are used, so days from {start.isoformat()} load on the next sync.") - report["problems"].append(f"{label}: {spec['label']}: waiting for tomorrow's report limit") - waits.append(report["problems"][-1]) - break - if bounded and _clock() >= deadline: - state, reason = "deferred", TIME_LIMIT_REASON - wait(label, spec, reason) - break - used += 1 - _spend_budget(con, today, used, cid) - created = stamp() - try: - if bounded: - report_id, rows = _run_report(key, table, start, end, rt, profile, cid, - max_wait=max(0.0, deadline - _clock()), now=_clock) - else: - report_id, rows = _run_report(key, table, start, end, rt, profile, cid) - except client.AdsReportUnavailable as exc: - state = "not_offered" - reason = _not_offered_reason(spec, exc) - break - except client.AdsError as exc: - if (bounded and isinstance(exc, client.AdsReportPending) and exc.report_id - and (stamp() - created).total_seconds() < client.REPORT_MAX_WAIT_SECONDS): - _write_pending(con, table, cid, exc.report_id, start, end, created) - if table_name(table) not in checkpointed: - checkpointed.append(table_name(table)) - blocked = True - state, reason = "deferred", PREPARING_REASON - wait(label, spec, reason) - break - state = "failed" - reason = str(exc)[:200] - report["problems"].append(f"{label}: {spec['label']}: {reason}") - break - pulled += _land(con, table, rows, report_id, start, end, profile, cid, label) - if table_name(table) not in checkpointed: - checkpointed.append(table_name(table)) - if state == "done": - _record_cadence(con, table, now, cid) - if state == "deferred" and reason in (PREPARING_REASON, TIME_LIMIT_REASON): - report["partial"] = True - if reason: - _record_limit(con, table, reason, cid, state=state) - shown = f"{label}: {reason}" if cid else reason - report["limits"][table] = " ".join(filter(None, (report["limits"].get(table), shown))) - elif state == "done": - _clear_limit(con, table, cid) - total = _count(con, table) - entry["tables"][table] = {"state": state, "pulled": pulled, "in_mirror": _count(con, table, cid)} - if state != "skipped": - amazon_store._record_state(con, _state_key(amazon_store.entity_key(mark), cid), - "done" if state == "done" else "partial", _count(con, table, cid), state) - previous = report["tables"].get(table, {}) - severity = {"skipped": -1, "done": 0, "not_offered": 1, "deferred": 2, "failed": 3} - aggregate = max((state, previous.get("state", state)), key=severity.get) - report["tables"][table] = {"state": aggregate, "pulled": previous.get("pulled", 0) + pulled, - "in_mirror": total} - if (pulled or origin_changed) and table_name(table) not in synced: + landed = _sync_table(p, table, entry, connect_day, now, legacy, table in requested, + resumed.get(table), log) + p.eligible.discard(table) + if landed and table_name(table) not in synced: synced.append(table_name(table)) - log(f" {table_name(table):<36} pulled {pulled:>7} mirror total {total:>7} {state}") - for table, state in report["tables"].items(): - if state["state"] == "skipped": - continue - amazon_store._record_state(con, amazon_store.entity_key(_marker_table(table)), - "done" if state["state"] == "done" else "partial", - state["in_mirror"], state["state"]) - if table in report["limits"]: - _record_limit(con, table, report["limits"][table], state=state["state"]) - else: - _clear_limit(con, table) - report["pendingReports"] = sum(_read_pending(con, table, cid) is not None - for cid, *_ in connections for table in TABLES) - if report["partial"]: - report["continueAfterSeconds"] = CONTINUE_AFTER_SECONDS + _record_tables(con, report, connections) if own and rt is not None and (synced or checkpointed): _persist(key, rt, con, list(dict.fromkeys(synced + checkpointed)), report, log) finally: @@ -604,10 +606,75 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None return report +_SEVERITY = {"skipped": -1, "done": 0, "not_offered": 1, "deferred": 2, "failed": 3} + + +def _sync_table(p, table, entry, connect_day, now, legacy, requested, prior, log): + """One table's turn in one connection's pass: run or skip it, record its state. -> True if it changed.""" + con, cid, label, report = p.con, p.cid, p.label, p.report + spec = schema.TABLE_SPECS[table] + _ensure_table(con, table) + origin_changed = _refresh_origin(con, table, cid, label, legacy) + state, reason, pulled = prior or ("done", "", 0) + pulled += p.pulled.pop(table, 0) + if state != "done" or not requested: + pass + elif not spec["offered"]: + state, reason = "not_offered", spec["not_offered_reason"] + elif not _cadence_due(con, table, now, cid): + state = "skipped" + elif p.blocked: + state, reason = "deferred", PREPARING_REASON + p.wait(spec, reason) + else: + state, reason, rows = _run_windows(p, table, connect_day) + pulled += rows + if state == "done": + _record_cadence(con, table, now, cid) + if state == "done" and not reason and p.notes.get(table): + reason = " ".join(p.notes[table]) + if state == "deferred" and reason in (PREPARING_REASON, TIME_LIMIT_REASON): + report["partial"] = True + if reason: + _record_limit(con, table, reason, cid, state=state) + shown = f"{label}: {reason}" if cid else reason + report["limits"][table] = " ".join(filter(None, (report["limits"].get(table), shown))) + elif state == "done": + _clear_limit(con, table, cid) + total = _count(con, table) + entry["tables"][table] = {"state": state, "pulled": pulled, "in_mirror": _count(con, table, cid)} + if state != "skipped": + amazon_store._record_state(con, _state_key(amazon_store.entity_key(_marker_table(table)), cid), + "done" if state == "done" else "partial", _count(con, table, cid), state) + previous = report["tables"].get(table, {}) + report["tables"][table] = {"state": max((state, previous.get("state", state)), key=_SEVERITY.get), + "pulled": previous.get("pulled", 0) + pulled, "in_mirror": total} + log(f" {table_name(table):<36} pulled {pulled:>7} mirror total {total:>7} {state}") + return bool(pulled or origin_changed) + + +def _record_tables(con, report, connections): + """Write each table's aggregate state and limit across connections, and the pass's pending count.""" + for table, state in report["tables"].items(): + if state["state"] == "skipped": + continue + amazon_store._record_state(con, amazon_store.entity_key(_marker_table(table)), + "done" if state["state"] == "done" else "partial", + state["in_mirror"], state["state"]) + if table in report["limits"]: + _record_limit(con, table, report["limits"][table], state=state["state"]) + else: + _clear_limit(con, table) + report["pendingReports"] = sum(_read_pending(con, table, cid) is not None + for cid, *_ in connections for table in TABLES) + if report["partial"]: + report["continueAfterSeconds"] = CONTINUE_AFTER_SECONDS + + def _not_offered_reason(spec, exc): return (f"Amazon Reporting v1 does not offer {spec['label']} to this advertiser " - f"({str(exc)[:160]}). The table keeps the days it has. To ask for it, " - f"contact Amazon Ads support with the report type {spec['report_type']}.") + f"({str(exc)[:client.ERROR_TEXT_LIMIT]}). The table keeps the days it has. To ask for it, " + f"contact Amazon Ads support and quote Amazon's message.") def _persist(tenant_key, rt, con, synced, report, log): diff --git a/platform/harness/amazon_client.py b/platform/harness/amazon_client.py index e23212c763efad0f852ff17ec45788958e394af6..e20946f56b1df78ff28148fc4ef15aaed350ac5e 100644 --- a/platform/harness/amazon_client.py +++ b/platform/harness/amazon_client.py @@ -450,11 +450,8 @@ REPORT_TYPES = { "report": "GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT", "history": {"from": "2025-04-01", "evidence": "Monthly FATAL, the period may fall outside the available data range, for 2021-09, " - "2024-10, 2024-11, 2025-01 and 2025-03; 2025-04 onward landed (2026-11-14)", - "periods": {"WEEK": {"from": "2025-07-13", "provisional": True, - "evidence": "2026-11-14 probe refused 2025-07-06 and served 2025-10-05. " - "The intervening twelve weeks are unasked. 2025-07-13 is the " - "first unasked Sunday, not a proved earliest served floor"}}}, + "2024-10, 2024-11, 2025-01 and 2025-03; 2025-04 onward landed. Weekly FATAL through the " + "week of 2025-07-06, DONE 2025-10-05 (2026-11-14)"}, "role": "Brand Analytics", "validates_against": "Amazon dataByAsin row count and ASIN purchases per batch vs stored rows for the latest month", "json_root": "dataByAsin", "monthly": True, @@ -1127,7 +1124,7 @@ for _tables, _passes, _daily, _intraday in ( from harness import amazon_ads_schema as _ads_schema CADENCE.update({ - "ads_" + table: {"perDay": 4, "lookbackDays": 21, "intradayLookbackDays": 5, "period": "day"} + "ads_" + table: {"perDay": 4, "lookbackDays": 60, "intradayLookbackDays": 5, "period": "day"} for table in _ads_schema.TABLE_SPECS }) del _table, _tables, _spec, _passes, _daily, _intraday, _ads_schema diff --git a/platform/harness/amazon_schema.py b/platform/harness/amazon_schema.py index 3bb62ba261fee6cdf8f38e2e32538e815988f0c0..aee3e372d128fbbedf4f630973a3c3cd07d97bfe 100644 --- a/platform/harness/amazon_schema.py +++ b/platform/harness/amazon_schema.py @@ -95,8 +95,6 @@ PROVENANCE = { "search_query_performance": "measured_at_sync", "sku_keywords": "derived", "keyword_ranks": "apify_dataset", - "competitor_snapshots": "competitor_run", - "competitor_images": "competitor_run", "fee_estimates_api": "measured_at_sync", } @@ -107,13 +105,10 @@ PROVENANCE.update(dict.fromkeys((t + "_history" for t in connector_spec.AMAZON_H REVIEW_UNAVAILABLE_NOTE = ( "Amazon's API does not supply written reviews, average star rating or review count. " "Read them on the product page or in Seller Central.") -SQP_LIMIT_NOTE = "Amazon lists at most 100 searches a month for each product." #: Human copy for the disclosure screen, per provenance. ⛔ NO EM DASH OR EN DASH: this reaches a #: screen and standing rule 2 governs it. PROVENANCE_COPY = { - "competitor_run": ("Daily competitor facts saved by the competitor run. Every snapshot day is kept. " - "Image rows keep each distinct variant and link, dated when first saved."), "apify_dataset": ("Apify Amazon search dataset joined to each SKU's latest matching keyword row on or before the capture date. " "Sponsored rank uses the vendor rank or ordered sponsored placements in the captured dataset. " "Global position is separate. Unmatched observations remain in the raw mirror."), @@ -308,8 +303,8 @@ def catalogue(): descriptor = descriptors[key] label, native = descriptor.label, descriptor.native table[key] = {"label": label, "native": native} - if provenance in {"derived", "apify_dataset", "competitor_run"}: - table["columns"] = [dict(field) for field in AMAZON_FIELDS.get(key, ()) + if provenance in {"derived", "apify_dataset"}: + table["columns"] = [dict(field) for field in AMAZON_FIELDS[key] if field["name"] != STORE] table["note"] = PROVENANCE_COPY[provenance] table["local_columns"] = [dict(field) for field in AMAZON_FIELDS.get(key, ()) @@ -319,7 +314,7 @@ def catalogue(): def columns_for(table): """The pinned column list for one table, as `[(name, type)]`. Empty when measured at sync.""" - if PROVENANCE.get(str(table)) in {"derived", "apify_dataset", "competitor_run"}: + if PROVENANCE.get(str(table)) in {"derived", "apify_dataset"}: return [] spec = (catalogue().get("tables") or {}).get(str(table)) or {} return [(c["name"], c.get("type") or "VARCHAR") for c in (spec.get("columns") or [])] @@ -382,17 +377,17 @@ def _table_spec(table, identity, relationships=(), *, grain="entity", line_discr fields = AMAZON_FIELDS.get(DATABASE_PREFIX + table, ()) spec["columns"].extend(dict(field) for field in fields if field["name"] != STORE) spec["local_columns"] = [dict(field) for field in fields if field["name"] == STORE] - local_sources = { - "derived": ("listing_items", "search_query_performance"), - "apify_dataset": ("apify:khadinakbar/amazon-search-scraper", "sku_keywords"), - "competitor_run": ("amazon_rivals.run_quotes", "amazon_quotes_loop.run_tenant"), - } - provenance = PROVENANCE[table] - if provenance in local_sources: + if PROVENANCE[table] == "derived": + spec["columns_source"] = { + "path": "platform/core/connector_spec.py", "reader": "connector_spec.AMAZON_FIELDS", + "provenance": "derived", "assumption": PROVENANCE_COPY["derived"], + "sources": ("listing_items", "search_query_performance"), + } + if PROVENANCE[table] == "apify_dataset": spec["columns_source"] = { "path": "platform/core/connector_spec.py", "reader": "connector_spec.AMAZON_FIELDS", - "provenance": provenance, "assumption": PROVENANCE_COPY[provenance], - "sources": local_sources[provenance], + "provenance": "apify_dataset", "assumption": PROVENANCE_COPY["apify_dataset"], + "sources": ("apify:khadinakbar/amazon-search-scraper", "sku_keywords"), } return spec @@ -575,13 +570,9 @@ TABLE_SPECS.update({ "browse_node_review_topics": _table_spec("browse_node_review_topics", ("browse_node_id", "marketplace_id", "sort_by", "sentiment", "topic"), grain="browse_node_sort_sentiment_topic"), "sku_keywords": _table_spec("sku_keywords", ("keyword_id",), - (("sku", "listing_items", "sku", "many_to_one"),), grain="sku_period_asin_query"), + (("sku", "listing_items", "sku", "many_to_one"),), grain="sku_month_asin_query"), "keyword_ranks": _table_spec("keyword_ranks", ("rank_id",), (("keyword_id", "sku_keywords", "keyword_id", "many_to_one"),), grain="keyword_placement_per_sku"), - "competitor_snapshots": _table_spec("competitor_snapshots", ("asin", "snapshot_date"), - grain="asin_day"), - "competitor_images": _table_spec("competitor_images", ("asin", "variant", "link"), - grain="asin_variant_link"), }) # A history table is one row per version, identified by its version key and first valid date. diff --git a/platform/harness/amazon_store.py b/platform/harness/amazon_store.py index 67b28d1ef525ab432cddcf99bec5ae3830761134..bfc706768046d164133cf70140c1039b6560d79a 100644 --- a/platform/harness/amazon_store.py +++ b/platform/harness/amazon_store.py @@ -57,7 +57,6 @@ import threading import uuid from contextlib import closing, contextmanager from datetime import datetime, timedelta, timezone -from decimal import Decimal, InvalidOperation from pathlib import Path _HERE = Path(__file__).resolve().parent @@ -92,7 +91,7 @@ DERIVED_TABLES = ("sku_keywords", "keyword_ranks") #: Snapshot reports whose changes are kept as versions in `amzn__history`. HISTORY_SOURCES = tuple(connector_spec.AMAZON_HISTORY_SOURCES) HISTORY_TABLES = tuple(t + "_history" for t in HISTORY_SOURCES) -DATABASE_TABLES = TABLES + DERIVED_TABLES + HISTORY_TABLES + ("competitor_snapshots", "competitor_images") +DATABASE_TABLES = TABLES + DERIVED_TABLES + HISTORY_TABLES PERIOD_TABLES = MARGIN_TABLES + GROUP_A_TABLES LEGACY_SNAPSHOT_TABLES = ("listings", "inventory") @@ -2021,35 +2020,18 @@ def replace_keys_for(table): if c not in _ORIGIN_TYPES) -@contextmanager -def _report_transaction(con): - con.execute("BEGIN TRANSACTION") - try: - yield - con.execute("COMMIT") - except Exception: - con.execute("ROLLBACK") - raise - - -def replace_window(con, table, cols, types, rows, keys, after=None, before=None): +def replace_window(con, table, cols, types, rows, keys, after=None): """Replace incoming key groups atomically, or report why only insertion is possible. `after(con)` runs inside the same transaction, after the insert and before the commit. """ missing = [c for c in keys if c not in cols] - if missing or not keys or not rows: + if missing: note = (f"{table}: window replacement unavailable; missing identity columns: " - f"{', '.join(missing)}; rows inserted; use a report carrying these columns") if missing else None - if before is None and after is None: - return _replace(con, table, cols, types, rows), note - with _report_transaction(con): - if before is not None: - before(con) - count = _replace(con, table, cols, types, rows) - if after is not None: - after(con) - return count, note + f"{', '.join(missing)}; rows inserted; use a report carrying these columns") + return _replace(con, table, cols, types, rows), note + if not keys or not rows: + return _replace(con, table, cols, types, rows), None quote = lambda name: '"' + str(name).replace('"', '""') + '"' target = quote(table) # The window's rows are staged in batches inside the transaction, typed as the mirror and @@ -2060,9 +2042,8 @@ def replace_window(con, table, cols, types, rows, keys, after=None, before=None) try: predicate = " AND ".join( f'{target}.{quote(c)} IS NOT DISTINCT FROM "k".{quote(c)}' for c in keys) - with _report_transaction(con): - if before is not None: - before(con) + con.execute("BEGIN TRANSACTION") + try: count = _replace(con, temporary, cols, types, rows, source_table=table) inserted = cols if table == table_name("brand_analytics") and {"_period_start", "_period_end"} <= set(keys): @@ -2074,6 +2055,10 @@ def replace_window(con, table, cols, types, rows, keys, after=None, before=None) con.execute(f"INSERT INTO {target} ({inserted_columns}) SELECT {inserted_columns} FROM {temporary}") if after is not None: after(con) + con.execute("COMMIT") + except Exception: + con.execute("ROLLBACK") + raise return count, None finally: con.execute(f"DROP TABLE IF EXISTS {temporary}") @@ -2617,11 +2602,7 @@ def sqp_progress_sentence(detail): if any(type(n) is not int or n < 0 for n in steps): raise ValueError("Invalid saved Search Query Performance progress") months_done, months, done, total, eta = steps - weeks_done, weeks = progress.get("weeksDone", 0), progress.get("weeksTotal", 0) - if any(type(n) is not int or n < 0 for n in (weeks_done, weeks)): - raise ValueError("Invalid saved Search Query Performance weekly progress") - weekly = f", {weeks_done:,} of {weeks:,} weeks complete" if weeks else "" - sentence = f"{months_done:,} of {months:,} months complete{weekly}, {done:,} of {total:,} batches." + sentence = f"{months_done:,} of {months:,} months complete, {done:,} of {total:,} batches." if eta: hours, minutes = round(eta / 3600), max(1, round(eta / 60)) left = (f"{hours:,} hours" if hours >= 2 else "1 minute" if minutes == 1 else f"{minutes} minutes") @@ -2649,7 +2630,7 @@ def _sqp_month(con, start, end): "period": period, "start": _utc(start).date().isoformat(), "end": (_utc(end) - timedelta(days=1)).date().isoformat(), - "batches": {}, "asins": {}, "refusedAsins": {}, "retries": {}} + "batches": {}, "asins": {}, "retries": {}} def _sqp_save_month(con, start, marker): @@ -2658,97 +2639,17 @@ def _sqp_save_month(con, start, marker): def _sqp_done(con, start, end, batch): - return not _sqp_missing_asins(_sqp_month(con, start, end), batch.split()) - - -def _sqp_missing_asins(marker, asins): - """Successful ASIN summaries and durable named exclusions complete a period independently of batching.""" - done = set(marker["asins"]) | set(marker.get("refusedAsins", {})) - for batch, retry in marker["retries"].items(): - done.update(set(batch.split()) - set(retry["asins"])) - return [asin for asin in asins if asin not in done] - - -def _sqp_owed_tasks(con, tasks): - """Batch only each period's missing ASINs, retaining unfinished retry identities across catalogue changes.""" - windows = {} - for task in tasks: - if task[0] == SQP_TABLE: - windows.setdefault(task[1:3], []).extend(task[3].split()) - result = [] - for task in tasks: - if task[0] != SQP_TABLE: - result.append(task) - continue - asins = windows.pop(task[1:3], None) - if asins is None: - continue - marker = _sqp_month(con, *task[1:3]) - missing = _sqp_missing_asins(marker, list(dict.fromkeys(asins))) - remaining, batches = set(missing), [] - for batch, retry in marker["retries"].items(): - requested = set(retry["asins"]) - if requested and requested <= remaining: - batches.append(batch) - remaining.difference_update(requested) - batches.extend(sqp_batches([asin for asin in missing if asin in remaining], - _client().REPORT_TYPES[SQP_TABLE]["asin_option_max_chars"])) - result.extend((SQP_TABLE, *task[1:3], batch) for batch in batches) - return result + return batch in _sqp_month(con, start, end)["batches"] def _sqp_week_tasks(start, end, batches): - """Schedule every completed week newest first through the same ASIN batch queue.""" + """Schedule the latest completed week through the same ASIN batch queue.""" if "WEEK" not in _client().REPORT_TYPES[SQP_TABLE].get("periods", ()): return [] - return [(SQP_TABLE, w0, w1, batch) for w0, w1 in reversed(_week_windows(start, end)) + return [(SQP_TABLE, w0, w1, batch) for w0, w1 in _week_windows(start, end)[-1:] for batch in batches] -def _sqp_history_done(con, end): - """Every admitted period and current catalogue ASIN needs its own durable completion evidence.""" - asins = _sqp_catalogue(con) - tasks = first_load_plan([SQP_TABLE], history_floor(con, SQP_TABLE, end), end, - sqp_asins=asins, starts=history_starts(con, [SQP_TABLE], end)) - return bool(asins) and all(_sqp_done(con, task[1], task[2], task[3]) for task in tasks) - - -def _sqp_progress(con, months, weeks, asins, batches, end): - """Count durable per-ASIN completion separately for admitted MONTH and WEEK windows.""" - from harness.amazon_client import REPORT_SUSTAINED_SECONDS - - progress = {} - for period, windows, unit, label in (("MONTH", months, "month", "%Y-%m"), - ("WEEK", weeks, "week", "%Y-%m-%d")): - floor = history_floor(con, SQP_TABLE, end, period=period) - rows = [] - for w0, w1 in windows: - if w0 < floor: - continue - missing = set(_sqp_missing_asins(_sqp_month(con, w0, w1), asins)) - rows.append({unit: w0.strftime(label), "total": len(batches), - "done": sum(not missing.intersection(batch.split()) for batch in batches)}) - progress[unit + "s"] = rows - progress[unit + "sDone"] = sum(r["done"] >= r["total"] for r in rows) - progress[unit + "sTotal"] = len(rows) - rows = progress["months"] + progress["weeks"] - done, total = sum(r["done"] for r in rows), sum(r["total"] for r in rows) - progress.update(batchesDone=done, batchesTotal=total, - etaSeconds=int((total - done) * REPORT_SUSTAINED_SECONDS)) - return progress if rows and batches else None - - -def _sqp_first_load_status(con, cohort): - """Project current period floors and per-ASIN evidence through the shared SQP progress shape.""" - asins = _sqp_catalogue(con) - spec = _client().REPORT_TYPES[SQP_TABLE] - months = _month_windows(history_floor(con, SQP_TABLE, cohort), max(cohort, datetime.now(timezone.utc))) - weeks = (_week_windows(history_floor(con, SQP_TABLE, cohort, period="WEEK"), cohort) - if "WEEK" in spec.get("periods", ()) else []) - progress = _sqp_progress(con, months, weeks, asins, sqp_batches(asins, spec["asin_option_max_chars"]), cohort) - return {"progress": progress, "sentence": sqp_progress_sentence({"progress": progress})} if progress else None - - class SQPBudgetDeferred(RuntimeError): """A batch or its single retry remains durable for the next bounded pass.""" @@ -2757,42 +2658,6 @@ class SQPPassDeferred(SQPBudgetDeferred): """A scheduled pass spent `SQP_PASS_REPORTS`; only SQP stops, other tables keep running.""" -def _sqp_document_rows(text, spec, requested, marker, meta): - """Validate the source document before flattening rows and computing per-ASIN totals.""" - text = _document_text(text) - doc = json.loads(text) - if not isinstance(doc, dict): - raise ValueError("Search Query Performance document must be a JSON object") - raw = doc.get(spec["json_root"]) - if not isinstance(raw, list) or any(not isinstance(r, dict) for r in raw): - raise ValueError("Search Query Performance document has no dataByAsin row array") - totals = {asin: {"count": 0, "purchases": 0} for asin in requested} - for row in raw: - asin = row.get("asin") - if not isinstance(asin, str): - raise ValueError("Search Query Performance ASIN must be a string") - if asin not in totals or row.get("startDate") != marker["start"] or row.get("endDate") != marker["end"]: - raise ValueError("Search Query Performance returned an ASIN or period outside the requested batch") - purchase_data = row.get("purchaseData") - if not isinstance(purchase_data, dict): - raise ValueError("Search Query Performance purchaseData must be a JSON object") - purchase = purchase_data.get("asinPurchaseCount") - if type(purchase) is not int or purchase < 0: - raise ValueError("Search Query Performance omitted a nonnegative ASIN purchase count") - totals[asin]["count"] += 1 - totals[asin]["purchases"] += purchase - columns, rows = parse_json_report(text, spec["json_root"], - document_id=meta.get("reportDocumentId"), report_type=spec["report"]) - seen = set() - for row in rows: - key = amazon_schema.row_identity_for(SQP_TABLE, row) - if key in seen: - raise ValueError("Search Query Performance returned a duplicate month, ASIN and search query") - seen.add(key) - row["_report_period"] = marker.get("period", "MONTH") - return columns, rows, totals, len(raw) - - def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem): """One completed period and batch using the shared report transport and pass budget. @@ -2823,7 +2688,6 @@ def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem): reason = reason.replace("\u2014", "; ").replace("\u2013", " to ") for asin in refused: limit["refusedAsins"][asin] = reason - marker.setdefault("refusedAsins", {})[asin] = reason problem(sqp_refusal_message(asin, reason)) if not refused: limit["errors"] = [reason] @@ -2843,15 +2707,39 @@ def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem): _sqp_save_month(con, start, marker) return 0 - columns, rows, totals, source_count = _sqp_document_rows(text, spec, requested, marker, meta) + # Read source evidence BEFORE flattening or typing. Missing/malformed JSON is never empty success. + text = _document_text(text) + doc = json.loads(text) + raw = doc.get(spec["json_root"]) + if not isinstance(raw, list) or any(not isinstance(r, dict) for r in raw): + raise ValueError("Search Query Performance document has no dataByAsin row array") + totals = {asin: {"count": 0, "purchases": 0} for asin in requested} + for row in raw: + asin = row.get("asin") + if asin not in totals or row.get("startDate") != marker["start"] or row.get("endDate") != marker["end"]: + raise ValueError("Search Query Performance returned an ASIN or period outside the requested batch") + purchase = (row.get("purchaseData") or {}).get("asinPurchaseCount") + if type(purchase) is not int or purchase < 0: + raise ValueError("Search Query Performance omitted a nonnegative ASIN purchase count") + totals[asin]["count"] += 1 + totals[asin]["purchases"] += purchase + columns, rows = parse_json_report(text, spec["json_root"], + document_id=meta.get("reportDocumentId"), report_type=spec["report"]) identity = amazon_schema.TABLE_SPECS[SQP_TABLE]["row_identity"] + seen = set() + for row in rows: + key = amazon_schema.row_identity_for(SQP_TABLE, row) + if key in seen: + raise ValueError("Search Query Performance returned a duplicate month, ASIN and search query") + seen.add(key) + row["_report_period"] = marker.get("period", "MONTH") columns = list(dict.fromkeys([*columns, *identity, "_report_period", "purchase_data_asin_purchase_count", *_ORIGIN_TYPES])) mirror = table_name(SQP_TABLE) types = _ensure(con, mirror, columns, rows, preserve_rows=True, declared={"purchase_data_asin_purchase_count": "BIGINT"}) marker["asins"].update(totals) - marker["batches"][batch] = {"asins": requested, "count": source_count, + marker["batches"][batch] = {"asins": requested, "count": len(raw), "purchases": sum(t["purchases"] for t in totals.values()), "reportDocumentId": meta["reportDocumentId"]} con.execute("BEGIN TRANSACTION") @@ -2895,24 +2783,6 @@ def _validate_sqp(con): return {"state": "refused", "reason": f"Search Query Performance reconciliation failed: {exc}"} -def _sku_keyword_source_fields(columns): - from harness.amazon_sqp_reads import report_period_sql - - fields = {"start_date": "start_date", "end_date": "end_date", "asin": "asin", - "search_query": "search_query_data_search_query", - "impressions": "impression_data_asin_impression_count", - "clicks": "click_data_asin_click_count", "cart_adds": "cart_add_data_asin_cart_add_count", - "purchases": "purchase_data_asin_purchase_count"} - expressions = [] - for name, source in fields.items(): - numeric = name in ("impressions", "clicks", "cart_adds", "purchases") - kind, cast = ("BIGINT", "TRY_CAST") if numeric else ("VARCHAR", "CAST") - expressions.append(f'{cast}("{source}" AS {kind}) AS {name}' if source in columns else - f'NULL::{kind} AS {name}') - period = report_period_sql(columns) if {"start_date", "end_date"} <= columns else "NULL::VARCHAR" - return ", ".join([*expressions, f"{period} AS _report_period"]) - - def refresh_sku_keywords(con): """Bind a local derived view to listing summaries and raw SQP without another source copy.""" listing_columns = _columns_of(con, "amzn_listing_items") @@ -2940,7 +2810,19 @@ def refresh_sku_keywords(con): output_meta = ', "_connection_id", store' if scoped else "" backend_meta = ', l."_connection_id", l.store' if scoped else "" backend_match = ' AND m."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else "" - select = _sku_keyword_source_fields(raw_columns) + raw_fields = { + "start_date": "start_date", "end_date": "end_date", "asin": "asin", + "search_query": "search_query_data_search_query", + "impressions": "impression_data_asin_impression_count", + "clicks": "click_data_asin_click_count", "cart_adds": "cart_add_data_asin_cart_add_count", + "purchases": "purchase_data_asin_purchase_count", + } + select = ", ".join( + f'{"TRY_CAST" if name in ("impressions", "clicks", "cart_adds", "purchases") else "CAST"}' + f'("{source}" AS {"BIGINT" if name in ("impressions", "clicks", "cart_adds", "purchases") else "VARCHAR"}) AS {name}' + if source in raw_columns else + f'NULL::{"BIGINT" if name in ("impressions", "clicks", "cart_adds", "purchases") else "VARCHAR"} AS {name}' + for name, source in raw_fields.items()) raw_from = " FROM amzn_search_query_performance" if raw_columns else " WHERE FALSE" con.execute(f""" CREATE OR REPLACE VIEW amzn_sku_keywords AS @@ -2956,25 +2838,23 @@ def refresh_sku_keywords(con): SELECT asin{account_group}, COUNT(DISTINCT sku) AS mapped_sku_count FROM mapping GROUP BY asin{account_group} ), raw AS (SELECT {select}{raw_meta}{raw_from}), queries AS ( SELECT *, CAST(to_json([start_date, end_date, asin, search_query{grain_meta}]) AS VARCHAR) AS source_grain - FROM raw WHERE _report_period IS NOT NULL + FROM raw ), matched AS ( SELECT l.sku, {query_columns}, l.backend_keywords, c.mapped_sku_count{matched_meta} FROM listings l JOIN mapping m ON m.sku=l.sku{mapping_join} JOIN queries q ON q.asin=m.asin{query_join} JOIN mapped_counts c ON c.asin=m.asin{count_join} ) SELECT md5(CAST(to_json([sku, source_grain]) AS VARCHAR)) AS keyword_id, - sku, asin, CASE WHEN _report_period='MONTH' THEN substr(start_date, 1, 7) || '-01' END AS month, - start_date, end_date, search_query, + sku, asin, substr(start_date, 1, 7) || '-01' AS month, start_date, end_date, search_query, impressions, clicks, cart_adds, purchases, backend_keywords, source_grain, mapped_sku_count, 'Amazon ASIN/query metrics shared across ' || CAST(mapped_sku_count AS VARCHAR) || - ' listing SKUs. No SKU allocation. Aggregate distinct SQP source grains.' AS source_basis, - _report_period{output_meta} + ' listing SKUs. No SKU allocation. Aggregate distinct SQP source grains.' AS source_basis{output_meta} FROM matched UNION ALL SELECT md5(CAST(to_json([l.sku, 'backend_keywords'{', l."_connection_id"' if scoped else ''}]) AS VARCHAR)), l.sku, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, l.backend_keywords, NULL, NULL, - 'Listing backend keywords only. No matching Search Query Performance rows.', NULL::VARCHAR{backend_meta} + 'Listing backend keywords only. No matching Search Query Performance rows.'{backend_meta} FROM listings l WHERE NOT EXISTS (SELECT 1 FROM matched m WHERE m.sku=l.sku{backend_match}) """) refresh_keyword_ranks(con) @@ -3071,56 +2951,43 @@ def _validate_keyword_ranks(con): def _validate_sku_keywords(con): - """Tie distinct derived grains and purchases to raw rows separately for every MONTH and WEEK.""" - from harness.amazon_sqp_reads import _independent_period - + """Compare each month's distinct derived SQP grains with independently grouped raw source rows.""" basis = amazon_schema.PROVENANCE_COPY["derived"] - columns = _columns_of(con, table_name(SQP_TABLE)) - if not columns: + if not _columns_of(con, table_name(SQP_TABLE)): return {"state": "unproven", "reason": "Search Query Performance source is absent", "source_basis": basis} try: - declared = "_report_period" if "_report_period" in columns else "NULL::VARCHAR" - account = ', "_connection_id"' if amazon_schema.CONNECTION_ID in columns else "" - derived_account = ', "_connection_id"' if amazon_schema.CONNECTION_ID in _columns_of(con, "amzn_sku_keywords") else "" - raw = con.execute(f""" - SELECT CAST(start_date AS VARCHAR), CAST(end_date AS VARCHAR), list(DISTINCT {declared}), - COUNT(*), COUNT(DISTINCT (asin, search_query_data_search_query{account})), + raw = con.execute(""" + SELECT substr(CAST(start_date AS VARCHAR), 1, 7), + COUNT(*), COUNT(DISTINCT (start_date, end_date, asin, search_query_data_search_query)), SUM(purchase_data_asin_purchase_count), COUNT(purchase_data_asin_purchase_count), COUNT(*) FILTER (WHERE purchase_data_asin_purchase_count < 0 OR start_date IS NULL OR end_date IS NULL OR asin IS NULL OR search_query_data_search_query IS NULL) - FROM amzn_search_query_performance GROUP BY ALL + FROM amzn_search_query_performance GROUP BY 1 """).fetchall() - derived = con.execute(f""" - SELECT start_date, end_date, _report_period, COUNT(*), SUM(purchases), SUM(invalid) + derived = con.execute(""" + SELECT substr(month, 1, 7), COUNT(*), SUM(purchases), SUM(invalid) FROM ( - SELECT start_date, end_date, _report_period, asin, search_query{derived_account}, MIN(purchases) AS purchases, + SELECT month, start_date, end_date, asin, search_query, MIN(purchases) AS purchases, CASE WHEN COUNT(purchases) <> COUNT(*) OR MIN(purchases) <> MAX(purchases) - OR COUNT(*) <> COUNT(DISTINCT sku) - OR COUNT(*) FILTER (WHERE month IS DISTINCT FROM CASE WHEN _report_period='MONTH' - THEN substr(start_date, 1, 7) || '-01' END) > 0 THEN 1 ELSE 0 END AS invalid + OR COUNT(*) <> COUNT(DISTINCT sku) THEN 1 ELSE 0 END AS invalid FROM amzn_sku_keywords WHERE source_grain IS NOT NULL - GROUP BY ALL - ) GROUP BY ALL + GROUP BY month, start_date, end_date, asin, search_query + ) GROUP BY 1 """).fetchall() - actual = {(start, end, period): {"grains": n, "purchases": purchases, "invalid": invalid} - for start, end, period, n, purchases, invalid in derived} - held, months = {}, {} - for start, end, declarations, count, distinct, purchases, present, invalid in raw: - labels = {_independent_period(start, end, declared) for declared in declarations} - period = labels.pop() if len(labels) == 1 else None - key = (start, end, period) - observed = actual.get(key, {"grains": 0, "purchases": 0, "invalid": 0}) - good = (period is not None and count == distinct == present and not invalid and not observed["invalid"] + actual = {month: {"grains": n, "purchases": purchases, "invalid": invalid} + for month, n, purchases, invalid in derived} + months = {} + for month, count, distinct, purchases, present, invalid in raw: + observed = actual.get(month, {"grains": 0, "purchases": 0, "invalid": 0}) + good = (count == distinct == present and not invalid and not observed["invalid"] and observed["grains"] == distinct and observed["purchases"] == purchases) - held[key] = {"state": "ok" if good else "refused", "start": start, "end": end, "period": period, - "expected": {"grains": distinct, "purchases": purchases}, "actual": observed} - if period == "MONTH": - months[start[:7]] = held[key] - extra = set(actual) - set(held) - good = bool(held) and not extra and all(row["state"] == "ok" for row in held.values()) - return {"state": "ok" if good else "refused" if held or extra else "unproven", - "months": months, "periods": list(held.values()), "source_basis": basis, - "reason": ("Distinct ASIN/query grains reconcile to raw SQP for every period" if good else + months[month] = {"state": "ok" if good else "refused", + "expected": {"grains": distinct, "purchases": purchases}, "actual": observed} + extra = set(actual) - set(months) + good = bool(months) and not extra and all(row["state"] == "ok" for row in months.values()) + return {"state": "ok" if good else "refused" if months or extra else "unproven", + "months": months, "source_basis": basis, + "reason": ("Distinct ASIN/query grains reconcile to raw SQP for every month" if good else "SQP coverage or purchases differ. Unmapped ASINs, duplicate grains and missing counts cannot reconcile")} except Exception as exc: return {"state": "refused", "source_basis": basis, @@ -3156,12 +3023,8 @@ def first_load_plan(tables, start, end, *, recent_days=None, size=MAX_WINDOW_DAY for table in tables: spec = _client().REPORT_TYPES[table] table_start = starts.get(table, start) if starts is not None else start - if table == SQP_TABLE: - rule = _history_rule(spec, "WEEK") - week_start = _date_utc(rule["from"]) if rule.get("from") else table_start - week_start = starts.get((table, "WEEK"), week_start) if starts is not None else week_start - weekly_history.extend(_sqp_week_tasks(week_start, end, sqp_batches( - sqp_asins, spec["asin_option_max_chars"]))) + if table == SQP_TABLE and table_start < end: + recent.extend(_sqp_week_tasks(table_start, end, sqp_batches(sqp_asins, spec["asin_option_max_chars"]))) if table == "settlements": continue if spec.get("snapshot"): @@ -4096,13 +3959,6 @@ def standing_refusals(report): return sorted(holds["standing"]) if holds else [] -def _deferred_table(reason, refusal, *, current=False): - """Keep a seller-wide refusal visible when a scheduling hold prevents another request.""" - if refusal and refusal.get("state") == "not_offered": - return {"state": "refused", "amazonStatus": "NOT_OFFERED", "reason": refusal["reason"]} - return {"state": "deferred", "reason": reason, "current": current} - - class _AmazonMirrorStore(_mirrors.MirrorStore): def __init__(self): self.connector, self.spec = "amazon", _mirror_spec("amazon") @@ -4180,10 +4036,10 @@ class _AmazonMirrorStore(_mirrors.MirrorStore): """ rows = report.get("tables") or {} problems = [str(problem) for key in ("problems", "audit_problems") for problem in report.get(key) or []] - # Landed SQP batches and dated periods publish even while older history stays owed. + # Landed SQP batches and landed months count as finished work, so a chunk of only that progress still publishes. if report.get("state") in {"partial", "failed"} and not any( isinstance(row, dict) and (row.get("state") != "deferred" or name == SQP_TABLE - or bool(row.get("landedMonths") or row.get("landedWeeks"))) + or bool(row.get("landedMonths"))) and _mirrors._table_sync_complete("amazon", row) for name, row in rows.items()): return None unfinished = {name[len(PREFIX):] if name.startswith(PREFIX) else name for name in waiting} @@ -4208,7 +4064,7 @@ class _AmazonMirrorStore(_mirrors.MirrorStore): unfinished |= set(partner) for names in scoped.values(): names |= {pairs[name] for name in list(names) if name in pairs and pairs[name] not in unfinished} - if not unfinished and not scoped and not problems and report.get("state") not in {"partial", "failed"}: + if not unfinished and not scoped and not problems: return None owners = {key for name in set(rows) | unfinished for key in (name, table_name(name))} account_owned = {str(problem) for cid, result in accounts.items() @@ -5255,10 +5111,8 @@ def _restore_first_load_wait(tenant_key, con): return due -def _loaded_refresh_hold(con, table, now, *, slot_pass=False, history=False): +def _loaded_refresh_hold(con, table, now, *, slot_pass=False): """Return a loaded table's cadence hold while its first load continues, or None when due.""" - if history and table == "brand_analytics" and os.environ.get("AIOS_AMAZON_WEEKLY_HISTORY") == "1": - return None now = _utc(now) tick = _read_state(con, entity_key("sync_tick_" + table)) # A queued resync is a repair: only Amazon's own floor holds it. @@ -5550,8 +5404,6 @@ def needs_first_load(tenant_key, rt=None, *, completed_tables=None, repair_table floor = read_backfill_floor(con, table) if cohort is None: return False - if table == SQP_TABLE: - return _sqp_history_done(con, cohort) # The same per-table floor first_load_plan uses, so a load that lands every window completes. return floor is not None and floor <= history_floor(con, table, cohort) # ⛔ HOTFIX 2026-09-29 (D-3496): the same rule for a load that STOPPED. A load left @@ -5621,8 +5473,6 @@ def first_load_status(tenant_key, rt=None): "nextDue": _iso(_first_load_due(con)) if _first_load_due(con) else None} if job: out["job"] = job - if SQP_TABLE in enabled_tables: - out["sqp"] = _sqp_first_load_status(con, cohort) return out finally: con.close() @@ -6444,22 +6294,9 @@ def first_load_job_summary(docs, *, table_status=None): "remaining": record.get("remaining", last.get("remainingAfter")), "tablesPublished": list(last.get("tablesPublished") or []), "end": last.get("end"), "operation": last.get("operation"), "resume_at": last.get("resume_at"), - "sqp": last.get("sqp"), "tables": {**(last.get("tables") or {}), **(table_status or {})}} -def _checkpoint_first_load_sqp(tenant_key, rt, cohort, record, save): - """Checkpoint derived SQP progress after publication without making it another completion authority.""" - if cohort is None or not read_database_enablement(rt)["databases"].get(amazon_schema.DATABASE_PREFIX + SQP_TABLE): - return - con, _path = _connect_tenant_mirror(tenant_key, rt) - try: - record["sqp"] = _sqp_first_load_status(con, cohort) - finally: - con.close() - save(lambda doc: doc.update(sqp=record["sqp"])) - - def _record_generation(docs, row): def add(doc): rows = [r for r in (doc.get("generations") or []) if r.get("generation") != row["generation"]] @@ -6469,18 +6306,6 @@ def _record_generation(docs, row): docs.update(FIRST_LOAD_GENERATIONS_KEY, add, flush="sync") -def _record_brand_week_write(record, task, wall_seconds, cpu_seconds, rows): - """Accumulate measured writes while retaining each week's own CPU and wall time.""" - timing = record.setdefault("brandAnalytics", {"wallSeconds": 0, "cpuSeconds": 0, "periods": []}) - week = {"account": task["account"], "_period_start": task["start"][:10], - "_period_end": task["end"][:10], "wallSeconds": round(wall_seconds, 3), - "cpuSeconds": round(cpu_seconds, 3)} - timing["periods"].append(week) - for field in ("wallSeconds", "cpuSeconds"): - timing[field] = round(sum(item[field] for item in timing["periods"]), 3) - timing["rows"] = rows - - def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_seconds=5 * 3600, log=print, docs=None, plan=None, client=None, write=None, delegate=None, counts=None, holder=None, now=time.time, sleep=time.sleep, poll_seconds=None, job_id=None): @@ -6546,13 +6371,12 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon if cursor.get("end") and end and cursor["end"] != _iso(end): # A new cohort end means a new first load; the old cursor's windows describe another plan. cursor = {} - cohort = end or (_utc(cursor["end"]) if cursor.get("end") else None) finished = dict(cursor.get("finished") or {}) opened = dict(cursor.get("open") or {}) throttled = dict(cursor.get("throttled") or {}) failed = dict(cursor.get("failed") or {}) downloads = dict(cursor.get("downloads") or {}) - save(lambda doc: doc.update(tenant=key, end=_iso(cohort) if cohort else None, backfillDays=DEFAULT_BACKFILL_DAYS, + save(lambda doc: doc.update(tenant=key, end=_iso(end) if end else None, backfillDays=DEFAULT_BACKFILL_DAYS, generation=int(generation), finished=finished, open=opened, throttled=throttled, failed=failed, downloads=downloads)) @@ -6561,7 +6385,7 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon record["remainingBefore"] = sum(owed(t) for t in tasks) mine = _pace_weekly_history([t for t in tasks if t.get("prefetch")], owed) - # A write that finishes no window waits for the next generation, including unserved documents. + # A window the writer served but did not finish (deferred, held) waits for the next generation. stalled = set() own_tables = sorted({t["table"] for t in mine if owed(t)}) ready = {} @@ -6678,8 +6502,6 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon prefix.append(task) if not prefix: continue - if table == "brand_analytics": - prefix = prefix[:1] # Each week needs its own measured write interval. sibling = SHARED_REPORT_SIBLINGS.get(table) pair = [t for t in by_table.get(sibling, []) if owed(t)][:len(prefix)] if sibling else [] units = [table] + ([sibling] if pair and [t["key"] for t in pair] == [t["key"] for t in prefix] @@ -6692,30 +6514,28 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon log(f"{table}: the first-load write failed ({type(exc).__name__}); " "its windows stay owed and the next generation downloads them again.") for t in prefix: - stalled.add(t["id"]) ready.pop(t["key"], None) continue - wall_seconds, cpu_seconds = time.time() - t0, time.process_time() - c0 + if table == "brand_analytics": + record["brandAnalytics"] = {"wallSeconds": round(time.time() - t0, 1), + "cpuSeconds": round(time.process_time() - c0, 1), + "rows": (result["tables"].get(table) or {}).get("rows")} stamp_done = _epoch_iso(now()) for unit in units: row = result["tables"].get(unit) or {} - ok = result["published"] and (row.get("state", "done") not in { + ok = result["published"] and row.get("state", "done") not in { "failed", "refused", "throttled", "pending", "held", "deferred", "disabled"} - or unit == "brand_analytics" and row.get("state") == "deferred" - and row.get("current") and row.get("landedWeeks", 0) > 0) if not ok: continue for t in (prefix if unit == table else pair): if t["key"] in result["served"]: - if unit == "brand_analytics": - _record_brand_week_write(record, t, wall_seconds, cpu_seconds, row.get("rows")) finished[t["id"]] = {"at": stamp_done, "generation": int(generation), "reportId": (opened.get(t["key"]) or {}).get("reportId")} record["windowsWritten"] += 1 if unit not in record["tablesPublished"]: record["tablesPublished"].append(unit) for t in prefix + pair: - if t["id"] not in finished: + if t["id"] not in finished and t["key"] in result["served"]: stalled.add(t["id"]) ready.pop(t["key"], None) if t["id"] in finished or t["key"] not in result["served"]: @@ -6755,8 +6575,6 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon break record["otherTablesPasses"] += 1 result = result if isinstance(result, dict) else {} - if plan is None: - _checkpoint_first_load_sqp(key, rt, cohort, record, save) if result.get("state") != "partial" or not result.get("landedTasks"): break try: @@ -6765,16 +6583,12 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon after = tasks record["remainingAfter"] = sum(t["id"] not in finished and not t.get("landed") and t["id"] not in failed for t in after) - record["brandAnalyticsPeriods"] = sorted({t["start"][:10] for t in tasks + after - if t["table"] == "brand_analytics" and t.get("start") - and (t.get("landed") or t["id"] in finished)}) try: row_counts = counts(list(dict.fromkeys(EMPTY_REASON_TABLES + BEHIND_TABLES))) except Exception: # noqa: BLE001 row_counts = {} table_status = None if plan is None: - _checkpoint_first_load_sqp(key, rt, cohort, record, save) con, _path = _connect_tenant_mirror(key, rt) try: table_status = {"inventory": inventory_table_status(con)} @@ -6816,129 +6630,6 @@ _AMAZON_FLOOR_TEXT = re.compile( _FLOOR_MIN_AGE = timedelta(days=45) -def _sqp_depth_start(value, period): - """Admit a complete MONTH or Sunday-based WEEK boundary.""" - if period not in _client().SQP_PERIOD: - raise ValueError("SQP depth supports MONTH and WEEK only") - if isinstance(value, str) and re.fullmatch(r"\d{4}-\d{2}-\d{2}", value): - value += "T00:00:00Z" - day = _utc(value) - boundary = _grain_floor(day, period.lower()) - if day != boundary: - raise ValueError("SQP depth needs the first day of a complete period") - return day - - -def _sqp_depth_shift(start, period, count): - if period == "WEEK": - return start + timedelta(weeks=count) - return _months_before(start, -count) - - -def _sqp_depth_eligible(row, now): - """Only Amazon's old out-of-range refusal can bound served history.""" - return (row["state"] == "out_of_range" and row.get("amazonStatus") in ("FATAL", 400) - and bool(_AMAZON_FLOOR_TEXT.search(row.get("amazon") or "")) - and _date_utc(row["end"]) + timedelta(days=1) <= now - _FLOOR_MIN_AGE) - - -def _sqp_depth_observation(request, asin, period, start, now): - cx = _client() - end = _sqp_depth_shift(start, period, 1) - row = {"period": period, "start": start.date().isoformat(), - "end": (end - timedelta(days=1)).date().isoformat(), "reportId": None, - "state": "empty", "hasRows": False, "floorEligible": False} - try: - text, meta = request(cx.REPORT_TYPES[SQP_TABLE]["report"], start=_iso(start), - end=_iso(end - timedelta(seconds=1)), - options={"reportPeriod": period, "asin": asin}) - except cx.AmazonError as exc: - words, status = str(getattr(exc, "vendor_reason", "") or exc), getattr(exc, "status", None) - if status not in ("FATAL", 400) or not _AMAZON_FLOOR_TEXT.search(words): - raise - row.update(state="out_of_range", amazonStatus=status, amazon=words, - reportId=getattr(exc, "report_id", None)) - row["floorEligible"] = _sqp_depth_eligible(row, now) - return row - _columns, _rows, _totals, count = _sqp_document_rows( - text, cx.REPORT_TYPES[SQP_TABLE], [asin], row, meta) - row.update(state="served" if count else "empty", hasRows=bool(count), - reportId=meta.get("reportId"), reportDocumentId=meta.get("reportDocumentId"), amazonStatus="DONE") - return row - - -def _sqp_depth_linear(rows, start, period): - refusals = 0 - while start.date().isoformat() in rows: - row = rows[start.date().isoformat()] - refusals = refusals + 1 if row["floorEligible"] else 0 - if refusals >= 2: - return None - start = _sqp_depth_shift(start, period, -1) - return start - - -def _sqp_depth_next(rows, newest, period): - served = sorted(day for day, row in rows.items() if row["state"] == "served") - if not served: - return _sqp_depth_linear(rows, newest, period) - first = _sqp_depth_start(served[0], period) - refused = sorted(day for day, row in rows.items() if row["floorEligible"] and day < served[0]) - if refused: - low = _sqp_depth_start(refused[-1], period) - distance = ((first - low).days // 7 if period == "WEEK" else - (first.year - low.year) * 12 + first.month - low.month) - if distance > 1: - middle = _sqp_depth_shift(low, period, distance // 2) - if middle.date().isoformat() not in rows: - return middle - return _sqp_depth_linear(rows, _sqp_depth_shift(first, period, -1), period) - - -def _sqp_depth_result(rows, period, newest, reports, *, done=False): - served = sorted(day for day, row in rows.items() if row["state"] == "served") - first = _sqp_depth_start(served[0], period) if served else None - bounded = bool(first) and all(rows.get(_sqp_depth_shift(first, period, -n).date().isoformat(), {}).get( - "floorEligible", False) for n in (1, 2)) - return {"period": period, "newest": newest.date().isoformat(), - "earliestServed": served[0] if served else None, - "floor": served[0] if done and bounded else None, - "state": ("measured" if bounded else "unproven") if done else "running", - "reportsRequested": reports, "observations": [rows[day] for day in sorted(rows, reverse=True)]} - - -def sqp_depth_walk(request, asin, period, newest, known=(), *, now=None, checkpoint=None): - """Find one ASIN's served boundary, retaining empty periods and requiring two adjacent old refusals.""" - if not isinstance(asin, str) or not re.fullmatch(r"[A-Z0-9]{10}", asin): - raise ValueError("SQP depth needs one ASIN") - newest, now = _sqp_depth_start(newest, period), _utc(now) - if _sqp_depth_shift(newest, period, 1) > now: - raise ValueError("SQP depth needs a completed newest period") - rows = {} - for saved in known: - row = dict(saved) - start = _sqp_depth_start(row["start"], period) - if row.get("period") != period or start > newest or row.get("state") not in ( - "served", "empty", "out_of_range"): - raise ValueError("SQP depth known observations must belong to this period and window") - end = (_sqp_depth_shift(start, period, 1) - timedelta(days=1)).date().isoformat() - if row.get("end") != end or (row["state"] == "served" and row.get("hasRows") is not True): - raise ValueError("SQP depth known observations need matching windows and row evidence") - row["floorEligible"] = _sqp_depth_eligible(row, now) - rows[row["start"]] = row - reports = 0 - while (start := _sqp_depth_next(rows, newest, period)) is not None: - row = _sqp_depth_observation(request, asin, period, start, now) - rows[row["start"]] = row - reports += 1 - if checkpoint: - checkpoint(_sqp_depth_result(rows, period, newest, reports)) - result = _sqp_depth_result(rows, period, newest, reports, done=True) - if checkpoint: - checkpoint(result) - return result - - def _dated_table(table): """A table whose first load plans dated history windows.""" spec = _client().REPORT_TYPES.get(table) @@ -7046,23 +6737,20 @@ def _seller_floor(con, end): f"{DEFAULT_BACKFILL_DAYS} day fallback.") -def _history_limit_key(table, period=None): - """Keep the legacy MONTH key and isolate every other period's observed history floor.""" - suffix = "" if period in (None, "MONTH") else "_" + period - return entity_key(f"limit_{table}_history{suffix}") +def _history_limit_key(table): + """Amazon's observed floor for one table; `limit_` family, so a table reset clears it too.""" + return entity_key(f"limit_{table}_history") -def _observed_history_limit(con, table, period=None): +def _observed_history_limit(con, table): """-> (floor, reason) Amazon showed by refusing an older window, or None.""" - state = _read_state(con, _history_limit_key(table, period)) + state = _read_state(con, _history_limit_key(table)) try: detail = json.loads(state["detail"]) if state and state.get("detail") else {} if detail.get("floor"): return _utc(detail["floor"]), str(detail.get("reason") or "") except (ValueError, TypeError, AttributeError): pass - if period not in (None, "MONTH"): - return None legacy = _read_state(con, entity_key("limit_" + table)) try: reason = json.loads(legacy["detail"]).get("reason", "") if legacy and legacy.get("detail") else "" @@ -7074,13 +6762,7 @@ def _observed_history_limit(con, table, period=None): return None -def _history_rule(spec, period=None): - """A period-specific rule replaces the table default without inheriting another period's floor.""" - rule = spec.get("history") or {} - return rule.get("periods", {}).get(period, rule) - - -def history_bound(con, table, end, *, now=None, period=None): +def history_bound(con, table, end, *, now=None): """-> (floor, cause): the oldest history start for `table`, and why when Amazon or the fallback sets it. max(seller floor, the table's `history` rule, Amazon's observed limit), on the table's window grain. @@ -7088,9 +6770,9 @@ def history_bound(con, table, end, *, now=None, period=None): `end` is the frozen cohort end (seller floor); `now` is the clock a rolling Amazon rule moves with. """ spec = _client().REPORT_TYPES.get(table) or {} - grain = "week" if period == "WEEK" else _history_grain(spec) + grain = _history_grain(spec) now = _utc(now) if now is not None else datetime.now(timezone.utc) - rule = _history_rule(spec, period) + rule = spec.get("history") or {} if rule.get("from") or rule.get("seller_floor") is False: # Brand-level data (Brand Analytics) predates the seller's first order, so the seller month # does not bound it: a `from` rule is the floor, else history walks back until Amazon refuses. @@ -7112,36 +6794,29 @@ def history_bound(con, table, end, *, now=None, period=None): elif rule.get("from"): bound = _grain_ceil(_date_utc(rule["from"]), grain) rule_cause = f"Amazon serves this data back to {bound.date().isoformat()} only." - if rule.get("provisional"): - rule_cause = (f"Weekly history starts at the provisional planning bound {bound.date().isoformat()}. " - "The earliest served week is unproved; an older range refusal sets the observed floor.") if bound is not None and bound > floor: floor, cause = bound, rule_cause - observed = _observed_history_limit(con, table, period) + observed = _observed_history_limit(con, table) if observed and observed[0] > floor: floor, cause = observed[0], observed[1] or ( f"Amazon keeps this data back to {observed[0].date().isoformat()}; older history is not available.") - if (spec.get("weekly") and not WEEKLY_HISTORY_PLANNED - and os.environ.get("AIOS_AMAZON_WEEKLY_HISTORY") != "1"): + if spec.get("weekly") and not WEEKLY_HISTORY_PLANNED: latest = _week_start(_utc(end)) - timedelta(days=7) if latest > floor: older = f" Amazon serves weeks back to {floor.date().isoformat()}." if bound is not None else "" floor, cause = latest, ("The latest completed week loads first; older weeks are not loaded yet, " - "because weekly history requires AIOS_AMAZON_WEEKLY_HISTORY=1." + older) + "because the weekly sync defers a past week until its next weekly turn." + older) return floor, cause -def history_floor(con, table, end, *, now=None, period=None): +def history_floor(con, table, end, *, now=None): """The oldest history start `first_load_plan` and `needs_first_load` use for `table`.""" - return history_bound(con, table, end, now=now, period=period)[0] + return history_bound(con, table, end, now=now)[0] def history_starts(con, tables, end, *, now=None): """{table: history_floor} for each dated table; the one dict every first-load planner passes.""" - starts = {table: history_floor(con, table, end, now=now) for table in tables if _dated_table(table)} - if SQP_TABLE in starts and "WEEK" in _client().REPORT_TYPES[SQP_TABLE].get("periods", ()): - starts[SQP_TABLE, "WEEK"] = history_floor(con, SQP_TABLE, end, now=now, period="WEEK") - return starts + return {table: history_floor(con, table, end, now=now) for table in tables if _dated_table(table)} def history_status(con, tables, end): @@ -7167,7 +6842,8 @@ def _history_reopened(con, table, cohort): return False bound = history_floor(con, table, cohort) if table == SQP_TABLE: - return bool(_sqp_catalogue(con)) and not _sqp_history_done(con, cohort) + return bool(_sqp_catalogue(con)) and any( + not _sqp_month(con, w0, w1)["batches"] for w0, w1 in _month_windows(bound, cohort)) floor = read_backfill_floor(con, table) return floor is not None and floor > bound @@ -7468,12 +7144,11 @@ def _report_document(con, table, fields, report_type, **kwargs): return result -def _newer_history_landed(con, table, end, period=None): +def _newer_history_landed(con, table, end): """A window of `table` newer than `end` has landed, so a refusal at `end` is a floor, not an outage.""" if table == SQP_TABLE: - windows = _week_windows if period == "WEEK" else _month_windows return any(any(not batch.get("refused") for batch in _sqp_month(con, w0, w1)["batches"].values()) - for w0, w1 in windows(end, datetime.now(timezone.utc))) + for w0, w1 in _month_windows(end, datetime.now(timezone.utc))) floor = read_backfill_floor(con, table) return floor is not None and end <= floor @@ -7488,11 +7163,10 @@ def _close_history_floor(con, table, exc, start, end): return None now = datetime.now(timezone.utc) status = getattr(exc, "status", None) - period = _sqp_period(start, end) if table == SQP_TABLE else None words = str(getattr(exc, "vendor_reason", "") or exc) floor = read_backfill_floor(con, table) said = (bool(_AMAZON_FLOOR_TEXT.search(words)) and (status in ("FATAL", 400) or " answered 400" in str(exc)) - and end <= now - _FLOOR_MIN_AGE and _newer_history_landed(con, table, end, period)) + and end <= now - _FLOOR_MIN_AGE and _newer_history_landed(con, table, end)) unexplained = (table != SQP_TABLE and status == "FATAL" and not getattr(exc, "has_error_document", True) and floor is not None and end <= floor and start < now - timedelta(days=365) and bool(_read_state(con, entity_key("reconcile_" + table)))) @@ -7502,25 +7176,14 @@ def _close_history_floor(con, table, exc, start, end): reason = (f"Amazon keeps this report back to {stop.date().isoformat()}; older history is not available. " "Keep the connection enabled for newer data; ask Seller Support for older history.") amazon = " ".join(words.replace("\u2014", "; ").replace("\u2013", " to ").split())[:300] - _record_state(con, _history_limit_key(table, period), "done", detail=json.dumps({ - "floor": _iso(stop), "period": period, "reason": reason, "amazon": amazon, - "window": [_iso(start), _iso(end)]})) + _record_state(con, _history_limit_key(table), "done", detail=json.dumps({ + "floor": _iso(stop), "reason": reason, "amazon": amazon, "window": [_iso(start), _iso(end)]})) if table != SQP_TABLE: # SQP's limit_ key is its progress document; its floor lives in the history marker only. _record_state(con, entity_key("limit_" + table), "done", detail=json.dumps({"reason": reason})) return reason -def _history_task_period(task): - return _sqp_period(task[1], task[2]) if task[0] == SQP_TABLE and task[1] is not None else None - - -def _history_cut_tasks(tasks, table, period, end): - """Only the refused period's windows at or below its floor are no longer owed.""" - return (task for task in tasks if task[0] == table and task[2] is not None - and task[2] <= end and _history_task_period(task) == period) - - def _record_reconciliation(con, table, marker): """Keep the most recent landed period, even while backfill travels backwards.""" key = entity_key("reconcile_" + table) @@ -9146,156 +8809,6 @@ def _order_metric_totals(records, *, flattened=False): return totals -ORDER_LINE_PROOF = "reconcile_order_items_" - - -def _order_report_months(start, end): - first = _utc(start).replace(day=1, hour=0, minute=0, second=0, microsecond=0) - while first < _utc(end): - following = (first.replace(day=28) + timedelta(days=4)).replace(day=1) - yield first, following - first = following - - -def _order_previous_months(con, orders): - if not {"amazon_order_id", "purchase_date"} <= _columns_of(con, "amzn_order_items"): - return set() - result = con.execute("SELECT DISTINCT strftime(TRY_CAST(purchase_date AS TIMESTAMPTZ), '%Y-%m-01') " - "FROM amzn_order_items WHERE amazon_order_id IN (SELECT unnest(?))", [list(orders)]) - return {month for (month,) in result.fetchall()} - - -def _order_line_summary(summary, row): - status = str(row.get("order_status") or "").strip().lower() - if status in ("cancelled", "canceled"): - return - summary["count"] += 1 - origin = f"{row['_report_document_id']}:{row['_report_row_number']}".encode() - summary["origin_xor"] = str(int(summary["origin_xor"]) ^ int.from_bytes( - hashlib.md5(origin, usedforsecurity=False).digest(), "little")) - currency = row.get("currency") - if not status or not currency or (summary["currency"] and summary["currency"] != currency): - summary["invalid"] = True - summary["currency"] = currency - try: - price = Decimal(str(row.get("item_price"))) - if not price.is_finite() or price < 0: - raise ValueError("invalid order line price") - summary["sales"] = str(Decimal(summary["sales"]) + price) - except (InvalidOperation, ValueError): - summary["invalid"] = True - - -def _order_report_observations(con, cols, rows, meta, report_type, start, end): - """Measure raw report windows before order grouping, typing or insertion.""" - spec = _client().REPORT_TYPES["order_items"] - required = {"amazon_order_id", "purchase_date", "asin", "order_status", "item_price", "currency"} - proof = {"reportType": report_type, "reportDocumentId": meta.get("reportDocumentId"), - "start": _iso(start), "end": _iso(end), "months": {}, "touched": set()} - if report_type == spec["report"]: - for first, following in _order_report_months(start, end): - proof["months"][first.date().isoformat()] = { - "start": _iso(max(first, start)), "end": _iso(min(following, end)), "asins": {}, - "invalid": bool(cols and not required <= set(cols))} - orders = set() - for row in rows: - orders.add(str(row.get("amazon_order_id") or "")) - if len(orders) >= 1000: - proof["touched"].update(_order_previous_months(con, orders)) - orders.clear() - try: - purchased = _utc(row["purchase_date"]) - if not row["purchase_date"]: - raise ValueError("missing purchase date") - except (KeyError, TypeError, ValueError, AttributeError): - proof["touched"].add(None) - continue - month = purchased.strftime("%Y-%m-01") - proof["touched"].add(month) - if report_type != spec["report"] or not start <= purchased < end: - continue - window = proof["months"][month] - if not row.get("asin") or not row.get("amazon_order_id"): - window["invalid"] = True - summary = window["asins"].setdefault(str(row.get("asin") or ""), - {"count": 0, "sales": "0", "currency": None, "invalid": False, "origin_xor": "0"}) - _order_line_summary(summary, row) - proof["touched"].update(_order_previous_months(con, orders)) - return proof - - -def _order_report_totals(windows): - totals = {} - for window in windows: - for asin, raw in window["asins"].items(): - total = totals.setdefault(asin, {"count": 0, "sales": "0", "currency": None, - "invalid": False, "origin_xor": "0"}) - total["count"] += raw["count"] - total["sales"] = str(Decimal(total["sales"]) + Decimal(raw["sales"])) - total["origin_xor"] = str(int(total["origin_xor"]) ^ int(raw["origin_xor"])) - total["invalid"] |= raw["invalid"] or bool( - raw["count"] and total["currency"] and total["currency"] != raw["currency"]) - if raw["count"]: - total["currency"] = raw["currency"] - return totals - - -def _record_order_report(con, proof): - """Publish raw line totals with disjoint purchase-date windows or an explicit invalidation.""" - prefix = entity_key(ORDER_LINE_PROOF) - touched = proof["touched"] | proof["months"].keys() - unknown = None in touched - if unknown: - touched.update(row[0][len(prefix):].replace("_", "-") + "-01" for row in con.execute( - "SELECT entity FROM _sync_state WHERE starts_with(entity, ?)", [prefix]).fetchall()) - for month in sorted(touched - {None}): - key = prefix + month[:7].replace("-", "_") - saved = _read_state(con, key) or {} - try: - previous = json.loads(saved.get("detail") or "{}") - windows = previous.get("windows", []) - if not isinstance(windows, list): - raise ValueError("invalid order source windows") - window = proof["months"].get(month) - windows = [old for old in windows if window and - (old["end"] <= window["start"] or old["start"] >= window["end"])] - except (ValueError, TypeError, KeyError, AttributeError): - windows, window = [], proof["months"].get(month) - if window and not unknown: - windows.append({**window, "reportDocumentId": proof["reportDocumentId"]}) - first = _utc(month + "T00:00:00Z") - following = (first.replace(day=28) + timedelta(days=4)).replace(day=1) - marker = {"version": 1, "source": _client().REPORT_TYPES["order_items"]["validates_against"], - "reportType": _client().REPORT_TYPES["order_items"]["report"], "cancelled_excluded": True, - "start": month, "end": following.date().isoformat(), "windows": windows, - "asins": _order_report_totals(windows), - "reportDocumentIds": list(dict.fromkeys(w["reportDocumentId"] for w in windows))} - if not windows: - marker["reason"] = "Order updates require purchase-month report proof to be refreshed" - marker["invalidated_by"] = {key: proof[key] for key in - ("reportType", "reportDocumentId", "start", "end")} - _record_state(con, key, "done", detail=json.dumps(marker)) - - -def _reset_order_report(con): - if _columns_of(con, "amzn_order_items"): - con.execute("DELETE FROM amzn_order_items") - con.execute("DELETE FROM _sync_state WHERE starts_with(entity, ?)", [entity_key(ORDER_LINE_PROOF)]) - - -def _write_order_report_window(con, cols, types, rows, proof, *, reset=False): - before = _reset_order_report if reset else None - after = lambda target: _record_order_report(target, proof) - if not cols: - with _report_transaction(con): - if before: - before(con) - after(con) - return 0, None - return replace_window(con, "amzn_order_items", cols, types, rows, replace_keys_for("order_items"), - before=before, after=after) - - def _order_status_observations(con, rows, meta): """Retain status bounds for the current Sales window in the existing account sync state.""" saved = _read_state(con, entity_key("reconcile_order_metrics")) @@ -10249,18 +9762,14 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri and (table in completed or (read_watermark(con, table) and table not in repairs))] due_loaded = [] for table in loaded: - history = table == "brand_analytics" and table in reopened - hold = _loaded_refresh_hold(con, table, end, slot_pass=slot_pass, history=history) + hold = _loaded_refresh_hold(con, table, end, slot_pass=slot_pass) if hold is None: - if not history: - due_loaded.append(table) + due_loaded.append(table) continue next_due, reason = hold report["tables"][table] = {"state": "deferred", "reason": reason, "nextDue": _iso(next_due), "current": current_table(table)} log(f"{table}: DEFERRED: {reason}") - if history: - want.remove(table) if due_loaded: # Refreshes use their own floors; the history partition retains its bounded budget. con.close() @@ -10292,13 +9801,12 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri or table in completed or table in FEEDBACK_TABLES or (table == "settlements" and not refresh_loaded)): continue - tick = _read_state(con, entity_key("sync_tick_" + table)) - reason = ("This table has no recorded sync yet. Its first load is still running." if not tick else - "This table's first load is still running. Its previous rows were kept.") - report["tables"][table] = _deferred_table(reason, prior_refusals.get(table)) + report["tables"][table] = {"state": "deferred", "reason": + "The first load is continuing in bounded passes on the resident sync thread."} # The planner waits for the next phase on this hold. The card's tick stays as it was. _record_state(con, entity_key(PHASE_HOLD_PREFIX + table), "deferred") - if table == SQP_TABLE and tick and tick.get("state") == "throttled": + tick = _read_state(con, entity_key("sync_tick_" + table)) if table == SQP_TABLE else None + if tick and tick.get("state") == "throttled": # The first load owns SQP now; a passed SQP deadline must not make it due every minute. _record_state(con, entity_key("sync_tick_" + table), "deferred", tick.get("rows") or 0, tick.get("detail")) want = [table for table in want if (read_watermark(con, table) @@ -10327,7 +9835,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri plan = sync_plan(ticks, want, now=end, slot_pass=slot_pass) want = plan["tables"] for table, reason in plan["deferred"].items(): - report["tables"][table] = _deferred_table(reason, prior_refusals.get(table), current=current_table(table)) + report["tables"][table] = {"state": "deferred", "reason": reason, + "current": current_table(table)} log(f"{table}: DEFERRED: {reason}") elif not first_load: # The vendor's report floor also applies to an operator's Sync now click. @@ -10339,7 +9848,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri reason = _interval_reason(table, _read_state(con, entity_key("sync_tick_" + table)), datetime.now(timezone.utc)) if reason: - report["tables"][table] = _deferred_table(reason, prior_refusals.get(table), current=current_table(table)) + report["tables"][table] = {"state": "deferred", "reason": reason, + "current": current_table(table)} want.remove(table) # Heads go first even when the caller listed lines first. Scheduled plans still select one. @@ -10367,10 +9877,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri if prepare_brand_analytics(con): changed_tables.add(table_name("brand_analytics")) - def write_window(table, cols, types, rows, after=None, order_proof=None, reset=False): - if table == "order_items": - count, note = _write_order_report_window(con, cols, types, rows, order_proof, reset=reset) - elif cx.REPORT_TYPES[table].get("weekly"): + def write_window(table, cols, types, rows, after=None): + if cx.REPORT_TYPES[table].get("weekly"): count, note = write_week(con, table, cols, types, rows, tenant_key, report, log) else: count, note = replace_window(con, table_name(table), cols, types, rows, @@ -10441,7 +9949,20 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri report["nextDueSeconds"] = min(report.get("nextDueSeconds") or SQP_RESUME_SECONDS, SQP_RESUME_SECONDS) def sqp_progress(): - sqp_limit["progress"] = _sqp_progress(con, sqp_months, sqp_weeks, sqp_asins, batches, end) + """Batches done of total per planned month, from the durable month markers.""" + rows = [] + for w0, w1 in sqp_months: + marker = _sqp_month(con, w0, w1) + rows.append({"month": w0.strftime("%Y-%m"), "total": len(batches), + "done": sum(batch in marker["batches"] for batch in batches)}) + done, total = sum(r["done"] for r in rows), sum(r["total"] for r in rows) + weekly = sum(not _sqp_done(con, w0, w1, batch) for table, w0, w1, batch in work + if table == SQP_TABLE and w0 is not None and _sqp_period(w0, w1) == "WEEK") + from harness.amazon_client import REPORT_SUSTAINED_SECONDS + sqp_limit["progress"] = { + "months": rows, "batchesDone": done, "batchesTotal": total, + "monthsDone": sum(r["done"] >= r["total"] for r in rows), "monthsTotal": len(rows), + "etaSeconds": int((total - done + weekly) * REPORT_SUSTAINED_SECONDS)} if rows and batches else None try: measure_end = _utc(load["detail"]) if first_load and load and load.get("detail") else end @@ -10458,15 +9979,16 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri plans = {table: sync_window(table, None, now=end, floor=starts.get(table)) for table in pending} undocumented = repairs.intersection(pending) floors = {table: None if table in undocumented else read_backfill_floor(con, table) for table in pending} - wanted = [table for table in pending if (table not in completed or table in exhausted) and (table == SQP_TABLE or not read_watermark(con, table) + wanted = [table for table in pending if (table not in completed or table in exhausted) and (not read_watermark(con, table) or table in undocumented or table in reopened or read_watermark(con, table) < plans[table]["end"])] planned = first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end, sqp_asins=sqp_asins, starts=starts) - for entry in _sqp_owed_tasks(con, _sqp_after_cohort(planned, end, sqp_asins)): + for entry in _sqp_after_cohort(planned, end, sqp_asins): task = entry if len(entry) == 4 else (*entry, None) table, w0, w1, batch = task if table == SQP_TABLE: - work.append(task) + if not _sqp_done(con, w0, w1, batch): + work.append(task) elif _after_cohort(table, w0, end): # A month after the frozen end has its own marker: the backfill floor only moves backwards. if not _cohort_month_done(con, table, w0): @@ -10511,22 +10033,16 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri if not recent <= mark <= end: mark = recent plan = plans[table] = sync_window(table, mark, now=end, since=since, - floor=history_floor(con, table, end), intraday=_lookback_intraday(con, table, end, slot_pass=slot_pass)) if table == SQP_TABLE: - # Monthly history precedes every completed week's ASIN batches. - start = plan["start"] if since is not None else history_floor(con, table, end) - months = list(reversed(_month_windows(start, plan["end"]))) - tasks = [(table, w0, w1, batch) for w0, w1 in months for batch in batches] - weekly_tasks = _sqp_week_tasks( - _utc(since) if since is not None else history_floor(con, table, end, period="WEEK"), - end, batches) - owed = [*tasks, *weekly_tasks] - if since is None: - owed = _sqp_owed_tasks(con, owed) - work.extend(owed) - if not any(_sqp_period(w0, w1) == "MONTH" for _table, w0, w1, _batch in owed): - work.append((table, None, None, None)) + # Closed months land newest first, before the latest week's batches. + months = list(reversed(_month_windows(plan["start"], plan["end"]))) + tasks = [(table, w0, w1, batch) for w0, w1 in months for batch in batches + if since is not None or not _sqp_done(con, w0, w1, batch)] + weekly_tasks = [task for task in _sqp_week_tasks( + _utc(since) if since is not None else end - timedelta(days=7), end, batches) + if since is not None or not _sqp_done(con, task[1], task[2], task[3])] + work.extend([*tasks, *weekly_tasks] if tasks else [*weekly_tasks, (table, None, None, None)]) else: work.append((table, None, None, None)) @@ -10562,15 +10078,12 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri sqp_limit["weeklyStart"] = (_week_start(end) - timedelta(days=7)).date().isoformat() if old_limit.get("weeklyStart") == sqp_limit["weeklyStart"]: sqp_limit["weeklyCause"] = old_limit.get("weeklyCause", "") - plan = plans.get(SQP_TABLE) or sync_window(SQP_TABLE, read_watermark(con, SQP_TABLE), now=end, since=since, - floor=history_floor(con, SQP_TABLE, end)) + plan = plans.get(SQP_TABLE) or sync_window(SQP_TABLE, read_watermark(con, SQP_TABLE), now=end, since=since) # A first load also owes the months that closed after its frozen cohort end. sqp_months = sorted({*_month_windows(plan["start"], plan["end"]), *(_month_windows(plan["end"], datetime.now(timezone.utc)) if first_load else ()), *((w0, w1) for name, w0, w1, _batch in work if name == SQP_TABLE and w0 is not None and _sqp_period(w0, w1) == "MONTH")}) - sqp_weeks = (_week_windows(history_floor(con, SQP_TABLE, end, period="WEEK"), end) - if "WEEK" in cx.REPORT_TYPES[SQP_TABLE].get("periods", ()) else []) sqp_limit["periods"] = [w0.strftime("%Y-%m") for w0, _w1 in sqp_months] sqp_progress() sqp_limit["deferredReason"] = ( @@ -10578,12 +10091,11 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri if not sqp_asins else sqp_limit["partialMonthReason"] if not sqp_limit["periods"] else "") _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit)) - history_cut = {} # Each table and period owns its observed floor. + history_cut = {} # table -> Amazon's floor observed this pass; older tasks are already counted landed. for index, (table, task_start, task_end, task_batch) in enumerate(work): if (first_load or table == SQP_TABLE) and table in failed: continue - history_key = (table, _history_task_period((table, task_start, task_end, task_batch))) - if history_cut.get(history_key) is not None and task_end is not None and task_end <= history_cut[history_key]: + if history_cut.get(table) is not None and task_end is not None and task_end <= history_cut[table]: continue if ((first_load or table == SQP_TABLE) and isinstance(report_budget, int) and documents >= max(0, report_budget) and (task_end is not None or table == "settlements" @@ -10621,7 +10133,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri pending_reports.add(table) report["nextDueSeconds"] = min(report.get("nextDueSeconds") or 900, 900) _record_state(con, entity_key(table), "pending", 0, weekly_pending["reason"]) - except cx.AmazonError as exc: + except Exception as exc: if isinstance(exc, cx.AmazonThrottled): problem = _sync_problem(exc, table, exc.operation or "report_document", tenant_key, rt, report, log) report["tables"][table] = problem @@ -10632,34 +10144,16 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri report["nextDueSeconds"] = min(report.get("nextDueSeconds") or wait, wait) _record_state(con, entity_key(table), "throttled", detail=problem["reason"]) continue - limit_reason = _close_history_floor(con, table, exc, task_start, task_end) - if limit_reason: - history_cut[history_key] = task_end - landed.update(_history_cut_tasks(work, table, "WEEK", task_end)) - sqp_limit["weeklyCause"] = limit_reason - sqp_limit["errors"] = [] - sqp_progress() - _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit)) - log(limit_reason) - continue causes.append(str(exc)) - if not _AMAZON_FLOOR_TEXT.search(str(getattr(exc, "vendor_reason", "") or exc)): + if isinstance(exc, cx.AmazonError) and not isinstance(exc, (cx.AmazonReportPending, cx.AmazonThrottled)): marker = _sqp_month(con, task_start, task_end) marker["batches"][task_batch] = {"refused": True, "reason": str(exc), "asins": []} - marker.setdefault("refusedAsins", {}).update({a: str(exc) for a in task_batch.split()}) _record_state(con, _sqp_month_key(task_start, "WEEK"), "refused", detail=json.dumps(marker)) landed.add((table, task_start, task_end, task_batch)) - else: - failed.add(table) - report["tables"][table] = {"state": "deferred", "reason": str(exc)} - except ValueError as exc: - # Unreadable source content remains owed without creating an ASIN refusal or history floor. - causes.append(str(exc)) if causes and not weekly_cause_logged: sqp_limit["weeklyCause"] = "Amazon weekly Search Query Performance: " + "; ".join(dict.fromkeys(causes)) sqp_problem(sqp_limit["weeklyCause"]) weekly_cause_logged = True - sqp_progress() _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit)) if weekly_deferred: break @@ -10898,7 +10392,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri floors[table] = w0 stop = _note_history_window(con, table, w0, count) if stop: - history_cut[history_key] = w0 + history_cut[table] = w0 landed.update(task for task in work if task[0] == table and task[2] is not None and task[2] <= w0) log(stop) @@ -10984,32 +10478,28 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri declared = {} if cols and (weekly or "_period_start" in replace_keys_for(table)): cols, rows = stamp_week(cols, rows, w0, w1) - order_proof = (_order_report_observations(con, cols, rows, meta, rtype, w0, w1) - if "order_items" in (table, sibling) else None) if not cols: if first_load: # Even an empty recent window clears old rows exactly once. A weekly # table never clears: R18's retention keeps watched history. for empty_table in (table, sibling): - if empty_table and empty_table != "order_items" and floors[empty_table] is None and not weekly: + if empty_table and floors[empty_table] is None and not weekly: if con.execute("SELECT 1 FROM information_schema.tables WHERE table_name=?", [table_name(empty_table)]).fetchone(): con.execute(f"DELETE FROM {table_name(empty_table)} WHERE 1=1") - cleared = table != "order_items" - elif order_proof is None: + cleared = True + else: completed_windows.append(window) continue if sibling: sibling_mirror = table_name(sibling) sibling_types = _ensure(con, sibling_mirror, cols, rows, declared=declared, preserve_rows=True) if cols else {} - sibling_reset = not sibling_cleared and not sibling_plan["incremental"] - if cols and sibling_reset and sibling != "order_items": + if cols and not sibling_cleared and not sibling_plan["incremental"]: con.execute(f"DELETE FROM {sibling_mirror} WHERE 1=1") - if cols or order_proof is not None: - sibling_written += write_window(sibling, cols, sibling_types, rows, - order_proof=order_proof, reset=sibling_reset) sibling_cleared = True + if cols: + sibling_written += write_window(sibling, cols, sibling_types, rows) order_observations = None if table == "orders": rows = _dedupe_orders(rows) @@ -11026,17 +10516,13 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri preserve_rows=True) if cols else {} # Explicit windows and first loads replace the mirror; incremental passes keep history. # A weekly table replaces by its period identity instead (D-2675). - order_reset = not cleared and not plan["incremental"] - if order_reset and not weekly and table != "order_items": + if not cleared and not plan["incremental"] and not weekly: con.execute(f"DELETE FROM {mirror} WHERE 1=1") cleared = True - if cols or table == "order_items": + if cols: after = (lambda target: _record_state(target, entity_key("reconcile_orders"), "done", detail=json.dumps(order_observations))) if order_observations else None - written += write_window(table, cols, types, rows, after=after, - order_proof=order_proof, reset=order_reset) - if table == "order_items": - cleared = True + written += write_window(table, cols, types, rows, after=after) if _periodic(cx.REPORT_TYPES[table]): _record_reconciliation(con, table, {"start": _iso(w0), "end": _iso(w1), "count": len(rows)}) @@ -11052,7 +10538,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri floors[reached] = w0 stop = _note_history_window(con, table, w0, len(rows) if cols else 0) if stop: - history_cut[history_key] = w0 + history_cut[table] = w0 landed.update(task for task in work if task[0] == table and task[2] is not None and task[2] <= w0) log(stop) @@ -11217,14 +10703,14 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri report["tables"][table] = {"state": "deferred", "reason": str(exc)} break except Exception as e: # noqa: BLE001 - limit_reason = (_close_history_floor(con, table, e, task_start, task_end) - if first_load or table == SQP_TABLE else None) + limit_reason = _close_history_floor(con, table, e, task_start, task_end) if first_load else None if limit_reason and table == SQP_TABLE: - history_cut[history_key] = task_end - landed.update(_history_cut_tasks(work, table, "MONTH", task_end)) + # Only months at or below Amazon's floor stop; newer months and the weekly batches continue. + history_cut[table] = task_end + landed.update(task for task in work if task[0] == table and task[2] is not None + and task[2] <= task_end) if sqp_limit is not None: sqp_limit["errors"] = [] - sqp_progress() _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit)) report["tables"][table] = {"state": "deferred", "current": True, "reason": limit_reason} log(limit_reason) @@ -11318,20 +10804,15 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri sqp_progress() _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit)) for table in pending if first_load else (): - # Landed periods publish with their markers while the remaining history stays owed. + # Landed months publish with their markers, as SQP batches do; the months still owed run next pass. months = sum(task[0] == table and task[2] is not None for task in landed) row = report["tables"].get(table) - if (table == SQP_TABLE or not _periodic(cx.REPORT_TYPES[table]) or not months + if (table == SQP_TABLE or not cx.REPORT_TYPES[table].get("monthly") or not months or (row is not None and not (isinstance(row, dict) and row.get("state") in {"deferred", "throttled", "pending"}))): continue - grain = "week" if cx.REPORT_TYPES[table].get("weekly") else "month" - row = row or {"reason": f"{months} {grain}{'' if months == 1 else 's'} landed. The remaining {grain}s continue next pass."} - count_key = "landedWeeks" if grain == "week" else "landedMonths" - row_count = (con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0] - if _columns_of(con, table_name(table)) else 0) - report["tables"][table] = {**row, "state": "deferred", "current": True, - count_key: months, "rows": row_count} + row = row or {"reason": f"{months} month{'' if months == 1 else 's'} landed. The remaining months continue next pass."} + report["tables"][table] = {**row, "state": "deferred", "current": True, "landedMonths": months} if first_load: refusals ={row["table"]: row for row in _refused_reports(con)} not_offered = {table for table, row in refusals.items() if row["state"] == "not_offered"} @@ -13453,11 +12934,6 @@ def _sqp_resume_legs(ck, runtime, asins): con.executemany("INSERT INTO amzn_listings VALUES (?)", [(a,) for a in asins]) if watermark: record_watermark(con, SQP_TABLE, _utc(watermark)) - # A fixture watermark's earlier months need the same per-ASIN evidence as a real completed load. - for w0, w1 in _month_windows(history_floor(con, SQP_TABLE, _utc(watermark)), _utc(watermark)): - marker = _sqp_month(con, w0, w1) - marker["asins"] = {a: {"count": 0, "purchases": 0} for a in asins} - _sqp_save_month(con, w0, marker) calls.clear() return con diff --git a/platform/harness/datastore.py b/platform/harness/datastore.py index f4e8f89a4877ebe1d79a2ccebf34cafa9cd4e417..a0f53d9d4b07a53ef3d7d17f9638e9e6c61fc1b7 100644 --- a/platform/harness/datastore.py +++ b/platform/harness/datastore.py @@ -3732,7 +3732,6 @@ REPLACE_EACH_PASS = { "product_tag": {"domain": []}, } REPLACE_PAGE = 5000 -REPLACE_READ_ATTEMPTS = 3 _connector_spec.register("odoo", _connector_spec.tables_for("odoo"), mirror=_connector_spec.ConnectorMirrorSpec(tuple(ENTITIES))) @@ -4019,13 +4018,13 @@ def _base_domain(spec): return list(spec.get("domain") or ()) + ([("active", "in", [True, False])] if spec["archivable"] else []) -def sync_entity(key, max_batches=200, batch=BATCH, log=print, report=None): +def sync_entity(key, max_batches=200, batch=BATCH, log=print): """Publish a single requested entity through the same source boundary as a full pass.""" if source_paused(): return _sync_entity(key, max_batches=max_batches, batch=batch, log=log) candidate = getattr(_SYNC_TLS, "candidate", None) if candidate is not None: - phase, pulled = _sync_one(key, max_batches=max_batches, batch=batch, log=log, report=report) + phase, pulled = _sync_one(key, max_batches=max_batches, batch=batch, log=log) # A recurring refresh may report changed pages to a bespoke cursor, but a # tenant pass cannot claim completion until those private pages finish too. pending = _connection_pending(candidate["con"], key) @@ -4034,7 +4033,7 @@ def sync_entity(key, max_batches=200, batch=BATCH, log=print, report=None): return phase, pulled rt = _resident_odoo_runtime() if rt is None: - return _sync_one(key, max_batches=max_batches, batch=batch, log=log, report=report) + return _sync_one(key, max_batches=max_batches, batch=batch, log=log) report = sync_all(entities=[key], max_batches=max_batches, log=log, batch=batch) return report[key]["phase"], report[key]["pulled"] @@ -4184,17 +4183,16 @@ def _sync_entity(key, max_batches=200, batch=BATCH, log=print): con.close() -def _sync_one(key, max_batches=200, batch=BATCH, log=print, report=None): +def _sync_one(key, max_batches=200, batch=BATCH, log=print): """One entity's pass: the cursor sync, the full replace, or the cursor sync then the replace.""" _connection_model(ENTITIES[key]) plan = REPLACE_EACH_PASS.get(key) if plan is None or source_paused(): return _sync_entity(key, max_batches=max_batches, batch=batch, log=log) if plan.get("scope") != "internal": - return _replace_entity(key, log=log, report=report) + return _replace_entity(key, log=log) phase, pulled = _sync_entity(key, max_batches=max_batches, batch=batch, log=log) - replaced, changed = _replace_entity(key, log=log, report=report) - return (phase if replaced == "live" else replaced), pulled + changed + return phase, pulled + _replace_entity(key, log=log)[1] def _replace_scope(con, key, plan): @@ -4208,44 +4206,37 @@ def _replace_scope(con, key, plan): return "(" + " OR ".join(terms) + ") AND " + _connection_where() -def _read_replace_rows(key, spec, domain): - """Read uncapped pages again if neither surrounding count matches, with a fixed attempt bound.""" - fields = spec["fields"] + list(spec.get("m2m") or []) - for _ in range(REPLACE_READ_ATTEMPTS): - expected = int(O.get_odoo().search_count(spec["model"], domain)) - rows, last = [], 0 - while True: - recs = O.search_read(spec["model"], domain + [("id", ">", last)], ["id"] + fields, - limit=REPLACE_PAGE, order="id asc") - if any(field not in rec for rec in recs for field in fields): - raise RuntimeError(f"{key}: full read omitted a requested field. The mirror keeps its previous rows.") - rows += _flatten_records(recs, spec["fields"], spec.get("m2m") or (), - named_m2m=spec.get("named_m2m"), raw_m2m=spec.get("raw_m2m"), spec=spec) - # Only an empty page ends a read, since Odoo may return short pages before the end. - if not recs: - break - last = recs[-1]["id"] - if len(rows) == expected: - return rows - after = int(O.get_odoo().search_count(spec["model"], domain)) - if len(rows) == after: - return rows - return None +def _replace_entity(key, log=print): + """Read every row the plan's domain matches, then swap the mirror's scope for it in one transaction. - -def _replace_entity(key, log=print, report=None): - """Replace a complete read atomically, or hold a moving entity before changing rows or state.""" + Returns (phase, changed_rows). A failed or short read raises before any delete. + """ if source_paused(): _paused_notice(f"full read of {key}", log) return PAUSED_PHASE, 0 spec, plan = ENTITIES[key], REPLACE_EACH_PASS[key] - rows = _read_replace_rows(key, spec, list(plan["domain"])) - if rows is None: - if report is not None: - report["cause"] = {"classifier": "odoo_entity_moving", "entity": key, - "attempts": REPLACE_READ_ATTEMPTS} - log(f" {key}: Odoo changed during all {REPLACE_READ_ATTEMPTS} full reads. Previous rows are kept.") - return "backfill", 0 + domain = list(plan["domain"]) + fields = spec["fields"] + list(spec.get("m2m") or []) + expected = int(O.get_odoo().search_count(spec["model"], domain)) + rows, last = [], 0 + while True: + recs = O.search_read(spec["model"], domain + [("id", ">", last)], ["id"] + fields, + limit=REPLACE_PAGE, order="id asc") + if any(field not in rec for rec in recs for field in fields): + raise RuntimeError(f"{key}: full read omitted a requested field. The mirror keeps its previous rows.") + rows += _flatten_records(recs, spec["fields"], spec.get("m2m") or (), + named_m2m=spec.get("named_m2m"), raw_m2m=spec.get("raw_m2m"), spec=spec) + # Stop on an EMPTY page: a server-side page cap returns short pages before the end. + if not recs: + break + last = recs[-1]["id"] + # Rows moved during the read when the read matches a second count. Refuse only a read + # that disagrees with both counts, before any delete. + if len(rows) != expected: + after = int(O.get_odoo().search_count(spec["model"], domain)) + if len(rows) != after: + raise RuntimeError(f"{key}: full read returned {len(rows)} rows, counts were " + f"{expected} and {after}. The mirror keeps its previous rows.") con = connect() try: _ensure_table(con, key, spec) @@ -4337,7 +4328,6 @@ def sync_all(entities=None, max_batches=200, log=print, after_entity=None, batch with binding: for key in selected: t0 = time.time() - row = out.setdefault(key, {"phase": "live", "pulled": 0, "secs": 0}) marker = _connection_state_key(key + ".pass") previous = candidate["con"].execute("SELECT phase FROM _sync_state WHERE entity=?", [marker]).fetchone() if entry else None if entry and candidate.get("resuming") and previous == ("done",): @@ -4346,11 +4336,11 @@ def sync_all(entities=None, max_batches=200, log=print, after_entity=None, batch if entry: candidate["con"].execute("DELETE FROM _sync_state WHERE entity=?", [marker]) phase, pulled = sync_entity(key, max_batches=max_batches, log=log, - **({"batch": batch} if batch != BATCH else {}), - **({"report": row} if candidate is not None else {})) + **({"batch": batch} if batch != BATCH else {})) if entry and phase == "live": candidate["con"].execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", [marker, "done", 0, None, 0, time.strftime("%Y-%m-%d %H:%M:%S")]) + row = out.setdefault(key, {"phase": "live", "pulled": 0, "secs": 0}) if phase != "live": row["phase"] = phase row["pulled"] += pulled diff --git a/platform/harness/mirror_store.py b/platform/harness/mirror_store.py index 0119483696b1e6cbccce7d4973869ea4b7e85be0..26ced67ab9742fa4f743be549afb2259e17ca574 100644 --- a/platform/harness/mirror_store.py +++ b/platform/harness/mirror_store.py @@ -355,8 +355,7 @@ def _sync_result(connector, report): completed = all(isinstance(row, dict) and row.get("phase") in {"live", "backfill"} for row in report.values()) return completed, completed and all(row["phase"] == "live" for row in report.values()) - if (report.get("problems") or report.get("audit_problems") or (connector == "amazon" - and any(row.get("state") == "refused" for row in (report.get("tables") or {}).values()))): + if report.get("problems") or report.get("audit_problems"): # A problem owned by one held table publishes the rest; it never makes the pass current. holds = for_connector(connector)._publication_holds(report, ()) if connector == "amazon" else None if holds is None or holds["blocked"]: @@ -381,34 +380,16 @@ def _sync_result(connector, report): return completed, current -def odoo_moving_entities(report): - """Return only typed, exhausted moving-entity holds whose cause matches their entity.""" - if not isinstance(report, dict): - return {} - return {name: row["cause"] for name, row in (report or {}).items() - if re.fullmatch(r"[a-z][a-z0-9_]{0,79}", str(name)) and isinstance(row, dict) - and row.get("phase") == "backfill" and isinstance(row.get("cause"), dict) - and row["cause"].get("classifier") == "odoo_entity_moving" - and row["cause"].get("entity") == name - and type(row["cause"].get("attempts")) is int and row["cause"]["attempts"] == 3} - - -def record_sync_attempt(connector, rt): - """Record writer activity without claiming a completed pass or replacing a previous refusal.""" - with _CATCH_UP_LOCK: - state = _catch_up_source(rt, connector) - if not state["current"]: - state.update(pass_pending=True, attempt_at=time.time()) - - def tracked_sync(connector, runtime, pull, *args, _catch_up_complete=True, **kwargs): """Count completed passes and require a clean result before clearing startup catch-up.""" connector = connector_spec.CONNECTOR_SPEC_KEYS.get(connector, connector) rt = runtime if rt is None: return pull(*args, **kwargs) - record_sync_attempt(connector, rt) with _CATCH_UP_LOCK: + state = _catch_up_source(rt, connector) + if not state["current"]: + state.update(pass_pending=True, attempt_at=time.time()) active = (connector, _tenant_key(rt.key)) _ACTIVE_PULLS[active] = _ACTIVE_PULLS.get(active, 0) + 1 try: @@ -540,99 +521,6 @@ def _sql_literal(value): return "'" + str(value).replace("'", "''") + "'" -def _merge_publication_entries(base, local, stored): - """Merge keyed observations against their common base, naming conflicting local changes.""" - missing = object() - merged, conflicts = {}, [] - for name in sorted(set(base) | set(local) | set(stored)): - before, ours, theirs = (entries.get(name, missing) for entries in (base, local, stored)) - if _same_publication_entry(ours, before): - value = theirs - elif _same_publication_entry(ours, theirs): - value = theirs - elif _same_publication_entry(theirs, before): - value = ours - else: - conflicts.append(name) - continue - if value is not missing: - merged[name] = value - return merged, conflicts - - -def _same_publication_entry(left, right): - """A content-addressed table retains its identity when only its write timestamp differs.""" - if isinstance(left, dict) and isinstance(right, dict): - return {k: v for k, v in left.items() if k != "written_at"} == { - k: v for k, v in right.items() if k != "written_at"} - return left == right - - -def _marker_freshness(row): - """An invalid observation time cannot outrank a valid source-state observation.""" - try: - return _utc(row[5]) - except (ValueError, OverflowError): - return datetime.min.replace(tzinfo=timezone.utc) - - -def _merge_publication_markers(base, local, stored): - """Merge source state independently of observation time, retaining its newest valid freshness.""" - versions = [{row[0]: tuple(row) for row in rows} for rows in (base, local, stored)] - states = [{name: row[:5] for name, row in version.items()} for version in versions] - merged, conflicts = _merge_publication_entries(*states) - for name, state in merged.items(): - matching = [version[name] for version in versions - if name in version and version[name][:5] == state] - merged[name] = max(matching, key=_marker_freshness) - return merged, conflicts - - -def _candidate_checkpoint(con): - """Read the private candidate's common-base manifest and authoritative local markers.""" - if not con.execute("SELECT 1 FROM information_schema.tables " - "WHERE table_name='_candidate_checkpoint'").fetchone(): - return None - row = con.execute("SELECT document FROM _candidate_checkpoint").fetchone() - return json.loads(row[0]) if row else None - - -def _candidate_base_tables(checkpoint): - """Keep initial local content provenance distinct from an actual saved manifest.""" - return {**checkpoint.get("local_tables", {}), **checkpoint["base"].get("tables", {})} - - -def _save_candidate_checkpoint(con, base, markers, *, local_tables=None): - """Keep per-table base provenance separate from the installed serving publication.""" - con.execute("CREATE TABLE IF NOT EXISTS _candidate_checkpoint (document VARCHAR)") - con.execute("DELETE FROM _candidate_checkpoint") - con.execute("INSERT INTO _candidate_checkpoint VALUES (?)", [json.dumps({ - "base": base, "markers": markers, "local_tables": local_tables or {}})]) - - -def _save_initial_candidate_checkpoint(store, rt, candidate, published): - """Capture pre-manifest local table content before private source work changes it.""" - base = datastore._publication(published, store.connector) - local_tables = {} - missing = store._tables(published) - set(base.get("tables", {})) - if missing: - sealed = store._sealed_mode(rt) - part_limit = store.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0) - with tempfile.TemporaryDirectory(prefix="mirror-local-base-") as tmp: - for name in sorted(missing): - entry, _fresh = store._checkpoint_table(published, name, {}, set(), tmp, - sealed=sealed, part_limit=part_limit) - local_tables[name] = entry - _save_candidate_checkpoint(candidate, base, store._state_rows(published), local_tables=local_tables) - - -def _table_marker_rows(store, rows, tables): - """Pair held data markers and direct table status with their table's rows.""" - names = {row[0] for row in store._held_state_rows(rows, tables)} - names.update(row[0] for row in rows if row[0].split("__", 1)[-1] in tables) - return [row for row in rows if row[0] in names] - - class MirrorStore: """One connector policy, with state cached separately for each tenant mirror.""" @@ -1410,232 +1298,162 @@ class MirrorStore: return entry, fresh - def _rebase_publication(self, tenant_key, rt, con, base, manifest, rows, *, allowed, report): - """Keep stored changes and their markers, holding overlapping private changes for retry.""" - raw = self._download(rt, _MANIFEST_KEY) - if raw is None: - # A local snapshot identity alone does not prove a saved manifest existed. - if "tables" in base: - raise PublicationHeld(f"{self.label} checkpoint held: the saved base publication is missing.") - return rows, False - stored = json.loads(raw) - if stored.get("tenant") != _tenant_key(tenant_key): - raise ValueError("Mirror manifest belongs to another tenant") - has_base = con.execute("SELECT 1 FROM information_schema.tables " - "WHERE table_name='_candidate_base'").fetchone() - candidate_base = con.execute("SELECT identity FROM _candidate_base").fetchone() if has_base else None - base_id = candidate_base[0] if candidate_base else base.get("snapshot_id", "unknown") - if base_id != base.get("snapshot_id", "unknown"): - raise PublicationHeld(f"{self.label} checkpoint held: private rows have a different base publication.") - if (stored.get("snapshot_id", "unknown") == base_id - and stored.get("tables") == base.get("tables") - and stored.get("sync_state") == base.get("sync_state")): - return rows, False - checkpoint = _candidate_checkpoint(con) - tables, conflicts = _merge_publication_entries( - _candidate_base_tables(checkpoint) if checkpoint else base.get("tables", {}), - manifest["tables"], stored.get("tables", {})) - if not allowed: - names = ", ".join(conflicts) or "the source's held tables" - raise PublicationHeld(f"{self.label} checkpoint held: a newer stored publication changed {names}. " - "Private rows keep their previous base for the next pass.") - with tempfile.TemporaryDirectory(prefix="mirror-rebase-") as tmp: - self._before_restore(tenant_key, rt) - stored, stored_rows, _paths, _total, _refused = self._validated_snapshot( - tenant_key, rt, con, raw, tmp) - if checkpoint is None: - raise PublicationHeld(f"{self.label} checkpoint held: common-base markers are unavailable.") - base_rows = checkpoint["markers"] - markers, marker_conflicts = _merge_publication_markers(base_rows, rows, stored_rows) - versions = base_rows + rows + stored_rows - conflicts = sorted(set(conflicts) | { - name for name in report.get("held_tables", ()) - if set(marker_conflicts) & {row[0] for row in _table_marker_rows(self, versions, {name})}}) - held_markers = {row[0] for row in _table_marker_rows(self, versions, set(conflicts))} - if set(marker_conflicts) - held_markers: - raise PublicationHeld(f"{self.label} checkpoint held: a newer stored publication changed markers " - + ", ".join(marker_conflicts) + ". Private rows stay for the next pass.") - if conflicts: - report["conflicts"] = conflicts - report["problems"].append(f"{self.label} checkpoint held tables: a newer stored publication changed " - + ", ".join(conflicts) + ". Private rows keep their previous base for the next pass.") - tables.update({name: stored["tables"][name] for name in conflicts if name in stored["tables"]}) - markers = {name: row for name, row in markers.items() if name not in held_markers} - markers.update({row[0]: tuple(row) for row in stored_rows if row[0] in held_markers}) - manifest["tables"] = tables - return list(markers.values()), True - - - def _checkpoint_table(self, con, name, prior, named, tmp, *, sealed, part_limit): - """Prepare changed immutable content, reusing every part the base already names.""" - path = Path(tmp) / (name + ".parquet") - size = None - if "parts" not in prior: - con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)}) TO ? " - "(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)]) - size = path.stat().st_size - if size is None or size > part_limit: - return self._chunked_entry(con, name, Path(tmp), prior, named, sealed=sealed) - metadata = _blob_metadata(path.read_bytes()) - if (metadata["sha256"] == prior.get("sha256") and prior.get("key") - and (".sealed-v1." in prior["key"]) == sealed): - return None, [] - suffix = ".sealed-v1.parquet" if sealed else ".parquet" - key = _content_key(name, suffix, metadata) - metadata.update(rows=con.execute(f"SELECT count(*) FROM {_quoted_identifier(name)}").fetchone()[0], - written_at=_iso(datetime.now(timezone.utc)), key=key) - if key in named and named[key] != metadata["sha256"]: - raise ValueError(f"{self.label} mirror content key collides with a published blob") - return metadata, [] if key in named else [(key, path)] - - - def _checkpoint_tables(self, rt, con, manifest, dirty, named, tmp, *, sealed, part_limit): - """Prepare local changes and migrate legacy remote entries without dropping refused tables.""" - tables, changed, pending = self._tables(con), [], [] - legacy = {name for name, meta in manifest["tables"].items() - if (not meta.get("key") and "parts" not in meta) - or any((".sealed-v1." in key) != sealed for key, _ in self._entry_blob_keys(name, meta))} - for name in sorted(legacy - tables): - prior, migrated = manifest["tables"][name], [] - for index, (key, metadata) in enumerate(self._entry_blob_keys(name, prior)): - if (".sealed-v1." in key) == sealed: - migrated.append(dict(metadata)) - continue - path = Path(tmp) / f"{name}.migrate{index}.parquet" - path.write_bytes(self._checked_mirror_blob(rt, key, metadata)) - new_key = key.replace(".parquet", ".sealed-v1.parquet") - migrated.append({**metadata, "key": new_key}) - pending.append((new_key, path)) - manifest["tables"][name] = ({**prior, "parts": migrated} if "parts" in prior - else {**prior, **migrated[0]}) - changed.append(name) - for name in sorted((dirty | legacy | (tables - set(manifest["tables"]))) & tables): - entry, fresh = self._checkpoint_table(con, name, manifest["tables"].get(name, {}), named, tmp, - sealed=sealed, part_limit=part_limit) - if entry is not None: - manifest["tables"][name] = entry - changed.append(name) - pending.extend(fresh) - return changed, pending - - - def _upload_checkpoint_parts(self, tenant_key, rt, pending, part_limit, progress): - """Ledger immutable keys before uploading bounded parts, retaining byte progress on failure.""" - if any(path.stat().st_size > part_limit for _key, path in pending): - raise ValueError("mirror part exceeds the byte limit") - if pending: - self._ledger_add(rt, tenant_key, [key for key, _path in pending]) - for key, path in pending: - data = path.read_bytes() - self._upload(rt, key, data) - progress["uploaded"] += len(data) - - - def _publish_checkpoint(self, tenant_key, rt, con, cache, manifest, changed, pending, tmp, *, - publication, hold, rebase_allowed, report, log, progress): - """Publish one complete manifest after content upload and a fresh stored-base comparison.""" - checkpoint = _candidate_checkpoint(con) - old = checkpoint["base"] if checkpoint else cache["manifest"] - sealed = bool(manifest.get("blob_envelope_version")) - part_limit = self.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0) - rows = self._state_rows(con) - if not self._state_complete(rows): - return None - suffix = ".sealed-v1" if sealed else "" - state_meta, _blobs = self._state_entry(rows, part_limit, suffix) - if not (publication or changed or state_meta != old.get("sync_state") - or manifest["tables"] != old.get("tables", {}) or old.get("version") != MANIFEST_VERSION): - return None - self._upload_checkpoint_parts(tenant_key, rt, pending, part_limit, progress) - rows, report["rebased"] = self._rebase_publication( - tenant_key, rt, con, old, manifest, rows, - allowed=rebase_allowed and publication is not None and hold is None, report=report) - state_meta, blobs = self._state_entry(rows, part_limit, suffix) - pending, named = [], self._manifest_keys(old) - for index, (key, state) in enumerate(blobs): - if key not in named: - path = Path(tmp) / f"sync_state.{index}.json" - path.write_bytes(state) - pending.append((key, path)) - self._upload_checkpoint_parts(tenant_key, rt, pending, part_limit, progress) - stamp = _iso(datetime.now(timezone.utc)) - manifest.update(sync_state=state_meta, writer=_writer(), written_at=stamp, **(publication or {})) - if publication is not None: - manifest["completed_at"] = stamp - if hold is not None: - manifest["hold"] = hold - self._upload(rt, _HELD_KEY, _mirror_json(manifest)) - return manifest - superseded = _superseded_hold(self, tenant_key, rt) - if superseded: - manifest["superseded_hold"] = superseded - published = self._upload(rt, _MANIFEST_KEY, _mirror_json(manifest)) - progress["uploaded"] += len(published) - if superseded is not None: - try: - rt.delete_path(self.prefix + _HELD_KEY) - except Exception as exc: - log(f"{self.label} superseded hold cleanup deferred ({type(exc).__name__}); " - "the next publication retries") - if publication is None: - datastore._write_publication(con, self.connector, manifest) - cache["manifest"] = manifest - cache["problems"].clear() - self._retire_blobs(tenant_key, rt, published, named, log, orphans=cache.get("state") != "refused") - return manifest - - def persist_mirror(self, tenant_key, rt, con, changed_tables, *, report, log=print, - _publication=None, _hold=None, _rebase_allowed=True): - """Checkpoint immutable parts and markers before publishing one manifest, retaining private work on failure.""" + _publication=None, _hold=None): + """Checkpoint changed tables and the connector's markers, publishing the manifest last. + + Candidates include partially written tables. Hashing their parquet locally avoids reuploading + identical content; no previous blobs are downloaded on a sync pass. Failed candidates remain + dirty for retry on the next pass, while landed local rows and cursors stay usable. + + S67-T32 (D-3481), the order that makes a killed checkpoint harmless: (1) every new blob gets a + CONTENT key, and a key the current manifest names is never written; (2) the key ledger lists + every new key before its upload; (3) the blobs upload; (4) ONE manifest upload, a single-row + upsert, publishes them; (5) only then are superseded blobs deleted. Dying anywhere before (4) + leaves the previous manifest naming only complete, untouched blobs. + """ candidate = getattr(datastore._SYNC_TLS, "candidate", None) if candidate is not None and _publication is None: if candidate["tenant"] != tenant_key or candidate["connector"] != self.connector: raise ValueError("Checkpoint belongs to another source candidate") - return - progress = {"uploaded": 0} + return # The enclosing source pass owns publication after every table completes. + uploaded = 0 with self.lock: cache = self.restore_mirror(tenant_key, rt, log=log) cache["dirty"].update(changed_tables) try: if cache.get("state") == "refused" and not self._refused_may_publish(tenant_key, rt, con, cache): - raise PublicationHeld( - f"{self.label} mirror checkpoint held: this process could not restore the saved snapshot " - "at start. The saved copy is kept; restore tenant-store access and restart to " - "restore it before checkpointing.") - checkpoint = _candidate_checkpoint(con) - old = checkpoint["base"] if checkpoint else cache["manifest"] + message = (f"{self.label} mirror checkpoint held: this process could not restore the saved snapshot " + "at start. The saved copy is kept; restore tenant-store access and restart to " + "restore it before checkpointing.") + report["problems"].append(message) + log(message) + return + old = cache["manifest"] sealed = self._sealed_mode(rt) - if old.get("blob_envelope_version") and not sealed: - raise tenant_keys.TenantKeyLocked("A sealed mirror cannot publish without its tenant key.") + if old.get('blob_envelope_version') and not sealed: + raise tenant_keys.TenantKeyLocked('A sealed mirror cannot publish without its tenant key.') + suffix = '.sealed-v1' if sealed else '' + part_limit = self.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0) if old.get("snapshot_id") and _publication is None: raise ValueError("An identified source requires the complete sync publication boundary") - tables = self._tables(con) | set(cache.get("refused_tables", ())) + named = self._manifest_keys(old) + tables = self._tables(con) + refused_tables = set(cache.get("refused_tables", ())) manifest = {"version": MANIFEST_VERSION, "tenant": _tenant_key(tenant_key), - "tables": {name: meta for name, meta in old.get("tables", {}).items() if name in tables}} + "tables": {name: meta for name, meta in old.get("tables", {}).items() + if name in tables or name in refused_tables}} if sealed: - manifest["blob_envelope_version"] = be.VERSION - part_limit = self.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0) + manifest['blob_envelope_version'] = be.VERSION + changed, pending = [], [] with tempfile.TemporaryDirectory(prefix="mirror-persist-") as tmp: - changed, pending = self._checkpoint_tables( - rt, con, manifest, cache["dirty"], self._manifest_keys(old), tmp, - sealed=sealed, part_limit=part_limit) - saved = self._publish_checkpoint(tenant_key, rt, con, cache, manifest, changed, pending, tmp, - publication=_publication, hold=_hold, rebase_allowed=_rebase_allowed, - report=report, log=log, progress=progress) - if _hold is not None: - return saved - if saved or not changed: + # A legacy entry (no content key) is a candidate too: the first checkpoint migrates. + legacy = {name for name, meta in manifest["tables"].items() + if (not meta.get("key") and "parts" not in meta) + or any((".sealed-v1." in key) != sealed + for key, _ in self._entry_blob_keys(name, meta))} + # Refused local tables still name good remote parts: migrate those bytes too. + for name in sorted(legacy - tables): + prior = manifest['tables'][name] + migrated = [] + for index, (key, metadata) in enumerate(self._entry_blob_keys(name, prior)): + if ('.sealed-v1.' in key) == sealed: + migrated.append(dict(metadata)) + continue + data = self._checked_mirror_blob(rt, key, metadata) + path = Path(tmp) / f'{name}.migrate{index}.parquet' + path.write_bytes(data) + new_key = key.replace('.parquet', '.sealed-v1.parquet') + migrated.append({**metadata, 'key': new_key}) + pending.append((new_key, path)) + manifest['tables'][name] = ({**prior, 'parts': migrated} if 'parts' in prior + else {**prior, **migrated[0]}) + changed.append(name) + for name in sorted((cache["dirty"] | legacy | (tables - set(manifest["tables"]))) & tables): + if not self._valid_table(name): + raise ValueError(f"{self.label} mirror table name is invalid") + prior = manifest["tables"].get(name, {}) + path = Path(tmp) / (name + ".parquet") + size = None + if "parts" not in prior: + con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)}) TO ? " + "(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)]) + size = path.stat().st_size + if size is None or size > part_limit: + entry, fresh = self._chunked_entry(con, name, Path(tmp), prior, named, sealed=sealed) + if entry is None: + continue + manifest["tables"][name] = entry + changed.append(name) + pending.extend(fresh) + continue + data = path.read_bytes() + metadata = _blob_metadata(data) + if (metadata["sha256"] == prior.get("sha256") and prior.get("key") + and (".sealed-v1." in prior["key"]) == sealed): + continue + key = _content_key(name, suffix + ".parquet", metadata) + metadata.update(rows=con.execute(f"SELECT count(*) FROM {_quoted_identifier(name)}").fetchone()[0], + written_at=_iso(datetime.now(timezone.utc)), key=key) + manifest["tables"][name] = metadata + changed.append(name) + if key in named: + # The same content under its own name is already stored and published. + if named[key] != metadata["sha256"]: + raise ValueError(f"{self.label} mirror content key collides with a published blob") + continue + pending.append((key, path)) + rows = self._state_rows(con) + state_meta, state_blobs = self._state_entry(rows, part_limit, suffix) + published = None + if self._state_complete(rows) and (_publication or changed or state_meta != old.get("sync_state") + or manifest["tables"] != old.get("tables", {}) + or old.get("version") != MANIFEST_VERSION): + for index, (state_key, state) in enumerate(state_blobs): + if state_key not in named: + state_path = Path(tmp) / f"sync_state.{index}.json" + state_path.write_bytes(state) + pending.append((state_key, state_path)) + if any(path.stat().st_size > part_limit for _key, path in pending): + raise ValueError("mirror part exceeds the byte limit") + if pending: + self._ledger_add(rt, tenant_key, [key for key, _path in pending]) + for key, path in pending: + data = path.read_bytes() + self._upload(rt, key, data) + uploaded += len(data) + stamp = _iso(datetime.now(timezone.utc)) + manifest.update(sync_state=state_meta, writer=_writer(), written_at=stamp, + **(_publication or {})) + if _publication is not None: + manifest["completed_at"] = stamp + if _hold is not None: + manifest["hold"] = _hold + self._upload(rt, _HELD_KEY, _mirror_json(manifest)) + return manifest + superseded = _superseded_hold(self, tenant_key, rt) + if superseded: + manifest["superseded_hold"] = superseded + published = self._upload(rt, _MANIFEST_KEY, _mirror_json(manifest)) + uploaded += len(published) + if superseded is not None: + # A newer publication replaces a stored hold. Nobody has to release it. + try: + rt.delete_path(self.prefix + _HELD_KEY) + except Exception: # noqa: BLE001 + log(f"{self.label} superseded hold pointer stays until the next publication") + if _publication is None: + datastore._write_publication(con, self.connector, manifest) + cache["manifest"] = manifest + cache["problems"].clear() + if published: + self._retire_blobs(tenant_key, rt, published, named, log, orphans=cache.get("state") != "refused") + if published or not changed: cache["dirty"].clear() - log(f"{self.label} mirror persisted to the store: {len(changed)} changed tables, " - f"{progress['uploaded']:,} bytes uploaded") - return saved - except PublicationHeld as exc: - report["problems"].append(str(exc)) - log(str(exc)) + log(f"{self.label} mirror persisted to the store: {len(changed)} changed tables, {uploaded:,} bytes uploaded") + return manifest if published else None except Exception as exc: - message = (f"{self.label} mirror persistence failed ({type(exc).__name__}); " - f"{progress['uploaded']:,} bytes uploaded. " + message = (f"{self.label} mirror persistence failed ({type(exc).__name__}); {uploaded:,} bytes uploaded. " "Local rows were kept; restore tenant-store access and resync before restarting.") report["problems"].append(message) log(message) @@ -1820,20 +1638,8 @@ def _export_tables(store, con, directory): return paths -def _candidate_matches(store, candidate, published, previous): - """Resume a known same-tenant common base even after another publication was installed.""" - if not previous: - return False - installed = datastore._publication(published, store.connector) - checkpoint = _candidate_checkpoint(candidate) - if checkpoint is None: - return previous[0] == installed.get("snapshot_id", "unknown") - base = checkpoint["base"] - return previous[0] == base.get("snapshot_id", "unknown") and base.get("tenant") == installed.get("tenant") - - def _open_candidate(store, tenant_key, rt, published): - """Resume known common-base provenance or seed private rows from the serving copy.""" + """Resume private rows only while their base publication still matches the serving copy.""" path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb") candidate = datastore.apply_limits(datastore.duckdb.connect(str(path))) datastore._ensure_sync_state(candidate) @@ -1843,9 +1649,7 @@ def _open_candidate(store, tenant_key, rt, published): candidate.execute("CREATE TABLE IF NOT EXISTS _candidate_held (name VARCHAR PRIMARY KEY)") previous = candidate.execute("SELECT * FROM _candidate_base").fetchone() base = datastore._publication(published, store.connector).get("snapshot_id", "unknown") - if _candidate_matches(store, candidate, published, previous): - if _candidate_checkpoint(candidate) is None: - _save_initial_candidate_checkpoint(store, rt, candidate, published) + if previous and previous[0] == base: return candidate, previous[1] started = _iso(datetime.now(timezone.utc)) try: @@ -1860,7 +1664,6 @@ def _open_candidate(store, tenant_key, rt, published): candidate.execute("DELETE FROM _candidate_pending") candidate.execute("DELETE FROM _candidate_held") candidate.execute("INSERT INTO _candidate_base VALUES (?,?)", [base, started]) - _save_initial_candidate_checkpoint(store, rt, candidate, published) return candidate, started except BaseException: candidate.close() @@ -1880,7 +1683,7 @@ def pending_sync_entities(connector, tenant_key, rt): candidate = datastore.apply_limits(datastore.duckdb.connect(str(path))) try: base = candidate.execute("SELECT identity FROM _candidate_base").fetchone() - if not _candidate_matches(store, candidate, published, base): + if not base or base[0] != datastore._publication(published, store.connector).get("snapshot_id", "unknown"): return None pending = {row[0] for row in candidate.execute("SELECT name FROM _candidate_pending").fetchall()} if store.connector == "odoo": @@ -1898,13 +1701,6 @@ def _holds_rows(holds): return holds is not None and bool(holds["tables"] or holds.get("accounts")) -def _held_tables(holds): - """Retain whole private tables and their original bases even when only an account is held.""" - if holds is None: - return set() - return set(holds["tables"]).union(*(hold["tables"] for hold in (holds.get("accounts") or {}).values())) - - def _hold_previous(store, candidate, published, tables): """Give each held table its published rows and data markers, inside the candidate's open transaction.""" prior = store._tables(published) @@ -1925,36 +1721,6 @@ def _hold_previous(store, candidate, published, tables): candidate.executemany("INSERT INTO _sync_state VALUES (?,?,?,?,?,?)", previous) -def _retain_conflicting_candidate(store, candidate, published, manifest, conflicts): - """Refresh independent tables while retaining held private rows, markers and their original common bases.""" - checkpoint = _candidate_checkpoint(candidate) - tables = set(conflicts) - private = _table_marker_rows(store, store._state_rows(candidate), tables) - saved_rows = store._state_rows(published) - held_names = {row[0] for row in _table_marker_rows(store, checkpoint["markers"] + saved_rows, tables)} - base_rows = [row for row in saved_rows if row[0] not in held_names] - base_rows += [row for row in checkpoint["markers"] if row[0] in held_names] - base = {**manifest, "snapshot_id": checkpoint["base"].get("snapshot_id", "unknown"), - "tables": {name: entry for name, entry in manifest["tables"].items() if name not in tables}} - base["tables"].update({name: entry for name, entry in _candidate_base_tables(checkpoint).items() - if name in tables}) - candidate.execute("BEGIN TRANSACTION") - try: - _hold_previous(store, candidate, published, (store._tables(candidate) | store._tables(published)) - tables) - owned = store._state_rows(candidate) - if owned: - candidate.executemany("DELETE FROM _sync_state WHERE entity=?", [[row[0]] for row in owned]) - rows = [row for row in saved_rows if row[0] not in held_names] + private - if rows: - candidate.executemany("INSERT INTO _sync_state VALUES (?,?,?,?,?,?)", rows) - _save_candidate_checkpoint(candidate, base, base_rows) - candidate.executemany("INSERT OR IGNORE INTO _candidate_held VALUES (?)", [[name] for name in conflicts]) - candidate.execute("COMMIT") - except BaseException: - candidate.execute("ROLLBACK") - raise - - def _publication_line(store, report, done, holds, *, snapshot_id=None, why=None): """One log line: what a pass published and what it held, each with its reason.""" rows = report.get("tables") or {} @@ -2124,7 +1890,7 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub prior_snapshot_id = datastore._publication(published, store.connector).get("snapshot_id", "unknown") metadata.pop("rows") metadata["completed_at"] = _iso(datetime.now(timezone.utc)) - result = {"problems": [], "held_tables": _held_tables(holds)} + result = {"problems": []} save_started = time.monotonic() with tempfile.TemporaryDirectory(prefix="mirror-publish-") as tmp: exporting = time.monotonic() @@ -2142,18 +1908,9 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub report["problems"].extend(problems) return raise RuntimeError("; ".join(problems)) - if report is not None and "problems" in report: - report["problems"].extend(result["problems"]) - retained = set(result.get("conflicts", ())) | result["held_tables"] - kept = bool(retained) try: installing = time.monotonic() - rows = store._state_rows(candidate) - if result.get("rebased"): - store._before_restore(tenant_key, rt) - _saved, rows, paths, _total, _refused = store._validated_snapshot( - tenant_key, rt, published, _mirror_json(manifest), tmp) - store._install_snapshot(published, manifest, rows, paths) + store._install_snapshot(published, manifest, store._state_rows(candidate), paths) installed = True store._after_publish(tenant_key, manifest) except BaseException: @@ -2163,19 +1920,20 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub note("install", installing) for listener in list(_PUBLISH_LISTENERS): try: - listener(store.connector, tenant_key, manifest, rows, paths) + listener(store.connector, tenant_key, manifest, store._state_rows(candidate), paths) except Exception as exc: # The snapshot is saved and installed here. A failed hand-off only delays readers. log(f"{store.label} mirror hand-off failed ({type(exc).__name__}: {exc})") if _holds_rows(holds): candidate.execute("ROLLBACK") - if retained: - _retain_conflicting_candidate(store, candidate, published, manifest, retained) - if odoo and holds is not None: + candidate.execute("DELETE FROM _candidate_base") + candidate.execute("INSERT INTO _candidate_base VALUES (?,?)", [manifest["snapshot_id"], started]) + if odoo: # D-11329: a held entity keeps its private progress but never narrows the next pass. names = [[name] for name in sorted(holds["held"])] candidate.executemany("DELETE FROM _candidate_pending WHERE name=?", names) candidate.executemany("INSERT OR IGNORE INTO _candidate_held VALUES (?)", names) + kept = True if holds is not None: log(_publication_line(store, report, done, holds, snapshot_id=manifest["snapshot_id"])) if odoo: diff --git a/platform/modules/amazon_profit.py b/platform/modules/amazon_profit.py index 59d6316871b54a6954a4ae6bbebafe8eeb97a348..b4eec1ea7621e45881b0ce3995aa653bc8d9d074 100644 --- a/platform/modules/amazon_profit.py +++ b/platform/modules/amazon_profit.py @@ -31,11 +31,9 @@ import re import shutil import tempfile import threading -from decimal import Decimal, ROUND_HALF_UP from pathlib import Path from modules import amazon_sku_metrics as M -from modules import amazon_part_dates as _part_dates import core.user_tables as _registry from core import module_tables from core import amazon_rivals as _rivals @@ -121,58 +119,14 @@ def _field(key, label, ftype, description, *, computed, **extra): return out -def _progress_fields(description, *, key="projects", label="Projects", empty_label="No projects"): +def _progress_fields(description): """Project progress and its completed-count label, written by the Projects doors.""" - return (_field(key, label, "pct", description, computed=True, - format={"display": "progress", "labelKey": key + "_label", "emptyLabel": empty_label}), - _field(key + "_label", label + (" completed" if key == "projects" else " label"), "text", - "Completed projects of the counted ones." if key == "projects" else "Open PDP workflow stage and progress.", + return (_field("projects", "Projects", "pct", description, computed=True, + format={"display": "progress", "labelKey": "projects_label", "emptyLabel": "No projects"}), + _field("projects_label", "Projects completed", "text", "Completed projects of the counted ones.", computed=True)) -HISTORY_FIELD = _field("history", "History", "text", "Open the row's recorded changes.", computed=True) -ASIN_APP_KEYS = ("asin", "projects", "pdp_update", "trend_score", "search_trend", "market_share_trend", - "click_share_trend", "exposure", "funnel_score", "pdp_last_updated", "title_last_updated", - "bullets_last_updated", "images_last_updated", "aplus_last_updated", "history") -SEARCH_TRENDS = (("search", "search_trend", "Search trend", "int"), - ("market_share", "market_share_trend", "Market share trend", "pct"), - ("click_share", "click_share_trend", "Click share trend", "pct")) -EXPOSURES = {"under": "Under-exposed", "match": "Matched", "over": "Over-exposed", - "few": "Few", "none": "Not seen"} -PDP_PART_DATE_KEYS = tuple(part + "_last_updated" for part in ("title", "bullets", "images", "aplus")) -PDP_CELL_KEYS = ("pdp_update", "pdp_update_label", "pdp_update_more") + PDP_PART_DATE_KEYS -_PDP_PROGRESS_FIELDS = _progress_fields("Progress of the earliest open PDP workflow.", key="pdp_update", - label="PDP update", empty_label="No open workflow") -_SIGNAL_FIELDS = ( - _field("trend_score", "Trend score", "int", "Search trend score from 0 to 100.", computed=True), - *(_field(key, label, "pct", "Change against the same reported months last year.", computed=True) - for _, key, label, _ in SEARCH_TRENDS), - _field("exposure", "Exposure", "select", "Sales or click share against impression share.", - computed=True, options=list(EXPOSURES.values())), - _field("funnel_score", "Funnel score", "int", "Search-weighted funnel score, with 5 at the market.", - computed=True, format={"decimals": 1}), -) -_ASIN_DATE_FIELDS = tuple(_field(key, label, "date", "Latest recorded listing or part change.", computed=True) - for key, label in zip(("pdp_last_updated",) + PDP_PART_DATE_KEYS, - ("PDP last updated", "Title last updated", "Bullets last updated", - "Images last updated", "A+ last updated"))) -_SIGNAL_COMPANIONS = tuple(field for kind, key, label, now_type in SEARCH_TRENDS for field in ( - _field(key + "_series", label + " series", "text", "Every reported historical month, with raw values, ASIN gaps and reasons.", computed=True), - _field(key + "_now", label + " now", now_type, "Searches in the newest complete store month." if kind == "search" - else "Share aggregated over the twelve-month calculation window.", computed=True), - _field(key + "_reason", label + " reason", "text", "Why the year comparison is unavailable.", computed=True), - _field(key + "_now_reason", label + " now reason", "text", "Why the current value is unavailable.", computed=True), -)) + tuple(_field(key + "_reason", label + " reason", "text", "Why this signal is unavailable.", computed=True) - for key, label in (("trend_score", "Trend score"), ("funnel_score", "Funnel score"), ("exposure", "Exposure"))) -_ASIN_COMPANIONS = (_PDP_PROGRESS_FIELDS[1:] - + (_field("pdp_update_more", "More PDP workflows", "int", "Other open PDP workflows.", computed=True),) - + _SIGNAL_COMPANIONS - + (_field("exposure_label", "Exposure detail", "text", "Exposure multiple and comparison base.", computed=True), - _field("pdp_last_updated_reason", "PDP date reason", "text", "Why Amazon's listing date is unavailable.", computed=True))) -_ASIN_SOURCE_KEYS = tuple(f["key"] for f in _SIGNAL_FIELDS + _SIGNAL_COMPANIONS) + ( - "exposure_label", "pdp_last_updated", "pdp_last_updated_reason") - - def _metric_fields(skip=()): """The sibling metric module's columns, as computed fields of this module.""" return tuple(_field(key, label, ftype, description, computed=True, **dict(extra or {})) @@ -226,7 +180,7 @@ PARENT_COMPETITOR_FIELDS = tuple(_field(k, label, t, d, computed=True, **dict(x) for k, label, t, d, x in _rivals.PARENT_COLUMNS) COMPETITOR_KEYS = _rivals.ROW_KEYS + _rivals.PARENT_KEYS _SKU_PROJECT_FIELDS = _progress_fields("Average progress of the projects on this SKU, its ASIN and its parent ASIN.") -COMPUTED_FIELDS = (_ECON_FIELDS[:1] + _SKU_PROJECT_FIELDS[:1] + (HISTORY_FIELD, PICTURE_FIELD) + _ECON_FIELDS[1:] + _metric_fields() +COMPUTED_FIELDS = (_ECON_FIELDS[:1] + _SKU_PROJECT_FIELDS[:1] + (PICTURE_FIELD,) + _ECON_FIELDS[1:] + _metric_fields() + (HEALTH_NOTE_FIELD,) + _SKU_PROJECT_FIELDS[1:] + COMPETITOR_FIELDS) #: Owned columns: people edit them and a refresh never writes them. @@ -378,7 +332,7 @@ def cost_note(row, *, code=None, found=True): if source and row.get("cogs_per_unit") is None: return source if source.endswith(".") else source + "." return NO_COST_NOTE -_IDENTITY_KEYS = ("sku", "picture", "asin", "parent_asin", "title", "projects", "projects_label", "history") +_IDENTITY_KEYS = ("sku", "picture", "asin", "parent_asin", "title", "projects", "projects_label") _PERIOD_KEYS = tuple(k for k in COMPUTED_KEYS if k not in _IDENTITY_KEYS + COMPETITOR_KEYS) #: The profit block a rollup table carries; `_ECON_ADDITIVE` sums, margin and band recompute. @@ -397,12 +351,9 @@ def _grain_fields(key_column, grain): else (metric["parent_asin"], PICTURE_FIELD)) where = "this ASIN or its parent ASIN" if key_column == "asin" else "this parent ASIN" progress = _progress_fields(f"Average progress of the projects on {where}.") - app = (_PDP_PROGRESS_FIELDS[:1] + _SIGNAL_FIELDS + _ASIN_DATE_FIELDS + (HISTORY_FIELD,) - if key_column == "asin" else (HISTORY_FIELD,)) - companions = _ASIN_COMPANIONS if key_column == "asin" else () - return (ids[:1] + progress[:1] + app + ids[1:] + (econ["title"],) + rollup + tuple(econ[k] for k in _ROLLUP_ECON) + return (ids[:1] + progress[:1] + ids[1:] + (econ["title"],) + rollup + tuple(econ[k] for k in _ROLLUP_ECON) + _metric_fields(skip=("parent_asin",)) + (HEALTH_NOTE_FIELD,) - + progress[1:] + companions) + + progress[1:]) ASIN_FIELDS = _grain_fields("asin", "asin") @@ -448,7 +399,7 @@ PROJECT_COMPUTED_KEYS = tuple(f["key"] for f in PROJECT_COMPUTED_FIELDS) def table_specs(): """Every module table in creation order.""" - specs = ( + return ( {"key": TABLE_KEY, "label": TABLE_LABEL, "fields": COMPUTED_FIELDS + OWNED_FIELDS + TRACK_FIELDS, "owned": OWNED_KEYS, "keyColumn": "sku", "template": TEMPLATE_KEY}, {"key": ASIN_KEY, "label": ASIN_LABEL, "fields": ASIN_FIELDS + COMPETITOR_FIELDS, "owned": (), @@ -456,10 +407,6 @@ def table_specs(): {"key": PARENT_KEY, "label": PARENT_LABEL, "fields": PARENT_FIELDS + PARENT_COMPETITOR_FIELDS, "owned": (), "keyColumn": "parent_asin", "template": PARENT_TEMPLATE_KEY}, ) - for spec in specs: - fields = {f["key"]: f for f in spec["fields"]} - spec["fields"] = tuple(fields[key] for key in _project_field_order(fields)) - return specs # ---------------------------------------------------------------- period and threshold @@ -1760,7 +1707,7 @@ def _sync_cursor(tenant_key, rt): else: import duckdb cur = duckdb.connect(str(path), read_only=True) - if _mirror_has_economics(cur) or {'sku', 'attributes', 'summaries'} <= _columns(cur, 'amzn_listing_items'): + if _mirror_has_economics(cur): return cur, cur.close cur.close() for key in [k for k in _SNAPSHOT if k[0] == tenant_key]: @@ -1773,7 +1720,7 @@ def sync_matches(tenant_key, synced_tables, *, st=None, con=None, catalogue=None """Refresh the current period, apply decisions, then suggest for every unlinked SKU. Returns the report. It raises on failure; `on_amazon_sync` is the never-raise wrapper.""" synced = {str(t or "").strip().lower() for t in synced_tables or ()} - if not synced & {"economics", "listings", "listing_items"}: + if not synced & {"economics", "listings"}: return {"state": "skipped", "synced": sorted(synced)} if st is None: from harness import amazon_store @@ -1785,17 +1732,6 @@ def sync_matches(tenant_key, synced_tables, *, st=None, con=None, catalogue=None con, close = _sync_cursor(tenant_key, st) try: report = {"state": "done"} - if synced & {"listings", "listing_items"}: - try: - report["partDates"] = _part_dates.observe(st, con, day=today or _today()) - report["partDateCells"] = paint_asin_part_dates(st) - except Exception as exc: # quality-ok: Date storage failure must not block profit matching. - report['partDatesError'] = f'{type(exc).__name__}: {exc}' - _log(f"{tenant_key}: listing part dates failed ({report['partDatesError']}).") - if not synced & {'economics', 'listings'}: - return dict(report, state='failed', reason=report['partDatesError']) - if not synced & {"economics", "listings"}: - return report try: refreshed = refresh(st, con, by=WRITER, today=today, tenant=tenant_key) report["refresh"] = {k: refreshed.get(k) for k in ("skus", "linked", "refusal")} @@ -1871,16 +1807,12 @@ def _definition_fields(fields=None, owned_keys=None): def _project_field_order(keys): - """Pin App fields in schema defaults and keep companions last, without changing saved layouts.""" + """Place Projects beside a product identity in schema defaults, never saved user layouts.""" keys = list(keys) - if not keys or keys[0] not in ("sku", "asin", "parent_asin"): - return keys - pinned = ASIN_APP_KEYS if keys[0] == "asin" else (keys[0], "projects", "history") - companions = (("projects_label",) + tuple(f["key"] for f in _ASIN_COMPANIONS) - if keys[0] == "asin" else ()) - return ([key for key in pinned if key in keys] - + [key for key in keys if key not in pinned + companions] - + [key for key in companions if key in keys]) + if keys and keys[0] in ("sku", "asin", "parent_asin") and "projects" in keys: + keys.remove("projects") + keys.insert(1, "projects") + return keys def _format_repairs(fields, wanted): @@ -1900,9 +1832,9 @@ def _grain_word_repairs(fields, wanted): continue if field["key"] == "grain" and target.get("options") and field.get("options") != target["options"]: out.setdefault("grain", {})["options"] = list(target["options"]) - if field["key"] in ("projects", "pdp_update") and (field.get("type") != target["type"] or "link" in field + if field["key"] == "projects" and (field.get("type") != target["type"] or "link" in field or (field.get("format") or {}) != (target.get("format") or {})): - out[field["key"]] = {"type": target["type"], "format": dict(target.get("format") or {}), "link": None} + out["projects"] = {"type": target["type"], "format": dict(target.get("format") or {}), "link": None} return out @@ -2093,9 +2025,6 @@ def write_rows(st, computed, *, table_key=TABLE_KEY, key_column="sku", label=TAB for rid, row in table_rows(st, table_key).items() if str(row.get(key_column) or "").strip() not in computed} wanted = {rid: values for rid, values in wanted.items() if values} - for rid, row in table_rows(st, table_key).items(): - if row.get("history") != "View": - wanted.setdefault(rid, {})["history"] = "View" if wanted: import core.user_tables as ut @@ -2639,7 +2568,7 @@ def grain_rows(rows, key, *, thin=THIN_DEFAULT, period_label="", fields=()): flat["cost_note"] = _grain_cost_note(flat, children) flat["period"] = period_label out[value] = {k: stored_cell(_FIELD_BY_KEY.get(k), flat.get(k)) for k in keys - if k not in ("projects", "projects_label") + PDP_CELL_KEYS + _ASIN_SOURCE_KEYS} + if k not in ("projects", "projects_label")} return out @@ -2871,117 +2800,8 @@ def _withhold_untied(by_grain, checks): def _grain_period_keys(fields, key_column): return tuple(f["key"] for f in fields - if f["key"] not in (key_column, "picture", "parent_asin", "title", "projects", "projects_label", "history") - + COMPETITOR_KEYS + PDP_CELL_KEYS + _ASIN_SOURCE_KEYS) - - -def _listing_dates(con): - """Read the newest Amazon lastUpdatedDate per ASIN from raw listing summaries.""" - if "summaries" not in _columns(con, "amzn_listing_items"): - return {} - rows = con.execute("""SELECT json_extract_string(s.value, '$.asin'), - CAST(MAX(TRY_CAST(json_extract_string(s.value, '$.lastUpdatedDate') AS TIMESTAMPTZ)) - AT TIME ZONE 'UTC' AS DATE) - FROM amzn_listing_items l, json_each(TRY_CAST(l.summaries AS JSON)) s GROUP BY 1""").fetchall() - return {asin: date.isoformat() for asin, date in rows if asin and date is not None} - - -def _exposure_detail(exposure): - """Format the recorded exposure multiple with the prototype's one-decimal rounding.""" - if not exposure or exposure["multiple"] is None: - return None - multiple = Decimal.from_float(float(exposure["multiple"])).quantize(Decimal("0.1"), rounding=ROUND_HALF_UP) - return f"{multiple}x {exposure['base']} share" - - -def _asin_signal_cells(signal, history): - """Store scalar cells and raw monthly bars for every reported historical month.""" - out = {key: stored_cell(_FIELD_BY_KEY[key], signal[key]) for key in ("trend_score", "funnel_score")} - out.update({key + "_reason": value for key, value in signal["reasons"].items() - if key in ("trend_score", "funnel_score", "exposure")}) - exposure = signal["exposure"] - out["exposure"] = EXPOSURES[exposure["kind"]] if exposure else None - out["exposure_label"] = _exposure_detail(exposure) - for kind, key, _label, _now_type in SEARCH_TRENDS: - trend = signal["trends"][kind] - points = {point["month"]: point for point in trend["series"]} - series = [points[month] for month in history] - latest = series[-1] if series else {"value": None, "reason": signal["note"] or "No search report"} - now = latest["value"] if kind == "search" else signal[kind] - out.update({key: None if trend["change"] is None else str(trend["change"] * 100), - key + "_series": json.dumps(series, separators=(",", ":")), - key + "_now": (stored_cell(_FIELD_BY_KEY[key + "_now"], now) if kind == "search" - else None if now is None else str(now * 100)), - key + "_reason": trend["reason"], - key + "_now_reason": (signal["note"] or latest["reason"] if kind == "search" - else signal["reasons"][kind])}) - return out - - -def paint_asin_signals(st, con, *, today): - """Refresh source signals and Amazon's date for every stored ASIN, independent of the profit period.""" - from modules import amazon_search_signals as signals - asins = [str(row["asin"]).strip() for row in table_rows(st, ASIN_KEY).values() if row.get("asin")] - result = signals.signals(con, asins, today=today) - dates = _listing_dates(con) - parts = _part_dates.cells(st, asins) - computed = {asin: dict(_asin_signal_cells(signal, result["history"]), **parts[asin], pdp_last_updated=dates.get(asin), - pdp_last_updated_reason=None if asin in dates else "No listing lastUpdatedDate reported") - for asin, signal in result["asins"].items()} - return write_rows(st, computed, table_key=ASIN_KEY, key_column="asin", label=ASIN_LABEL, period_keys=()) - - -def paint_asin_part_dates(st): - """Refresh recorded part dates without profit validation, search reports or Odoo matching.""" - asins = [str(row['asin']).strip() for row in table_rows(st, ASIN_KEY).values() if row.get('asin')] - if not asins: - return {'rows': 0} - return write_rows(st, _part_dates.cells(st, asins), table_key=ASIN_KEY, - key_column='asin', label=ASIN_LABEL, period_keys=()) - - -def asin_signal_checks(st, con, *, today=None): - """Tie stored signal cells to the signal reader's independently validated raw monthly inputs.""" - from modules import amazon_search_signals as signals - rows = {str(row["asin"]).strip(): row for row in table_rows(st, ASIN_KEY).values() if row.get("asin")} - result = signals.signals(con, rows, today=today) - verified = signals.validate(con, result, today=today) - differences = sum(any(rows[asin].get(key) != value for key, value in _asin_signal_cells(signal, result["history"]).items()) - for asin, signal in result["asins"].items()) - return [{"check": "amazon_profit: stored ASIN search signals tie to raw monthly reports", - "ok": verified["ok"] and not differences, "rows": len(rows), "differences": differences, - "reason": verified["reason"]}] - - -def listing_date_checks(st, con): - """Independently tie stored PDP dates to the newest raw summary timestamp per ASIN.""" - newest = {} - available = "summaries" in _columns(con, "amzn_listing_items") - if available: - cursor = con.execute("SELECT summaries FROM amzn_listing_items") - for batch in iter(lambda: cursor.fetchmany(2048), []): - for (raw,) in batch: - try: - summaries = json.loads(raw) if isinstance(raw, str) else raw - except (TypeError, ValueError): - continue - for summary in summaries if isinstance(summaries, list) else (): - if not isinstance(summary, dict) or not summary.get("asin"): - continue - try: - stamp = dt.datetime.fromisoformat(str(summary.get("lastUpdatedDate")).replace("Z", "+00:00")) - except ValueError: - continue - stamp = stamp.replace(tzinfo=stamp.tzinfo or dt.timezone.utc).astimezone(dt.timezone.utc) - asin = summary["asin"] - if asin not in newest or stamp > newest[asin]: - newest[asin] = stamp - rows = list(table_rows(st, ASIN_KEY).values()) - differences = sum(row.get("pdp_last_updated") != (newest[row["asin"]].date().isoformat() - if row.get("asin") in newest else None) for row in rows) - return [{"check": "amazon_profit: PDP last updated ties to newest listing lastUpdatedDate per ASIN", - "ok": available and not differences, "rows": len(rows), "differences": differences, - "reason": None if available else "Listing summaries are unavailable"}] + if f["key"] not in (key_column, "picture", "parent_asin", "title", "projects", "projects_label") + + COMPETITOR_KEYS) def _date(value): @@ -3322,7 +3142,6 @@ def refresh(st, con, *, period=None, thin=None, by="", cost_reader=None, today=N report["parent"] = write_rows(st, by_grain["parent_asin"], table_key=PARENT_KEY, key_column="parent_asin", label=PARENT_LABEL, period_keys=_grain_period_keys(PARENT_FIELDS, "parent_asin")) - report["asinSignals"] = paint_asin_signals(st, con, today=day) report["blankReasons"] = blank_check(stored_tables(st)) bands = {sku: values.get("band") for sku, values in computed.items()} report["owned"] = apply_owned(st, bands, today=day, owner=owner_label) @@ -3468,7 +3287,6 @@ def _mirror_has_economics(con): def snapshot_cursor(tenant=TENANT_ZERO, tables=(ECONOMICS, SETTLEMENTS, LISTINGS, "amzn_orders", - "amzn_listing_items", "amzn_search_query_performance", "amzn_ads_sp_advertised_products", "amzn_ads_sb_advertised_products", "amzn_ads_sd_advertised_products")): """An in-memory DuckDB over the stored mirror snapshot. Downloads only; writes nothing back.""" @@ -3630,10 +3448,10 @@ def stored_grain_checks(st, con): def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=None, typed_rows=None, - typed_values=None, typed_period=None, st=None, today=None): + typed_values=None, typed_period=None, st=None): """Tie net proceeds components to independent Amazon sources within 2% over settled days. Receipt COGS ties to Odoo; typed COGS (`typed_rows`) is listed apart and ties to its stored values. - With `st`, stored grain figures, search signals and listing dates tie to their sources.""" + With `st`, the stored ASIN and parent figures tie per row to the mirror and every blank says why.""" month = month or last_complete_month() bounds = period_bounds(month) start, end = bounds["start"], bounds["end"] @@ -3693,8 +3511,6 @@ def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=No checks.extend(typed_cogs_checks(con, typed_rows, typed_values, cost_usage, typed_period or bounds)) if st is not None: checks.extend(stored_grain_checks(st, con)) - checks.extend(listing_date_checks(st, con)) - checks.extend(asin_signal_checks(st, con, today=today)) return checks diff --git a/platform/modules/amazon_profit_account.py b/platform/modules/amazon_profit_account.py index 3db4617a8cb6a943296fbaf65f79adf36ad5bacf..d42a19ca1622ba1d3f7573adf878b1e17f56b321 100644 --- a/platform/modules/amazon_profit_account.py +++ b/platform/modules/amazon_profit_account.py @@ -555,124 +555,6 @@ def validate_pace(con, bounds, pace, elapsed, month_days, reason): "checks": checks, "sourceTables": [ECONOMICS, "_sync_state"]} -class _SourceRows: - def __init__(self, rows): - self.rows = rows - - def fetchone(self): - return self.rows[0] if self.rows else None - - def fetchall(self): - return list(self.rows) - - -def _settlement_source(sql): - """Parse distinct settlement labels and referenced order dates without losing source lines or null keys.""" - expressions = {"posted": ("posted_date", f"CAST({profit._moment_sql('s.posted_date')} AS DATE)")} - for name, field in (("description", "amount_description"), ("kind", "amount_type"), - ("transaction", "transaction_type")): - expressions[name] = field, f"lower(regexp_replace(COALESCE({field}, ''), '[^a-zA-Z0-9]', '', 'g'))" - fields = [field for field, _expression in expressions.values()] - projection = ", ".join(expression.replace("s.posted_date", "posted_date") + " AS " + name - for name, (_field, expression) in expressions.items()) - keys = ", ".join(f"{field} AS source_{field}" for field in fields) - labels = (f"labels AS MATERIALIZED (SELECT {keys}, {projection} " - f"FROM (SELECT DISTINCT {', '.join(fields)} FROM {profit.SETTLEMENTS})), ") - for name, (_field, expression) in expressions.items(): - sql = sql.replace(expression + " AS " + name, "label." + name + " AS " + name) - join = " AND ".join(f"s.{field} IS NOT DISTINCT FROM label.source_{field}" for field in fields) - sql = sql.replace(f"FROM {profit.SETTLEMENTS} s JOIN streams", - f"FROM {profit.SETTLEMENTS} s JOIN labels label ON {join} JOIN streams") - sql = sql.replace("FROM amzn_orders GROUP BY amazon_order_id", - f"FROM amzn_orders WHERE amazon_order_id IN (SELECT trim(order_id) " - f"FROM {profit.SETTLEMENTS}) GROUP BY amazon_order_id") - return sql.replace("WITH orders AS (", "WITH " + labels + "orders AS (", 1) - - -class _ValidationSources: - """Share settlement source preparation across this request's independent window sums.""" - - def __init__(self, con, bounds): - self.con, self.results, self.spans = con, {}, {} - self.bounds = [span for bound in bounds for span in (bound, *_comparisons(bound).values())] - - def _spans(self, currency=None): - if currency in self.spans: - return self.spans[currency] - spans = {(bound["start"], bound["end"]) for bound in self.bounds} - if currency is not None: - for start, end in tuple(spans): - covered = profit.settlement_coverage(self, start, end, currency) - spans.add((covered["start"] or start, covered["end"] or end)) - self.spans[currency] = sorted(spans) - return self.spans[currency] - - def _components(self, sql, params): - marker = "SELECT component, COALESCE(SUM(currency_amount), 0)" - prefix, select = sql.split(marker, 1) - aggregates, select = (marker + select).split("FROM dated WHERE", 1) - aggregates = aggregates.removeprefix("SELECT component, ") - # The grouping keys retain every input to the original window predicate and its bad-row counts. - prefix += (" , prepared(component, basis, posted, dates, bad_dates, amount, source_rows, " - "bad_amounts, unmatched_orders) AS (SELECT component, basis, posted, dates, bad_dates, " - + aggregates + " FROM dated GROUP BY component, basis, posted, dates, bad_dates) ") - spans = self._spans(params[0]) - for bound in ("lo", "hi", "lo", "hi"): - select = select.replace("CAST(? AS DATE)", "span." + bound, 1) - ranges = ", ".join(f"({index}, CAST(? AS DATE), CAST(? AS DATE))" for index in range(len(spans))) - select = f"FROM prepared JOIN (VALUES {ranges}) AS span(id, lo, hi) ON" + select - select = select.replace("GROUP BY component ORDER BY component", - "GROUP BY span.id, component ORDER BY span.id, component") - bindings = list(params[:2]) + [value for span in spans for value in span] - rows = self.con.execute(_settlement_source(prefix) + "SELECT span.id, component, " - "COALESCE(SUM(amount), 0), SUM(source_rows), SUM(bad_amounts), SUM(unmatched_orders) " - + select, bindings).fetchall() - for index, span in enumerate(spans): - self.results[(sql, tuple(params[:2]) + span * 2)] = [row[1:] for row in rows if row[0] == index] - - def _unscoped(self, sql): - marker = "AND (CAST(" - source, _date_filter = sql.split(marker, 1) - source = source.replace("SELECT count(*)", f"SELECT CAST({profit._moment_sql('posted_date')} AS DATE) AS posted", 1) - spans = self._spans(profit.economics_currency(self.con)) - sums = ["count(*) FILTER (WHERE posted BETWEEN CAST(? AS DATE) AND CAST(? AS DATE) OR posted IS NULL)" - for _span in spans] - counts = self.con.execute("SELECT " + ", ".join(sums) + " FROM (" + source + ")", - [value for span in spans for value in span]).fetchone() - for index, count in enumerate(counts): - self.results[(sql, spans[index])] = [(count,)] - - def execute(self, sql, params=()): - params = tuple(params or ()) - key = sql, params - if key in self.results: - return _SourceRows(self.results[key]) - if sql == "SELECT column_name FROM information_schema.columns WHERE table_name = ?": - self.results[key] = self.con.execute(sql, params).fetchall() - return _SourceRows(self.results[key]) - components = (sql.startswith("WITH orders AS (") and len(params) == 6 - and sql.endswith(" GROUP BY component ORDER BY component")) - unscoped = (sql.startswith(f"SELECT count(*) FROM {profit.SETTLEMENTS} s\n") - and "WHERE h.settlement_id IS NULL" in sql and len(params) == 2) - if components: - self._components(sql, params) - elif unscoped: - self._unscoped(sql) - if key in self.results: - return _SourceRows(self.results[key]) - return self.con.execute(sql, params) - - -def _account_bounds(chosen, today): - bounds = [chosen, _bounds(today.replace(day=1), today - dt.timedelta(days=1), "mtd")] - current = today.replace(day=1) - for _ in range(2): - last = current - dt.timedelta(days=1) - current = last.replace(day=1) - bounds.append(_bounds(current, last, current.strftime("%Y-%m"))) - return bounds - - def account_view(con, period=None, *, stored=None, settings=None, today=None): """Return selected, comparison and monthly account windows without refreshing stored inputs.""" today = today or profit._today() @@ -692,8 +574,6 @@ def account_view(con, period=None, *, stored=None, settings=None, today=None): "TRY_CAST(left(CAST(start_date AS VARCHAR),10) AS DATE) IS NULL").fetchone()[0]: raise profit.SourceUnavailable("Economics dates unavailable. Resync Amazon economics.") stored, settings = stored or {}, settings or {} - periods = _account_bounds(chosen, today) - con = _ValidationSources(con, periods) cache = {} def window(bounds): key = bounds["start"], bounds["end"] @@ -702,8 +582,13 @@ def account_view(con, period=None, *, stored=None, settings=None, today=None): return {**cache[key], "period": bounds} def compared(bounds): return {**window(bounds), "comparisons": {key: window(span) for key, span in _comparisons(bounds).items()}} - mtd = compared(periods[1]) - months = [compared(bounds) for bounds in periods[2:]] + current = today.replace(day=1) + mtd = compared(_bounds(current, today - dt.timedelta(days=1), "mtd")) + months = [] + for _ in range(2): + last = current - dt.timedelta(days=1) + current = last.replace(day=1) + months.append(compared(_bounds(current, last, current.strftime("%Y-%m")))) pace = {} elapsed = mtd["period"]["days"] month_days = calendar.monthrange(today.year, today.month)[1] diff --git a/platform/modules/competitor_watch.py b/platform/modules/competitor_watch.py index ac3a76e4eaed32434799b49073048bb504e1868f..92712d6fc7a053bac09cf4678c754d26a06c387e 100644 --- a/platform/modules/competitor_watch.py +++ b/platform/modules/competitor_watch.py @@ -300,7 +300,7 @@ class _TermIndex(_DocumentIndex): self.sql("SELECT (value ->> 'activeTotal')::bigint FROM {tbl} WHERE key = %s")) total_params = ([self.slug, *([display_term(search)] if search else [])] if search or tracked else [self.key(KEY_TERM_INDEX)]) - tail = ("), " + statement = self.term_sql("WITH totals(total) AS (" + total + "), " "bounds AS (SELECT total, CASE WHEN %s >= total THEN " "greatest(0, floor((total - 1)::numeric / %s)::bigint * %s) ELSE %s END AS skip FROM totals) " "SELECT total, skip, EXISTS(SELECT 1 FROM {terms} WHERE tenant = %s AND term = %s AND " + membership + "), " @@ -308,12 +308,6 @@ class _TermIndex(_DocumentIndex): + ("AND strpos(lower(label), %s) > 0 " if search else "") + 'ORDER BY ' + order + ' LIMIT %s OFFSET (SELECT skip FROM bounds)) ' "FROM bounds") - if isinstance(total, str): - statement = self.term_sql("WITH totals(total) AS (" + total + tail) - else: - # PostgreSQL's `self.sql` returns a psycopg Composed: compose it, never concatenate it as text. - from psycopg import sql as psql - statement = psql.SQL("WITH totals(total) AS (") + total + psql.SQL(self.term_sql(tail)) params = [*total_params, offset, limit, limit, offset, self.slug, selected, self.slug, *([display_term(search)] if search else []), limit] with self.connection() as con: @@ -469,25 +463,15 @@ def _ba_columns(cur): def _sqp(cur, term=None): - """Monthly SQP rows as (term, asin, impressions, end_date) for our ASINs.""" + """SQP rows as (term, asin, impressions, end_date). SQP reports our ASINs only.""" need = {"search_query_data_search_query", "asin"} cols = _columns(cur, SQP_TABLE) if cur is not None else set() if not need <= cols: return [] - if "_report_period" in cols: - period = '"_report_period" = \'MONTH\'' - elif {"start_date", "end_date"} <= cols: - start = 'TRY_CAST("start_date" AS DATE)' - end_date = 'TRY_CAST("end_date" AS DATE)' - period = f'day({start}) = 1 AND {end_date} = last_day({start})' - else: - return [] imp = ('TRY_CAST("impression_data_asin_impression_count" AS DOUBLE)' if "impression_data_asin_impression_count" in cols else "0") end = 'CAST("end_date" AS VARCHAR)' if "end_date" in cols else "''" - where = f" WHERE {period}" - if term is not None: - where += f' AND {_term_sql("search_query_data_search_query")} = ?' + where = f' WHERE {_term_sql("search_query_data_search_query")} = ?' if term is not None else "" rows = cur.execute( f'SELECT "search_query_data_search_query", "asin", {imp}, {end} FROM {SQP_TABLE}{where}', [term] if term is not None else []).fetchall() diff --git a/requirements.txt b/requirements.txt index bb0118a83c34c480c0e9c495e3be5262671d4081..45716e5201551b4cc934ae5d6e0d24f861185128 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,6 @@ duckdb==1.5.4 # duckdb's pandas bridge imports pytz for timezone values; pandas 3 no longer installs it. pytz>=2024.1 pyyaml>=6.0 -jsonschema==4.26.0 pillow>=10.0 beautifulsoup4>=4.12 lxml>=5.0 diff --git a/web/package-lock.json b/web/package-lock.json index 29b264c9661800ff3cda10dcc3ced14ed65a8e49..5ce70eacdce18de5fa9d988738a65780c7c06adf 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -25,7 +25,6 @@ "@types/react": "^18.3.12", "@types/react-dom": "^18.3.1", "@vitejs/plugin-react": "^6.0.3", - "esbuild": "^0.27.0", "openapi-typescript": "^7.13.0", "oxlint": "^1.71.0", "typescript": "~6.0.2", @@ -390,448 +389,6 @@ "integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==", "license": "MIT" }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", - "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", - "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", - "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", - "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", - "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", - "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", - "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", - "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", - "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", - "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", - "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", - "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", - "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", - "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", - "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", - "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", - "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", - "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", - "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", - "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", - "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", - "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", - "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", - "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", - "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", - "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, "node_modules/@glideapps/glide-data-grid": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/@glideapps/glide-data-grid/-/glide-data-grid-6.0.3.tgz", @@ -2292,48 +1849,6 @@ "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", "license": "MIT" }, - "node_modules/esbuild": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", - "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.27.7", - "@esbuild/android-arm": "0.27.7", - "@esbuild/android-arm64": "0.27.7", - "@esbuild/android-x64": "0.27.7", - "@esbuild/darwin-arm64": "0.27.7", - "@esbuild/darwin-x64": "0.27.7", - "@esbuild/freebsd-arm64": "0.27.7", - "@esbuild/freebsd-x64": "0.27.7", - "@esbuild/linux-arm": "0.27.7", - "@esbuild/linux-arm64": "0.27.7", - "@esbuild/linux-ia32": "0.27.7", - "@esbuild/linux-loong64": "0.27.7", - "@esbuild/linux-mips64el": "0.27.7", - "@esbuild/linux-ppc64": "0.27.7", - "@esbuild/linux-riscv64": "0.27.7", - "@esbuild/linux-s390x": "0.27.7", - "@esbuild/linux-x64": "0.27.7", - "@esbuild/netbsd-arm64": "0.27.7", - "@esbuild/netbsd-x64": "0.27.7", - "@esbuild/openbsd-arm64": "0.27.7", - "@esbuild/openbsd-x64": "0.27.7", - "@esbuild/openharmony-arm64": "0.27.7", - "@esbuild/sunos-x64": "0.27.7", - "@esbuild/win32-arm64": "0.27.7", - "@esbuild/win32-ia32": "0.27.7", - "@esbuild/win32-x64": "0.27.7" - } - }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", diff --git a/web/package-lock.linux.sha256 b/web/package-lock.linux.sha256 index e63f7a0b4481302fb298fbe769e767faac24f540..daa34af70b65b397fc83a12fdcec3ae911ff70a3 100644 --- a/web/package-lock.linux.sha256 +++ b/web/package-lock.linux.sha256 @@ -1,2 +1,2 @@ -9cd826154f5fecd9784f6632ad62e9da94db63b1ed343ab137fbd29ce5690c07 package-lock.json -# resolved on linux/x64 by HF job 6afd11b54643bac7ac631093 on 2026-11-18, npm ci passed there before upload. Regenerate ONLY with lockfile_linux.py (W-170). +31cb494538043db42bdf698846c2262f5eb234fb686aa03d9cf3ab1cbfdbe2b8 package-lock.json +# resolved on linux/x64 in the SDLC 72 lane A HF container on 2026-10-09 (S72-T05, npm install --save-exact @phosphor-icons/react@2.1.10); npm ci --ignore-scripts passed there (228 packages). Regenerate ONLY with lockfile_linux.py (W-170). diff --git a/web/package.json b/web/package.json index a8eadcac73854da83c0cdf4d02f6865087b0889d..899f0c9da672b6ff210e4762a6a020881243a048 100644 --- a/web/package.json +++ b/web/package.json @@ -39,7 +39,6 @@ "@types/react": "^18.3.12", "@types/react-dom": "^18.3.1", "@vitejs/plugin-react": "^6.0.3", - "esbuild": "^0.27.0", "openapi-typescript": "^7.13.0", "oxlint": "^1.71.0", "typescript": "~6.0.2", diff --git a/web/public/AIOS_DEPLOY_RECEIPT b/web/public/AIOS_DEPLOY_RECEIPT index 88db683d5ed0811f9cf08add51149a98ec14f66b..6d9415b8a9829b4b9a49fcda032a3a4ff75353d3 100644 --- a/web/public/AIOS_DEPLOY_RECEIPT +++ b/web/public/AIOS_DEPLOY_RECEIPT @@ -1 +1 @@ -4f29d08d01160aa6ccd46623bfc075cb2ca87d826499ba6a6cc4196a66b79c1d +55137f24b4cb4023d1609a3ec3179bd65aba9f362ca4a4a7403d2839fd270873 diff --git a/web/src/apps/amazon-profit/AppView.tsx b/web/src/apps/amazon-profit/AppView.tsx index d3d8af591b0667c26e358f81ee86f602a34c1ec7..a3d252bd168235097df3c1d09f37ed6ca4efd040 100644 --- a/web/src/apps/amazon-profit/AppView.tsx +++ b/web/src/apps/amazon-profit/AppView.tsx @@ -2,7 +2,7 @@ import { useCallback, useEffect, useMemo, useState } from "react"; import { API_V1, CREDENTIALS, ROWS_STALE_EVENT, signal } from "../../apiContract"; import { ACCESS_REFUSED_MESSAGE, AppTableGrid, clearTopicRowsCache, dropTableRowsCache, OverlayProvider, LP_BLUE_DEEP, LP_GREEN_DEEP } from "../../customer-grid/seam"; -import type { AppCellDraw, AppTableHost, AppViewProps, Row } from "../../customer-grid/seam"; +import type { AppTableHost, AppViewProps } from "../../customer-grid/seam"; import { AMAZON_DATA_ENTRY_TABLE, AMAZON_PROFIT_TABLE, AmazonToolbar, COMMAND_CENTER_TABLES, CommandCenterTabStrip, CommandCenterTableContext } from "./AmazonPeriodPicker"; import { LogoLoader } from "../../ui/LogoLoader"; @@ -11,36 +11,12 @@ import PicturePopup from "./PicturePopup"; import { ProjectsPopup, ProjectsPage, projectsTargetFromRow } from "./Projects"; import type { ProjectsTarget } from "./Projects"; import { CompetitorsPopup, competitorsTargetFromRow } from "./Competitors"; -import GridFilter, { SIGNAL_SORTS } from "./pdp/GridFilter"; -import HistoryPopup from "./pdp/HistoryPopup"; -import PdpUpdatePopup, { type PdpListing, type PdpNavigation } from "./pdp/PdpUpdatePopup"; -import WorkflowWindow from "./pdp/WorkflowWindow"; -import type { PdpProjectNavigation } from "./pdp/ContinueWorkflow"; -import type { PdpProjectPane } from "./pdp/contract"; import "../../styles/sdlc81/command-center-app.css"; -import "../../styles/sdlc86/project-manager-pdp.css"; const PROJECT_PROGRESS_COLORS = { inProgress: LP_BLUE_DEEP, completed: LP_GREEN_DEEP }; const PROJECT_PROGRESS_TABLES = new Set([AMAZON_PROFIT_TABLE, "ut_amazon_asin_profit", "ut_amazon_parent_profit"]); /** The App's 56 px rows (R5). Table keeps its own default. */ const COMMAND_CENTER_COLUMNS = { rowHeight: 56 }; -const ASIN_COLUMNS = { rowHeight: 56, headerTone: { projects: "primary", pdp_update: "primary", history: "primary" }, - widths: { asin: 150, projects: 210, pdp_update: 170, trend_score: 150, search_trend: 210, - market_share_trend: 240, click_share_trend: 240, exposure: 190, funnel_score: 150, history: 150 } } as const; -const noReport = (reasonKey: string) => ({ reasonKey, equals: "No search report", text: "No search report" }); -const signalTrend = (key: string, share: boolean): AppCellDraw => ({ kind: "trend", seriesKey: key + "_series", - valueKey: key + "_now", changeKey: key, format: share ? "percent" : "number", showValue: share, - omitMissingValue: true, valuePrecision: "adaptive", changeDecimals: 0, changeNeutralBand: .5, fitLabels: true, barTone: "primary", - unavailable: noReport(key + "_reason") }); -const SIGNAL_CELLS: AppTableHost["cellRenderers"] = { - trend_score: { draw: { kind: "ring", labelPosition: "after", unavailable: noReport("trend_score_reason") } }, - search_trend: { draw: signalTrend("search_trend", false) }, - market_share_trend: { draw: signalTrend("market_share_trend", true) }, - click_share_trend: { draw: signalTrend("click_share_trend", true) }, - exposure: { draw: { kind: "badge", subKey: "exposure_label", emptyText: "None", unavailable: noReport("exposure_reason") } }, - funnel_score: { draw: { kind: "score", neutralRange: [4.5, 5.5], outerRange: [3, 7], emptyText: "Few", - unavailable: noReport("funnel_score_reason") } }, -}; /** Admission belongs to the selected tab's fresh probe. */ type TableProbe = { table: string; status: "ready"; tables: Record } @@ -48,17 +24,12 @@ type TableProbe = { table: string; status: "ready"; tables: Record(null); const [picture, setPicture] = useState<{ ref: string; label: string } | null>(null); - const [projects, setProjects] = useState<(ProjectsTarget & { - selectedProjectId?: string; originListing?: PdpListing; initialPane?: PdpProjectPane; - }) | null>(null); + const [projects, setProjects] = useState(null); const [competitors, setCompetitors] = useState(null); - const [history, setHistory] = useState(null); - const [pdp, setPdp] = useState(null); - const [workflow, setWorkflow] = useState<{ id: string; listing: PdpListing; resumeRevision?: number } | null>(null); const [gridRevision, setGridRevision] = useState(0); const [entryEmpty, setEntryEmpty] = useState<{ table: string; empty: boolean } | null>(null); const openTable = useCallback((next: string) => { @@ -69,9 +40,6 @@ export default function ProjectManagementView(props: AppViewProps & PdpNavigatio setPicture(null); setProjects(null); setCompetitors(null); - setHistory(null); - setPdp(null); - setWorkflow(null); setTable(next); }, [table]); const reloadGrid = useCallback(() => { @@ -120,37 +88,12 @@ export default function ProjectManagementView(props: AppViewProps & PdpNavigatio }, [dataEntry, admitted, table]); const entryChecked = !dataEntry || notBuilt || entryEmpty?.table === table; const allComplete = dataEntry && admitted && entryEmpty?.table === table && entryEmpty.empty; - const openProject = (id: string, asin: string, listing: PdpListing, initialPane?: PdpProjectPane) => { - if (props.onOpenProject) { props.onOpenProject(id, asin); return; } - setPdp(null); - setProjects({ grain: "asin", rowKey: asin, title: listing.title, selectedProjectId: id, originListing: listing, initialPane }); - }; - const resumeWorkflow = () => { - setProjects(null); - setWorkflow(current => current ? { ...current, resumeRevision: (current.resumeRevision ?? 0) + 1 } : current); - }; - const projectNavigation: PdpProjectNavigation = { onOpenWorkflow: (next, item, target) => { - if (workflow?.id === next.id && workflow.listing.asin === item.asin) { - resumeWorkflow(); - return; - } - setProjects(null); - if (props.onOpenWorkflow) { props.onOpenWorkflow(next.id, item.asin); return; } - setWorkflow({ id: next.id, listing: { asin: item.asin, title: target?.title ?? "", sku: target?.sku ?? null, brand: null } }); - } }; // C2: progress colours, button doors and the Picture popup, declared once per table. // Only the Projects table keeps New record (`append`). const cellRenderers = useMemo(() => ({ - ...(table === "ut_amazon_asin_profit" ? SIGNAL_CELLS : {}), "*": { progressColors: PROJECT_PROGRESS_TABLES.has(table) ? PROJECT_PROGRESS_COLORS : undefined }, projects: { button: true, activate: row => setProjects(projectsTargetFromRow(table, row)) }, competitors: { button: true, activate: row => setCompetitors(competitorsTargetFromRow(table, row)) }, - history: { button: true, activate: row => setHistory(row) }, - pdp_update: { button: true, activate: row => { - if (!row.asin) return false; - setPdp({ asin: String(row.asin), title: String(row.title || ""), - sku: row.sku ? String(row.sku) : null, brand: row.brand ? String(row.brand) : null }); - } }, "type:image": { activate: (record, field) => { const ref = String(record[field.key] ?? "").trim(); if (!ref) return false; @@ -172,35 +115,17 @@ export default function ProjectManagementView(props: AppViewProps & PdpNavigatio : table === "ut_amazon_projects" ?
- +
: !entryChecked ? : : undefined} - sortChoices={table === "ut_amazon_asin_profit" ? SIGNAL_SORTS : undefined} + columns={COMMAND_CENTER_COLUMNS} cellRenderers={cellRenderers} toolbar={} />} {picture && setPicture(null)} />} - {projects && setProjects(null)} onChanged={reloadGrid} - backLabel={projects.initialPane ? "Back to workflow" : projects.originListing ? "Back to projects" : undefined} - onBack={projects.initialPane ? resumeWorkflow : projects.originListing - ? () => { setPdp(projects.originListing ?? null); setProjects(null); setWorkflow(null); } : undefined} />} + {projects && setProjects(null)} onChanged={reloadGrid} />} {competitors && setCompetitors(null)} onChanged={reloadGrid} />} - {history && setHistory(null)} onChanged={reloadGrid} />} - {pdp && setPdp(null)} onChanged={reloadGrid} - onOpenWorkflow={props.onOpenWorkflow ?? ((id, asin) => { - setWorkflow({ id, listing: { ...pdp, asin } }); setPdp(null); - })} onAddWorkflow={props.onAddWorkflow} - onOpenProject={(id, asin) => openProject(id, asin, pdp)} />} - {workflow && setWorkflow(null)} onChanged={reloadGrid} - onBack={() => { setPdp(workflow.listing); setWorkflow(null); }} - onOpenProject={() => openProject(workflow.id, workflow.listing.asin, workflow.listing)} - onOpenProjectPane={pane => openProject(workflow.id, workflow.listing.asin, workflow.listing, pane)} />} ; } diff --git a/web/src/apps/amazon-profit/ProjectOverview.tsx b/web/src/apps/amazon-profit/ProjectOverview.tsx index 7bbf8cf74fa4e814215c9c259d84ce773a6ac55a..46d7a996abb4aa50c68be723a073429ad3e87265 100644 --- a/web/src/apps/amazon-profit/ProjectOverview.tsx +++ b/web/src/apps/amazon-profit/ProjectOverview.tsx @@ -11,8 +11,6 @@ import { X } from "@phosphor-icons/react/dist/csr/X"; import { useEscapeClose } from "../../ui/useEscapeClose"; import ProjectTasks from "./ProjectTasks"; import ProjectSkuPicker from "./ProjectSkuPicker"; -import { PdpApprovalRow, PdpOverviewRows, PdpStep, type ProjectPdp } from "./pdp/ContinueWorkflow"; -import { removeItems } from "./pdp/api"; import { EMPTY_FILTER, GRAINS, ProductToolbar, SelectBox, TABLES, normalizeProducts, number, readPage, scopedNodes, type Field, type FilterTree, type Row, type Sorts, @@ -238,12 +236,11 @@ function ProjectDeleteDialog({ name, busy, error, onCancel, onConfirm }: { } export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChanged, onDeleted, onCancel, - error, setError, setDetail, setTick, formId, active, rowScope = null, onClearRowScope, pdp }: { + error, setError, setDetail, setTick, formId, active, rowScope = null, onClearRowScope }: { id: string; detail: Detail; canEdit: boolean; onChanged: () => void; onDeleted: () => void; onCancel: () => void; error: string; setError: Dispatch>; setDetail: Dispatch>; setTick: Dispatch>; formId: string; active: boolean; rowScope?: RowScope | null; onClearRowScope?: () => void; - pdp?: ProjectPdp; }) { const [draft, setDraft] = useState(() => overviewDraftOf(detail.project)); const [people, setPeople] = useState([]); @@ -321,12 +318,6 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange setBusy(true); const fallback = "Products were not removed."; try { - if (pdp) { - if (!pdp.detail?.canEdit) throw new Error("The workflow is unavailable. Reload the project."); - pdp.accept((await removeItems(id, pairs.map(pair => pair.key))).workflow); - setTick(value => value + 1); - return { refused: [] }; - } const answer = await call(`/${enc(id)}/targets/remove`, { method: "POST", body: { targets: pairs } }, fallback); if (!answer?.project || !Array.isArray(answer.targets)) throw new Error("The remove answer is incomplete. Reload the project."); @@ -352,10 +343,10 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange writing.current = false; setBusy(false); } - }, [detail?.canEdit, id, mayEdit, onChanged, setDetail, setError, setTick, pdp]); + }, [detail?.canEdit, id, mayEdit, onChanged, setDetail, setError, setTick]); const { project, targets } = detail; - const canEdit = mayEdit && detail.canEdit && (!pdp || Boolean(pdp.detail?.canEdit)); + const canEdit = mayEdit && detail.canEdit; const deleteOpen = confirming && canEdit && active; useEffect(() => { if (!canEdit || !active) { setConfirming(false); setAdding(false); } @@ -469,7 +460,6 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
- {project.type === "pdp" && pdp ? : null} {canEdit ? set("status", value)} /> @@ -492,7 +482,6 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange onChange={values => { set("ownerUsername", values[0] ?? null); set("owner", null); }} /> : project.owner || project.ownerUsername ? : No owner} - {project.type === "pdp" && pdp ? : null}
{contributors.map(person => @@ -566,7 +555,7 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange : null}
{canEdit && active && adding - ? setAdding(false)} pdp={pdp} + ? setAdding(false)} onAdded={answer => { setDetail(current => current ? { ...current, project: answer.project, targets: answer.targets, cause: answer.cause } : current); @@ -577,7 +566,7 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange : null} {targets.length ? + onRemove={removeTargets} /> : null}
@@ -665,10 +654,9 @@ async function readLinked(table: string, grain: ProjectGrain, tree: FilterTree, } } -function ProjectSkuList({ targets: linked, scope, canEdit, busy, active, onRemove, pdp }: { +function ProjectSkuList({ targets: linked, scope, canEdit, busy, active, onRemove }: { targets: Target[]; scope: (RowScope & { clear: () => void }) | null; canEdit: boolean; busy: boolean; active: boolean; onRemove: (pairs: TargetPair[]) => Promise; - pdp?: ProjectPdp; }) { const id = useId(); const [chosen, setChosen] = useState("sku"); @@ -698,14 +686,13 @@ function ProjectSkuList({ targets: linked, scope, canEdit, busy, active, onRemov
: null} target.grain === grain)} canEdit={canEdit} busy={busy} active={active} - onRemove={onRemove} pdp={pdp} /> + onRemove={onRemove} /> ; } -function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, onRemove, pdp }: { +function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, onRemove }: { grain: ProjectGrain; panelId: string; tabId: string | null; targets: Target[]; canEdit: boolean; busy: boolean; active: boolean; onRemove: (pairs: TargetPair[]) => Promise; - pdp?: ProjectPdp; }) { const table = TABLES[grain]; const label = GRAIN_LABEL[grain]; @@ -799,7 +786,7 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o : new Set(Array.from(current).filter(key => shown.has(key)))); }, [term, blocked, shownSignature]); const columns = LIST_COLUMNS[grain]; - const span = (canEdit ? 1 : 0) + 1 + columns.length + (pdp ? 2 : 0); + const span = (canEdit ? 1 : 0) + 1 + columns.length; const resetView = () => { setSelected(new Set()); setRefused([]); setRemoveError(""); }; const clearView = () => { resetView(); setFilters(EMPTY_FILTER); setSorts([]); setGroup(null); setSearch(""); }; const viewSet = filtered || sorts.length > 0 || group !== null || selected.size > 0 || search !== ""; @@ -864,7 +851,6 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o {canEdit ?
: null} {columns.map(column => )} - {pdp ? <> : null} {groups.map(([value, members]) => {group !== null ? ; })} - {pdp ? : null} ; })} )} @@ -913,14 +898,6 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o ; } -function PdpSkuCells({ pdp, target }: { pdp: ProjectPdp; target: Target }) { - const asin = target.grain === "asin" ? target.key : target.asin; - const item = pdp.detail?.workflow.items.find(item => item.asin === asin); - return <> - ; -} - export function ProjectForm({ targets, onCreated, onCancel, onForbidden }: { targets: Array<{ grain: ProjectGrain; key: string }>; onCreated: (project: Project) => void; onCancel: () => void; onForbidden: () => void; diff --git a/web/src/apps/amazon-profit/ProjectSkuPicker.tsx b/web/src/apps/amazon-profit/ProjectSkuPicker.tsx index b8bff2a7eea1f0e4dfbc31652e7d9895a81c3999..9e2f4f7d2981e043d432174fad3a15fc14c74c3f 100644 --- a/web/src/apps/amazon-profit/ProjectSkuPicker.tsx +++ b/web/src/apps/amazon-profit/ProjectSkuPicker.tsx @@ -11,9 +11,6 @@ import { FilterBuilderPanel, Popover, SortBuilder, countConditions, import { CenteredPopup, GRAIN_LABEL, ProjectRequestError, call, enc, messageOf, pictureSrc, text, type Project, type ProjectGrain, type Target } from "./ProjectsShared"; import "../../styles/sdlc85/project-sku-picker.css"; -import { projectClaimNames, readProjectClaims, type ProjectPdp } from "./pdp/ContinueWorkflow"; -import { addItems, PdpRequestError } from "./pdp/api"; -import type { PdpClaim } from "./pdp/contract"; export type FilterTree = FilterBuilderPanelProps["filters"]; type FilterNode = FilterTree["nodes"][number]; @@ -175,7 +172,7 @@ export type TaskTargetSource = { name: string; targets?: ReadonlyArray<{ grain: string; key: string }> | null; targetFilter?: { grain: string; filter: unknown } | null; completionFilter?: unknown; }; -type PickerBase = { projectId: string; canEdit: boolean; onClose: () => void; pdp?: ProjectPdp }; +type PickerBase = { projectId: string; canEdit: boolean; onClose: () => void }; export type SkuPickerProps = PickerBase & ( | { mode: "add"; existing: Target[]; onAdded: (answer: ProjectSkuAddAnswer) => void } | { mode: "task"; task: TaskTargetSource; onAssign: (change: TaskTargetChange) => Promise }); @@ -204,12 +201,11 @@ function taskStart(task: TaskTargetSource | null) { done: treeOf(task?.completionFilter), picked, assigned: !!saved || Object.keys(picked).length > 0 }; } -export default function ProjectSkuPicker({ projectId, existing, canEdit, onAdded, onClose, pdp }: { +export default function ProjectSkuPicker({ projectId, existing, canEdit, onAdded, onClose }: { projectId: string; existing: Target[]; canEdit: boolean; onAdded: (answer: ProjectSkuAddAnswer) => void; onClose: () => void; - pdp?: ProjectPdp; }) { - return ; + return ; } /** One picker body: `add` links products to the project, `task` sets one task's targets and completion filter. */ @@ -217,15 +213,11 @@ export function SkuPicker(props: SkuPickerProps) { const { projectId, canEdit, onClose } = props; const task = props.mode === "task" ? props : null; const taskMode = task !== null; - const pdp = props.mode === "add" ? props.pdp : undefined; - const pdpMode = Boolean(pdp); - const workflow = pdp?.detail?.workflow; - const [taken, setTaken] = useState([]); const existing = props.mode === "add" ? props.existing : NO_TARGETS; const onAdded = props.mode === "add" ? props.onAdded : ignoreAnswer; const id = useId(); const [start] = useState(() => taskStart(task?.task ?? null)); - const [grain, setGrain] = useState(pdp ? "asin" : start.grain); + const [grain, setGrain] = useState(start.grain); const [query, setQuery] = useState(""); const [search, setSearch] = useState(""); const [filters, setFilters] = useState(start.filter); @@ -269,17 +261,15 @@ export function SkuPicker(props: SkuPickerProps) { const order = useMemo(() => group ? [{ colId: group, dir: "asc" }, ...sorts.filter(sort => sort.colId !== group)] : sorts, [group, sorts]); const available = Math.max(0, (match?.total ?? 0) - (match?.linked ?? 0)); - const holder = (key: string) => workflow ? projectClaimNames(taken, workflow, key) : ""; - const selectedCount = allMatching ? Math.max(0, available - excluded.size) - : workflow ? Array.from(selected).filter(key => !holder(key)).length : selected.size; - const ready = canEdit && (!pdp || Boolean(pdp.detail?.canEdit)) && !forbidden && !loading && !busy && query === search && match !== null && !error; + const selectedCount = allMatching ? Math.max(0, available - excluded.size) : selected.size; + const ready = canEdit && !forbidden && !loading && !busy && query === search && match !== null && !error; const isSelected = (key: string) => linked.has(key) || (allMatching ? !excluded.has(key) : selected.has(key)); - const pageKeys = rows.map(row => text(row[grain])).filter(key => key && !linked.has(key) && !holder(key)); + const pageKeys = rows.map(row => text(row[grain])).filter(key => key && !linked.has(key)); const checkedOnPage = pageKeys.filter(isSelected).length; const preview = taskMode && kind === "filter"; const picks = selected.size + Object.entries(held) .reduce((sum, [value, keys]) => value === grain ? sum : sum + (keys?.length ?? 0), 0); - const grains = pdp ? ["asin"] as const : taskMode ? TASK_GRAINS : GRAINS; + const grains = taskMode ? TASK_GRAINS : GRAINS; const resetSelection = () => { setSelected(new Set()); setAllMatching(false); setExcluded(new Set()); }; const resetRead = () => { @@ -335,11 +325,6 @@ export function SkuPicker(props: SkuPickerProps) { if (counted && counted.total > 0 && !products.length) { throw new Error("Matching products did not load. Retry to refresh the matches."); } - if (pdpMode) { - const claims = await readProjectClaims(products.map(row => text(row.asin)), controller.signal); - if (controller.signal.aborted) return; - setTaken(current => cursor === null ? claims : [...current, ...claims]); - } setFields(page.fields); setRows(current => { if (counted?.total === 0) return []; @@ -370,7 +355,7 @@ export function SkuPicker(props: SkuPickerProps) { }) .finally(() => { if (!controller.signal.aborted) setLoading(false); }); return () => controller.abort(); - }, [projectId, canEdit, grain, table, tree, order, cursor, attempt, linkedSignature, taskMode, pdpMode]); + }, [projectId, canEdit, grain, table, tree, order, cursor, attempt, linkedSignature, taskMode]); const toggleKeys = (keys: string[], checked: boolean) => { if (allMatching) setExcluded(current => { @@ -389,11 +374,6 @@ export function SkuPicker(props: SkuPickerProps) { if (!ready || !selectedCount || submitting.current) return; submitting.current = true; setBusy(true); setRefused([]); try { - if (pdp) { - pdp.accept((await addItems(projectId, Array.from(selected).filter(key => !holder(key)))).workflow); - onClose(); - return; - } // Exclusions apply after server ranking, so they cannot promote a different top-N row. const complete = allMatching && excluded.size ? await matchingKeys(projectId, grain, tree, match!.total) : null; @@ -413,11 +393,7 @@ export function SkuPicker(props: SkuPickerProps) { } else onClose(); } catch (reason) { setError(messageOf(reason, "Products were not added.")); - if (reason instanceof PdpRequestError && reason.status === 409) { - try { setTaken(await readProjectClaims(rows.map(row => text(row.asin)))); } - catch (refreshReason) { setError(messageOf(refreshReason, "Workflow claims did not load. Retry.")); } - } - if ((reason instanceof ProjectRequestError || reason instanceof PdpRequestError) && [401, 403].includes(reason.status)) { + if (reason instanceof ProjectRequestError && [401, 403].includes(reason.status)) { setForbidden(true); setRows([]); setFields([]); setMatch(null); } } finally { submitting.current = false; setBusy(false); } @@ -438,10 +414,7 @@ export function SkuPicker(props: SkuPickerProps) { try { const every = await matchingKeys(projectId, grain, tree, current.total); if (every.total !== current.total) throw new Error("Products changed. Retry to refresh the matches."); - const claims = workflow ? await readProjectClaims(every.keys) : []; - if (workflow) setTaken(claims); - const eligible = workflow ? every.keys.filter(key => !linked.has(key) && !projectClaimNames(claims, workflow, key)) : every.keys; - setSelected(previous => new Set([...previous, ...eligible])); + setSelected(previous => new Set([...previous, ...every.keys])); } catch (reason) { refuse(reason, "Matches did not load."); } finally { submitting.current = false; setBusy(false); } }; @@ -535,7 +508,7 @@ export function SkuPicker(props: SkuPickerProps) { {match?.linked ? ` · ${number(match.linked)} already linked` : ""} {match && available > 0 && !preview ? - {preview ? null : }
{boxFor(visibleKeys, `Select all shown ${plural}`)}Product{column.label}StepWorkflow
@@ -887,7 +873,6 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o return {shown || Not set}
{item ? : Unavailable}
toggleKeys([key], checked)} />
{picture ? :
{key} {grain === "parent_asin" diff --git a/web/src/apps/amazon-profit/Projects.tsx b/web/src/apps/amazon-profit/Projects.tsx index 1b9c0d06f6cdaf70bfb46b9f32273561a32bb611..d3bc220537c431e800719352a63fc55b87703d0d 100644 --- a/web/src/apps/amazon-profit/Projects.tsx +++ b/web/src/apps/amazon-profit/Projects.tsx @@ -1,16 +1,12 @@ -import { useCallback, useEffect, useId, useMemo, useRef, useState, type ReactNode } from "react"; +import { useCallback, useEffect, useId, useMemo, useRef, useState } from "react"; import { - CenteredPopup, GRAIN_LABEL, LoadingRows, ProjectHeader, call, enc, messageOf, + CenteredPopup, GRAIN_LABEL, LoadingRows, call, enc, messageOf, type Detail, type ProjectGrain, type ProjectList, } from "./ProjectsShared"; import ProjectResults from "./ProjectResults"; import { AddExisting, ProjectBrowser } from "./ProjectsList"; -import ProjectOverview, { ProjectForm, RowScopeChip, inRowScope, type RowScope } from "./ProjectOverview"; +import ProjectOverview, { ProjectForm, RowScopeChip, type RowScope } from "./ProjectOverview"; import ProjectActivities from "./ProjectActivities"; -import PdpActivities from "./pdp/PdpActivities"; -import BrandRules from "./pdp/BrandRules"; -import ContinueWorkflow, { useProjectPdp, type PdpProjectNavigation } from "./pdp/ContinueWorkflow"; -import type { PdpProjectPane } from "./pdp/contract"; export { CenteredPopup, projectsTargetFromRow } from "./ProjectsShared"; export type { CenteredPopupProps, Project, ProjectGrain, ProjectsTarget } from "./ProjectsShared"; @@ -23,29 +19,16 @@ const PROJECT_PANES = [ { key: "results", label: "Dashboard" }, { key: "log", label: "Activities" }, ] as const; -type ExtraPane = { key: string; label: string; content: ReactNode }; -type ProjectExtras = { - pdp?: PdpProjectNavigation; backLabel?: string; onBack?: () => void; - initialPane?: PdpProjectPane; - headerExtra?: ReactNode; extraPanes?: ExtraPane[]; -}; +type ProjectPane = (typeof PROJECT_PANES)[number]["key"]; const RESULT_GRAIN = { sku: "sku", asin: "asin", parent_asin: "parent" } as const; -function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged, onDeleted, onCancel, - pdp: navigation, headerExtra, extraPanes, initialPane }: { +function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged, onDeleted, onCancel }: { id: string; canEdit: boolean; rowScope?: RowScope; onChanged: () => void; onDeleted: () => void; onCancel: () => void; -} & ProjectExtras) { +}) { const [detail, setDetail] = useState(null); const [error, setError] = useState(""); const [tick, setTick] = useState(0); - const [pane, setPane] = useState(initialPane ?? "setup"); - const projectChanged = () => { setTick(value => value + 1); onChanged(); }; - const pdp = useProjectPdp(detail?.project, detail?.targets ?? [], tick, mayEdit && Boolean(detail?.canEdit), projectChanged, navigation); - const extras = detail?.project.type === "pdp" ? [ - { key: "brand", label: "Brand and rules", content: }, ...(extraPanes ?? []), - ] : []; - const panes = extras.length ? [...PROJECT_PANES, ...extras] : PROJECT_PANES; + const [pane, setPane] = useState("setup"); // The row a Projects cell opened from scopes the Overview SKU list and the Dashboard until it is cleared. const [rowScope, setRowScope] = useState(() => openedFrom?.key.trim() ? { grain: openedFrom.grain, key: openedFrom.grain === "sku" ? openedFrom.key.trim() : openedFrom.key.trim().toUpperCase() } @@ -81,20 +64,20 @@ function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged, const rail =