loopable / api /connectors_amazon.py
fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
daae64a verified
Raw History Blame
47 kB
"""connectors_amazon.py β€” the API-layer FACE of the Amazon connector: the liveness probe.
β›”β›” **THE TRANSPORT IS NOT HERE. IT IS `platform/harness/amazon_client.py`, AND THAT IS A LAYER
RULE, NOT A PREFERENCE.** `harness/amazon_store.py` needs the same client this probe needs, and
nothing under `platform/` may import an API-layer module -- layers point one way. The first version
of this connector put the transport here and had the store reach up for it; `verify_meta`'s
architecture leg caught it in one line. Everything the client exports is re-exported below, so a
caller that imports it from here still works.
β›” **THE PROBER IS REGISTERED, NEVER IMPORTED BY `core`.** `core` never imports up
(ARCHITECTURE.md), and the tempting shortcut -- a `sys.path.insert` plus an `import` from inside
`core/keychain.py` -- is an architecture inversion no gate was watching for until wave 31.
`keychain.register_prober` is the slot; `datastore.set_paused_probe` is the same idiom one layer
down.
β›” **READ-ONLY.** See the client's header for the promise and its two fixed POST wrappers.
"""
from __future__ import annotations
import os
import logging
import sys
import time
from pathlib import Path
from fastapi import APIRouter, HTTPException, Request
_PLATFORM = Path(__file__).resolve().parents[2] / "platform"
if str(_PLATFORM) not in sys.path:
sys.path.insert(0, str(_PLATFORM))
from core.request_timing import raise_if_resource_error
# ⭐ RE-EXPORTED, NOT REDEFINED. One definition of the credential shape, the endpoints, the report
# map and the transport; this module adds the probe and the registration and nothing else.
from harness.amazon_client import ( # noqa: F401,E402
AWD_INVENTORY_PAGE_SIZE, COMPETITIVE_SUMMARY_BATCH_SIZE,
CREDENTIAL_FIELDS, DEFAULT_REGION, ENDPOINTS, KEY, LABEL, LWA_TOKEN_URL, OPTIONAL_FIELDS,
REPORT_TYPES, SANDBOX_ENDPOINTS, base_for, is_sandbox,
ROLES_REFUSED, ROLES_TAKEN, US_MARKETPLACE_ID, AmazonError, _create_report,
_competitive_summary_batch, _COMPETITIVE_SUMMARY_BATCH_PATH,
_COMPETITIVE_SUMMARY_ITEM_URI, _download_document, _post_form, _REPORTS_CREATE_PATH,
_request, _safe, _TIMEOUT, _AWD_INVENTORY_PATH, access_token, awd_inventory,
competitive_summaries, marketplace_participations, report_document,
)
#: ⚠ MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data
#: structure on a screen about their Amazon account. That was the wave-31 Meta defect and the
#: comments in `keychain.py` warn about it in as many words.
ENTRY_TYPE = "amazon_sp"
class AmazonStatusSafetyError(ValueError):
"""A status response contains a forbidden credential shape."""
class AmazonStatusRowsRefused(AmazonStatusSafetyError):
"""S67-T31: some status ROWS were refused; the rest are carried, already guarded.
Still an `AmazonStatusSafetyError`, so every caller that refuses on the base class keeps
refusing. `safe_status` holds only rows that passed the guard and `refused` only table NAMES:
neither carries the refused content, so a caller may degrade those rows and keep the others.
"""
def __init__(self, safe_status, refused):
super().__init__('Amazon status contains a forbidden credential shape.')
self.safe_status = safe_status
self.refused = tuple(refused)
#: The status guard's scalar ceiling, and the sentence a row wears when its status is refused.
STATUS_SCALAR_LIMIT = 512
STATUS_UNAVAILABLE = "Sync status is unavailable for this database."
def trim_status(value, limit=STATUS_SCALAR_LIMIT):
"""Cut every string over `limit` to a word-bounded excerpt that fits, recursively.
S67-T31: a long, honest cause (staging's Search Query Performance cause measured 935
characters) used to trip the length branch of the guard and take the whole connector modal
down with a 500. β›” Call this only AFTER the credential-shape scan of the UNTRIMMED value
(`screen_status_rows`): cutting first could slice a token shape out of a string and accept
the rest, which would trim a refusal into acceptance.
"""
if isinstance(value, dict):
return {key: trim_status(item, limit) for key, item in value.items()}
if isinstance(value, list):
return [trim_status(item, limit) for item in value]
if isinstance(value, tuple):
return tuple(trim_status(item, limit) for item in value)
if isinstance(value, str) and len(value) > limit:
head = value[:limit - 3]
cut = head.rsplit(" ", 1)[0] if " " in head else head
return cut.rstrip(" ,;:") + "..."
return value
def screen_status_rows(status):
"""`(safe, refused)` for a `{table: row}` status: each row judged on its own.
A row carrying a credential key, an `Atzr|` token or a bearer shape ANYWHERE in its untrimmed
form is refused whole (NC: never trimmed into acceptance, its value never returned). A row that
is only too long is trimmed, then held to the full guard again.
"""
import json
safe, refused = {}, []
for name, row in (status or {}).items():
try:
whole = json.dumps(row, ensure_ascii=False, allow_nan=False)
if status_payload_has_secret(whole, max_length=None):
refused.append(name)
continue
trimmed = trim_status(row)
if status_payload_has_secret(json.dumps(trimmed, ensure_ascii=False, allow_nan=False)):
refused.append(name)
continue
except (TypeError, ValueError, RecursionError):
refused.append(name)
continue
safe[name] = trimmed
return safe, refused
def status_payload_has_secret(serialized_payload, *, max_length=STATUS_SCALAR_LIMIT):
"""Scan serialized JSON for C2's credential keys, token shapes and oversized values.
Pure and usable without a server. Invalid JSON is unsafe too; never echo rejected content.
The credential key roster is the same one the connector accepts at connect time.
`max_length=None` scans for credential shapes only (the pre-trim pass of `screen_status_rows`).
"""
import json
import re
try:
pending = [json.loads(serialized_payload, object_pairs_hook=lambda pairs: pairs)]
except (TypeError, ValueError, UnicodeError, RecursionError):
return True
credential_keys = {key.casefold() for key in CREDENTIAL_FIELDS}
while pending:
value = pending.pop()
if isinstance(value, tuple):
key, item = value
if key.casefold() in credential_keys:
return True
pending.extend((key, item))
elif isinstance(value, list):
pending.extend(value)
elif isinstance(value, str):
if ((max_length is not None and len(value) > max_length) or 'Atzr|' in value
or re.search(r'\bbearer\s+\S+', value, flags=re.IGNORECASE)):
return True
elif (value is not None and max_length is not None
and len(str(value)) > max_length):
return True
return False
def assert_safe_status(payload):
"""Return a status only after scanning its serialized form, or raise without its content."""
import json
try:
serialized = json.dumps(payload, ensure_ascii=False, allow_nan=False)
except (TypeError, ValueError, RecursionError):
raise AmazonStatusSafetyError('Amazon status could not be safely serialized.') from None
if status_payload_has_secret(serialized):
raise AmazonStatusSafetyError('Amazon status contains a forbidden credential shape.')
return payload
def first_load_status(tenant_key, rt=None):
"""Writer-side progress helper; request status uses sync_state's read-only cursor."""
from harness import amazon_store
return assert_safe_status(amazon_store.first_load_status(tenant_key, rt=rt))
def _stored_table_cap():
"""The row ceiling a materialised `ut_*` table can hold; `amazon_relational.plan` applies it."""
try:
from core import user_tables
return int(getattr(user_tables, "MAX_ROWS", 60_000))
except Exception: # noqa: BLE001
return 60_000
def _mirror_counts(cur, az):
"""{table: row count} for each Amazon mirror table that exists. Read only; absent = never synced."""
counts = {}
for table in az.DATABASE_TABLES:
try:
counts[table] = int(cur.execute(
f"SELECT count(*) FROM {az.table_name(table)}").fetchone()[0])
except Exception as exc: # noqa: BLE001
raise_if_resource_error(exc)
continue
return counts
NOTIFICATION_SECRET_SLOT = "amazon_sp_notifications"
NOTIFICATION_SECRET_HEADER = "x-loopable-amazon-secret"
NOTIFICATION_DEDUPE_SECONDS = 600
NOTIFICATION_TARGETING_CONTRACT = "amazon-table-events-v1"
notification_router = APIRouter(prefix="/api/v1/connectors/amazon_sp")
def _notification_secret(rt):
"""Read the tenant's named generic keychain slot, without an environment fallback."""
from core import keychain
rows = [row for row in keychain.list_entries(rt)
if row.get("type") == "generic" and row.get("label") == NOTIFICATION_SECRET_SLOT]
if len(rows) != 1:
return None
fields = keychain.read_fields(rt, rows[0]["id"]) or {}
secret = fields.get("api_key")
return secret if isinstance(secret, str) and secret else None
def _authenticate_notification(rt, supplied):
"""Authenticate before reading or parsing any event bytes."""
import hmac
try:
secret = _notification_secret(rt)
except Exception:
secret = None
if not secret or not supplied or not hmac.compare_digest(
secret.encode("utf-8"), supplied.encode("utf-8")):
raise HTTPException(status_code=401, detail="Amazon notification authentication failed.")
def notification_target(event):
"""Resolve an approved event to one known source table and its vendor event ID."""
from harness.amazon_client import NOTIFICATION_TYPES, REPORT_TYPES
if not isinstance(event, dict):
raise ValueError("Amazon notification must be an object.")
detail = event.get("detail", event)
if not isinstance(detail, dict):
raise ValueError("Amazon notification detail must be an object.")
event_type = detail.get("notificationType", detail.get("NotificationType"))
if event_type not in NOTIFICATION_TYPES:
raise ValueError("Amazon notification type is not approved for this connector.")
metadata = detail.get("notificationMetadata", detail.get("NotificationMetadata", {}))
if not isinstance(metadata, dict):
raise ValueError("Amazon notification metadata must be an object.")
event_id = metadata.get("notificationId", metadata.get("NotificationId")) or event.get("id")
if not isinstance(event_id, str) or not event_id.strip() or len(event_id) > 256:
raise ValueError("Amazon notification requires a vendor event ID.")
declared = detail.get("table")
if event_type == "REPORT_PROCESSING_FINISHED":
payload = detail.get("payload", detail.get("Payload", {}))
if not isinstance(payload, dict):
raise ValueError("Amazon report notification payload must be an object.")
report = payload.get("reportProcessingFinishedNotification",
payload.get("ReportProcessingFinishedNotification", {}))
if not isinstance(report, dict):
raise ValueError("Amazon report notification must describe its report.")
report_type = report.get("reportType", report.get("ReportType"))
candidates = [table for table, spec in REPORT_TYPES.items()
if report_type and report_type in
[spec.get(key) for key in ("report", "report_incremental", "report_discover")]]
if declared is None and len(candidates) == 1:
declared = candidates[0]
if not isinstance(declared, str) or declared not in candidates:
raise ValueError("Amazon report notification requires one matching known table.")
if report.get("processingStatus", report.get("ProcessingStatus")) != "DONE":
return event_id, None
table = declared
else:
table = NOTIFICATION_TYPES[event_type]["table"]
if declared is not None and declared != table:
raise ValueError("Amazon notification table does not match its event type.")
return event_id, table
def notification_sync_options(receipt):
"""Keep explicit table targets narrow and exclude them from full-source catch-up completion."""
from harness import amazon_store
tables = receipt.get("tables")
if tables is None:
return {}
if (not isinstance(tables, (list, tuple)) or not tables
or any(not isinstance(table, str) or table not in amazon_store.TABLES for table in tables)):
raise ValueError("amazon_notification_target_invalid")
return {"tables": list(dict.fromkeys(tables)), "_catch_up_complete": False}
def _notification_sync_available(desk):
"""The integrator publishes this marker only after producer and resident consumer support targeting."""
import inspect
return (getattr(desk, "NOTIFICATION_TARGETING_CONTRACT", None) == NOTIFICATION_TARGETING_CONTRACT
and {"tables", "event_id", "rt"} <= set(inspect.signature(desk.request_sync).parameters))
@notification_router.post("/notifications", status_code=202)
async def amazon_notifications(request: Request):
"""Queue an authenticated event through the durable targeted sync desk."""
from harness import runtime
from core import connector_sync
tenant = request.query_params.get("tenant")
if not tenant or tenant not in runtime.known_tenants():
raise HTTPException(status_code=401, detail="Amazon notification authentication failed.")
rt = runtime.get_runtime(tenant)
if rt.key != tenant:
raise HTTPException(status_code=401, detail="Amazon notification authentication failed.")
_authenticate_notification(rt, request.headers.get(NOTIFICATION_SECRET_HEADER))
try:
event_id, table = notification_target(await request.json())
except (ValueError, TypeError, UnicodeError):
raise HTTPException(status_code=400, detail="Amazon notification has no approved table target.") from None
if table is None:
return {"accepted": False, "reason": "Amazon report did not finish successfully."}
if not _notification_sync_available(connector_sync):
raise HTTPException(status_code=503, detail="amazon_notification_targeting_unavailable")
receipt = connector_sync.request_sync("amazon_sp", tenant, requested_by="amazon notification",
rt=rt, tables=[table], event_id=event_id)
if receipt.get("tables") != [table] or receipt.get("eventId") != event_id:
raise HTTPException(status_code=503, detail="amazon_notification_target_contract_violated")
return {"accepted": receipt.get("accepted") is True, "table": table,
"duplicate": receipt.get("duplicate") is True}
def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
"""Each Amazon table's sync state, or None when this mirror cannot be read.
Read only: never _connect_tenant_mirror or first_load_status, which issue DDL.
Missing bookkeeping is unknown, not a measured zero. First-load numbers describe the
whole connection, because that is the scope of the producer's durable progress marker.
"""
import json
from datetime import datetime, timezone
import duckdb
from harness import amazon_store as az
from harness.amazon_client import DECLARED_DATASETS
from routes_odoo_tables import _sync_entities, stuck_verdict
# S66-T46 (lead): an unreadable deletion ledger is flagged once per row as
# `purgeScheduleUnknown`, never written into every table's cause, which would push out the
# reason each table already names (Amazon's refusal, the empty period, the declared reason).
purge_unknown = False
try:
purge_plan = az.scheduled_purge(tenant_key, rt, now=now)
purge_message = az.deletion_schedule_message(purge_plan) if purge_plan else ""
except Exception as exc:
raise_if_resource_error(exc)
purge_plan, purge_message, purge_unknown = None, "", True
def finish(status):
for row in status.values():
# A key only when there is something to say: a declared dataset's row stays exactly
# its declaration while no purge is scheduled (the group (a) status contract).
if purge_plan:
row["purgeScheduledAt"] = purge_plan["purgeAt"]
if purge_unknown and not row.get("declared"):
row["purgeScheduleUnknown"] = True
if purge_message:
cause = row.get("unknownCause") or ""
# Status scalars have a 512-character safety ceiling. Keep the full existing
# cause separately if the deletion sentence needs a shorter screen excerpt.
budget = 512 - len(purge_message) - 1
if len(cause) > budget:
row["syncCause"] = cause
cause = cause[:budget - 3].rsplit(" ", 1)[0] + "..."
row["unknownCause"] = " ".join(part for part in (cause, purge_message) if part)
# S67-T31: judged PER ROW. An over-long cause is trimmed (staging's 500 was
# `search_query_performance.unknownCause` at 935 characters); a credential shape still
# refuses, now naming only its rows so the caller degrades those and keeps the rest.
safe, refused = screen_status_rows(status)
if refused:
raise AmazonStatusRowsRefused(assert_safe_status(safe), refused)
return assert_safe_status(safe)
declared = {name: {"rowsLoaded": None, "declared": True, "unknownCause": spec["reason"]}
for name, spec in DECLARED_DATASETS.items()}
unavailable = {table: {"rowsLoaded": None, "lastSuccess": None, "inProgress": False,
"unknownCause": "This container has no readable sync status. Sync this database to load its rows."}
for table in az.DATABASE_TABLES}
unavailable.update(declared)
def stamp(value):
if not value:
return None
try:
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
return (parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)).astimezone(timezone.utc)
except (ValueError, TypeError, OverflowError):
return None
def iso(value):
return value.isoformat().replace("+00:00", "Z") if value else None
cadence = cadence_seconds if cadence_seconds is not None else 1800
try:
path = az._tenant_mirror_path(tenant_key, rt=rt)
if not path.is_file():
return finish(unavailable)
# β›”β›” A SECOND `duckdb.connect(read_only=True)` ON A FILE THIS PROCESS ALREADY HOLDS RAISES
# `ConnectionException: Can't open a connection to same database file with a different
# configuration`. On the DEFAULT tenant layout the Amazon mirror IS the process-global
# analytical mirror (`datastore.DB_PATH`), so the swallowed refusal below turned this whole
# screen into a permanent "sync status is unknown" on exactly the deployment the ticket is
# for. Measured on staging 2026-09-14, W60-T35. `datastore.ro_con()` is the shipped
# per-thread READ cursor on that same instance, and it takes no lock against the writer.
from harness import datastore as _ds
if path.resolve() == _ds.DB_PATH.resolve():
# `ro_cursor()` is a cursor on the instance this process ALREADY holds: no second
# connection, no lock contention with the writer, and no DDL. Deliberately NOT
# `ro_con()`, which refuses until `ready()` and would report the whole screen as
# unknown whenever the Odoo backfill is mid-flight, and NOT `datastore.connect()`,
# which issues `CREATE TABLE IF NOT EXISTS`.
_cur = _ds.ro_cursor()
entities = _sync_entities(_cur, cadence)
mirrored = _mirror_counts(_cur, az)
else:
with _ds.bounded_connection(duckdb.connect(str(path), read_only=True)) as cur:
entities = _sync_entities(cur, cadence)
mirrored = _mirror_counts(cur, az)
except Exception as exc: # noqa: BLE001
raise_if_resource_error(exc)
# No exception text: paths, provider payloads and credentials never become status copy.
return finish(unavailable)
if entities is None:
return finish(unavailable)
clock = stamp(now) if now is not None else datetime.now(timezone.utc)
states, bad_ticks = {}, set()
for table in az.TABLES:
tick = entities.get(az.entity_key("sync_tick_" + table))
if tick:
updated = stamp(tick.get("updatedAt"))
if updated is None or tick.get("phase") not in ("attempted", "done", "running"):
bad_ticks.add(table)
else:
states[table] = {"state": tick["phase"], "updated_at": iso(updated)}
# An unreadable input cannot support a cadence promise for any table in the rotation.
plan = az.sync_plan(states, now=clock) if not bad_ticks and clock is not None else None
first_load, load_unknown = None, ""
markers = {key: {"state": value.get("phase"), "detail": value.get("cursor")}
for key, value in entities.items()}
refused_reports = az.refused_reports(markers)
enabled = az.read_database_enablement(rt)["databases"]
load = entities.get(az.entity_key(az.FIRST_LOAD_STATUS_KEY)) or {}
if load:
try:
progress = json.loads(load.get("cursor") or "")
remaining, eta = progress["remaining"], progress["etaSeconds"]
if (type(remaining) is not int or not 0 <= remaining <= 2**53 - 1
or type(eta) is not int or not 0 <= eta <= 2**53 - 1
or (load.get("phase") == "done" and remaining > 0)):
raise ValueError("Invalid first-load progress")
if remaining > 0:
first_load = {"remaining": remaining, "etaSeconds": eta,
"cause": az.first_load_eta_cause(remaining),
"needsAttention": load.get("phase") == "failed"}
if refused_reports:
first_load = {**(first_load or {"remaining": remaining, "etaSeconds": eta}),
"refused": refused_reports}
if first_load:
first_load = {**(first_load or {}), **az.first_load_payload(markers, remaining, eta,
state=load.get("phase"), tables=[table for table in az.TABLES
if enabled.get(az.amazon_schema.DATABASE_PREFIX + table)])}
except (ValueError, TypeError, KeyError, OverflowError, RecursionError):
load_unknown = "First-load progress is unknown because its saved marker could not be read. Retry sync to refresh it."
elif (entities.get(az.entity_key(az.FIRST_LOAD_END_KEY)) or {}).get("phase") not in (None, "done"):
load_unknown = "First-load progress is not available yet. Keep the connection enabled and check again after this pass."
if first_load is None and refused_reports and not load:
first_load = az.first_load_payload(markers, 0, 0, state="failed",
tables=[table for table in az.TABLES if enabled.get(az.amazon_schema.DATABASE_PREFIX + table)])
first_load["etaDays"] = None
answer = dict(declared)
for table in az.TABLES:
landed = entities.get(az.entity_key(table)) or {}
tick = entities.get(az.entity_key("sync_tick_" + table)) or {}
watermark = entities.get(az.entity_key(az.WATERMARK_PREFIX + table)) or {}
rows = mirrored.get(table, landed.get("rows"))
if type(rows) is not int or rows < 0:
rows = None
# Hotfix 2026-09-24: during the first load no per-table marker exists yet, so the screen
# read "Not loaded yet" beside a mirror already holding thousands of rows. The mirror's
# own count is the measured answer.
if rows is None and table in mirrored:
rows = mirrored[table]
# A refused pass closes its tick (S66-T42); only the watermark can then prove a success.
refused_now = landed.get("phase") in ("refused", "failed", "throttled")
successes = [when for marker in ((watermark,) if refused_now else (landed, tick, watermark))
if marker.get("phase") == "done"
and (when := stamp(marker.get("updatedAt"))) is not None]
stuck, stuck_cause = tick.get("stuck"), tick.get("stuckCause") or ""
if tick.get("phase") == "attempted":
# The shared stall helper calls an active phase 'running'; Amazon calls it 'attempted'.
updated = stamp(tick.get("updatedAt"))
stuck, stuck_cause = stuck_verdict(
"running", iso(updated), cadence,
now=clock.replace(tzinfo=None) if clock else None)
reason = plan["deferred"].get(table) if plan else None
# Only the planner's explicit due stamp is a due date. Minimum stagger spacing is not
# a promise about when a later resident tick will select this table.
next_due = stamp(reason.partition("next due ")[2]) if reason else None
if first_load and load.get("phase") in ("running", "partial"):
wait_marker = entities.get(az.entity_key(az.FIRST_LOAD_DUE_KEY)) or {}
due = stamp(wait_marker.get("cursor")) if wait_marker.get("phase") == "pending" else None
vendor_wait = az._interval_wait(table, states.get(table), clock)[1] if clock else None
if landed.get("phase") == "pending" or rows is None:
next_due = vendor_wait or due or clock
if not reason:
reason = (str(landed.get("cursor")) if landed.get("phase") == "pending" else
"The first load is queued on the resident sync thread; keep the connection enabled.")
unknown = [load_unknown] if load_unknown else []
proof = entities.get(az.entity_key("reconcile_" + table)) or {}
try:
period = json.loads(proof.get("cursor") or "{}")
if not isinstance(period, dict):
period = {}
except (ValueError, TypeError):
period = {}
if rows is None:
unknown.append("This table has not loaded yet. Sync it to request rows from Amazon.")
if landed.get("phase") in ("refused", "failed", "throttled"):
unknown = [str(landed.get("cursor") or "Amazon refused this dataset. Retry sync to refresh its reason.")]
elif rows == 0 or period.get("count") == 0:
span = " to ".join(dict.fromkeys(str(period[k]) for k in ("start", "end") if period.get(k)))
unknown.append("Amazon returned no rows for " + (span or "the last requested period") + ".")
# Standing rule 1's second half: a limit that cannot be removed is REPORTED, with cause
# and fix. `amazon_relational.plan` refuses a table over MAX_ROWS and said so only in the
# server log, so the owner saw an empty database and no reason.
over = mirrored.get(table)
if over is not None and over > _stored_table_cap():
unknown.append(
f"The mirror holds {over:,} rows, more than a stored table can hold "
f"({_stored_table_cap():,}). This table is served read-through from the mirror "
"with no row cap.")
# S66-T42/T43 (lead): a table `amazon_relational.apply_plan` could not create because the
# workspace is at the store's table cap says so here, with the cause and the fix.
not_stored = entities.get(az.entity_key("not_stored_" + table)) or {}
if not_stored.get("phase") == "not_stored":
unknown.append(str(not_stored.get("cursor") or
"This table is not in this workspace yet because the workspace is at "
"its database limit. Its rows are kept in the Amazon mirror."))
if plan is None and landed.get("phase") not in ("refused", "failed", "throttled"):
unknown.append("The sync schedule is unknown because a saved timestamp could not be read. Retry sync to refresh it.")
limit = None
if (table in az.GROUP_A_TABLES and table != "catalog_items") or table == "transactions":
marker = entities.get(az.entity_key("limit_" + table)) or {}
try:
limit = json.loads(marker.get("cursor") or "{}").get("reason")
except (ValueError, TypeError, AttributeError):
limit = "History progress is unreadable. Retry sync to refresh its saved date."
if limit:
unknown.append(limit)
if table == "brand_analytics":
# R18's weekly retention is a code fact: its copy never waits for a weekly pull's marker.
limit = az.BRAND_ANALYTICS_LIMIT
unknown.append(limit)
if table == "catalog_items":
marker = entities.get(az.entity_key("limit_" + table)) or {}
try:
limit = json.loads(marker.get("cursor") or "{}").get("reason")
except (ValueError, TypeError, AttributeError):
limit = "Catalog request progress is unreadable. Sync Amazon listings and retry."
limit = limit or ("Amazon Catalog allows 20 ASINs per search and 2 requests per second. "
"Every own ASIN and page is followed; sync Amazon listings first.")
unknown.append(limit)
if table == "listing_items":
limit = ("Amazon Listings allows 1,000 SKUs per query and 5 requests per second. "
"Date ranges are split and every page followed. Keep the connection enabled.")
unknown.append(limit)
if table == az.SQP_TABLE:
marker = entities.get(az.SQP_LIMIT_KEY) or {}
if marker.get("cursor"):
try:
detail = json.loads(marker["cursor"])
limit = az.sqp_limit_sentence(detail)
unknown.append(limit)
# S66-T41 (lead): one sentence PER REASON, so a whole refused batch (18
# ASINs) stays inside `assert_safe_status`'s 512-character scalar ceiling and
# every refused ASIN is still named on screen.
by_reason = {}
for asin, refusal in detail.get("refusedAsins", {}).items():
by_reason.setdefault(str(refusal), []).append(str(asin))
for refusal, asins in by_reason.items():
unknown.append(az.sqp_refusal_message(asins[0], refusal) if len(asins) == 1
else f"Amazon refused ASINs {', '.join(sorted(asins))} for "
f"Search Query Performance: {refusal}")
unknown.extend(detail.get("errors", []))
if detail.get("deferredReason"):
reason = detail["deferredReason"]
if detail.get("partialMonthReason"):
unknown.append(detail["partialMonthReason"])
except (ValueError, TypeError, KeyError, AttributeError):
unknown.append("Search Query Performance limits are unknown because the saved marker could not be read. Retry sync to refresh it.")
if table in az.GROUP_A_TABLES and len(" ".join(unknown)) > 512:
# Preserve the dataset's refusal/empty-period and vendor limit. Generic schedule
# diagnostics can otherwise overflow the status guard and hide every database.
unknown = [part for part in unknown if part != load_unknown
and not part.startswith(("The sync schedule is unknown", "This table has not loaded yet."))]
answer[table] = {
"rowsLoaded": rows,
"lastSuccess": iso(max(successes)) if successes else None,
"inProgress": tick.get("phase") == "attempted" and stuck is False,
"deferred": {"reason": reason} if reason else None,
"nextDue": iso(next_due),
"firstLoad": first_load,
"stuck": stuck, "stuckCause": stuck_cause,
"unknownCause": " ".join(unknown),
}
# A source-internal mismatch published the table. Account markers carry a connection prefix.
warned = az.entity_key("warning_" + table)
warning = next((str(value.get("cursor") or "") for key, value in entities.items()
if (key == warned or key.endswith("__" + warned)) and value.get("cursor")), "")
if warning:
answer[table]["warning"] = warning
if table == az.SQP_TABLE or table in az.GROUP_A_TABLES or table in ("transactions", "brand_analytics"):
answer[table]["limit"] = limit
for table in (*az.DERIVED_TABLES, "competitor_snapshots", "competitor_images"):
answer[table] = {
"rowsLoaded": mirrored.get(table), "lastSuccess": None, "inProgress": False,
"unknownCause": az.amazon_schema.PROVENANCE_COPY[az.amazon_schema.PROVENANCE[table]],
}
# A history table is written in its source's transaction, so it shares the source's last success.
for table in az.HISTORY_TABLES:
rows = mirrored.get(table)
answer[table] = {
"rowsLoaded": rows,
"lastSuccess": (answer.get(table.removesuffix("_history")) or {}).get("lastSuccess"),
"inProgress": False,
"unknownCause": (az.amazon_schema.PROVENANCE_COPY["history"] if rows is not None
else "History starts with the next sync of the source table."),
}
return finish(answer)
#: W55-T19 / R7 / C10 β€” AMAZON'S OWN ENABLEMENT DOCUMENT. This is deliberately not Odoo's
#: model catalogue or connector config: Amazon publishes report schemas, not ``fields_get``, and
#: its seven databases are enabled at the provider boundary rather than masquerading as Odoo
#: models. The document lives on the request's tenant runtime, so two tenants can make opposite
#: choices without a process-global map ever seeing either answer.
ENABLEMENT_KEY = "amazon_connector_enablement"
ENABLEMENT_ROUTE = "/connectors/amazon_sp/databases/{databaseKey}/enablement"
class EnablementConfigError(ValueError):
"""The tenant's stored Amazon enablement document is malformed."""
def database_keys():
"""The authored Amazon database keys, including derived mirror views."""
from harness.amazon_store import DATABASE_TABLES
return tuple("ut_amzn_" + str(name) for name in DATABASE_TABLES)
def read_database_enablement(rt):
"""Use the sync engine's fail-closed evaluator for this tenant's database choices."""
from harness import amazon_store as az
return az.read_database_enablement(rt)
def set_database_enabled(rt, database_key, enabled):
"""Persist one tenant's Amazon database choice without touching its materialised rows."""
key = str(database_key or "").strip()
known = set(database_keys())
if key not in known:
raise KeyError(key)
if type(enabled) is not bool:
raise ValueError("enabled must be true or false")
def _set(current):
if current is None:
current = {}
if not isinstance(current, dict):
raise EnablementConfigError("the stored Amazon enablement document is malformed")
stored = current.get("databases", {})
if not isinstance(stored, dict):
raise EnablementConfigError("the stored Amazon database choices are malformed")
if any(type(value) is not bool for value in stored.values()):
raise EnablementConfigError("the stored Amazon database choices are malformed")
if set(stored) - known:
logging.getLogger(__name__).warning("Ignoring unknown Amazon database enablement keys")
# A fresh document contains the whole current roster. That makes the stored answer
# inspectable while retaining the migration rule above for future database additions.
choices = {name: stored.get(name, True) for name in database_keys()}
choices[key] = enabled
return {**current, "databases": choices}
rt.update(ENABLEMENT_KEY, _set, flush="sync")
return read_database_enablement(rt)
def enablement_capability(rt, is_admin):
"""The provider-aware capability served by Amazon's connector detail response."""
state = read_database_enablement(rt)
can_manage = bool(is_admin) and state["available"]
return {"provider": ENTRY_TYPE,
"mode": "database",
"canManage": can_manage,
# A member receives no write URL. The route still enforces administrator status; this
# is the matching affordance wall, not the permission boundary.
"updatePath": ENABLEMENT_ROUTE if can_manage else "",
# Disabling changes only this config document. No mirror or ``ut_amzn_*`` row is
# removed, so re-enabling resumes against the same materialised database.
"preservesData": True,
"available": state["available"]}
def probe_credential(fields, timeout=8, *, fetch_participations=None):
"""Return the guarded Amazon status; a disclosure violation must raise at this boundary."""
return assert_safe_status(_probe_credential_result(
fields, timeout, fetch_participations=fetch_participations))
def _probe_credential_result(fields, timeout=8, *, fetch_participations=None):
"""`{ok, message}` for a stored `amazon_sp` credential: the keychain's liveness probe.
β›” IT LIVES HERE, IN THE API LAYER, AND IS REGISTERED INTO `core.keychain` RATHER THAN IMPORTED
BY IT. `core` never imports up (ARCHITECTURE.md); `keychain.register_prober` is the slot, and
`connectors_meta` established the idiom after a first draft did `sys.path.insert` plus an
`import` from inside `core`.
⚠ ZERO marketplace participations is reported as NOT ok, deliberately, for the same reason Meta
reports zero ad accounts that way: a credential that authenticates but reaches nothing is an
authorization answer, and `ok: true` would hide the one thing the admin has to fix.
"""
f = fields if isinstance(fields, dict) else {}
missing = [key for key in CREDENTIAL_FIELDS
if not isinstance(f.get(key), str) or not f[key].strip()]
if missing:
return {"ok": False,
"message": "Missing " + ", ".join(missing) + ". Enter every required field."}
try:
# Bind only when explicitly tested. Shape gates can inject a local response function.
fetch = fetch_participations if fetch_participations is not None else marketplace_participations
parts = fetch(f, timeout=timeout)
live = [part for part in parts if isinstance(part, dict)
and isinstance(part.get("participation"), dict)
and part["participation"].get("isParticipating") is True]
if not live:
return {"ok": False,
"message": "Amazon reported no active marketplace access. Check authorization and test again."}
except Exception: # noqa: BLE001
# AmazonError includes provider-supplied text and an invalid region can echo user input.
return {"ok": False,
"message": "Amazon did not confirm marketplace access. Check the five fields and test again."}
return {"ok": True, "message": "Amazon confirmed marketplace access."}
def register():
"""Fill the keychain's probe slot for this connector.
Called at import by the API layer's connector registry (`routes_connectors._register_probers`),
so any process that can serve the connector directory has also registered the prober. Never
raises: a probe slot that could not be filled costs a liveness CHECK, never the ability to
store a credential.
"""
try:
import core.keychain as kc
return kc.register_prober(ENTRY_TYPE, probe_credential)
except Exception: # noqa: BLE001
return False
def _env_credential():
"""A credential assembled from the environment, for `--check` only.
β›” NOT a runtime path. The product reads Amazon credentials from the per-tenant keychain and
from nowhere else; this exists so an operator can prove a freshly minted refresh token works
before pasting it into the app.
"""
# ⚠ `OPTIONAL_FIELDS` joins the read so `AMAZON_SP_SANDBOX=true` reaches `is_sandbox` and the
# probe talks to the sandbox host. Absent still means PRODUCTION, which is the safe default.
f = {k: (os.environ.get("AMAZON_SP_" + k.upper()) or "").strip()
for k in tuple(CREDENTIAL_FIELDS) + OPTIONAL_FIELDS}
return {k: v for k, v in f.items() if v}
def probe_report(fields, report_type=None, days=30, log=print):
"""Drive the WHOLE report chain once and print the MEASURED header. Writes to no store.
`createReport -> poll -> getReportDocument -> download -> decompress -> parse`, then print the
columns the document actually carried. **This is the only way to learn a flat-file report's
schema short of a production sync**, and it is exactly what the connector cannot know from
Amazon's published artifacts: five of our seven tables have none.
β›” NOTHING IS PERSISTED β€” no mirror table, no `ut_amzn_*`, no `_sync_state` row. That is what
makes it safe to point at the SANDBOX. `amazon_store.sync` refuses a sandbox credential so mock
rows can never reach a grid; this path sidesteps that question entirely by writing nothing.
⚠⚠ SANDBOX REALITY, MEASURED 2026-09-04, AND IT IS NOT WHAT "the sandbox mirrors production"
SUGGESTS: the static sandbox's `getReportDocument` **always returns
`GET_MERCHANT_LISTINGS_ALL_DATA`, whatever report type you asked for** (amzn/selling-partner-
api-models#231). So this probe proves the CHAIN for any report type, and yields a genuine header
for exactly ONE of our seven tables β€” `listings`. The other six still need production.
⚠ And the parse goes through `amazon_store.parse_flat_file`, the SAME function `sync` uses.
A second reader here would prove something production does not do.
"""
from datetime import datetime, timedelta, timezone
from harness.amazon_store import parse_flat_file
rt = report_type or REPORT_TYPES["listings"]["report"]
sbx = is_sandbox(fields)
end = datetime.now(timezone.utc)
start = end - timedelta(days=days)
log(f"host : {'SANDBOX' if sbx else 'PRODUCTION'} {base_for(fields)}")
log(f"report : {rt}")
if sbx and rt != "GET_MERCHANT_LISTINGS_ALL_DATA":
log("WARNING: the static sandbox returns GET_MERCHANT_LISTINGS_ALL_DATA whatever you ask "
"for, so the document below is NOT this report type. The chain is still proven.")
t0 = time.time()
text, meta = report_document(
fields, rt,
start=start.strftime("%Y-%m-%dT%H:%M:%SZ"),
end=end.strftime("%Y-%m-%dT%H:%M:%SZ"), log=log)
log(f"document : {len(text):,} chars, compression={meta.get('compression')} "
f"({time.time() - t0:.1f}s)")
cols, rows = parse_flat_file(text)
log("")
log(f"MEASURED HEADER: {len(cols)} columns, {len(rows)} data rows")
for i, c in enumerate(cols):
log(f" {i + 1:>3}. {c}")
if rows:
# ⚠ TRUNCATED, and only the first row. A report document is the SELLER'S data; a probe
# prints enough to prove the shape and no more.
log("")
log("first row, values truncated to 40 chars:")
for c in cols[:12]:
log(f" {c:<34} {str(rows[0].get(c))[:40]}")
return cols, rows
def main(argv):
"""`--check` proves a credential; `--probe-report` proves the whole report chain."""
if "--check" not in argv and "--probe-report" not in argv:
print("usage: connectors_amazon.py --check prove the credential")
print(" connectors_amazon.py --probe-report [--report TYPE]")
print(" drive the full report chain and print")
print(" the MEASURED header. Writes nothing.")
print(" reads AMAZON_SP_CLIENT_ID / _CLIENT_SECRET / _REFRESH_TOKEN / _SELLER_ID /")
print(" _MARKETPLACE_ID, plus AMAZON_SP_SANDBOX=true for the sandbox host")
return 2
f = _env_credential()
have = [k for k in CREDENTIAL_FIELDS if f.get(k)]
print("credential fields present: " + (", ".join(have) if have else "NONE")
+ " (values are never printed)")
if not {"client_id", "client_secret", "refresh_token"} <= set(have):
print("REFUSED: client_id, client_secret and refresh_token are all required")
return 2
t0 = time.time()
res = probe_credential(f, timeout=_TIMEOUT)
print(f"{'OK' if res.get('ok') else 'REFUSED'}: {res.get('message')} "
f"({time.time() - t0:.1f}s)")
if not res.get("ok"):
return 1
print("\nreport types this connector will pull (owner ruling R4):")
for table, spec in REPORT_TYPES.items():
rep = spec.get("report") or spec.get("report_discover") or "NO CONFIRMED REPORT FLAVOR (see O5)"
print(f" {table:<16} {rep}")
if "--probe-report" in argv:
which = None
for i, a in enumerate(argv):
if a == "--report" and i + 1 < len(argv):
which = argv[i + 1]
print("")
try:
probe_report(f, report_type=which)
except Exception: # noqa: BLE001
# ⚠ REPORTED, NOT RAISED. The probe is a diagnostic: a refusal is a RESULT the operator
# needs to read (a role they lack, a throttle, an unsupported sandbox report type), not
# a traceback that buries it.
print("REPORT PROBE REFUSED: Amazon did not confirm the report. Check authorization and try again.")
return 1
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))