Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Download api/connectors_amazon.py from fsanyoto/runloopable: direct link, hf CLI and curl.
- Browser
- Download file 47 kB
-
https://huggingface.co/spaces/fsanyoto/runloopable/resolve/main/api/connectors_amazon.py
- Command line
-
hf download hf://spaces/fsanyoto/runloopable/api/connectors_amazon.py
-
curl -L -o connectors_amazon.py https://huggingface.co/spaces/fsanyoto/runloopable/resolve/main/api/connectors_amazon.py
47 kB
| """connectors_amazon.py β the API-layer FACE of the Amazon connector: the liveness probe. | |
| ββ **THE TRANSPORT IS NOT HERE. IT IS `platform/harness/amazon_client.py`, AND THAT IS A LAYER | |
| RULE, NOT A PREFERENCE.** `harness/amazon_store.py` needs the same client this probe needs, and | |
| nothing under `platform/` may import an API-layer module -- layers point one way. The first version | |
| of this connector put the transport here and had the store reach up for it; `verify_meta`'s | |
| architecture leg caught it in one line. Everything the client exports is re-exported below, so a | |
| caller that imports it from here still works. | |
| β **THE PROBER IS REGISTERED, NEVER IMPORTED BY `core`.** `core` never imports up | |
| (ARCHITECTURE.md), and the tempting shortcut -- a `sys.path.insert` plus an `import` from inside | |
| `core/keychain.py` -- is an architecture inversion no gate was watching for until wave 31. | |
| `keychain.register_prober` is the slot; `datastore.set_paused_probe` is the same idiom one layer | |
| down. | |
| β **READ-ONLY.** See the client's header for the promise and its two fixed POST wrappers. | |
| """ | |
| from __future__ import annotations | |
| import os | |
| import logging | |
| import sys | |
| import time | |
| from pathlib import Path | |
| from fastapi import APIRouter, HTTPException, Request | |
| _PLATFORM = Path(__file__).resolve().parents[2] / "platform" | |
| if str(_PLATFORM) not in sys.path: | |
| sys.path.insert(0, str(_PLATFORM)) | |
| from core.request_timing import raise_if_resource_error | |
| # β RE-EXPORTED, NOT REDEFINED. One definition of the credential shape, the endpoints, the report | |
| # map and the transport; this module adds the probe and the registration and nothing else. | |
| from harness.amazon_client import ( # noqa: F401,E402 | |
| AWD_INVENTORY_PAGE_SIZE, COMPETITIVE_SUMMARY_BATCH_SIZE, | |
| CREDENTIAL_FIELDS, DEFAULT_REGION, ENDPOINTS, KEY, LABEL, LWA_TOKEN_URL, OPTIONAL_FIELDS, | |
| REPORT_TYPES, SANDBOX_ENDPOINTS, base_for, is_sandbox, | |
| ROLES_REFUSED, ROLES_TAKEN, US_MARKETPLACE_ID, AmazonError, _create_report, | |
| _competitive_summary_batch, _COMPETITIVE_SUMMARY_BATCH_PATH, | |
| _COMPETITIVE_SUMMARY_ITEM_URI, _download_document, _post_form, _REPORTS_CREATE_PATH, | |
| _request, _safe, _TIMEOUT, _AWD_INVENTORY_PATH, access_token, awd_inventory, | |
| competitive_summaries, marketplace_participations, report_document, | |
| ) | |
| #: β MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data | |
| #: structure on a screen about their Amazon account. That was the wave-31 Meta defect and the | |
| #: comments in `keychain.py` warn about it in as many words. | |
| ENTRY_TYPE = "amazon_sp" | |
| class AmazonStatusSafetyError(ValueError): | |
| """A status response contains a forbidden credential shape.""" | |
| class AmazonStatusRowsRefused(AmazonStatusSafetyError): | |
| """S67-T31: some status ROWS were refused; the rest are carried, already guarded. | |
| Still an `AmazonStatusSafetyError`, so every caller that refuses on the base class keeps | |
| refusing. `safe_status` holds only rows that passed the guard and `refused` only table NAMES: | |
| neither carries the refused content, so a caller may degrade those rows and keep the others. | |
| """ | |
| def __init__(self, safe_status, refused): | |
| super().__init__('Amazon status contains a forbidden credential shape.') | |
| self.safe_status = safe_status | |
| self.refused = tuple(refused) | |
| #: The status guard's scalar ceiling, and the sentence a row wears when its status is refused. | |
| STATUS_SCALAR_LIMIT = 512 | |
| STATUS_UNAVAILABLE = "Sync status is unavailable for this database." | |
| def trim_status(value, limit=STATUS_SCALAR_LIMIT): | |
| """Cut every string over `limit` to a word-bounded excerpt that fits, recursively. | |
| S67-T31: a long, honest cause (staging's Search Query Performance cause measured 935 | |
| characters) used to trip the length branch of the guard and take the whole connector modal | |
| down with a 500. β Call this only AFTER the credential-shape scan of the UNTRIMMED value | |
| (`screen_status_rows`): cutting first could slice a token shape out of a string and accept | |
| the rest, which would trim a refusal into acceptance. | |
| """ | |
| if isinstance(value, dict): | |
| return {key: trim_status(item, limit) for key, item in value.items()} | |
| if isinstance(value, list): | |
| return [trim_status(item, limit) for item in value] | |
| if isinstance(value, tuple): | |
| return tuple(trim_status(item, limit) for item in value) | |
| if isinstance(value, str) and len(value) > limit: | |
| head = value[:limit - 3] | |
| cut = head.rsplit(" ", 1)[0] if " " in head else head | |
| return cut.rstrip(" ,;:") + "..." | |
| return value | |
| def screen_status_rows(status): | |
| """`(safe, refused)` for a `{table: row}` status: each row judged on its own. | |
| A row carrying a credential key, an `Atzr|` token or a bearer shape ANYWHERE in its untrimmed | |
| form is refused whole (NC: never trimmed into acceptance, its value never returned). A row that | |
| is only too long is trimmed, then held to the full guard again. | |
| """ | |
| import json | |
| safe, refused = {}, [] | |
| for name, row in (status or {}).items(): | |
| try: | |
| whole = json.dumps(row, ensure_ascii=False, allow_nan=False) | |
| if status_payload_has_secret(whole, max_length=None): | |
| refused.append(name) | |
| continue | |
| trimmed = trim_status(row) | |
| if status_payload_has_secret(json.dumps(trimmed, ensure_ascii=False, allow_nan=False)): | |
| refused.append(name) | |
| continue | |
| except (TypeError, ValueError, RecursionError): | |
| refused.append(name) | |
| continue | |
| safe[name] = trimmed | |
| return safe, refused | |
| def status_payload_has_secret(serialized_payload, *, max_length=STATUS_SCALAR_LIMIT): | |
| """Scan serialized JSON for C2's credential keys, token shapes and oversized values. | |
| Pure and usable without a server. Invalid JSON is unsafe too; never echo rejected content. | |
| The credential key roster is the same one the connector accepts at connect time. | |
| `max_length=None` scans for credential shapes only (the pre-trim pass of `screen_status_rows`). | |
| """ | |
| import json | |
| import re | |
| try: | |
| pending = [json.loads(serialized_payload, object_pairs_hook=lambda pairs: pairs)] | |
| except (TypeError, ValueError, UnicodeError, RecursionError): | |
| return True | |
| credential_keys = {key.casefold() for key in CREDENTIAL_FIELDS} | |
| while pending: | |
| value = pending.pop() | |
| if isinstance(value, tuple): | |
| key, item = value | |
| if key.casefold() in credential_keys: | |
| return True | |
| pending.extend((key, item)) | |
| elif isinstance(value, list): | |
| pending.extend(value) | |
| elif isinstance(value, str): | |
| if ((max_length is not None and len(value) > max_length) or 'Atzr|' in value | |
| or re.search(r'\bbearer\s+\S+', value, flags=re.IGNORECASE)): | |
| return True | |
| elif (value is not None and max_length is not None | |
| and len(str(value)) > max_length): | |
| return True | |
| return False | |
| def assert_safe_status(payload): | |
| """Return a status only after scanning its serialized form, or raise without its content.""" | |
| import json | |
| try: | |
| serialized = json.dumps(payload, ensure_ascii=False, allow_nan=False) | |
| except (TypeError, ValueError, RecursionError): | |
| raise AmazonStatusSafetyError('Amazon status could not be safely serialized.') from None | |
| if status_payload_has_secret(serialized): | |
| raise AmazonStatusSafetyError('Amazon status contains a forbidden credential shape.') | |
| return payload | |
| def first_load_status(tenant_key, rt=None): | |
| """Writer-side progress helper; request status uses sync_state's read-only cursor.""" | |
| from harness import amazon_store | |
| return assert_safe_status(amazon_store.first_load_status(tenant_key, rt=rt)) | |
| def _stored_table_cap(): | |
| """The row ceiling a materialised `ut_*` table can hold; `amazon_relational.plan` applies it.""" | |
| try: | |
| from core import user_tables | |
| return int(getattr(user_tables, "MAX_ROWS", 60_000)) | |
| except Exception: # noqa: BLE001 | |
| return 60_000 | |
| def _mirror_counts(cur, az): | |
| """{table: row count} for each Amazon mirror table that exists. Read only; absent = never synced.""" | |
| counts = {} | |
| for table in az.DATABASE_TABLES: | |
| try: | |
| counts[table] = int(cur.execute( | |
| f"SELECT count(*) FROM {az.table_name(table)}").fetchone()[0]) | |
| except Exception as exc: # noqa: BLE001 | |
| raise_if_resource_error(exc) | |
| continue | |
| return counts | |
| NOTIFICATION_SECRET_SLOT = "amazon_sp_notifications" | |
| NOTIFICATION_SECRET_HEADER = "x-loopable-amazon-secret" | |
| NOTIFICATION_DEDUPE_SECONDS = 600 | |
| NOTIFICATION_TARGETING_CONTRACT = "amazon-table-events-v1" | |
| notification_router = APIRouter(prefix="/api/v1/connectors/amazon_sp") | |
| def _notification_secret(rt): | |
| """Read the tenant's named generic keychain slot, without an environment fallback.""" | |
| from core import keychain | |
| rows = [row for row in keychain.list_entries(rt) | |
| if row.get("type") == "generic" and row.get("label") == NOTIFICATION_SECRET_SLOT] | |
| if len(rows) != 1: | |
| return None | |
| fields = keychain.read_fields(rt, rows[0]["id"]) or {} | |
| secret = fields.get("api_key") | |
| return secret if isinstance(secret, str) and secret else None | |
| def _authenticate_notification(rt, supplied): | |
| """Authenticate before reading or parsing any event bytes.""" | |
| import hmac | |
| try: | |
| secret = _notification_secret(rt) | |
| except Exception: | |
| secret = None | |
| if not secret or not supplied or not hmac.compare_digest( | |
| secret.encode("utf-8"), supplied.encode("utf-8")): | |
| raise HTTPException(status_code=401, detail="Amazon notification authentication failed.") | |
| def notification_target(event): | |
| """Resolve an approved event to one known source table and its vendor event ID.""" | |
| from harness.amazon_client import NOTIFICATION_TYPES, REPORT_TYPES | |
| if not isinstance(event, dict): | |
| raise ValueError("Amazon notification must be an object.") | |
| detail = event.get("detail", event) | |
| if not isinstance(detail, dict): | |
| raise ValueError("Amazon notification detail must be an object.") | |
| event_type = detail.get("notificationType", detail.get("NotificationType")) | |
| if event_type not in NOTIFICATION_TYPES: | |
| raise ValueError("Amazon notification type is not approved for this connector.") | |
| metadata = detail.get("notificationMetadata", detail.get("NotificationMetadata", {})) | |
| if not isinstance(metadata, dict): | |
| raise ValueError("Amazon notification metadata must be an object.") | |
| event_id = metadata.get("notificationId", metadata.get("NotificationId")) or event.get("id") | |
| if not isinstance(event_id, str) or not event_id.strip() or len(event_id) > 256: | |
| raise ValueError("Amazon notification requires a vendor event ID.") | |
| declared = detail.get("table") | |
| if event_type == "REPORT_PROCESSING_FINISHED": | |
| payload = detail.get("payload", detail.get("Payload", {})) | |
| if not isinstance(payload, dict): | |
| raise ValueError("Amazon report notification payload must be an object.") | |
| report = payload.get("reportProcessingFinishedNotification", | |
| payload.get("ReportProcessingFinishedNotification", {})) | |
| if not isinstance(report, dict): | |
| raise ValueError("Amazon report notification must describe its report.") | |
| report_type = report.get("reportType", report.get("ReportType")) | |
| candidates = [table for table, spec in REPORT_TYPES.items() | |
| if report_type and report_type in | |
| [spec.get(key) for key in ("report", "report_incremental", "report_discover")]] | |
| if declared is None and len(candidates) == 1: | |
| declared = candidates[0] | |
| if not isinstance(declared, str) or declared not in candidates: | |
| raise ValueError("Amazon report notification requires one matching known table.") | |
| if report.get("processingStatus", report.get("ProcessingStatus")) != "DONE": | |
| return event_id, None | |
| table = declared | |
| else: | |
| table = NOTIFICATION_TYPES[event_type]["table"] | |
| if declared is not None and declared != table: | |
| raise ValueError("Amazon notification table does not match its event type.") | |
| return event_id, table | |
| def notification_sync_options(receipt): | |
| """Keep explicit table targets narrow and exclude them from full-source catch-up completion.""" | |
| from harness import amazon_store | |
| tables = receipt.get("tables") | |
| if tables is None: | |
| return {} | |
| if (not isinstance(tables, (list, tuple)) or not tables | |
| or any(not isinstance(table, str) or table not in amazon_store.TABLES for table in tables)): | |
| raise ValueError("amazon_notification_target_invalid") | |
| return {"tables": list(dict.fromkeys(tables)), "_catch_up_complete": False} | |
| def _notification_sync_available(desk): | |
| """The integrator publishes this marker only after producer and resident consumer support targeting.""" | |
| import inspect | |
| return (getattr(desk, "NOTIFICATION_TARGETING_CONTRACT", None) == NOTIFICATION_TARGETING_CONTRACT | |
| and {"tables", "event_id", "rt"} <= set(inspect.signature(desk.request_sync).parameters)) | |
| async def amazon_notifications(request: Request): | |
| """Queue an authenticated event through the durable targeted sync desk.""" | |
| from harness import runtime | |
| from core import connector_sync | |
| tenant = request.query_params.get("tenant") | |
| if not tenant or tenant not in runtime.known_tenants(): | |
| raise HTTPException(status_code=401, detail="Amazon notification authentication failed.") | |
| rt = runtime.get_runtime(tenant) | |
| if rt.key != tenant: | |
| raise HTTPException(status_code=401, detail="Amazon notification authentication failed.") | |
| _authenticate_notification(rt, request.headers.get(NOTIFICATION_SECRET_HEADER)) | |
| try: | |
| event_id, table = notification_target(await request.json()) | |
| except (ValueError, TypeError, UnicodeError): | |
| raise HTTPException(status_code=400, detail="Amazon notification has no approved table target.") from None | |
| if table is None: | |
| return {"accepted": False, "reason": "Amazon report did not finish successfully."} | |
| if not _notification_sync_available(connector_sync): | |
| raise HTTPException(status_code=503, detail="amazon_notification_targeting_unavailable") | |
| receipt = connector_sync.request_sync("amazon_sp", tenant, requested_by="amazon notification", | |
| rt=rt, tables=[table], event_id=event_id) | |
| if receipt.get("tables") != [table] or receipt.get("eventId") != event_id: | |
| raise HTTPException(status_code=503, detail="amazon_notification_target_contract_violated") | |
| return {"accepted": receipt.get("accepted") is True, "table": table, | |
| "duplicate": receipt.get("duplicate") is True} | |
| def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None): | |
| """Each Amazon table's sync state, or None when this mirror cannot be read. | |
| Read only: never _connect_tenant_mirror or first_load_status, which issue DDL. | |
| Missing bookkeeping is unknown, not a measured zero. First-load numbers describe the | |
| whole connection, because that is the scope of the producer's durable progress marker. | |
| """ | |
| import json | |
| from datetime import datetime, timezone | |
| import duckdb | |
| from harness import amazon_store as az | |
| from harness.amazon_client import DECLARED_DATASETS | |
| from routes_odoo_tables import _sync_entities, stuck_verdict | |
| # S66-T46 (lead): an unreadable deletion ledger is flagged once per row as | |
| # `purgeScheduleUnknown`, never written into every table's cause, which would push out the | |
| # reason each table already names (Amazon's refusal, the empty period, the declared reason). | |
| purge_unknown = False | |
| try: | |
| purge_plan = az.scheduled_purge(tenant_key, rt, now=now) | |
| purge_message = az.deletion_schedule_message(purge_plan) if purge_plan else "" | |
| except Exception as exc: | |
| raise_if_resource_error(exc) | |
| purge_plan, purge_message, purge_unknown = None, "", True | |
| def finish(status): | |
| for row in status.values(): | |
| # A key only when there is something to say: a declared dataset's row stays exactly | |
| # its declaration while no purge is scheduled (the group (a) status contract). | |
| if purge_plan: | |
| row["purgeScheduledAt"] = purge_plan["purgeAt"] | |
| if purge_unknown and not row.get("declared"): | |
| row["purgeScheduleUnknown"] = True | |
| if purge_message: | |
| cause = row.get("unknownCause") or "" | |
| # Status scalars have a 512-character safety ceiling. Keep the full existing | |
| # cause separately if the deletion sentence needs a shorter screen excerpt. | |
| budget = 512 - len(purge_message) - 1 | |
| if len(cause) > budget: | |
| row["syncCause"] = cause | |
| cause = cause[:budget - 3].rsplit(" ", 1)[0] + "..." | |
| row["unknownCause"] = " ".join(part for part in (cause, purge_message) if part) | |
| # S67-T31: judged PER ROW. An over-long cause is trimmed (staging's 500 was | |
| # `search_query_performance.unknownCause` at 935 characters); a credential shape still | |
| # refuses, now naming only its rows so the caller degrades those and keeps the rest. | |
| safe, refused = screen_status_rows(status) | |
| if refused: | |
| raise AmazonStatusRowsRefused(assert_safe_status(safe), refused) | |
| return assert_safe_status(safe) | |
| declared = {name: {"rowsLoaded": None, "declared": True, "unknownCause": spec["reason"]} | |
| for name, spec in DECLARED_DATASETS.items()} | |
| unavailable = {table: {"rowsLoaded": None, "lastSuccess": None, "inProgress": False, | |
| "unknownCause": "This container has no readable sync status. Sync this database to load its rows."} | |
| for table in az.DATABASE_TABLES} | |
| unavailable.update(declared) | |
| def stamp(value): | |
| if not value: | |
| return None | |
| try: | |
| parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) | |
| return (parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)).astimezone(timezone.utc) | |
| except (ValueError, TypeError, OverflowError): | |
| return None | |
| def iso(value): | |
| return value.isoformat().replace("+00:00", "Z") if value else None | |
| cadence = cadence_seconds if cadence_seconds is not None else 1800 | |
| try: | |
| path = az._tenant_mirror_path(tenant_key, rt=rt) | |
| if not path.is_file(): | |
| return finish(unavailable) | |
| # ββ A SECOND `duckdb.connect(read_only=True)` ON A FILE THIS PROCESS ALREADY HOLDS RAISES | |
| # `ConnectionException: Can't open a connection to same database file with a different | |
| # configuration`. On the DEFAULT tenant layout the Amazon mirror IS the process-global | |
| # analytical mirror (`datastore.DB_PATH`), so the swallowed refusal below turned this whole | |
| # screen into a permanent "sync status is unknown" on exactly the deployment the ticket is | |
| # for. Measured on staging 2026-09-14, W60-T35. `datastore.ro_con()` is the shipped | |
| # per-thread READ cursor on that same instance, and it takes no lock against the writer. | |
| from harness import datastore as _ds | |
| if path.resolve() == _ds.DB_PATH.resolve(): | |
| # `ro_cursor()` is a cursor on the instance this process ALREADY holds: no second | |
| # connection, no lock contention with the writer, and no DDL. Deliberately NOT | |
| # `ro_con()`, which refuses until `ready()` and would report the whole screen as | |
| # unknown whenever the Odoo backfill is mid-flight, and NOT `datastore.connect()`, | |
| # which issues `CREATE TABLE IF NOT EXISTS`. | |
| _cur = _ds.ro_cursor() | |
| entities = _sync_entities(_cur, cadence) | |
| mirrored = _mirror_counts(_cur, az) | |
| else: | |
| with _ds.bounded_connection(duckdb.connect(str(path), read_only=True)) as cur: | |
| entities = _sync_entities(cur, cadence) | |
| mirrored = _mirror_counts(cur, az) | |
| except Exception as exc: # noqa: BLE001 | |
| raise_if_resource_error(exc) | |
| # No exception text: paths, provider payloads and credentials never become status copy. | |
| return finish(unavailable) | |
| if entities is None: | |
| return finish(unavailable) | |
| clock = stamp(now) if now is not None else datetime.now(timezone.utc) | |
| states, bad_ticks = {}, set() | |
| for table in az.TABLES: | |
| tick = entities.get(az.entity_key("sync_tick_" + table)) | |
| if tick: | |
| updated = stamp(tick.get("updatedAt")) | |
| if updated is None or tick.get("phase") not in ("attempted", "done", "running"): | |
| bad_ticks.add(table) | |
| else: | |
| states[table] = {"state": tick["phase"], "updated_at": iso(updated)} | |
| # An unreadable input cannot support a cadence promise for any table in the rotation. | |
| plan = az.sync_plan(states, now=clock) if not bad_ticks and clock is not None else None | |
| first_load, load_unknown = None, "" | |
| markers = {key: {"state": value.get("phase"), "detail": value.get("cursor")} | |
| for key, value in entities.items()} | |
| refused_reports = az.refused_reports(markers) | |
| enabled = az.read_database_enablement(rt)["databases"] | |
| load = entities.get(az.entity_key(az.FIRST_LOAD_STATUS_KEY)) or {} | |
| if load: | |
| try: | |
| progress = json.loads(load.get("cursor") or "") | |
| remaining, eta = progress["remaining"], progress["etaSeconds"] | |
| if (type(remaining) is not int or not 0 <= remaining <= 2**53 - 1 | |
| or type(eta) is not int or not 0 <= eta <= 2**53 - 1 | |
| or (load.get("phase") == "done" and remaining > 0)): | |
| raise ValueError("Invalid first-load progress") | |
| if remaining > 0: | |
| first_load = {"remaining": remaining, "etaSeconds": eta, | |
| "cause": az.first_load_eta_cause(remaining), | |
| "needsAttention": load.get("phase") == "failed"} | |
| if refused_reports: | |
| first_load = {**(first_load or {"remaining": remaining, "etaSeconds": eta}), | |
| "refused": refused_reports} | |
| if first_load: | |
| first_load = {**(first_load or {}), **az.first_load_payload(markers, remaining, eta, | |
| state=load.get("phase"), tables=[table for table in az.TABLES | |
| if enabled.get(az.amazon_schema.DATABASE_PREFIX + table)])} | |
| except (ValueError, TypeError, KeyError, OverflowError, RecursionError): | |
| load_unknown = "First-load progress is unknown because its saved marker could not be read. Retry sync to refresh it." | |
| elif (entities.get(az.entity_key(az.FIRST_LOAD_END_KEY)) or {}).get("phase") not in (None, "done"): | |
| load_unknown = "First-load progress is not available yet. Keep the connection enabled and check again after this pass." | |
| if first_load is None and refused_reports and not load: | |
| first_load = az.first_load_payload(markers, 0, 0, state="failed", | |
| tables=[table for table in az.TABLES if enabled.get(az.amazon_schema.DATABASE_PREFIX + table)]) | |
| first_load["etaDays"] = None | |
| answer = dict(declared) | |
| for table in az.TABLES: | |
| landed = entities.get(az.entity_key(table)) or {} | |
| tick = entities.get(az.entity_key("sync_tick_" + table)) or {} | |
| watermark = entities.get(az.entity_key(az.WATERMARK_PREFIX + table)) or {} | |
| rows = mirrored.get(table, landed.get("rows")) | |
| if type(rows) is not int or rows < 0: | |
| rows = None | |
| # Hotfix 2026-09-24: during the first load no per-table marker exists yet, so the screen | |
| # read "Not loaded yet" beside a mirror already holding thousands of rows. The mirror's | |
| # own count is the measured answer. | |
| if rows is None and table in mirrored: | |
| rows = mirrored[table] | |
| # A refused pass closes its tick (S66-T42); only the watermark can then prove a success. | |
| refused_now = landed.get("phase") in ("refused", "failed", "throttled") | |
| successes = [when for marker in ((watermark,) if refused_now else (landed, tick, watermark)) | |
| if marker.get("phase") == "done" | |
| and (when := stamp(marker.get("updatedAt"))) is not None] | |
| stuck, stuck_cause = tick.get("stuck"), tick.get("stuckCause") or "" | |
| if tick.get("phase") == "attempted": | |
| # The shared stall helper calls an active phase 'running'; Amazon calls it 'attempted'. | |
| updated = stamp(tick.get("updatedAt")) | |
| stuck, stuck_cause = stuck_verdict( | |
| "running", iso(updated), cadence, | |
| now=clock.replace(tzinfo=None) if clock else None) | |
| reason = plan["deferred"].get(table) if plan else None | |
| # Only the planner's explicit due stamp is a due date. Minimum stagger spacing is not | |
| # a promise about when a later resident tick will select this table. | |
| next_due = stamp(reason.partition("next due ")[2]) if reason else None | |
| if first_load and load.get("phase") in ("running", "partial"): | |
| wait_marker = entities.get(az.entity_key(az.FIRST_LOAD_DUE_KEY)) or {} | |
| due = stamp(wait_marker.get("cursor")) if wait_marker.get("phase") == "pending" else None | |
| vendor_wait = az._interval_wait(table, states.get(table), clock)[1] if clock else None | |
| if landed.get("phase") == "pending" or rows is None: | |
| next_due = vendor_wait or due or clock | |
| if not reason: | |
| reason = (str(landed.get("cursor")) if landed.get("phase") == "pending" else | |
| "The first load is queued on the resident sync thread; keep the connection enabled.") | |
| unknown = [load_unknown] if load_unknown else [] | |
| proof = entities.get(az.entity_key("reconcile_" + table)) or {} | |
| try: | |
| period = json.loads(proof.get("cursor") or "{}") | |
| if not isinstance(period, dict): | |
| period = {} | |
| except (ValueError, TypeError): | |
| period = {} | |
| if rows is None: | |
| unknown.append("This table has not loaded yet. Sync it to request rows from Amazon.") | |
| if landed.get("phase") in ("refused", "failed", "throttled"): | |
| unknown = [str(landed.get("cursor") or "Amazon refused this dataset. Retry sync to refresh its reason.")] | |
| elif rows == 0 or period.get("count") == 0: | |
| span = " to ".join(dict.fromkeys(str(period[k]) for k in ("start", "end") if period.get(k))) | |
| unknown.append("Amazon returned no rows for " + (span or "the last requested period") + ".") | |
| # Standing rule 1's second half: a limit that cannot be removed is REPORTED, with cause | |
| # and fix. `amazon_relational.plan` refuses a table over MAX_ROWS and said so only in the | |
| # server log, so the owner saw an empty database and no reason. | |
| over = mirrored.get(table) | |
| if over is not None and over > _stored_table_cap(): | |
| unknown.append( | |
| f"The mirror holds {over:,} rows, more than a stored table can hold " | |
| f"({_stored_table_cap():,}). This table is served read-through from the mirror " | |
| "with no row cap.") | |
| # S66-T42/T43 (lead): a table `amazon_relational.apply_plan` could not create because the | |
| # workspace is at the store's table cap says so here, with the cause and the fix. | |
| not_stored = entities.get(az.entity_key("not_stored_" + table)) or {} | |
| if not_stored.get("phase") == "not_stored": | |
| unknown.append(str(not_stored.get("cursor") or | |
| "This table is not in this workspace yet because the workspace is at " | |
| "its database limit. Its rows are kept in the Amazon mirror.")) | |
| if plan is None and landed.get("phase") not in ("refused", "failed", "throttled"): | |
| unknown.append("The sync schedule is unknown because a saved timestamp could not be read. Retry sync to refresh it.") | |
| limit = None | |
| if (table in az.GROUP_A_TABLES and table != "catalog_items") or table == "transactions": | |
| marker = entities.get(az.entity_key("limit_" + table)) or {} | |
| try: | |
| limit = json.loads(marker.get("cursor") or "{}").get("reason") | |
| except (ValueError, TypeError, AttributeError): | |
| limit = "History progress is unreadable. Retry sync to refresh its saved date." | |
| if limit: | |
| unknown.append(limit) | |
| if table == "brand_analytics": | |
| # R18's weekly retention is a code fact: its copy never waits for a weekly pull's marker. | |
| limit = az.BRAND_ANALYTICS_LIMIT | |
| unknown.append(limit) | |
| if table == "catalog_items": | |
| marker = entities.get(az.entity_key("limit_" + table)) or {} | |
| try: | |
| limit = json.loads(marker.get("cursor") or "{}").get("reason") | |
| except (ValueError, TypeError, AttributeError): | |
| limit = "Catalog request progress is unreadable. Sync Amazon listings and retry." | |
| limit = limit or ("Amazon Catalog allows 20 ASINs per search and 2 requests per second. " | |
| "Every own ASIN and page is followed; sync Amazon listings first.") | |
| unknown.append(limit) | |
| if table == "listing_items": | |
| limit = ("Amazon Listings allows 1,000 SKUs per query and 5 requests per second. " | |
| "Date ranges are split and every page followed. Keep the connection enabled.") | |
| unknown.append(limit) | |
| if table == az.SQP_TABLE: | |
| marker = entities.get(az.SQP_LIMIT_KEY) or {} | |
| if marker.get("cursor"): | |
| try: | |
| detail = json.loads(marker["cursor"]) | |
| limit = az.sqp_limit_sentence(detail) | |
| unknown.append(limit) | |
| # S66-T41 (lead): one sentence PER REASON, so a whole refused batch (18 | |
| # ASINs) stays inside `assert_safe_status`'s 512-character scalar ceiling and | |
| # every refused ASIN is still named on screen. | |
| by_reason = {} | |
| for asin, refusal in detail.get("refusedAsins", {}).items(): | |
| by_reason.setdefault(str(refusal), []).append(str(asin)) | |
| for refusal, asins in by_reason.items(): | |
| unknown.append(az.sqp_refusal_message(asins[0], refusal) if len(asins) == 1 | |
| else f"Amazon refused ASINs {', '.join(sorted(asins))} for " | |
| f"Search Query Performance: {refusal}") | |
| unknown.extend(detail.get("errors", [])) | |
| if detail.get("deferredReason"): | |
| reason = detail["deferredReason"] | |
| if detail.get("partialMonthReason"): | |
| unknown.append(detail["partialMonthReason"]) | |
| except (ValueError, TypeError, KeyError, AttributeError): | |
| unknown.append("Search Query Performance limits are unknown because the saved marker could not be read. Retry sync to refresh it.") | |
| if table in az.GROUP_A_TABLES and len(" ".join(unknown)) > 512: | |
| # Preserve the dataset's refusal/empty-period and vendor limit. Generic schedule | |
| # diagnostics can otherwise overflow the status guard and hide every database. | |
| unknown = [part for part in unknown if part != load_unknown | |
| and not part.startswith(("The sync schedule is unknown", "This table has not loaded yet."))] | |
| answer[table] = { | |
| "rowsLoaded": rows, | |
| "lastSuccess": iso(max(successes)) if successes else None, | |
| "inProgress": tick.get("phase") == "attempted" and stuck is False, | |
| "deferred": {"reason": reason} if reason else None, | |
| "nextDue": iso(next_due), | |
| "firstLoad": first_load, | |
| "stuck": stuck, "stuckCause": stuck_cause, | |
| "unknownCause": " ".join(unknown), | |
| } | |
| # A source-internal mismatch published the table. Account markers carry a connection prefix. | |
| warned = az.entity_key("warning_" + table) | |
| warning = next((str(value.get("cursor") or "") for key, value in entities.items() | |
| if (key == warned or key.endswith("__" + warned)) and value.get("cursor")), "") | |
| if warning: | |
| answer[table]["warning"] = warning | |
| if table == az.SQP_TABLE or table in az.GROUP_A_TABLES or table in ("transactions", "brand_analytics"): | |
| answer[table]["limit"] = limit | |
| for table in (*az.DERIVED_TABLES, "competitor_snapshots", "competitor_images"): | |
| answer[table] = { | |
| "rowsLoaded": mirrored.get(table), "lastSuccess": None, "inProgress": False, | |
| "unknownCause": az.amazon_schema.PROVENANCE_COPY[az.amazon_schema.PROVENANCE[table]], | |
| } | |
| # A history table is written in its source's transaction, so it shares the source's last success. | |
| for table in az.HISTORY_TABLES: | |
| rows = mirrored.get(table) | |
| answer[table] = { | |
| "rowsLoaded": rows, | |
| "lastSuccess": (answer.get(table.removesuffix("_history")) or {}).get("lastSuccess"), | |
| "inProgress": False, | |
| "unknownCause": (az.amazon_schema.PROVENANCE_COPY["history"] if rows is not None | |
| else "History starts with the next sync of the source table."), | |
| } | |
| return finish(answer) | |
| #: W55-T19 / R7 / C10 β AMAZON'S OWN ENABLEMENT DOCUMENT. This is deliberately not Odoo's | |
| #: model catalogue or connector config: Amazon publishes report schemas, not ``fields_get``, and | |
| #: its seven databases are enabled at the provider boundary rather than masquerading as Odoo | |
| #: models. The document lives on the request's tenant runtime, so two tenants can make opposite | |
| #: choices without a process-global map ever seeing either answer. | |
| ENABLEMENT_KEY = "amazon_connector_enablement" | |
| ENABLEMENT_ROUTE = "/connectors/amazon_sp/databases/{databaseKey}/enablement" | |
| class EnablementConfigError(ValueError): | |
| """The tenant's stored Amazon enablement document is malformed.""" | |
| def database_keys(): | |
| """The authored Amazon database keys, including derived mirror views.""" | |
| from harness.amazon_store import DATABASE_TABLES | |
| return tuple("ut_amzn_" + str(name) for name in DATABASE_TABLES) | |
| def read_database_enablement(rt): | |
| """Use the sync engine's fail-closed evaluator for this tenant's database choices.""" | |
| from harness import amazon_store as az | |
| return az.read_database_enablement(rt) | |
| def set_database_enabled(rt, database_key, enabled): | |
| """Persist one tenant's Amazon database choice without touching its materialised rows.""" | |
| key = str(database_key or "").strip() | |
| known = set(database_keys()) | |
| if key not in known: | |
| raise KeyError(key) | |
| if type(enabled) is not bool: | |
| raise ValueError("enabled must be true or false") | |
| def _set(current): | |
| if current is None: | |
| current = {} | |
| if not isinstance(current, dict): | |
| raise EnablementConfigError("the stored Amazon enablement document is malformed") | |
| stored = current.get("databases", {}) | |
| if not isinstance(stored, dict): | |
| raise EnablementConfigError("the stored Amazon database choices are malformed") | |
| if any(type(value) is not bool for value in stored.values()): | |
| raise EnablementConfigError("the stored Amazon database choices are malformed") | |
| if set(stored) - known: | |
| logging.getLogger(__name__).warning("Ignoring unknown Amazon database enablement keys") | |
| # A fresh document contains the whole current roster. That makes the stored answer | |
| # inspectable while retaining the migration rule above for future database additions. | |
| choices = {name: stored.get(name, True) for name in database_keys()} | |
| choices[key] = enabled | |
| return {**current, "databases": choices} | |
| rt.update(ENABLEMENT_KEY, _set, flush="sync") | |
| return read_database_enablement(rt) | |
| def enablement_capability(rt, is_admin): | |
| """The provider-aware capability served by Amazon's connector detail response.""" | |
| state = read_database_enablement(rt) | |
| can_manage = bool(is_admin) and state["available"] | |
| return {"provider": ENTRY_TYPE, | |
| "mode": "database", | |
| "canManage": can_manage, | |
| # A member receives no write URL. The route still enforces administrator status; this | |
| # is the matching affordance wall, not the permission boundary. | |
| "updatePath": ENABLEMENT_ROUTE if can_manage else "", | |
| # Disabling changes only this config document. No mirror or ``ut_amzn_*`` row is | |
| # removed, so re-enabling resumes against the same materialised database. | |
| "preservesData": True, | |
| "available": state["available"]} | |
| def probe_credential(fields, timeout=8, *, fetch_participations=None): | |
| """Return the guarded Amazon status; a disclosure violation must raise at this boundary.""" | |
| return assert_safe_status(_probe_credential_result( | |
| fields, timeout, fetch_participations=fetch_participations)) | |
| def _probe_credential_result(fields, timeout=8, *, fetch_participations=None): | |
| """`{ok, message}` for a stored `amazon_sp` credential: the keychain's liveness probe. | |
| β IT LIVES HERE, IN THE API LAYER, AND IS REGISTERED INTO `core.keychain` RATHER THAN IMPORTED | |
| BY IT. `core` never imports up (ARCHITECTURE.md); `keychain.register_prober` is the slot, and | |
| `connectors_meta` established the idiom after a first draft did `sys.path.insert` plus an | |
| `import` from inside `core`. | |
| β ZERO marketplace participations is reported as NOT ok, deliberately, for the same reason Meta | |
| reports zero ad accounts that way: a credential that authenticates but reaches nothing is an | |
| authorization answer, and `ok: true` would hide the one thing the admin has to fix. | |
| """ | |
| f = fields if isinstance(fields, dict) else {} | |
| missing = [key for key in CREDENTIAL_FIELDS | |
| if not isinstance(f.get(key), str) or not f[key].strip()] | |
| if missing: | |
| return {"ok": False, | |
| "message": "Missing " + ", ".join(missing) + ". Enter every required field."} | |
| try: | |
| # Bind only when explicitly tested. Shape gates can inject a local response function. | |
| fetch = fetch_participations if fetch_participations is not None else marketplace_participations | |
| parts = fetch(f, timeout=timeout) | |
| live = [part for part in parts if isinstance(part, dict) | |
| and isinstance(part.get("participation"), dict) | |
| and part["participation"].get("isParticipating") is True] | |
| if not live: | |
| return {"ok": False, | |
| "message": "Amazon reported no active marketplace access. Check authorization and test again."} | |
| except Exception: # noqa: BLE001 | |
| # AmazonError includes provider-supplied text and an invalid region can echo user input. | |
| return {"ok": False, | |
| "message": "Amazon did not confirm marketplace access. Check the five fields and test again."} | |
| return {"ok": True, "message": "Amazon confirmed marketplace access."} | |
| def register(): | |
| """Fill the keychain's probe slot for this connector. | |
| Called at import by the API layer's connector registry (`routes_connectors._register_probers`), | |
| so any process that can serve the connector directory has also registered the prober. Never | |
| raises: a probe slot that could not be filled costs a liveness CHECK, never the ability to | |
| store a credential. | |
| """ | |
| try: | |
| import core.keychain as kc | |
| return kc.register_prober(ENTRY_TYPE, probe_credential) | |
| except Exception: # noqa: BLE001 | |
| return False | |
| def _env_credential(): | |
| """A credential assembled from the environment, for `--check` only. | |
| β NOT a runtime path. The product reads Amazon credentials from the per-tenant keychain and | |
| from nowhere else; this exists so an operator can prove a freshly minted refresh token works | |
| before pasting it into the app. | |
| """ | |
| # β `OPTIONAL_FIELDS` joins the read so `AMAZON_SP_SANDBOX=true` reaches `is_sandbox` and the | |
| # probe talks to the sandbox host. Absent still means PRODUCTION, which is the safe default. | |
| f = {k: (os.environ.get("AMAZON_SP_" + k.upper()) or "").strip() | |
| for k in tuple(CREDENTIAL_FIELDS) + OPTIONAL_FIELDS} | |
| return {k: v for k, v in f.items() if v} | |
| def probe_report(fields, report_type=None, days=30, log=print): | |
| """Drive the WHOLE report chain once and print the MEASURED header. Writes to no store. | |
| `createReport -> poll -> getReportDocument -> download -> decompress -> parse`, then print the | |
| columns the document actually carried. **This is the only way to learn a flat-file report's | |
| schema short of a production sync**, and it is exactly what the connector cannot know from | |
| Amazon's published artifacts: five of our seven tables have none. | |
| β NOTHING IS PERSISTED β no mirror table, no `ut_amzn_*`, no `_sync_state` row. That is what | |
| makes it safe to point at the SANDBOX. `amazon_store.sync` refuses a sandbox credential so mock | |
| rows can never reach a grid; this path sidesteps that question entirely by writing nothing. | |
| β β SANDBOX REALITY, MEASURED 2026-09-04, AND IT IS NOT WHAT "the sandbox mirrors production" | |
| SUGGESTS: the static sandbox's `getReportDocument` **always returns | |
| `GET_MERCHANT_LISTINGS_ALL_DATA`, whatever report type you asked for** (amzn/selling-partner- | |
| api-models#231). So this probe proves the CHAIN for any report type, and yields a genuine header | |
| for exactly ONE of our seven tables β `listings`. The other six still need production. | |
| β And the parse goes through `amazon_store.parse_flat_file`, the SAME function `sync` uses. | |
| A second reader here would prove something production does not do. | |
| """ | |
| from datetime import datetime, timedelta, timezone | |
| from harness.amazon_store import parse_flat_file | |
| rt = report_type or REPORT_TYPES["listings"]["report"] | |
| sbx = is_sandbox(fields) | |
| end = datetime.now(timezone.utc) | |
| start = end - timedelta(days=days) | |
| log(f"host : {'SANDBOX' if sbx else 'PRODUCTION'} {base_for(fields)}") | |
| log(f"report : {rt}") | |
| if sbx and rt != "GET_MERCHANT_LISTINGS_ALL_DATA": | |
| log("WARNING: the static sandbox returns GET_MERCHANT_LISTINGS_ALL_DATA whatever you ask " | |
| "for, so the document below is NOT this report type. The chain is still proven.") | |
| t0 = time.time() | |
| text, meta = report_document( | |
| fields, rt, | |
| start=start.strftime("%Y-%m-%dT%H:%M:%SZ"), | |
| end=end.strftime("%Y-%m-%dT%H:%M:%SZ"), log=log) | |
| log(f"document : {len(text):,} chars, compression={meta.get('compression')} " | |
| f"({time.time() - t0:.1f}s)") | |
| cols, rows = parse_flat_file(text) | |
| log("") | |
| log(f"MEASURED HEADER: {len(cols)} columns, {len(rows)} data rows") | |
| for i, c in enumerate(cols): | |
| log(f" {i + 1:>3}. {c}") | |
| if rows: | |
| # β TRUNCATED, and only the first row. A report document is the SELLER'S data; a probe | |
| # prints enough to prove the shape and no more. | |
| log("") | |
| log("first row, values truncated to 40 chars:") | |
| for c in cols[:12]: | |
| log(f" {c:<34} {str(rows[0].get(c))[:40]}") | |
| return cols, rows | |
| def main(argv): | |
| """`--check` proves a credential; `--probe-report` proves the whole report chain.""" | |
| if "--check" not in argv and "--probe-report" not in argv: | |
| print("usage: connectors_amazon.py --check prove the credential") | |
| print(" connectors_amazon.py --probe-report [--report TYPE]") | |
| print(" drive the full report chain and print") | |
| print(" the MEASURED header. Writes nothing.") | |
| print(" reads AMAZON_SP_CLIENT_ID / _CLIENT_SECRET / _REFRESH_TOKEN / _SELLER_ID /") | |
| print(" _MARKETPLACE_ID, plus AMAZON_SP_SANDBOX=true for the sandbox host") | |
| return 2 | |
| f = _env_credential() | |
| have = [k for k in CREDENTIAL_FIELDS if f.get(k)] | |
| print("credential fields present: " + (", ".join(have) if have else "NONE") | |
| + " (values are never printed)") | |
| if not {"client_id", "client_secret", "refresh_token"} <= set(have): | |
| print("REFUSED: client_id, client_secret and refresh_token are all required") | |
| return 2 | |
| t0 = time.time() | |
| res = probe_credential(f, timeout=_TIMEOUT) | |
| print(f"{'OK' if res.get('ok') else 'REFUSED'}: {res.get('message')} " | |
| f"({time.time() - t0:.1f}s)") | |
| if not res.get("ok"): | |
| return 1 | |
| print("\nreport types this connector will pull (owner ruling R4):") | |
| for table, spec in REPORT_TYPES.items(): | |
| rep = spec.get("report") or spec.get("report_discover") or "NO CONFIRMED REPORT FLAVOR (see O5)" | |
| print(f" {table:<16} {rep}") | |
| if "--probe-report" in argv: | |
| which = None | |
| for i, a in enumerate(argv): | |
| if a == "--report" and i + 1 < len(argv): | |
| which = argv[i + 1] | |
| print("") | |
| try: | |
| probe_report(f, report_type=which) | |
| except Exception: # noqa: BLE001 | |
| # β REPORTED, NOT RAISED. The probe is a diagnostic: a refusal is a RESULT the operator | |
| # needs to read (a role they lack, a throttle, an unsupported sandbox report type), not | |
| # a traceback that buries it. | |
| print("REPORT PROBE REFUSED: Amazon did not confirm the report. Check authorization and try again.") | |
| return 1 | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main(sys.argv[1:])) | |