File size: 46,970 Bytes
e88a3a3 07bca1c e88a3a3 aa1490a e88a3a3 81a8217 e88a3a3 f1ae536 e88a3a3 07bca1c e88a3a3 07bca1c e88a3a3 992a41f a93637f 992a41f a93637f 992a41f a93637f 992a41f a93637f 992a41f dcd43e2 76338e3 81a8217 76338e3 f1ae536 76338e3 81a8217 dcd43e2 9a6cb9a dcd43e2 9a6cb9a f1ae536 9a6cb9a a93637f 9a6cb9a 81a8217 9a6cb9a dcd43e2 9a6cb9a dcd43e2 76338e3 dcd43e2 f1ae536 dcd43e2 76338e3 f1ae536 dcd43e2 9a6cb9a dcd43e2 9a6cb9a dcd43e2 868b75a dcd43e2 aa1490a 868b75a dcd43e2 868b75a dcd43e2 9a6cb9a dcd43e2 9a6cb9a dcd43e2 76338e3 9a6cb9a dcd43e2 aa1490a dcd43e2 9a6cb9a dcd43e2 9a6cb9a 76338e3 9a6cb9a dcd43e2 9a6cb9a 83e5118 9a6cb9a 83e5118 9a6cb9a dcd43e2 086cc6b cf09724 9a6cb9a 3e19b1e 4d24cf6 9a6cb9a dcd43e2 a83fcff 81a8217 a83fcff 85f794d a83fcff aa1490a a83fcff aa1490a a83fcff e88a3a3 992a41f e88a3a3 992a41f e88a3a3 992a41f e88a3a3 992a41f e88a3a3 992a41f e88a3a3 dcd43e2 e88a3a3 992a41f e88a3a3 992a41f e88a3a3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 | """connectors_amazon.py β the API-layer FACE of the Amazon connector: the liveness probe.
ββ **THE TRANSPORT IS NOT HERE. IT IS `platform/harness/amazon_client.py`, AND THAT IS A LAYER
RULE, NOT A PREFERENCE.** `harness/amazon_store.py` needs the same client this probe needs, and
nothing under `platform/` may import an API-layer module -- layers point one way. The first version
of this connector put the transport here and had the store reach up for it; `verify_meta`'s
architecture leg caught it in one line. Everything the client exports is re-exported below, so a
caller that imports it from here still works.
β **THE PROBER IS REGISTERED, NEVER IMPORTED BY `core`.** `core` never imports up
(ARCHITECTURE.md), and the tempting shortcut -- a `sys.path.insert` plus an `import` from inside
`core/keychain.py` -- is an architecture inversion no gate was watching for until wave 31.
`keychain.register_prober` is the slot; `datastore.set_paused_probe` is the same idiom one layer
down.
β **READ-ONLY.** See the client's header for the promise and its two fixed POST wrappers.
"""
from __future__ import annotations
import os
import logging
import sys
import time
from pathlib import Path
from fastapi import APIRouter, HTTPException, Request
_PLATFORM = Path(__file__).resolve().parents[2] / "platform"
if str(_PLATFORM) not in sys.path:
sys.path.insert(0, str(_PLATFORM))
from core.request_timing import raise_if_resource_error
# β RE-EXPORTED, NOT REDEFINED. One definition of the credential shape, the endpoints, the report
# map and the transport; this module adds the probe and the registration and nothing else.
from harness.amazon_client import ( # noqa: F401,E402
AWD_INVENTORY_PAGE_SIZE, COMPETITIVE_SUMMARY_BATCH_SIZE,
CREDENTIAL_FIELDS, DEFAULT_REGION, ENDPOINTS, KEY, LABEL, LWA_TOKEN_URL, OPTIONAL_FIELDS,
REPORT_TYPES, SANDBOX_ENDPOINTS, base_for, is_sandbox,
ROLES_REFUSED, ROLES_TAKEN, US_MARKETPLACE_ID, AmazonError, _create_report,
_competitive_summary_batch, _COMPETITIVE_SUMMARY_BATCH_PATH,
_COMPETITIVE_SUMMARY_ITEM_URI, _download_document, _post_form, _REPORTS_CREATE_PATH,
_request, _safe, _TIMEOUT, _AWD_INVENTORY_PATH, access_token, awd_inventory,
competitive_summaries, marketplace_participations, report_document,
)
#: β MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data
#: structure on a screen about their Amazon account. That was the wave-31 Meta defect and the
#: comments in `keychain.py` warn about it in as many words.
ENTRY_TYPE = "amazon_sp"
class AmazonStatusSafetyError(ValueError):
"""A status response contains a forbidden credential shape."""
class AmazonStatusRowsRefused(AmazonStatusSafetyError):
"""S67-T31: some status ROWS were refused; the rest are carried, already guarded.
Still an `AmazonStatusSafetyError`, so every caller that refuses on the base class keeps
refusing. `safe_status` holds only rows that passed the guard and `refused` only table NAMES:
neither carries the refused content, so a caller may degrade those rows and keep the others.
"""
def __init__(self, safe_status, refused):
super().__init__('Amazon status contains a forbidden credential shape.')
self.safe_status = safe_status
self.refused = tuple(refused)
#: The status guard's scalar ceiling, and the sentence a row wears when its status is refused.
STATUS_SCALAR_LIMIT = 512
STATUS_UNAVAILABLE = "Sync status is unavailable for this database."
def trim_status(value, limit=STATUS_SCALAR_LIMIT):
"""Cut every string over `limit` to a word-bounded excerpt that fits, recursively.
S67-T31: a long, honest cause (staging's Search Query Performance cause measured 935
characters) used to trip the length branch of the guard and take the whole connector modal
down with a 500. β Call this only AFTER the credential-shape scan of the UNTRIMMED value
(`screen_status_rows`): cutting first could slice a token shape out of a string and accept
the rest, which would trim a refusal into acceptance.
"""
if isinstance(value, dict):
return {key: trim_status(item, limit) for key, item in value.items()}
if isinstance(value, list):
return [trim_status(item, limit) for item in value]
if isinstance(value, tuple):
return tuple(trim_status(item, limit) for item in value)
if isinstance(value, str) and len(value) > limit:
head = value[:limit - 3]
cut = head.rsplit(" ", 1)[0] if " " in head else head
return cut.rstrip(" ,;:") + "..."
return value
def screen_status_rows(status):
"""`(safe, refused)` for a `{table: row}` status: each row judged on its own.
A row carrying a credential key, an `Atzr|` token or a bearer shape ANYWHERE in its untrimmed
form is refused whole (NC: never trimmed into acceptance, its value never returned). A row that
is only too long is trimmed, then held to the full guard again.
"""
import json
safe, refused = {}, []
for name, row in (status or {}).items():
try:
whole = json.dumps(row, ensure_ascii=False, allow_nan=False)
if status_payload_has_secret(whole, max_length=None):
refused.append(name)
continue
trimmed = trim_status(row)
if status_payload_has_secret(json.dumps(trimmed, ensure_ascii=False, allow_nan=False)):
refused.append(name)
continue
except (TypeError, ValueError, RecursionError):
refused.append(name)
continue
safe[name] = trimmed
return safe, refused
def status_payload_has_secret(serialized_payload, *, max_length=STATUS_SCALAR_LIMIT):
"""Scan serialized JSON for C2's credential keys, token shapes and oversized values.
Pure and usable without a server. Invalid JSON is unsafe too; never echo rejected content.
The credential key roster is the same one the connector accepts at connect time.
`max_length=None` scans for credential shapes only (the pre-trim pass of `screen_status_rows`).
"""
import json
import re
try:
pending = [json.loads(serialized_payload, object_pairs_hook=lambda pairs: pairs)]
except (TypeError, ValueError, UnicodeError, RecursionError):
return True
credential_keys = {key.casefold() for key in CREDENTIAL_FIELDS}
while pending:
value = pending.pop()
if isinstance(value, tuple):
key, item = value
if key.casefold() in credential_keys:
return True
pending.extend((key, item))
elif isinstance(value, list):
pending.extend(value)
elif isinstance(value, str):
if ((max_length is not None and len(value) > max_length) or 'Atzr|' in value
or re.search(r'\bbearer\s+\S+', value, flags=re.IGNORECASE)):
return True
elif (value is not None and max_length is not None
and len(str(value)) > max_length):
return True
return False
def assert_safe_status(payload):
"""Return a status only after scanning its serialized form, or raise without its content."""
import json
try:
serialized = json.dumps(payload, ensure_ascii=False, allow_nan=False)
except (TypeError, ValueError, RecursionError):
raise AmazonStatusSafetyError('Amazon status could not be safely serialized.') from None
if status_payload_has_secret(serialized):
raise AmazonStatusSafetyError('Amazon status contains a forbidden credential shape.')
return payload
def first_load_status(tenant_key, rt=None):
"""Writer-side progress helper; request status uses sync_state's read-only cursor."""
from harness import amazon_store
return assert_safe_status(amazon_store.first_load_status(tenant_key, rt=rt))
def _stored_table_cap():
"""The row ceiling a materialised `ut_*` table can hold; `amazon_relational.plan` applies it."""
try:
from core import user_tables
return int(getattr(user_tables, "MAX_ROWS", 60_000))
except Exception: # noqa: BLE001
return 60_000
def _mirror_counts(cur, az):
"""{table: row count} for each Amazon mirror table that exists. Read only; absent = never synced."""
counts = {}
for table in az.DATABASE_TABLES:
try:
counts[table] = int(cur.execute(
f"SELECT count(*) FROM {az.table_name(table)}").fetchone()[0])
except Exception as exc: # noqa: BLE001
raise_if_resource_error(exc)
continue
return counts
NOTIFICATION_SECRET_SLOT = "amazon_sp_notifications"
NOTIFICATION_SECRET_HEADER = "x-loopable-amazon-secret"
NOTIFICATION_DEDUPE_SECONDS = 600
NOTIFICATION_TARGETING_CONTRACT = "amazon-table-events-v1"
notification_router = APIRouter(prefix="/api/v1/connectors/amazon_sp")
def _notification_secret(rt):
"""Read the tenant's named generic keychain slot, without an environment fallback."""
from core import keychain
rows = [row for row in keychain.list_entries(rt)
if row.get("type") == "generic" and row.get("label") == NOTIFICATION_SECRET_SLOT]
if len(rows) != 1:
return None
fields = keychain.read_fields(rt, rows[0]["id"]) or {}
secret = fields.get("api_key")
return secret if isinstance(secret, str) and secret else None
def _authenticate_notification(rt, supplied):
"""Authenticate before reading or parsing any event bytes."""
import hmac
try:
secret = _notification_secret(rt)
except Exception:
secret = None
if not secret or not supplied or not hmac.compare_digest(
secret.encode("utf-8"), supplied.encode("utf-8")):
raise HTTPException(status_code=401, detail="Amazon notification authentication failed.")
def notification_target(event):
"""Resolve an approved event to one known source table and its vendor event ID."""
from harness.amazon_client import NOTIFICATION_TYPES, REPORT_TYPES
if not isinstance(event, dict):
raise ValueError("Amazon notification must be an object.")
detail = event.get("detail", event)
if not isinstance(detail, dict):
raise ValueError("Amazon notification detail must be an object.")
event_type = detail.get("notificationType", detail.get("NotificationType"))
if event_type not in NOTIFICATION_TYPES:
raise ValueError("Amazon notification type is not approved for this connector.")
metadata = detail.get("notificationMetadata", detail.get("NotificationMetadata", {}))
if not isinstance(metadata, dict):
raise ValueError("Amazon notification metadata must be an object.")
event_id = metadata.get("notificationId", metadata.get("NotificationId")) or event.get("id")
if not isinstance(event_id, str) or not event_id.strip() or len(event_id) > 256:
raise ValueError("Amazon notification requires a vendor event ID.")
declared = detail.get("table")
if event_type == "REPORT_PROCESSING_FINISHED":
payload = detail.get("payload", detail.get("Payload", {}))
if not isinstance(payload, dict):
raise ValueError("Amazon report notification payload must be an object.")
report = payload.get("reportProcessingFinishedNotification",
payload.get("ReportProcessingFinishedNotification", {}))
if not isinstance(report, dict):
raise ValueError("Amazon report notification must describe its report.")
report_type = report.get("reportType", report.get("ReportType"))
candidates = [table for table, spec in REPORT_TYPES.items()
if report_type and report_type in
[spec.get(key) for key in ("report", "report_incremental", "report_discover")]]
if declared is None and len(candidates) == 1:
declared = candidates[0]
if not isinstance(declared, str) or declared not in candidates:
raise ValueError("Amazon report notification requires one matching known table.")
if report.get("processingStatus", report.get("ProcessingStatus")) != "DONE":
return event_id, None
table = declared
else:
table = NOTIFICATION_TYPES[event_type]["table"]
if declared is not None and declared != table:
raise ValueError("Amazon notification table does not match its event type.")
return event_id, table
def notification_sync_options(receipt):
"""Keep explicit table targets narrow and exclude them from full-source catch-up completion."""
from harness import amazon_store
tables = receipt.get("tables")
if tables is None:
return {}
if (not isinstance(tables, (list, tuple)) or not tables
or any(not isinstance(table, str) or table not in amazon_store.TABLES for table in tables)):
raise ValueError("amazon_notification_target_invalid")
return {"tables": list(dict.fromkeys(tables)), "_catch_up_complete": False}
def _notification_sync_available(desk):
"""The integrator publishes this marker only after producer and resident consumer support targeting."""
import inspect
return (getattr(desk, "NOTIFICATION_TARGETING_CONTRACT", None) == NOTIFICATION_TARGETING_CONTRACT
and {"tables", "event_id", "rt"} <= set(inspect.signature(desk.request_sync).parameters))
@notification_router.post("/notifications", status_code=202)
async def amazon_notifications(request: Request):
"""Queue an authenticated event through the durable targeted sync desk."""
from harness import runtime
from core import connector_sync
tenant = request.query_params.get("tenant")
if not tenant or tenant not in runtime.known_tenants():
raise HTTPException(status_code=401, detail="Amazon notification authentication failed.")
rt = runtime.get_runtime(tenant)
if rt.key != tenant:
raise HTTPException(status_code=401, detail="Amazon notification authentication failed.")
_authenticate_notification(rt, request.headers.get(NOTIFICATION_SECRET_HEADER))
try:
event_id, table = notification_target(await request.json())
except (ValueError, TypeError, UnicodeError):
raise HTTPException(status_code=400, detail="Amazon notification has no approved table target.") from None
if table is None:
return {"accepted": False, "reason": "Amazon report did not finish successfully."}
if not _notification_sync_available(connector_sync):
raise HTTPException(status_code=503, detail="amazon_notification_targeting_unavailable")
receipt = connector_sync.request_sync("amazon_sp", tenant, requested_by="amazon notification",
rt=rt, tables=[table], event_id=event_id)
if receipt.get("tables") != [table] or receipt.get("eventId") != event_id:
raise HTTPException(status_code=503, detail="amazon_notification_target_contract_violated")
return {"accepted": receipt.get("accepted") is True, "table": table,
"duplicate": receipt.get("duplicate") is True}
def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
"""Each Amazon table's sync state, or None when this mirror cannot be read.
Read only: never _connect_tenant_mirror or first_load_status, which issue DDL.
Missing bookkeeping is unknown, not a measured zero. First-load numbers describe the
whole connection, because that is the scope of the producer's durable progress marker.
"""
import json
from datetime import datetime, timezone
import duckdb
from harness import amazon_store as az
from harness.amazon_client import DECLARED_DATASETS
from routes_odoo_tables import _sync_entities, stuck_verdict
# S66-T46 (lead): an unreadable deletion ledger is flagged once per row as
# `purgeScheduleUnknown`, never written into every table's cause, which would push out the
# reason each table already names (Amazon's refusal, the empty period, the declared reason).
purge_unknown = False
try:
purge_plan = az.scheduled_purge(tenant_key, rt, now=now)
purge_message = az.deletion_schedule_message(purge_plan) if purge_plan else ""
except Exception as exc:
raise_if_resource_error(exc)
purge_plan, purge_message, purge_unknown = None, "", True
def finish(status):
for row in status.values():
# A key only when there is something to say: a declared dataset's row stays exactly
# its declaration while no purge is scheduled (the group (a) status contract).
if purge_plan:
row["purgeScheduledAt"] = purge_plan["purgeAt"]
if purge_unknown and not row.get("declared"):
row["purgeScheduleUnknown"] = True
if purge_message:
cause = row.get("unknownCause") or ""
# Status scalars have a 512-character safety ceiling. Keep the full existing
# cause separately if the deletion sentence needs a shorter screen excerpt.
budget = 512 - len(purge_message) - 1
if len(cause) > budget:
row["syncCause"] = cause
cause = cause[:budget - 3].rsplit(" ", 1)[0] + "..."
row["unknownCause"] = " ".join(part for part in (cause, purge_message) if part)
# S67-T31: judged PER ROW. An over-long cause is trimmed (staging's 500 was
# `search_query_performance.unknownCause` at 935 characters); a credential shape still
# refuses, now naming only its rows so the caller degrades those and keeps the rest.
safe, refused = screen_status_rows(status)
if refused:
raise AmazonStatusRowsRefused(assert_safe_status(safe), refused)
return assert_safe_status(safe)
declared = {name: {"rowsLoaded": None, "declared": True, "unknownCause": spec["reason"]}
for name, spec in DECLARED_DATASETS.items()}
unavailable = {table: {"rowsLoaded": None, "lastSuccess": None, "inProgress": False,
"unknownCause": "This container has no readable sync status. Sync this database to load its rows."}
for table in az.DATABASE_TABLES}
unavailable.update(declared)
def stamp(value):
if not value:
return None
try:
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
return (parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)).astimezone(timezone.utc)
except (ValueError, TypeError, OverflowError):
return None
def iso(value):
return value.isoformat().replace("+00:00", "Z") if value else None
cadence = cadence_seconds if cadence_seconds is not None else 1800
try:
path = az._tenant_mirror_path(tenant_key, rt=rt)
if not path.is_file():
return finish(unavailable)
# ββ A SECOND `duckdb.connect(read_only=True)` ON A FILE THIS PROCESS ALREADY HOLDS RAISES
# `ConnectionException: Can't open a connection to same database file with a different
# configuration`. On the DEFAULT tenant layout the Amazon mirror IS the process-global
# analytical mirror (`datastore.DB_PATH`), so the swallowed refusal below turned this whole
# screen into a permanent "sync status is unknown" on exactly the deployment the ticket is
# for. Measured on staging 2026-09-14, W60-T35. `datastore.ro_con()` is the shipped
# per-thread READ cursor on that same instance, and it takes no lock against the writer.
from harness import datastore as _ds
if path.resolve() == _ds.DB_PATH.resolve():
# `ro_cursor()` is a cursor on the instance this process ALREADY holds: no second
# connection, no lock contention with the writer, and no DDL. Deliberately NOT
# `ro_con()`, which refuses until `ready()` and would report the whole screen as
# unknown whenever the Odoo backfill is mid-flight, and NOT `datastore.connect()`,
# which issues `CREATE TABLE IF NOT EXISTS`.
_cur = _ds.ro_cursor()
entities = _sync_entities(_cur, cadence)
mirrored = _mirror_counts(_cur, az)
else:
with _ds.bounded_connection(duckdb.connect(str(path), read_only=True)) as cur:
entities = _sync_entities(cur, cadence)
mirrored = _mirror_counts(cur, az)
except Exception as exc: # noqa: BLE001
raise_if_resource_error(exc)
# No exception text: paths, provider payloads and credentials never become status copy.
return finish(unavailable)
if entities is None:
return finish(unavailable)
clock = stamp(now) if now is not None else datetime.now(timezone.utc)
states, bad_ticks = {}, set()
for table in az.TABLES:
tick = entities.get(az.entity_key("sync_tick_" + table))
if tick:
updated = stamp(tick.get("updatedAt"))
if updated is None or tick.get("phase") not in ("attempted", "done", "running"):
bad_ticks.add(table)
else:
states[table] = {"state": tick["phase"], "updated_at": iso(updated)}
# An unreadable input cannot support a cadence promise for any table in the rotation.
plan = az.sync_plan(states, now=clock) if not bad_ticks and clock is not None else None
first_load, load_unknown = None, ""
markers = {key: {"state": value.get("phase"), "detail": value.get("cursor")}
for key, value in entities.items()}
refused_reports = az.refused_reports(markers)
enabled = az.read_database_enablement(rt)["databases"]
load = entities.get(az.entity_key(az.FIRST_LOAD_STATUS_KEY)) or {}
if load:
try:
progress = json.loads(load.get("cursor") or "")
remaining, eta = progress["remaining"], progress["etaSeconds"]
if (type(remaining) is not int or not 0 <= remaining <= 2**53 - 1
or type(eta) is not int or not 0 <= eta <= 2**53 - 1
or (load.get("phase") == "done" and remaining > 0)):
raise ValueError("Invalid first-load progress")
if remaining > 0:
first_load = {"remaining": remaining, "etaSeconds": eta,
"cause": az.first_load_eta_cause(remaining),
"needsAttention": load.get("phase") == "failed"}
if refused_reports:
first_load = {**(first_load or {"remaining": remaining, "etaSeconds": eta}),
"refused": refused_reports}
if first_load:
first_load = {**(first_load or {}), **az.first_load_payload(markers, remaining, eta,
state=load.get("phase"), tables=[table for table in az.TABLES
if enabled.get(az.amazon_schema.DATABASE_PREFIX + table)])}
except (ValueError, TypeError, KeyError, OverflowError, RecursionError):
load_unknown = "First-load progress is unknown because its saved marker could not be read. Retry sync to refresh it."
elif (entities.get(az.entity_key(az.FIRST_LOAD_END_KEY)) or {}).get("phase") not in (None, "done"):
load_unknown = "First-load progress is not available yet. Keep the connection enabled and check again after this pass."
if first_load is None and refused_reports and not load:
first_load = az.first_load_payload(markers, 0, 0, state="failed",
tables=[table for table in az.TABLES if enabled.get(az.amazon_schema.DATABASE_PREFIX + table)])
first_load["etaDays"] = None
answer = dict(declared)
for table in az.TABLES:
landed = entities.get(az.entity_key(table)) or {}
tick = entities.get(az.entity_key("sync_tick_" + table)) or {}
watermark = entities.get(az.entity_key(az.WATERMARK_PREFIX + table)) or {}
rows = mirrored.get(table, landed.get("rows"))
if type(rows) is not int or rows < 0:
rows = None
# Hotfix 2026-09-24: during the first load no per-table marker exists yet, so the screen
# read "Not loaded yet" beside a mirror already holding thousands of rows. The mirror's
# own count is the measured answer.
if rows is None and table in mirrored:
rows = mirrored[table]
# A refused pass closes its tick (S66-T42); only the watermark can then prove a success.
refused_now = landed.get("phase") in ("refused", "failed", "throttled")
successes = [when for marker in ((watermark,) if refused_now else (landed, tick, watermark))
if marker.get("phase") == "done"
and (when := stamp(marker.get("updatedAt"))) is not None]
stuck, stuck_cause = tick.get("stuck"), tick.get("stuckCause") or ""
if tick.get("phase") == "attempted":
# The shared stall helper calls an active phase 'running'; Amazon calls it 'attempted'.
updated = stamp(tick.get("updatedAt"))
stuck, stuck_cause = stuck_verdict(
"running", iso(updated), cadence,
now=clock.replace(tzinfo=None) if clock else None)
reason = plan["deferred"].get(table) if plan else None
# Only the planner's explicit due stamp is a due date. Minimum stagger spacing is not
# a promise about when a later resident tick will select this table.
next_due = stamp(reason.partition("next due ")[2]) if reason else None
if first_load and load.get("phase") in ("running", "partial"):
wait_marker = entities.get(az.entity_key(az.FIRST_LOAD_DUE_KEY)) or {}
due = stamp(wait_marker.get("cursor")) if wait_marker.get("phase") == "pending" else None
vendor_wait = az._interval_wait(table, states.get(table), clock)[1] if clock else None
if landed.get("phase") == "pending" or rows is None:
next_due = vendor_wait or due or clock
if not reason:
reason = (str(landed.get("cursor")) if landed.get("phase") == "pending" else
"The first load is queued on the resident sync thread; keep the connection enabled.")
unknown = [load_unknown] if load_unknown else []
proof = entities.get(az.entity_key("reconcile_" + table)) or {}
try:
period = json.loads(proof.get("cursor") or "{}")
if not isinstance(period, dict):
period = {}
except (ValueError, TypeError):
period = {}
if rows is None:
unknown.append("This table has not loaded yet. Sync it to request rows from Amazon.")
if landed.get("phase") in ("refused", "failed", "throttled"):
unknown = [str(landed.get("cursor") or "Amazon refused this dataset. Retry sync to refresh its reason.")]
elif rows == 0 or period.get("count") == 0:
span = " to ".join(dict.fromkeys(str(period[k]) for k in ("start", "end") if period.get(k)))
unknown.append("Amazon returned no rows for " + (span or "the last requested period") + ".")
# Standing rule 1's second half: a limit that cannot be removed is REPORTED, with cause
# and fix. `amazon_relational.plan` refuses a table over MAX_ROWS and said so only in the
# server log, so the owner saw an empty database and no reason.
over = mirrored.get(table)
if over is not None and over > _stored_table_cap():
unknown.append(
f"The mirror holds {over:,} rows, more than a stored table can hold "
f"({_stored_table_cap():,}). This table is served read-through from the mirror "
"with no row cap.")
# S66-T42/T43 (lead): a table `amazon_relational.apply_plan` could not create because the
# workspace is at the store's table cap says so here, with the cause and the fix.
not_stored = entities.get(az.entity_key("not_stored_" + table)) or {}
if not_stored.get("phase") == "not_stored":
unknown.append(str(not_stored.get("cursor") or
"This table is not in this workspace yet because the workspace is at "
"its database limit. Its rows are kept in the Amazon mirror."))
if plan is None and landed.get("phase") not in ("refused", "failed", "throttled"):
unknown.append("The sync schedule is unknown because a saved timestamp could not be read. Retry sync to refresh it.")
limit = None
if (table in az.GROUP_A_TABLES and table != "catalog_items") or table == "transactions":
marker = entities.get(az.entity_key("limit_" + table)) or {}
try:
limit = json.loads(marker.get("cursor") or "{}").get("reason")
except (ValueError, TypeError, AttributeError):
limit = "History progress is unreadable. Retry sync to refresh its saved date."
if limit:
unknown.append(limit)
if table == "brand_analytics":
# R18's weekly retention is a code fact: its copy never waits for a weekly pull's marker.
limit = az.BRAND_ANALYTICS_LIMIT
unknown.append(limit)
if table == "catalog_items":
marker = entities.get(az.entity_key("limit_" + table)) or {}
try:
limit = json.loads(marker.get("cursor") or "{}").get("reason")
except (ValueError, TypeError, AttributeError):
limit = "Catalog request progress is unreadable. Sync Amazon listings and retry."
limit = limit or ("Amazon Catalog allows 20 ASINs per search and 2 requests per second. "
"Every own ASIN and page is followed; sync Amazon listings first.")
unknown.append(limit)
if table == "listing_items":
limit = ("Amazon Listings allows 1,000 SKUs per query and 5 requests per second. "
"Date ranges are split and every page followed. Keep the connection enabled.")
unknown.append(limit)
if table == az.SQP_TABLE:
marker = entities.get(az.SQP_LIMIT_KEY) or {}
if marker.get("cursor"):
try:
detail = json.loads(marker["cursor"])
limit = az.sqp_limit_sentence(detail)
unknown.append(limit)
# S66-T41 (lead): one sentence PER REASON, so a whole refused batch (18
# ASINs) stays inside `assert_safe_status`'s 512-character scalar ceiling and
# every refused ASIN is still named on screen.
by_reason = {}
for asin, refusal in detail.get("refusedAsins", {}).items():
by_reason.setdefault(str(refusal), []).append(str(asin))
for refusal, asins in by_reason.items():
unknown.append(az.sqp_refusal_message(asins[0], refusal) if len(asins) == 1
else f"Amazon refused ASINs {', '.join(sorted(asins))} for "
f"Search Query Performance: {refusal}")
unknown.extend(detail.get("errors", []))
if detail.get("deferredReason"):
reason = detail["deferredReason"]
if detail.get("partialMonthReason"):
unknown.append(detail["partialMonthReason"])
except (ValueError, TypeError, KeyError, AttributeError):
unknown.append("Search Query Performance limits are unknown because the saved marker could not be read. Retry sync to refresh it.")
if table in az.GROUP_A_TABLES and len(" ".join(unknown)) > 512:
# Preserve the dataset's refusal/empty-period and vendor limit. Generic schedule
# diagnostics can otherwise overflow the status guard and hide every database.
unknown = [part for part in unknown if part != load_unknown
and not part.startswith(("The sync schedule is unknown", "This table has not loaded yet."))]
answer[table] = {
"rowsLoaded": rows,
"lastSuccess": iso(max(successes)) if successes else None,
"inProgress": tick.get("phase") == "attempted" and stuck is False,
"deferred": {"reason": reason} if reason else None,
"nextDue": iso(next_due),
"firstLoad": first_load,
"stuck": stuck, "stuckCause": stuck_cause,
"unknownCause": " ".join(unknown),
}
# A source-internal mismatch published the table. Account markers carry a connection prefix.
warned = az.entity_key("warning_" + table)
warning = next((str(value.get("cursor") or "") for key, value in entities.items()
if (key == warned or key.endswith("__" + warned)) and value.get("cursor")), "")
if warning:
answer[table]["warning"] = warning
if table == az.SQP_TABLE or table in az.GROUP_A_TABLES or table in ("transactions", "brand_analytics"):
answer[table]["limit"] = limit
for table in (*az.DERIVED_TABLES, "competitor_snapshots", "competitor_images"):
answer[table] = {
"rowsLoaded": mirrored.get(table), "lastSuccess": None, "inProgress": False,
"unknownCause": az.amazon_schema.PROVENANCE_COPY[az.amazon_schema.PROVENANCE[table]],
}
# A history table is written in its source's transaction, so it shares the source's last success.
for table in az.HISTORY_TABLES:
rows = mirrored.get(table)
answer[table] = {
"rowsLoaded": rows,
"lastSuccess": (answer.get(table.removesuffix("_history")) or {}).get("lastSuccess"),
"inProgress": False,
"unknownCause": (az.amazon_schema.PROVENANCE_COPY["history"] if rows is not None
else "History starts with the next sync of the source table."),
}
return finish(answer)
#: W55-T19 / R7 / C10 β AMAZON'S OWN ENABLEMENT DOCUMENT. This is deliberately not Odoo's
#: model catalogue or connector config: Amazon publishes report schemas, not ``fields_get``, and
#: its seven databases are enabled at the provider boundary rather than masquerading as Odoo
#: models. The document lives on the request's tenant runtime, so two tenants can make opposite
#: choices without a process-global map ever seeing either answer.
ENABLEMENT_KEY = "amazon_connector_enablement"
ENABLEMENT_ROUTE = "/connectors/amazon_sp/databases/{databaseKey}/enablement"
class EnablementConfigError(ValueError):
"""The tenant's stored Amazon enablement document is malformed."""
def database_keys():
"""The authored Amazon database keys, including derived mirror views."""
from harness.amazon_store import DATABASE_TABLES
return tuple("ut_amzn_" + str(name) for name in DATABASE_TABLES)
def read_database_enablement(rt):
"""Use the sync engine's fail-closed evaluator for this tenant's database choices."""
from harness import amazon_store as az
return az.read_database_enablement(rt)
def set_database_enabled(rt, database_key, enabled):
"""Persist one tenant's Amazon database choice without touching its materialised rows."""
key = str(database_key or "").strip()
known = set(database_keys())
if key not in known:
raise KeyError(key)
if type(enabled) is not bool:
raise ValueError("enabled must be true or false")
def _set(current):
if current is None:
current = {}
if not isinstance(current, dict):
raise EnablementConfigError("the stored Amazon enablement document is malformed")
stored = current.get("databases", {})
if not isinstance(stored, dict):
raise EnablementConfigError("the stored Amazon database choices are malformed")
if any(type(value) is not bool for value in stored.values()):
raise EnablementConfigError("the stored Amazon database choices are malformed")
if set(stored) - known:
logging.getLogger(__name__).warning("Ignoring unknown Amazon database enablement keys")
# A fresh document contains the whole current roster. That makes the stored answer
# inspectable while retaining the migration rule above for future database additions.
choices = {name: stored.get(name, True) for name in database_keys()}
choices[key] = enabled
return {**current, "databases": choices}
rt.update(ENABLEMENT_KEY, _set, flush="sync")
return read_database_enablement(rt)
def enablement_capability(rt, is_admin):
"""The provider-aware capability served by Amazon's connector detail response."""
state = read_database_enablement(rt)
can_manage = bool(is_admin) and state["available"]
return {"provider": ENTRY_TYPE,
"mode": "database",
"canManage": can_manage,
# A member receives no write URL. The route still enforces administrator status; this
# is the matching affordance wall, not the permission boundary.
"updatePath": ENABLEMENT_ROUTE if can_manage else "",
# Disabling changes only this config document. No mirror or ``ut_amzn_*`` row is
# removed, so re-enabling resumes against the same materialised database.
"preservesData": True,
"available": state["available"]}
def probe_credential(fields, timeout=8, *, fetch_participations=None):
"""Return the guarded Amazon status; a disclosure violation must raise at this boundary."""
return assert_safe_status(_probe_credential_result(
fields, timeout, fetch_participations=fetch_participations))
def _probe_credential_result(fields, timeout=8, *, fetch_participations=None):
"""`{ok, message}` for a stored `amazon_sp` credential: the keychain's liveness probe.
β IT LIVES HERE, IN THE API LAYER, AND IS REGISTERED INTO `core.keychain` RATHER THAN IMPORTED
BY IT. `core` never imports up (ARCHITECTURE.md); `keychain.register_prober` is the slot, and
`connectors_meta` established the idiom after a first draft did `sys.path.insert` plus an
`import` from inside `core`.
β ZERO marketplace participations is reported as NOT ok, deliberately, for the same reason Meta
reports zero ad accounts that way: a credential that authenticates but reaches nothing is an
authorization answer, and `ok: true` would hide the one thing the admin has to fix.
"""
f = fields if isinstance(fields, dict) else {}
missing = [key for key in CREDENTIAL_FIELDS
if not isinstance(f.get(key), str) or not f[key].strip()]
if missing:
return {"ok": False,
"message": "Missing " + ", ".join(missing) + ". Enter every required field."}
try:
# Bind only when explicitly tested. Shape gates can inject a local response function.
fetch = fetch_participations if fetch_participations is not None else marketplace_participations
parts = fetch(f, timeout=timeout)
live = [part for part in parts if isinstance(part, dict)
and isinstance(part.get("participation"), dict)
and part["participation"].get("isParticipating") is True]
if not live:
return {"ok": False,
"message": "Amazon reported no active marketplace access. Check authorization and test again."}
except Exception: # noqa: BLE001
# AmazonError includes provider-supplied text and an invalid region can echo user input.
return {"ok": False,
"message": "Amazon did not confirm marketplace access. Check the five fields and test again."}
return {"ok": True, "message": "Amazon confirmed marketplace access."}
def register():
"""Fill the keychain's probe slot for this connector.
Called at import by the API layer's connector registry (`routes_connectors._register_probers`),
so any process that can serve the connector directory has also registered the prober. Never
raises: a probe slot that could not be filled costs a liveness CHECK, never the ability to
store a credential.
"""
try:
import core.keychain as kc
return kc.register_prober(ENTRY_TYPE, probe_credential)
except Exception: # noqa: BLE001
return False
def _env_credential():
"""A credential assembled from the environment, for `--check` only.
β NOT a runtime path. The product reads Amazon credentials from the per-tenant keychain and
from nowhere else; this exists so an operator can prove a freshly minted refresh token works
before pasting it into the app.
"""
# β `OPTIONAL_FIELDS` joins the read so `AMAZON_SP_SANDBOX=true` reaches `is_sandbox` and the
# probe talks to the sandbox host. Absent still means PRODUCTION, which is the safe default.
f = {k: (os.environ.get("AMAZON_SP_" + k.upper()) or "").strip()
for k in tuple(CREDENTIAL_FIELDS) + OPTIONAL_FIELDS}
return {k: v for k, v in f.items() if v}
def probe_report(fields, report_type=None, days=30, log=print):
"""Drive the WHOLE report chain once and print the MEASURED header. Writes to no store.
`createReport -> poll -> getReportDocument -> download -> decompress -> parse`, then print the
columns the document actually carried. **This is the only way to learn a flat-file report's
schema short of a production sync**, and it is exactly what the connector cannot know from
Amazon's published artifacts: five of our seven tables have none.
β NOTHING IS PERSISTED β no mirror table, no `ut_amzn_*`, no `_sync_state` row. That is what
makes it safe to point at the SANDBOX. `amazon_store.sync` refuses a sandbox credential so mock
rows can never reach a grid; this path sidesteps that question entirely by writing nothing.
β β SANDBOX REALITY, MEASURED 2026-09-04, AND IT IS NOT WHAT "the sandbox mirrors production"
SUGGESTS: the static sandbox's `getReportDocument` **always returns
`GET_MERCHANT_LISTINGS_ALL_DATA`, whatever report type you asked for** (amzn/selling-partner-
api-models#231). So this probe proves the CHAIN for any report type, and yields a genuine header
for exactly ONE of our seven tables β `listings`. The other six still need production.
β And the parse goes through `amazon_store.parse_flat_file`, the SAME function `sync` uses.
A second reader here would prove something production does not do.
"""
from datetime import datetime, timedelta, timezone
from harness.amazon_store import parse_flat_file
rt = report_type or REPORT_TYPES["listings"]["report"]
sbx = is_sandbox(fields)
end = datetime.now(timezone.utc)
start = end - timedelta(days=days)
log(f"host : {'SANDBOX' if sbx else 'PRODUCTION'} {base_for(fields)}")
log(f"report : {rt}")
if sbx and rt != "GET_MERCHANT_LISTINGS_ALL_DATA":
log("WARNING: the static sandbox returns GET_MERCHANT_LISTINGS_ALL_DATA whatever you ask "
"for, so the document below is NOT this report type. The chain is still proven.")
t0 = time.time()
text, meta = report_document(
fields, rt,
start=start.strftime("%Y-%m-%dT%H:%M:%SZ"),
end=end.strftime("%Y-%m-%dT%H:%M:%SZ"), log=log)
log(f"document : {len(text):,} chars, compression={meta.get('compression')} "
f"({time.time() - t0:.1f}s)")
cols, rows = parse_flat_file(text)
log("")
log(f"MEASURED HEADER: {len(cols)} columns, {len(rows)} data rows")
for i, c in enumerate(cols):
log(f" {i + 1:>3}. {c}")
if rows:
# β TRUNCATED, and only the first row. A report document is the SELLER'S data; a probe
# prints enough to prove the shape and no more.
log("")
log("first row, values truncated to 40 chars:")
for c in cols[:12]:
log(f" {c:<34} {str(rows[0].get(c))[:40]}")
return cols, rows
def main(argv):
"""`--check` proves a credential; `--probe-report` proves the whole report chain."""
if "--check" not in argv and "--probe-report" not in argv:
print("usage: connectors_amazon.py --check prove the credential")
print(" connectors_amazon.py --probe-report [--report TYPE]")
print(" drive the full report chain and print")
print(" the MEASURED header. Writes nothing.")
print(" reads AMAZON_SP_CLIENT_ID / _CLIENT_SECRET / _REFRESH_TOKEN / _SELLER_ID /")
print(" _MARKETPLACE_ID, plus AMAZON_SP_SANDBOX=true for the sandbox host")
return 2
f = _env_credential()
have = [k for k in CREDENTIAL_FIELDS if f.get(k)]
print("credential fields present: " + (", ".join(have) if have else "NONE")
+ " (values are never printed)")
if not {"client_id", "client_secret", "refresh_token"} <= set(have):
print("REFUSED: client_id, client_secret and refresh_token are all required")
return 2
t0 = time.time()
res = probe_credential(f, timeout=_TIMEOUT)
print(f"{'OK' if res.get('ok') else 'REFUSED'}: {res.get('message')} "
f"({time.time() - t0:.1f}s)")
if not res.get("ok"):
return 1
print("\nreport types this connector will pull (owner ruling R4):")
for table, spec in REPORT_TYPES.items():
rep = spec.get("report") or spec.get("report_discover") or "NO CONFIRMED REPORT FLAVOR (see O5)"
print(f" {table:<16} {rep}")
if "--probe-report" in argv:
which = None
for i, a in enumerate(argv):
if a == "--report" and i + 1 < len(argv):
which = argv[i + 1]
print("")
try:
probe_report(f, report_type=which)
except Exception: # noqa: BLE001
# β REPORTED, NOT RAISED. The probe is a diagnostic: a refusal is a RESULT the operator
# needs to read (a role they lack, a throttle, an unsupported sandbox report type), not
# a traceback that buries it.
print("REPORT PROBE REFUSED: Amazon did not confirm the report. Check authorization and try again.")
return 1
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
|