Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/main.py +39 -0
- api/routes_connectors.py +2 -1
- platform/harness/amazon_client.py +89 -0
- platform/harness/amazon_store.py +747 -70
- platform/harness/mirror_store.py +69 -1
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "0046a8388",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
0046a8388
|
api/main.py
CHANGED
|
@@ -1269,6 +1269,39 @@ def _drain_pending_amazon_revocations():
|
|
| 1269 |
continue
|
| 1270 |
|
| 1271 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1272 |
def _pull_amazon(why, vendor=True):
|
| 1273 |
"""Produce Amazon rows in this container before projecting them, on the caller's thread.
|
| 1274 |
|
|
@@ -1294,6 +1327,8 @@ def _pull_amazon(why, vendor=True):
|
|
| 1294 |
print(f"[aios-api] amazon sync skipped ({why}): no stored amazon_sp credential "
|
| 1295 |
f"for {tenant_key}; the connector is idle, not broken")
|
| 1296 |
return False
|
|
|
|
|
|
|
| 1297 |
if _amazon.first_load_job_live(rt):
|
| 1298 |
print("[aios-api] the first-load job holds the Amazon writer lease", flush=True)
|
| 1299 |
return False
|
|
@@ -3397,6 +3432,10 @@ def _amazon_intraday_pass(tenant_key, previous_fingerprint):
|
|
| 3397 |
rt = runtime.get_runtime(tenant_key)
|
| 3398 |
if not amazon_store.credential(tenant_key, rt=rt):
|
| 3399 |
return skipped(f"no stored amazon_sp credential for {tenant_key}")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3400 |
if amazon_store.first_load_job_live(rt):
|
| 3401 |
return skipped("the first-load job holds the Amazon writer lease")
|
| 3402 |
plan = amazon_store.scheduled_sync_plan(tenant_key, rt=rt)
|
|
|
|
| 1269 |
continue
|
| 1270 |
|
| 1271 |
|
| 1272 |
+
_AMAZON_PUBLICATION_CHECKED = {}
|
| 1273 |
+
|
| 1274 |
+
|
| 1275 |
+
def _install_amazon_publication(tenant_key, rt, why, min_interval=0.0):
|
| 1276 |
+
"""Install an Amazon publication another writer made, then republish the grid registry. True when installed."""
|
| 1277 |
+
import time as _time
|
| 1278 |
+
from harness import amazon_store as _amazon
|
| 1279 |
+
import routes_odoo_tables as _rot
|
| 1280 |
+
# The intraday tick turns every minute. One manifest read per `min_interval` is enough.
|
| 1281 |
+
_now = _time.monotonic()
|
| 1282 |
+
if min_interval and _now - _AMAZON_PUBLICATION_CHECKED.get(tenant_key, float("-inf")) < min_interval:
|
| 1283 |
+
return False
|
| 1284 |
+
_AMAZON_PUBLICATION_CHECKED[tenant_key] = _now
|
| 1285 |
+
store = _amazon._MIRROR_STORE
|
| 1286 |
+
try:
|
| 1287 |
+
installed = store.install_newer_publication(tenant_key, rt, log=lambda message: print(
|
| 1288 |
+
f"[aios-api] amazon publication ({why}): {message}", flush=True))
|
| 1289 |
+
except Exception as exc: # noqa: BLE001
|
| 1290 |
+
# A refused install keeps the served copy; the pass that called it carries on.
|
| 1291 |
+
print(f"[aios-api] amazon publication ({why}) not installed: {type(exc).__name__}: "
|
| 1292 |
+
f"{str(exc)[:200]}", flush=True)
|
| 1293 |
+
return False
|
| 1294 |
+
if not installed:
|
| 1295 |
+
return False
|
| 1296 |
+
snapshot = (store.cache.get(store._mirror_cache_key(tenant_key, rt)) or {}).get("manifest", {}).get("snapshot_id")
|
| 1297 |
+
print(f"[aios-api] amazon publication installed ({why}): snapshot {snapshot or 'unknown'} "
|
| 1298 |
+
f"from another writer, without a restart", flush=True)
|
| 1299 |
+
# D-3495: grids read a registry built before the install until it is rebuilt.
|
| 1300 |
+
_rot.invalidate_sources()
|
| 1301 |
+
_rot._sources()
|
| 1302 |
+
return True
|
| 1303 |
+
|
| 1304 |
+
|
| 1305 |
def _pull_amazon(why, vendor=True):
|
| 1306 |
"""Produce Amazon rows in this container before projecting them, on the caller's thread.
|
| 1307 |
|
|
|
|
| 1327 |
print(f"[aios-api] amazon sync skipped ({why}): no stored amazon_sp credential "
|
| 1328 |
f"for {tenant_key}; the connector is idle, not broken")
|
| 1329 |
return False
|
| 1330 |
+
# D-13614: before the lease check, since the chain that holds the lease is the writer that publishes.
|
| 1331 |
+
_install_amazon_publication(tenant_key, rt, why)
|
| 1332 |
if _amazon.first_load_job_live(rt):
|
| 1333 |
print("[aios-api] the first-load job holds the Amazon writer lease", flush=True)
|
| 1334 |
return False
|
|
|
|
| 3432 |
rt = runtime.get_runtime(tenant_key)
|
| 3433 |
if not amazon_store.credential(tenant_key, rt=rt):
|
| 3434 |
return skipped(f"no stored amazon_sp credential for {tenant_key}")
|
| 3435 |
+
# D-13614: the intraday tick skips _pull_amazon while the chain holds the lease, so it installs here.
|
| 3436 |
+
if _install_amazon_publication(tenant_key, rt, "intraday", min_interval=300.0):
|
| 3437 |
+
state = _rebuild_amazon_relational("publication", rt, previous_fingerprint)
|
| 3438 |
+
previous_fingerprint = state["fingerprint"] if state.get("known") else None
|
| 3439 |
if amazon_store.first_load_job_live(rt):
|
| 3440 |
return skipped("the first-load job holds the Amazon writer lease")
|
| 3441 |
plan = amazon_store.scheduled_sync_plan(tenant_key, rt=rt)
|
api/routes_connectors.py
CHANGED
|
@@ -3267,7 +3267,8 @@ def _amazon_table_schedule(rt, connection_id=None, now=None):
|
|
| 3267 |
return value if isinstance(value, dict) else {}
|
| 3268 |
|
| 3269 |
def row_reason(table, state):
|
| 3270 |
-
|
|
|
|
| 3271 |
return None
|
| 3272 |
refusal = saved_json(prefix + az.entity_key("refused_" + table))
|
| 3273 |
parts = [amazon_hold_cause({"state": state, "amazonStatus": refusal.get("amazonStatus") or "",
|
|
|
|
| 3267 |
return value if isinstance(value, dict) else {}
|
| 3268 |
|
| 3269 |
def row_reason(table, state):
|
| 3270 |
+
# A mirror still restoring has no ticks at all, so never_run names no cause until one loads.
|
| 3271 |
+
if state == "done" or (state == "never_run" and not states):
|
| 3272 |
return None
|
| 3273 |
refusal = saved_json(prefix + az.entity_key("refused_" + table))
|
| 3274 |
parts = [amazon_hold_cause({"state": state, "amazonStatus": refusal.get("amazonStatus") or "",
|
platform/harness/amazon_client.py
CHANGED
|
@@ -305,6 +305,15 @@ SQP_PERIOD = ("MONTH", "WEEK")
|
|
| 305 |
#: ⚠ `report` is `None` where no bulk report flavor is confirmed. That is O5 of the ideation
|
| 306 |
#: record and it is NOT a licence to page an entity endpoint quietly: an unconfirmed row must
|
| 307 |
#: either find its report or REPORT the limit with cause, per standing rule 1.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 308 |
REPORT_TYPES = {
|
| 309 |
# ⭐⭐ R13 — BOTH ORDER REPORTS, BY ROLE. `report` seeds the one-time historical
|
| 310 |
# backfill because BY_ORDER_DATE is stable and pages cleanly by period; `report_incremental`
|
|
@@ -374,6 +383,10 @@ REPORT_TYPES = {
|
|
| 374 |
# page loop over a full catalogue is precisely what rule 1 forbids.
|
| 375 |
"pricing": {
|
| 376 |
"report": "GET_SALES_AND_TRAFFIC_REPORT",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 377 |
"json_root": "salesAndTrafficByAsin",
|
| 378 |
"report_covers": "buy box SHARE only; PRICE has no bulk path",
|
| 379 |
"price_path": "Product Pricing v2022-05-01 getCompetitiveSummary (batch 20, 0.033/sec)",
|
|
@@ -388,6 +401,11 @@ REPORT_TYPES = {
|
|
| 388 |
# latest week only; older weeks keep watched terms only (amazon_store.write_week).
|
| 389 |
"report": "GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT",
|
| 390 |
"weekly": True, "period": "WEEK",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 391 |
"json_root": "dataByDepartmentAndSearchTerm",
|
| 392 |
"validates_against": "report period totals vs our search-term rows",
|
| 393 |
"role": "Brand Analytics",
|
|
@@ -396,11 +414,15 @@ REPORT_TYPES = {
|
|
| 396 |
"api": "createQuery",
|
| 397 |
"validates_against": "DAY x MSKU sales, fees and net proceeds vs Amazon RANGE x MSKU totals per currency",
|
| 398 |
"role": "Selling Partner Insights",
|
|
|
|
|
|
|
| 399 |
},
|
| 400 |
"transactions": {
|
| 401 |
"api": "listTransactions",
|
| 402 |
"validates_against": "Amazon transaction count and totalAmount per currency vs stored items, and breakdowns vs transaction totals",
|
| 403 |
"role": "Finance and Accounting",
|
|
|
|
|
|
|
| 404 |
},
|
| 405 |
# snapshot forbids dataStartTime/dataEndTime; min_interval_s is a successful
|
| 406 |
# refresh floor (also enforced on manual sync). Monthly history is separate.
|
|
@@ -424,6 +446,10 @@ REPORT_TYPES = {
|
|
| 424 |
},
|
| 425 |
"search_query_performance": {
|
| 426 |
"report": "GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 427 |
"role": "Brand Analytics",
|
| 428 |
"validates_against": "Amazon dataByAsin row count and ASIN purchases per batch vs stored rows for the latest month",
|
| 429 |
"json_root": "dataByAsin", "monthly": True,
|
|
@@ -622,6 +648,10 @@ GROUP_A_TABLES = {
|
|
| 622 |
"label": "Amazon search catalog performance",
|
| 623 |
"report": "GET_BRAND_ANALYTICS_SEARCH_CATALOG_PERFORMANCE_REPORT",
|
| 624 |
"role": "Brand Analytics",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 625 |
"min_interval_s": 86400,
|
| 626 |
"cadence_note": "Complete calendar months; refresh no sooner than 86400 seconds.",
|
| 627 |
"monthly": True,
|
|
@@ -634,6 +664,9 @@ GROUP_A_TABLES = {
|
|
| 634 |
"label": "Amazon market basket",
|
| 635 |
"report": "GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT",
|
| 636 |
"role": "Brand Analytics",
|
|
|
|
|
|
|
|
|
|
| 637 |
"min_interval_s": 86400,
|
| 638 |
"cadence_note": "Complete calendar months; refresh no sooner than 86400 seconds.",
|
| 639 |
"monthly": True,
|
|
@@ -646,6 +679,10 @@ GROUP_A_TABLES = {
|
|
| 646 |
"label": "Amazon repeat purchase",
|
| 647 |
"report": "GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT",
|
| 648 |
"role": "Brand Analytics",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 649 |
"min_interval_s": 86400,
|
| 650 |
"cadence_note": "Complete calendar months; refresh no sooner than 86400 seconds.",
|
| 651 |
"monthly": True,
|
|
@@ -658,6 +695,9 @@ GROUP_A_TABLES = {
|
|
| 658 |
"label": "Amazon promotion performance",
|
| 659 |
"report": "GET_PROMOTION_PERFORMANCE_REPORT",
|
| 660 |
"role": "Selling Partner Insights",
|
|
|
|
|
|
|
|
|
|
| 661 |
"min_interval_s": 86400,
|
| 662 |
"cadence_note": "Date windows; refresh no sooner than 86400 seconds.",
|
| 663 |
"json_root": "promotions",
|
|
@@ -669,6 +709,9 @@ GROUP_A_TABLES = {
|
|
| 669 |
"label": "Amazon coupon performance",
|
| 670 |
"report": "GET_COUPON_PERFORMANCE_REPORT",
|
| 671 |
"role": "Brand Analytics",
|
|
|
|
|
|
|
|
|
|
| 672 |
"min_interval_s": 86400,
|
| 673 |
"cadence_note": "Date windows; refresh no sooner than 86400 seconds.",
|
| 674 |
"json_root": "coupons",
|
|
@@ -2340,6 +2383,52 @@ def group_a_records(fields, operation, *, start=None, end=None, asins=(),
|
|
| 2340 |
return rows
|
| 2341 |
|
| 2342 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2343 |
def _post_form(url, data, timeout=_TIMEOUT):
|
| 2344 |
# LWA exchanges credentials at api.amazon.com, not an SP-API operation. Its sole caller fixes
|
| 2345 |
# LWA_TOKEN_URL; the connector-wide gate checks that call separately from OPERATIONS.
|
|
|
|
| 305 |
#: ⚠ `report` is `None` where no bulk report flavor is confirmed. That is O5 of the ideation
|
| 306 |
#: record and it is NOT a licence to page an entity endpoint quietly: an unconfirmed row must
|
| 307 |
#: either find its report or REPORT the limit with cause, per standing rule 1.
|
| 308 |
+
#:
|
| 309 |
+
#: `history` is Amazon's own history floor for a dated table, one of:
|
| 310 |
+
#: {"rolling_days": N, "edge": "start"|"end", "evidence": str}: Amazon refuses a window whose
|
| 311 |
+
#: start (or end) is older than now minus N days. The floor moves with the clock.
|
| 312 |
+
#: {"from": "YYYY-MM-DD", "evidence": str}: Amazon serves nothing older than this date.
|
| 313 |
+
#: {"seller_floor": False, "evidence": str}: brand-level data; the seller's first month does not
|
| 314 |
+
#: bound it, so history walks back until Amazon refuses an older period.
|
| 315 |
+
#: A `from` rule also replaces the seller month, even when older. A rolling rule is max(seller, rule).
|
| 316 |
+
#: No `history` key: Amazon serves back to the seller's first data (`amazon_store.history_floor`).
|
| 317 |
REPORT_TYPES = {
|
| 318 |
# ⭐⭐ R13 — BOTH ORDER REPORTS, BY ROLE. `report` seeds the one-time historical
|
| 319 |
# backfill because BY_ORDER_DATE is stable and pages cleanly by period; `report_incremental`
|
|
|
|
| 383 |
# page loop over a full catalogue is precisely what rule 1 forbids.
|
| 384 |
"pricing": {
|
| 385 |
"report": "GET_SALES_AND_TRAFFIC_REPORT",
|
| 386 |
+
"history": {"rolling_days": 730, "edge": "start",
|
| 387 |
+
"evidence": "CANCELLED, Amazon's no data answer, for 2021-09, 2023-01 and 2024-11 (start 743 days "
|
| 388 |
+
"back, 17,584 orders that month); DONE with rows 2024-12 (start 713 days back), 2025-05, "
|
| 389 |
+
"2025-11 (2026-11-14)"},
|
| 390 |
"json_root": "salesAndTrafficByAsin",
|
| 391 |
"report_covers": "buy box SHARE only; PRICE has no bulk path",
|
| 392 |
"price_path": "Product Pricing v2022-05-01 getCompetitiveSummary (batch 20, 0.033/sec)",
|
|
|
|
| 401 |
# latest week only; older weeks keep watched terms only (amazon_store.write_week).
|
| 402 |
"report": "GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT",
|
| 403 |
"weekly": True, "period": "WEEK",
|
| 404 |
+
"history": {"from": "2025-08-10",
|
| 405 |
+
"evidence": "Weekly FATAL, the period may fall outside the available data range, for the weeks "
|
| 406 |
+
"of 2021-09-05, 2023-11-05, 2024-11-03, 2025-03-30, 2025-06-29 and 2025-08-03; DONE with "
|
| 407 |
+
"rows 2025-10-05, 2026-04-05 and 2026-07-05 (2026-11-14). The weeks between stop at "
|
| 408 |
+
"Amazon's first refusal. Search terms cover the marketplace, not the seller"},
|
| 409 |
"json_root": "dataByDepartmentAndSearchTerm",
|
| 410 |
"validates_against": "report period totals vs our search-term rows",
|
| 411 |
"role": "Brand Analytics",
|
|
|
|
| 414 |
"api": "createQuery",
|
| 415 |
"validates_against": "DAY x MSKU sales, fees and net proceeds vs Amazon RANGE x MSKU totals per currency",
|
| 416 |
"role": "Selling Partner Insights",
|
| 417 |
+
"history": {"rolling_days": 730, "edge": "start",
|
| 418 |
+
"evidence": "Data Kiosk FATAL: Start date must be no more than two years ago (2026-11-14)"},
|
| 419 |
},
|
| 420 |
"transactions": {
|
| 421 |
"api": "listTransactions",
|
| 422 |
"validates_against": "Amazon transaction count and totalAmount per currency vs stored items, and breakdowns vs transaction totals",
|
| 423 |
"role": "Finance and Accounting",
|
| 424 |
+
"history": {"rolling_days": 730, "edge": "start",
|
| 425 |
+
"evidence": "listTransactions 400: Start date cannot be before 2 years from now (2026-11-14)"},
|
| 426 |
},
|
| 427 |
# snapshot forbids dataStartTime/dataEndTime; min_interval_s is a successful
|
| 428 |
# refresh floor (also enforced on manual sync). Monthly history is separate.
|
|
|
|
| 446 |
},
|
| 447 |
"search_query_performance": {
|
| 448 |
"report": "GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT",
|
| 449 |
+
"history": {"from": "2025-04-01",
|
| 450 |
+
"evidence": "Monthly FATAL, the period may fall outside the available data range, for 2021-09, "
|
| 451 |
+
"2024-10, 2024-11, 2025-01 and 2025-03; 2025-04 onward landed. Weekly FATAL through the "
|
| 452 |
+
"week of 2025-07-06, DONE 2025-10-05 (2026-11-14)"},
|
| 453 |
"role": "Brand Analytics",
|
| 454 |
"validates_against": "Amazon dataByAsin row count and ASIN purchases per batch vs stored rows for the latest month",
|
| 455 |
"json_root": "dataByAsin", "monthly": True,
|
|
|
|
| 648 |
"label": "Amazon search catalog performance",
|
| 649 |
"report": "GET_BRAND_ANALYTICS_SEARCH_CATALOG_PERFORMANCE_REPORT",
|
| 650 |
"role": "Brand Analytics",
|
| 651 |
+
"history": {"rolling_days": 730, "edge": "end",
|
| 652 |
+
"evidence": "FATAL, the period may fall outside the available data range, for 2024-10 (end 744 "
|
| 653 |
+
"days back) and every month probed back to 2021-09; DONE with rows 2024-11 (end 714 days "
|
| 654 |
+
"back), 2025-03, 2025-04 (2026-11-14)"},
|
| 655 |
"min_interval_s": 86400,
|
| 656 |
"cadence_note": "Complete calendar months; refresh no sooner than 86400 seconds.",
|
| 657 |
"monthly": True,
|
|
|
|
| 664 |
"label": "Amazon market basket",
|
| 665 |
"report": "GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT",
|
| 666 |
"role": "Brand Analytics",
|
| 667 |
+
"history": {"from": "2019-08-01",
|
| 668 |
+
"evidence": "DONE with rows 2020-01, 2020-07 and 2021-07 (before the seller's first order "
|
| 669 |
+
"2021-08); DONE empty 2019-07 (2026-11-14)"},
|
| 670 |
"min_interval_s": 86400,
|
| 671 |
"cadence_note": "Complete calendar months; refresh no sooner than 86400 seconds.",
|
| 672 |
"monthly": True,
|
|
|
|
| 679 |
"label": "Amazon repeat purchase",
|
| 680 |
"report": "GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT",
|
| 681 |
"role": "Brand Analytics",
|
| 682 |
+
"history": {"from": "2023-02-01",
|
| 683 |
+
"evidence": "FATAL, the period may fall outside the available data range, for 2021-09 and 2023-01; "
|
| 684 |
+
"DONE with rows 2024-10, 2024-11, 2025-03, 2025-04 (2026-11-14). The months between "
|
| 685 |
+
"stop at Amazon's first refusal"},
|
| 686 |
"min_interval_s": 86400,
|
| 687 |
"cadence_note": "Complete calendar months; refresh no sooner than 86400 seconds.",
|
| 688 |
"monthly": True,
|
|
|
|
| 695 |
"label": "Amazon promotion performance",
|
| 696 |
"report": "GET_PROMOTION_PERFORMANCE_REPORT",
|
| 697 |
"role": "Selling Partner Insights",
|
| 698 |
+
"history": {"rolling_days": 730, "edge": "start",
|
| 699 |
+
"evidence": "FATAL: Requests with a promotionStartDateFrom of more than two years ago are not "
|
| 700 |
+
"supported (2021-09 window, 2026-11-14)"},
|
| 701 |
"min_interval_s": 86400,
|
| 702 |
"cadence_note": "Date windows; refresh no sooner than 86400 seconds.",
|
| 703 |
"json_root": "promotions",
|
|
|
|
| 709 |
"label": "Amazon coupon performance",
|
| 710 |
"report": "GET_COUPON_PERFORMANCE_REPORT",
|
| 711 |
"role": "Brand Analytics",
|
| 712 |
+
"history": {"rolling_days": 730, "edge": "start",
|
| 713 |
+
"evidence": "FATAL: Requests with a couponStartDateFrom of more than two years ago are not "
|
| 714 |
+
"supported (2021-09 window, 2026-11-14)"},
|
| 715 |
"min_interval_s": 86400,
|
| 716 |
"cadence_note": "Date windows; refresh no sooner than 86400 seconds.",
|
| 717 |
"json_root": "coupons",
|
|
|
|
| 2383 |
return rows
|
| 2384 |
|
| 2385 |
|
| 2386 |
+
#: getOrderMetrics refuses a Month interval over about two years ("2130D exceeded limit"); 731 days answers.
|
| 2387 |
+
ORDER_MONTHS_MAX = 24
|
| 2388 |
+
|
| 2389 |
+
|
| 2390 |
+
def _parse_utc(value):
|
| 2391 |
+
"""A datetime or ISO string as an aware UTC datetime; a bare date reads as UTC midnight."""
|
| 2392 |
+
if isinstance(value, str):
|
| 2393 |
+
value = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
| 2394 |
+
if value.tzinfo is None:
|
| 2395 |
+
value = value.replace(tzinfo=timezone.utc)
|
| 2396 |
+
return value.astimezone(timezone.utc)
|
| 2397 |
+
|
| 2398 |
+
|
| 2399 |
+
def order_months(fields, start, end, *, timeout=_TIMEOUT, sleep=time.sleep, now=time.monotonic):
|
| 2400 |
+
"""[(month_start "YYYY-MM-01", orderCount)] for [start, end) from getOrderMetrics granularity Month."""
|
| 2401 |
+
lower = _parse_utc(start).replace(day=1, hour=0, minute=0, second=0, microsecond=0)
|
| 2402 |
+
upper = _parse_utc(end).replace(day=1, hour=0, minute=0, second=0, microsecond=0)
|
| 2403 |
+
months = (upper.year - lower.year) * 12 + upper.month - lower.month
|
| 2404 |
+
if months <= 0:
|
| 2405 |
+
return []
|
| 2406 |
+
if months > ORDER_MONTHS_MAX:
|
| 2407 |
+
raise AmazonError(f"getOrderMetrics answers at most {ORDER_MONTHS_MAX} months per Month interval")
|
| 2408 |
+
marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
|
| 2409 |
+
operation = "getOrderMetrics"
|
| 2410 |
+
token = access_token(fields, timeout=timeout)
|
| 2411 |
+
_pace_entity(operation, ENTITY_INTERVALS[operation], sleep=sleep, now=now)
|
| 2412 |
+
got = _request(OPERATIONS[operation]["path"], token, region=str(fields.get("region") or DEFAULT_REGION),
|
| 2413 |
+
sandbox=is_sandbox(fields), timeout=timeout, marketplaceIds=marketplace,
|
| 2414 |
+
interval=f"{lower.strftime('%Y-%m-%dT%H:%M:%SZ')}--{upper.strftime('%Y-%m-%dT%H:%M:%SZ')}",
|
| 2415 |
+
granularity="Month", granularityTimeZone="UTC",
|
| 2416 |
+
throttle_fix=f"continue at Amazon's {1 / ENTITY_INTERVALS[operation]:g} requests per second")
|
| 2417 |
+
body = got.get("payload") if isinstance(got, dict) else None
|
| 2418 |
+
if not isinstance(body, list) or any(not isinstance(row, dict) for row in body):
|
| 2419 |
+
raise AmazonError("getOrderMetrics returned no valid month list; retry after Amazon restores its response")
|
| 2420 |
+
out = []
|
| 2421 |
+
for row in body:
|
| 2422 |
+
month = str(row.get("interval") or "").split("--", 1)[0][:7]
|
| 2423 |
+
count = row.get("orderCount")
|
| 2424 |
+
if not re.fullmatch(r"\d{4}-\d{2}", month) or type(count) is not int or count < 0:
|
| 2425 |
+
raise AmazonError("getOrderMetrics returned a month without an interval or a nonnegative orderCount")
|
| 2426 |
+
first = datetime.fromisoformat(month + "-01").replace(tzinfo=timezone.utc)
|
| 2427 |
+
if lower <= first < upper:
|
| 2428 |
+
out.append((month + "-01", count))
|
| 2429 |
+
return sorted(out)
|
| 2430 |
+
|
| 2431 |
+
|
| 2432 |
def _post_form(url, data, timeout=_TIMEOUT):
|
| 2433 |
# LWA exchanges credentials at api.amazon.com, not an SP-API operation. Its sole caller fixes
|
| 2434 |
# LWA_TOKEN_URL; the connector-wide gate checks that call separately from OPERATIONS.
|
platform/harness/amazon_store.py
CHANGED
|
@@ -227,6 +227,24 @@ def _refresh_account_status(con, entries, rt):
|
|
| 227 |
_record_state(con, key, phase, sum(s.get("rows", 0) for s in present), detail)
|
| 228 |
updated = min(s["updated_at"] for s in present)
|
| 229 |
con.execute("UPDATE _sync_state SET updated_at=? WHERE entity=?", [updated, key])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 230 |
loads, remaining, eta, waits = [], 0, 0, []
|
| 231 |
for entry in entries:
|
| 232 |
prefix = _account_prefix(entry["connection_id"])
|
|
@@ -671,9 +689,21 @@ def entity_key(table):
|
|
| 671 |
return PREFIX + str(table)
|
| 672 |
|
| 673 |
|
| 674 |
-
#:
|
| 675 |
-
#:
|
| 676 |
DEFAULT_BACKFILL_DAYS = int(os.environ.get("AMAZON_BACKFILL_DAYS") or 730)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 677 |
#: Per-table marker whose detail is the ISO UTC end of the last successfully landed window.
|
| 678 |
WATERMARK_PREFIX = "sync_mark_"
|
| 679 |
#: ⚠ AMAZON'S OWN CEILING, NOT OURS, and it is why a backfill is many reports rather than one.
|
|
@@ -685,6 +715,8 @@ FIRST_LOAD_RECENT_DAYS = MAX_WINDOW_DAYS
|
|
| 685 |
#: about 2.7 h. A first load therefore runs in BOUNDED passes, or it would hold the resident
|
| 686 |
#: thread for hours and the Odoo resync behind it would never run.
|
| 687 |
FIRST_LOAD_REPORTS_PER_PASS = 15
|
|
|
|
|
|
|
| 688 |
#: Rows one first-load pass writes before it takes no further document. A pacing bound, never a cap:
|
| 689 |
#: the pass publishes and the next continues, so Brand Analytics months (3.7M rows and about 25
|
| 690 |
#: minutes each) never hold the sync desk for hours (D-7081).
|
|
@@ -2614,7 +2646,9 @@ def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem):
|
|
| 2614 |
except (SQPBudgetDeferred, AmazonReportPending):
|
| 2615 |
raise
|
| 2616 |
except cx.AmazonError as exc:
|
| 2617 |
-
|
|
|
|
|
|
|
| 2618 |
reason = getattr(exc, "vendor_reason", str(exc))
|
| 2619 |
reason = reason.replace("\u2014", "; ").replace("\u2013", " to ")
|
| 2620 |
for asin in refused:
|
|
@@ -2926,41 +2960,51 @@ def _validate_sku_keywords(con):
|
|
| 2926 |
|
| 2927 |
|
| 2928 |
def first_load_plan(tables, start, end, *, recent_days=None, size=MAX_WINDOW_DAYS,
|
| 2929 |
-
sqp_asins=()):
|
| 2930 |
"""R18's request order: every recent month, then each table's history, oldest last.
|
| 2931 |
|
| 2932 |
Settlements have one discovery slot (None, None), never a createReport window.
|
| 2933 |
This planner is pure; durable floors filter its fixed windows when a pass resumes.
|
| 2934 |
SQP adds a fourth tuple member, its ASIN option, for each month x batch task.
|
|
|
|
|
|
|
| 2935 |
"""
|
| 2936 |
start, end = _utc(start), _utc(end)
|
| 2937 |
-
if
|
|
|
|
|
|
|
| 2938 |
return []
|
| 2939 |
size = min(float(size), MAX_WINDOW_DAYS)
|
| 2940 |
recent_days = FIRST_LOAD_RECENT_DAYS if recent_days is None else recent_days
|
| 2941 |
recent_days = min(float(recent_days), size)
|
| 2942 |
if size <= 0 or recent_days <= 0:
|
| 2943 |
raise ValueError("Amazon report windows must have a positive size.")
|
| 2944 |
-
recent_start = max(start, end - timedelta(days=recent_days))
|
| 2945 |
tables = list(dict.fromkeys(tables))
|
| 2946 |
# Hotfix 2026-09-24: the settlements slot goes LAST among the recent entries. It downloads
|
| 2947 |
# every new settlement document until the pass budget is spent and then ends the pass, so in
|
| 2948 |
# table order it starved inventory, pricing and brand_analytics of their recent month for
|
| 2949 |
# about four passes on live (52 fortnightly documents over 730 days).
|
| 2950 |
-
recent, history = [], []
|
| 2951 |
for table in tables:
|
| 2952 |
spec = _client().REPORT_TYPES[table]
|
| 2953 |
-
|
| 2954 |
-
|
|
|
|
| 2955 |
if table == "settlements":
|
| 2956 |
continue
|
| 2957 |
if spec.get("snapshot"):
|
| 2958 |
recent.append((table, None, None))
|
|
|
|
|
|
|
| 2959 |
elif spec.get("weekly"):
|
| 2960 |
-
# R18:
|
| 2961 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2962 |
elif spec.get("monthly"):
|
| 2963 |
-
months = list(reversed(_month_windows(
|
| 2964 |
if not months:
|
| 2965 |
continue
|
| 2966 |
tasks = ([(table, w0, w1, batch) for w0, w1 in months
|
|
@@ -2970,20 +3014,61 @@ def first_load_plan(tables, start, end, *, recent_days=None, size=MAX_WINDOW_DAY
|
|
| 2970 |
history.extend(task for task in tasks if task[1:3] != months[0])
|
| 2971 |
else:
|
| 2972 |
table_size = spec.get("max_window_days", size)
|
| 2973 |
-
table_recent = max(
|
| 2974 |
recent.append((table, table_recent, end))
|
| 2975 |
-
history.extend((table, w0, w1) for w0, w1 in reversed(_windows(
|
| 2976 |
recent.extend((table, None, None) for table in tables if table == "settlements")
|
| 2977 |
-
return recent + history
|
| 2978 |
|
| 2979 |
|
| 2980 |
def _sqp_after_cohort(tasks, end, asins):
|
| 2981 |
-
"""Insert
|
| 2982 |
-
|
| 2983 |
-
|
| 2984 |
-
|
| 2985 |
-
|
| 2986 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2987 |
|
| 2988 |
|
| 2989 |
def first_load_eta(report_count, *, stagger_seconds=None, burst=None, per_minute=None):
|
|
@@ -3916,9 +4001,10 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
|
|
| 3916 |
"""
|
| 3917 |
rows = report.get("tables") or {}
|
| 3918 |
problems = [str(problem) for key in ("problems", "audit_problems") for problem in report.get(key) or []]
|
| 3919 |
-
# SQP batches
|
| 3920 |
if report.get("state") in {"partial", "failed"} and not any(
|
| 3921 |
-
isinstance(row, dict) and (row.get("state") != "deferred" or name == SQP_TABLE
|
|
|
|
| 3922 |
and _mirrors._table_sync_complete("amazon", row) for name, row in rows.items()):
|
| 3923 |
return None
|
| 3924 |
unfinished = {name[len(PREFIX):] if name.startswith(PREFIX) else name for name in waiting}
|
|
@@ -5193,7 +5279,8 @@ def first_load_payload(states, remaining, eta_seconds, *, state="partial", table
|
|
| 5193 |
"refused": refused, "line": line, "etaDays": eta_days, "details": details}
|
| 5194 |
|
| 5195 |
|
| 5196 |
-
def needs_first_load(tenant_key, rt=None, *, completed_tables=None, repair_tables=None, invalid_proofs=None
|
|
|
|
| 5197 |
"""A fresh account or an unfinished durable load uses bounded passes, even after one table finishes."""
|
| 5198 |
rt = rt if rt is not None else _runtime_for(tenant_key)
|
| 5199 |
restored = restore_mirror(tenant_key, rt)
|
|
@@ -5237,6 +5324,10 @@ def needs_first_load(tenant_key, rt=None, *, completed_tables=None, repair_table
|
|
| 5237 |
if enabled.get(amazon_schema.DATABASE_PREFIX + table) and documented(table)}
|
| 5238 |
repairs = set()
|
| 5239 |
refusals = {row["table"]: row for row in _refused_reports(con)}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5240 |
|
| 5241 |
def terminal(table):
|
| 5242 |
state = _read_state(con, entity_key(table)) or {}
|
|
@@ -5256,11 +5347,10 @@ def needs_first_load(tenant_key, rt=None, *, completed_tables=None, repair_table
|
|
| 5256 |
if spec.get("snapshot"):
|
| 5257 |
return True
|
| 5258 |
floor = read_backfill_floor(con, table)
|
| 5259 |
-
|
| 5260 |
-
cohort = _utc(load["detail"]) if load and load.get("detail") else datetime.now(timezone.utc)
|
| 5261 |
-
except (ValueError, TypeError, AttributeError, OverflowError):
|
| 5262 |
return False
|
| 5263 |
-
|
|
|
|
| 5264 |
# ⛔ HOTFIX 2026-09-29 (D-3496): the same rule for a load that STOPPED. A load left
|
| 5265 |
# "failed" only because Amazon refused a report (a 403 for a report it does not offer this
|
| 5266 |
# seller) stayed "not done" forever, so every restart queued it again AND the scheduled
|
|
@@ -5277,6 +5367,13 @@ def needs_first_load(tenant_key, rt=None, *, completed_tables=None, repair_table
|
|
| 5277 |
repair_tables.update(repairs)
|
| 5278 |
if invalid_proofs is not None:
|
| 5279 |
invalid_proofs.update(invalid)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5280 |
missing = [table for table in TABLES
|
| 5281 |
if enabled.get(amazon_schema.DATABASE_PREFIX + table)
|
| 5282 |
and not read_watermark(con, table)
|
|
@@ -5284,7 +5381,7 @@ def needs_first_load(tenant_key, rt=None, *, completed_tables=None, repair_table
|
|
| 5284 |
owed = bool(load and load["state"] != "done" and missing)
|
| 5285 |
partially_refused = {name[len(PREFIX):] for name in restored.get("refused_tables", ())}
|
| 5286 |
return (bool(partially_refused.intersection(missing))
|
| 5287 |
-
or owed or bool(repairs)
|
| 5288 |
or (bool(missing) and not any(read_watermark(con, table) for table in TABLES))
|
| 5289 |
or any(table in missing for table in added))
|
| 5290 |
finally:
|
|
@@ -5308,10 +5405,16 @@ def first_load_status(tenant_key, rt=None):
|
|
| 5308 |
job["tables"]["inventory"] = inventory_table_status(con)
|
| 5309 |
if job and isinstance(job.get("remaining"), int) and state["state"] != "done":
|
| 5310 |
remaining = job["remaining"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5311 |
out = {"state": state["state"], **first_load_payload(markers, remaining,
|
| 5312 |
int(values["etaSeconds"]), state=state["state"],
|
| 5313 |
-
tables=
|
| 5314 |
-
|
| 5315 |
"nextDue": _iso(_first_load_due(con)) if _first_load_due(con) else None}
|
| 5316 |
if job:
|
| 5317 |
out["job"] = job
|
|
@@ -5513,11 +5616,17 @@ def first_load_task(account, fields, table, w0, w1, batch=None, *, landed=False)
|
|
| 5513 |
return task
|
| 5514 |
|
| 5515 |
|
| 5516 |
-
def _task_landed(con, table, w0, w1, batch):
|
| 5517 |
-
"""The mirror's own evidence that an earlier pass landed this window: watermark, floor or
|
|
|
|
|
|
|
|
|
|
| 5518 |
if table == SQP_TABLE and w0 is not None:
|
| 5519 |
return _sqp_done(con, w0, w1, batch)
|
| 5520 |
-
if
|
|
|
|
|
|
|
|
|
|
| 5521 |
return True
|
| 5522 |
floor = read_backfill_floor(con, table)
|
| 5523 |
return w1 is not None and floor is not None and w1 > floor
|
|
@@ -5530,8 +5639,9 @@ def first_load_job_plan(tenant_key, rt, *, log=print):
|
|
| 5530 |
before a first pass froze one: the job then runs one bounded pass first.
|
| 5531 |
"""
|
| 5532 |
entries = account_entries(tenant_key, rt)
|
| 5533 |
-
completed, repairs = set(), set()
|
| 5534 |
-
owed = needs_first_load(tenant_key, rt=rt, completed_tables=completed, repair_tables=repairs
|
|
|
|
| 5535 |
con, _path = _connect_tenant_mirror(tenant_key, rt)
|
| 5536 |
try:
|
| 5537 |
load = _read_state(con, entity_key(FIRST_LOAD_END_KEY))
|
|
@@ -5546,15 +5656,22 @@ def first_load_job_plan(tenant_key, rt, *, log=print):
|
|
| 5546 |
wanted = [t for t in tables if t not in completed or t in repairs] if single else tables
|
| 5547 |
sqp_asins = _sqp_catalogue(con) if SQP_TABLE in wanted else ()
|
| 5548 |
tasks, fields_by = [], {}
|
|
|
|
| 5549 |
for entry in entries:
|
| 5550 |
cid = entry["connection_id"]
|
| 5551 |
fields = fields_by[cid] = _sync_credential(tenant_key, rt, connection_id=cid)
|
| 5552 |
-
items = first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end, sqp_asins=sqp_asins
|
| 5553 |
-
|
|
|
|
| 5554 |
table, w0, w1 = item[:3]
|
| 5555 |
batch = item[3] if len(item) == 4 else None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5556 |
# Account-scoped floors live in each account's workspace; only one account reads them here.
|
| 5557 |
-
landed = single and _task_landed(con, table, w0, w1, batch)
|
| 5558 |
task = first_load_task(cid, fields, table, w0, w1, batch, landed=landed)
|
| 5559 |
# One sync writes every account, so a prefetch serves a single-account tenant only.
|
| 5560 |
task["prefetch"] = task["prefetch"] and single
|
|
@@ -6360,15 +6477,24 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon
|
|
| 6360 |
if not waiting_on and now() >= request_until:
|
| 6361 |
break
|
| 6362 |
sleep(poll_seconds if waiting_on else max(0.0, min(poll_seconds, next_token - now())))
|
| 6363 |
-
|
| 6364 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6365 |
try:
|
| 6366 |
-
delegate(own_tables,
|
| 6367 |
record["otherTablesPass"] = "done"
|
| 6368 |
except Exception as exc: # noqa: BLE001
|
| 6369 |
record["otherTablesPass"] = f"failed ({type(exc).__name__})"
|
| 6370 |
log(f"Amazon first-load pass for the other tables failed ({type(exc).__name__}); "
|
| 6371 |
"the next generation runs it again.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6372 |
try:
|
| 6373 |
after = plan_fn()
|
| 6374 |
except Exception: # noqa: BLE001
|
|
@@ -6412,8 +6538,280 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon
|
|
| 6412 |
pass
|
| 6413 |
|
| 6414 |
|
| 6415 |
-
|
| 6416 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6417 |
end = _utc(now)
|
| 6418 |
days = DEFAULT_BACKFILL_DAYS if backfill_days is None else backfill_days
|
| 6419 |
try:
|
|
@@ -6426,6 +6824,9 @@ def sync_window(table, mark, *, now, since=None, backfill_days=None, intraday=Fa
|
|
| 6426 |
elif mark is not None:
|
| 6427 |
start, incremental = mark, True
|
| 6428 |
cause = f"incremental window since watermark {_iso(mark)}"
|
|
|
|
|
|
|
|
|
|
| 6429 |
else:
|
| 6430 |
start, incremental = end - timedelta(days=days), False
|
| 6431 |
cause = f"first full load: {days} days of backfill"
|
|
@@ -6456,6 +6857,9 @@ def sync_window(table, mark, *, now, since=None, backfill_days=None, intraday=Fa
|
|
| 6456 |
if incremental or since is not None:
|
| 6457 |
start = _week_start(start)
|
| 6458 |
cause += "; completed Sunday to Saturday weeks only"
|
|
|
|
|
|
|
|
|
|
| 6459 |
else:
|
| 6460 |
start = end - timedelta(days=7)
|
| 6461 |
cause = "first load: the latest completed Sunday to Saturday week only (R18)"
|
|
@@ -6652,18 +7056,43 @@ def _report_document(con, table, fields, report_type, **kwargs):
|
|
| 6652 |
return result
|
| 6653 |
|
| 6654 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6655 |
def _close_history_floor(con, table, exc, start, end):
|
| 6656 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6657 |
floor = read_backfill_floor(con, table)
|
| 6658 |
-
|
| 6659 |
-
|
| 6660 |
-
|
| 6661 |
-
|
| 6662 |
-
|
|
|
|
| 6663 |
return None
|
| 6664 |
-
|
|
|
|
| 6665 |
"Keep the connection enabled for newer data; ask Seller Support for older history.")
|
| 6666 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6667 |
return reason
|
| 6668 |
|
| 6669 |
|
|
@@ -8666,9 +9095,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8666 |
raise RuntimeError("Amazon deletion is outstanding; sync cannot repopulate purged data")
|
| 8667 |
load = _read_state(con, entity_key(FIRST_LOAD_END_KEY))
|
| 8668 |
# Table completion evidence also admits empty histories behind a stale aggregate marker.
|
| 8669 |
-
completed, repairs, invalid_proofs = set(), set(), set()
|
| 8670 |
needs_first_load(tenant_key, rt=rt, completed_tables=completed, repair_tables=repairs,
|
| 8671 |
-
invalid_proofs=invalid_proofs)
|
| 8672 |
prior_refusals = {row["table"]: row for row in _refused_reports(con)}
|
| 8673 |
exhausted = {table for table in want if prior_refusals.get(table, {}).get("state") == "failed"}
|
| 8674 |
|
|
@@ -8702,7 +9131,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8702 |
for key in ("problems", "audit_problems"):
|
| 8703 |
report[key].extend(refreshed.get(key) or [])
|
| 8704 |
con, _path = _connect_tenant_mirror(tenant_key, rt)
|
| 8705 |
-
|
|
|
|
| 8706 |
due = _restore_first_load_wait(tenant_key, con)
|
| 8707 |
if due and due > end and not _client().prefetch_active():
|
| 8708 |
progress = first_load_status(tenant_key, rt)
|
|
@@ -8875,24 +9305,39 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8875 |
"monthsDone": sum(r["done"] >= r["total"] for r in rows), "monthsTotal": len(rows),
|
| 8876 |
"etaSeconds": int((total - done + weekly) * REPORT_SUSTAINED_SECONDS)} if rows and batches else None
|
| 8877 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8878 |
if first_load:
|
| 8879 |
# Freeze the cohort end across passes. Recomputing it would move every window boundary.
|
| 8880 |
end = _utc(load["detail"]) if load else _utc(_iso(end))
|
| 8881 |
_record_state(con, entity_key(FIRST_LOAD_END_KEY), "running", detail=_iso(end))
|
| 8882 |
-
|
|
|
|
|
|
|
| 8883 |
undocumented = repairs.intersection(pending)
|
| 8884 |
floors = {table: None if table in undocumented else read_backfill_floor(con, table) for table in pending}
|
| 8885 |
wanted = [table for table in pending if (table not in completed or table in exhausted) and (not read_watermark(con, table)
|
| 8886 |
-
or table in undocumented or read_watermark(con, table) < plans[table]["end"])]
|
| 8887 |
-
planned = first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end, sqp_asins=sqp_asins
|
| 8888 |
-
|
|
|
|
| 8889 |
task = entry if len(entry) == 4 else (*entry, None)
|
| 8890 |
table, w0, w1, batch = task
|
| 8891 |
if table == SQP_TABLE:
|
| 8892 |
if not _sqp_done(con, w0, w1, batch):
|
| 8893 |
work.append(task)
|
| 8894 |
-
elif
|
| 8895 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8896 |
# A failed enrichment after the final report retries completion without fetching again.
|
| 8897 |
for table in wanted:
|
| 8898 |
if not any(task[0] == table for task in work):
|
|
@@ -8984,9 +9429,12 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8984 |
if not sqp_asins else sqp_limit["partialMonthReason"] if not sqp_limit["periods"] else "")
|
| 8985 |
_record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
|
| 8986 |
|
|
|
|
| 8987 |
for index, (table, task_start, task_end, task_batch) in enumerate(work):
|
| 8988 |
if (first_load or table == SQP_TABLE) and table in failed:
|
| 8989 |
continue
|
|
|
|
|
|
|
| 8990 |
if ((first_load or table == SQP_TABLE) and isinstance(report_budget, int) and documents >= max(0, report_budget)
|
| 8991 |
and (task_end is not None or table == "settlements"
|
| 8992 |
or table in PERIOD_TABLES or table in LEGACY_SNAPSHOT_TABLES)):
|
|
@@ -9213,8 +9661,10 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9213 |
elif table in PERIOD_TABLES or table in LEGACY_SNAPSHOT_TABLES:
|
| 9214 |
operation = cx.REPORT_TYPES[table].get("api") or "report_document"
|
| 9215 |
spec = cx.REPORT_TYPES[table]
|
|
|
|
| 9216 |
if first_load:
|
| 9217 |
-
|
|
|
|
| 9218 |
[(None, None)] if spec.get("snapshot") else [])
|
| 9219 |
elif spec.get("snapshot"):
|
| 9220 |
margin_windows = [(None, None)]
|
|
@@ -9273,9 +9723,17 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9273 |
cleared = True
|
| 9274 |
if first_load and table in GROUP_A_TABLES:
|
| 9275 |
_record_state(con, entity_key("sync_tick_" + table), "done")
|
| 9276 |
-
if
|
|
|
|
|
|
|
| 9277 |
record_backfill_floor(con, table, w0)
|
| 9278 |
floors[table] = w0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9279 |
elif table != "settlements":
|
| 9280 |
period_spec = cx.REPORT_TYPES[table]
|
| 9281 |
windows = (([(task_start, task_end)] if task_end is not None else []) if first_load
|
|
@@ -9416,6 +9874,12 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9416 |
if reached:
|
| 9417 |
record_backfill_floor(con, reached, w0)
|
| 9418 |
floors[reached] = w0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9419 |
|
| 9420 |
if (first_load and table != "settlements" and table != SQP_TABLE
|
| 9421 |
and not cx.REPORT_TYPES[table].get("snapshot")
|
|
@@ -9502,8 +9966,13 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9502 |
(_read_state(con, entity_key("waiting_" + table)) or {}).get("detail") or "{}")
|
| 9503 |
elif period_repair_next is not None and landed_window:
|
| 9504 |
report["tables"][table]["nextWeek"] = _iso(period_repair_next)
|
| 9505 |
-
|
| 9506 |
-
_record_state(con,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9507 |
con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("resume_" + table)])
|
| 9508 |
if weekly and landed_window:
|
| 9509 |
# A landed week closes any earlier refusal of this table.
|
|
@@ -9518,6 +9987,16 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9518 |
mark = min(mark, waited_start)
|
| 9519 |
has_window = (cx.REPORT_TYPES[table].get("snapshot") or table in ("settlements", SQP_TABLE)
|
| 9520 |
or landed_window or (first_load and floors[table] is not None))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9521 |
if mark is not None and has_window:
|
| 9522 |
record_watermark(con, table, mark, total_rows)
|
| 9523 |
if not first_load and not resync:
|
|
@@ -9560,12 +10039,23 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9560 |
break
|
| 9561 |
except Exception as e: # noqa: BLE001
|
| 9562 |
limit_reason = _close_history_floor(con, table, e, task_start, task_end) if first_load else None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9563 |
if limit_reason:
|
| 9564 |
landed.update(task for task in work if task[0] == table)
|
| 9565 |
handled.add(table)
|
| 9566 |
total = con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0]
|
| 9567 |
_record_state(con, key, "done", total)
|
| 9568 |
-
record_watermark(con, table, plan["end"], total)
|
| 9569 |
report["tables"][table] = {"rows": total, "state": "done", "reason": limit_reason}
|
| 9570 |
log(limit_reason)
|
| 9571 |
continue
|
|
@@ -9648,8 +10138,18 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9648 |
"landedBatches": landed_batches}
|
| 9649 |
sqp_progress()
|
| 9650 |
_record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9651 |
if first_load:
|
| 9652 |
-
refusals =
|
| 9653 |
not_offered = {table for table, row in refusals.items() if row["state"] == "not_offered"}
|
| 9654 |
retrying = {table for table, row in refusals.items() if row["state"] == "retrying"}
|
| 9655 |
landed.update(task for task in work if task[0] in not_offered)
|
|
@@ -9681,10 +10181,11 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9681 |
# refresh floors must not re-queue itself every 30 seconds; the daily pass resumes it.
|
| 9682 |
report["landedTasks"] = len(landed)
|
| 9683 |
markers = {row[0]: {"state": row[1], "detail": row[3]} for row in _sync_state_rows(con)}
|
|
|
|
| 9684 |
progress = {"state": state, **first_load_payload(markers, remaining,
|
| 9685 |
max(first_load_eta(remaining, burst=0), floor_eta), state=state,
|
| 9686 |
-
tables=
|
| 9687 |
-
|
| 9688 |
due = (datetime.now(timezone.utc) + timedelta(seconds=report["nextDueSeconds"])) if (
|
| 9689 |
state == "partial" and not landed and report.get("nextDueSeconds")) else None
|
| 9690 |
_record_state(con, entity_key(FIRST_LOAD_DUE_KEY), "pending" if due else "done",
|
|
@@ -11572,7 +12073,10 @@ def sqp_selftest(log=print, check=None):
|
|
| 11572 |
patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_: (con.cursor(), None)), \
|
| 11573 |
patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \
|
| 11574 |
patch.object(module, "_sync_event"), patch.object(cx, "report_document", side_effect=document), \
|
|
|
|
|
|
|
| 11575 |
patch.dict(cx.REPORT_TYPES[SQP_TABLE], {"periods": ("MONTH",)}):
|
|
|
|
| 11576 |
def run(**kwargs):
|
| 11577 |
result = sync(Runtime.key, tables=[SQP_TABLE], rt=Runtime(), log=lambda *_: None, **kwargs)
|
| 11578 |
problems.extend(result["problems"])
|
|
@@ -12503,6 +13007,173 @@ def catalog_parent_selftest(log=print, check=None):
|
|
| 12503 |
return 1 if failures else 0
|
| 12504 |
|
| 12505 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12506 |
def selftest(log=print):
|
| 12507 |
"""Prove the parsers, the typing, the naming rule and validate's refusal. No network, no store.
|
| 12508 |
|
|
@@ -12511,6 +13182,8 @@ def selftest(log=print):
|
|
| 12511 |
"""
|
| 12512 |
fails = []
|
| 12513 |
checks = 0
|
|
|
|
|
|
|
| 12514 |
|
| 12515 |
def ck(name, cond, detail=""):
|
| 12516 |
"""⛔ THE DETAIL IS ASCII-ESCAPED, AND THAT IS NOT COSMETIC.
|
|
@@ -12842,6 +13515,7 @@ def selftest(log=print):
|
|
| 12842 |
ck("NC-AZ35 a large synthetic report parses and writes in bounded memory", bounded, detail)
|
| 12843 |
ck("NC-AZ38 brand analytics weekly wait and two week history fixtures", weekly_wait_selftest(log) == 0)
|
| 12844 |
ck("NC-AZ39 catalog picture fill fixtures", catalog_picture_selftest(log) == 0)
|
|
|
|
| 12845 |
log(f"\n{'GREEN' if not fails else 'RED'}: {checks - len(fails)}/{checks}")
|
| 12846 |
return 0 if not fails else 1
|
| 12847 |
|
|
@@ -12987,7 +13661,10 @@ def weekly_wait_selftest(log=print, check=None):
|
|
| 12987 |
|
| 12988 |
def accounts(tenant_key, **options):
|
| 12989 |
return _sync_candidate(tenant_key, **options)
|
| 12990 |
-
|
|
|
|
|
|
|
|
|
|
| 12991 |
patch.object(module, "account_entries", return_value=[{"connection_id": "c", "label": "c"}]),
|
| 12992 |
patch.object(module, "restore_mirror", return_value={"dirty": set(), "problems": []}),
|
| 12993 |
patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_: (state.con.cursor(), None)),
|
|
|
|
| 227 |
_record_state(con, key, phase, sum(s.get("rows", 0) for s in present), detail)
|
| 228 |
updated = min(s["updated_at"] for s in present)
|
| 229 |
con.execute("UPDATE _sync_state SET updated_at=? WHERE entity=?", [updated, key])
|
| 230 |
+
# H2: tenant-level readers (needs_first_load, status) see the oldest account floor once every account has one.
|
| 231 |
+
floor_keys = [(entity_key(SELLER_FIRST_MONTH_KEY), "month"), (entity_key(HISTORY_FLOOR_CAUSE_KEY), None),
|
| 232 |
+
*((_history_limit_key(t), "floor") for t in TABLES)]
|
| 233 |
+
for key, field in floor_keys:
|
| 234 |
+
present = [s for s in (_read_state(con, _account_prefix(e["connection_id"]) + key) for e in entries) if s]
|
| 235 |
+
con.execute("DELETE FROM _sync_state WHERE entity=?", [key])
|
| 236 |
+
if not present or (field and len(present) != len(entries)):
|
| 237 |
+
continue
|
| 238 |
+
try:
|
| 239 |
+
details = [json.loads(s["detail"]) for s in present if s.get("detail")]
|
| 240 |
+
except (ValueError, TypeError):
|
| 241 |
+
continue
|
| 242 |
+
if not details:
|
| 243 |
+
continue
|
| 244 |
+
dated = [d for d in details if field and d.get(field)]
|
| 245 |
+
# A measured account with no orders never hides another account's older month.
|
| 246 |
+
chosen = min(dated, key=lambda d: str(d[field])) if dated else details[0]
|
| 247 |
+
_record_state(con, key, present[0]["state"], detail=json.dumps(chosen))
|
| 248 |
loads, remaining, eta, waits = [], 0, 0, []
|
| 249 |
for entry in entries:
|
| 250 |
prefix = _account_prefix(entry["connection_id"])
|
|
|
|
| 689 |
return PREFIX + str(table)
|
| 690 |
|
| 691 |
|
| 692 |
+
#: FALLBACK ONLY: the history floor while the seller's first order month is unmeasured (R30 revoked
|
| 693 |
+
#: by the owner 2026-11-14). `history_floor` is the rule; a backfill is sliced (`_windows`).
|
| 694 |
DEFAULT_BACKFILL_DAYS = int(os.environ.get("AMAZON_BACKFILL_DAYS") or 730)
|
| 695 |
+
#: `_sync_state` marker: the seller's first month with orders, from getOrderMetrics.
|
| 696 |
+
SELLER_FIRST_MONTH_KEY = "seller_first_month"
|
| 697 |
+
#: `_sync_state` marker: why the seller floor is unmeasured; a failed measurement retries the next UTC day.
|
| 698 |
+
HISTORY_FLOOR_CAUSE_KEY = "history_floor_cause"
|
| 699 |
+
#: Measurement walks back no further than this.
|
| 700 |
+
SELLER_FLOOR_EARLIEST = datetime(2005, 1, 1, tzinfo=timezone.utc)
|
| 701 |
+
#: A brand-level table (`seller_floor: False`) never plans before this.
|
| 702 |
+
BRAND_FLOOR_EARLIEST = datetime(2015, 1, 1, tzinfo=timezone.utc)
|
| 703 |
+
#: A brand-level table stops after this many landed-empty windows older than its oldest window with rows.
|
| 704 |
+
BRAND_EMPTY_STOP = 12
|
| 705 |
+
#: A rolling Amazon floor is read one day late, so a window planned now is still valid when requested.
|
| 706 |
+
ROLLING_FLOOR_MARGIN = timedelta(days=1)
|
| 707 |
#: Per-table marker whose detail is the ISO UTC end of the last successfully landed window.
|
| 708 |
WATERMARK_PREFIX = "sync_mark_"
|
| 709 |
#: ⚠ AMAZON'S OWN CEILING, NOT OURS, and it is why a backfill is many reports rather than one.
|
|
|
|
| 715 |
#: about 2.7 h. A first load therefore runs in BOUNDED passes, or it would hold the resident
|
| 716 |
#: thread for hours and the Odoo resync behind it would never run.
|
| 717 |
FIRST_LOAD_REPORTS_PER_PASS = 15
|
| 718 |
+
#: A generation's repeated other-tables pass: about an hour of reports at Amazon's rate, then it publishes.
|
| 719 |
+
FIRST_LOAD_DELEGATE_REPORTS = 60
|
| 720 |
#: Rows one first-load pass writes before it takes no further document. A pacing bound, never a cap:
|
| 721 |
#: the pass publishes and the next continues, so Brand Analytics months (3.7M rows and about 25
|
| 722 |
#: minutes each) never hold the sync desk for hours (D-7081).
|
|
|
|
| 2646 |
except (SQPBudgetDeferred, AmazonReportPending):
|
| 2647 |
raise
|
| 2648 |
except cx.AmazonError as exc:
|
| 2649 |
+
# A period outside Amazon's data range is a history floor, never an ASIN refusal.
|
| 2650 |
+
out_of_range = bool(_AMAZON_FLOOR_TEXT.search(str(getattr(exc, "vendor_reason", "") or exc)))
|
| 2651 |
+
refused = [] if out_of_range else cx.sqp_refused_asins(exc, requested)
|
| 2652 |
reason = getattr(exc, "vendor_reason", str(exc))
|
| 2653 |
reason = reason.replace("\u2014", "; ").replace("\u2013", " to ")
|
| 2654 |
for asin in refused:
|
|
|
|
| 2960 |
|
| 2961 |
|
| 2962 |
def first_load_plan(tables, start, end, *, recent_days=None, size=MAX_WINDOW_DAYS,
|
| 2963 |
+
sqp_asins=(), starts=None):
|
| 2964 |
"""R18's request order: every recent month, then each table's history, oldest last.
|
| 2965 |
|
| 2966 |
Settlements have one discovery slot (None, None), never a createReport window.
|
| 2967 |
This planner is pure; durable floors filter its fixed windows when a pass resumes.
|
| 2968 |
SQP adds a fourth tuple member, its ASIN option, for each month x batch task.
|
| 2969 |
+
`starts` gives a table its own oldest bound (`history_starts`); a table it omits uses `start`.
|
| 2970 |
+
With `starts`, a weekly table also plans its older weeks, after every other table's history.
|
| 2971 |
"""
|
| 2972 |
start, end = _utc(start), _utc(end)
|
| 2973 |
+
if starts is not None:
|
| 2974 |
+
starts = {table: _utc(value) for table, value in starts.items() if value is not None}
|
| 2975 |
+
elif start >= end:
|
| 2976 |
return []
|
| 2977 |
size = min(float(size), MAX_WINDOW_DAYS)
|
| 2978 |
recent_days = FIRST_LOAD_RECENT_DAYS if recent_days is None else recent_days
|
| 2979 |
recent_days = min(float(recent_days), size)
|
| 2980 |
if size <= 0 or recent_days <= 0:
|
| 2981 |
raise ValueError("Amazon report windows must have a positive size.")
|
|
|
|
| 2982 |
tables = list(dict.fromkeys(tables))
|
| 2983 |
# Hotfix 2026-09-24: the settlements slot goes LAST among the recent entries. It downloads
|
| 2984 |
# every new settlement document until the pass budget is spent and then ends the pass, so in
|
| 2985 |
# table order it starved inventory, pricing and brand_analytics of their recent month for
|
| 2986 |
# about four passes on live (52 fortnightly documents over 730 days).
|
| 2987 |
+
recent, history, weekly_history = [], [], []
|
| 2988 |
for table in tables:
|
| 2989 |
spec = _client().REPORT_TYPES[table]
|
| 2990 |
+
table_start = starts.get(table, start) if starts is not None else start
|
| 2991 |
+
if table == SQP_TABLE and table_start < end:
|
| 2992 |
+
recent.extend(_sqp_week_tasks(table_start, end, sqp_batches(sqp_asins, spec["asin_option_max_chars"])))
|
| 2993 |
if table == "settlements":
|
| 2994 |
continue
|
| 2995 |
if spec.get("snapshot"):
|
| 2996 |
recent.append((table, None, None))
|
| 2997 |
+
elif table_start >= end:
|
| 2998 |
+
continue
|
| 2999 |
elif spec.get("weekly"):
|
| 3000 |
+
# R18: the latest completed week lands first. Older weeks keep watched terms only, so
|
| 3001 |
+
# with per-table floors they plan after every other table's history, newest first.
|
| 3002 |
+
weeks = _week_windows(table_start, end)
|
| 3003 |
+
recent.extend((table, w0, w1) for w0, w1 in weeks[-1:])
|
| 3004 |
+
if starts is not None and table in starts:
|
| 3005 |
+
weekly_history.extend((table, w0, w1) for w0, w1 in reversed(weeks[:-1]))
|
| 3006 |
elif spec.get("monthly"):
|
| 3007 |
+
months = list(reversed(_month_windows(table_start, end)))
|
| 3008 |
if not months:
|
| 3009 |
continue
|
| 3010 |
tasks = ([(table, w0, w1, batch) for w0, w1 in months
|
|
|
|
| 3014 |
history.extend(task for task in tasks if task[1:3] != months[0])
|
| 3015 |
else:
|
| 3016 |
table_size = spec.get("max_window_days", size)
|
| 3017 |
+
table_recent = max(table_start, end - timedelta(days=min(recent_days, table_size)))
|
| 3018 |
recent.append((table, table_recent, end))
|
| 3019 |
+
history.extend((table, w0, w1) for w0, w1 in reversed(_windows(table_start, table_recent, table_size)))
|
| 3020 |
recent.extend((table, None, None) for table in tables if table == "settlements")
|
| 3021 |
+
return recent + history + weekly_history
|
| 3022 |
|
| 3023 |
|
| 3024 |
def _sqp_after_cohort(tasks, end, asins):
|
| 3025 |
+
"""Insert each monthly table's months closed after the frozen cohort end, newest first, before its older tasks.
|
| 3026 |
+
|
| 3027 |
+
SQP takes one task per month and ASIN batch; every other monthly table takes one (table, w0, w1) per month.
|
| 3028 |
+
"""
|
| 3029 |
+
cx = _client()
|
| 3030 |
+
later = list(reversed(_month_windows(end, datetime.now(timezone.utc))))
|
| 3031 |
+
out, seen = [], set()
|
| 3032 |
+
for task in tasks:
|
| 3033 |
+
table = task[0]
|
| 3034 |
+
if later and table not in seen:
|
| 3035 |
+
seen.add(table)
|
| 3036 |
+
spec = cx.REPORT_TYPES.get(table) or {}
|
| 3037 |
+
if table == SQP_TABLE:
|
| 3038 |
+
out.extend((table, w0, w1, batch) for w0, w1 in later
|
| 3039 |
+
for batch in sqp_batches(asins, spec["asin_option_max_chars"]))
|
| 3040 |
+
elif spec.get("monthly"):
|
| 3041 |
+
out.extend((table, w0, w1) for w0, w1 in later)
|
| 3042 |
+
out.append(task)
|
| 3043 |
+
return out
|
| 3044 |
+
|
| 3045 |
+
|
| 3046 |
+
#: A periodic table's pass that landed no window says so on its done markers.
|
| 3047 |
+
NOTHING_NEW_DETAIL = "No new completed period landed in this pass."
|
| 3048 |
+
|
| 3049 |
+
|
| 3050 |
+
def _after_cohort(table, w0, end):
|
| 3051 |
+
"""A non-SQP monthly window that starts in or after the frozen cohort end's month."""
|
| 3052 |
+
if w0 is None or end is None or table == SQP_TABLE:
|
| 3053 |
+
return False
|
| 3054 |
+
if not (_client().REPORT_TYPES.get(table) or {}).get("monthly"):
|
| 3055 |
+
return False
|
| 3056 |
+
return _utc(w0) >= _utc(end).replace(day=1, hour=0, minute=0, second=0, microsecond=0)
|
| 3057 |
+
|
| 3058 |
+
|
| 3059 |
+
def _cohort_month_key(table, w0):
|
| 3060 |
+
return entity_key(f"after_cohort_{table}_{_utc(w0).strftime('%Y-%m')}")
|
| 3061 |
+
|
| 3062 |
+
|
| 3063 |
+
def _landed_nothing(table, landed_window, landed):
|
| 3064 |
+
"""A periodic table, not SQP, whose block landed no window in this pass."""
|
| 3065 |
+
return (table != SQP_TABLE and _periodic(_client().REPORT_TYPES.get(table)) and not landed_window
|
| 3066 |
+
and not any(task[0] == table and task[2] is not None for task in landed))
|
| 3067 |
+
|
| 3068 |
+
|
| 3069 |
+
def _cohort_month_done(con, table, w0):
|
| 3070 |
+
"""The durable marker of one landed month after the cohort end; the backfill floor never moves for it."""
|
| 3071 |
+
return (_read_state(con, _cohort_month_key(table, w0)) or {}).get("state") == "done"
|
| 3072 |
|
| 3073 |
|
| 3074 |
def first_load_eta(report_count, *, stagger_seconds=None, burst=None, per_minute=None):
|
|
|
|
| 4001 |
"""
|
| 4002 |
rows = report.get("tables") or {}
|
| 4003 |
problems = [str(problem) for key in ("problems", "audit_problems") for problem in report.get(key) or []]
|
| 4004 |
+
# Landed SQP batches and landed months count as finished work, so a chunk of only that progress still publishes.
|
| 4005 |
if report.get("state") in {"partial", "failed"} and not any(
|
| 4006 |
+
isinstance(row, dict) and (row.get("state") != "deferred" or name == SQP_TABLE
|
| 4007 |
+
or bool(row.get("landedMonths")))
|
| 4008 |
and _mirrors._table_sync_complete("amazon", row) for name, row in rows.items()):
|
| 4009 |
return None
|
| 4010 |
unfinished = {name[len(PREFIX):] if name.startswith(PREFIX) else name for name in waiting}
|
|
|
|
| 5279 |
"refused": refused, "line": line, "etaDays": eta_days, "details": details}
|
| 5280 |
|
| 5281 |
|
| 5282 |
+
def needs_first_load(tenant_key, rt=None, *, completed_tables=None, repair_tables=None, invalid_proofs=None,
|
| 5283 |
+
reopened_tables=None):
|
| 5284 |
"""A fresh account or an unfinished durable load uses bounded passes, even after one table finishes."""
|
| 5285 |
rt = rt if rt is not None else _runtime_for(tenant_key)
|
| 5286 |
restored = restore_mirror(tenant_key, rt)
|
|
|
|
| 5324 |
if enabled.get(amazon_schema.DATABASE_PREFIX + table) and documented(table)}
|
| 5325 |
repairs = set()
|
| 5326 |
refusals = {row["table"]: row for row in _refused_reports(con)}
|
| 5327 |
+
try:
|
| 5328 |
+
cohort = _utc(load["detail"]) if load and load.get("detail") else datetime.now(timezone.utc)
|
| 5329 |
+
except (ValueError, TypeError, AttributeError, OverflowError):
|
| 5330 |
+
cohort = None
|
| 5331 |
|
| 5332 |
def terminal(table):
|
| 5333 |
state = _read_state(con, entity_key(table)) or {}
|
|
|
|
| 5347 |
if spec.get("snapshot"):
|
| 5348 |
return True
|
| 5349 |
floor = read_backfill_floor(con, table)
|
| 5350 |
+
if cohort is None:
|
|
|
|
|
|
|
| 5351 |
return False
|
| 5352 |
+
# The same per-table floor first_load_plan uses, so a load that lands every window completes.
|
| 5353 |
+
return floor is not None and floor <= history_floor(con, table, cohort)
|
| 5354 |
# ⛔ HOTFIX 2026-09-29 (D-3496): the same rule for a load that STOPPED. A load left
|
| 5355 |
# "failed" only because Amazon refused a report (a 403 for a report it does not offer this
|
| 5356 |
# seller) stayed "not done" forever, so every restart queued it again AND the scheduled
|
|
|
|
| 5367 |
repair_tables.update(repairs)
|
| 5368 |
if invalid_proofs is not None:
|
| 5369 |
invalid_proofs.update(invalid)
|
| 5370 |
+
# H2: a loaded table whose history floor moved earlier owes only its history; recent data keeps landing.
|
| 5371 |
+
reopened = {table for table in TABLES
|
| 5372 |
+
if enabled.get(amazon_schema.DATABASE_PREFIX + table) and cohort is not None
|
| 5373 |
+
and table not in completed and table not in repairs and table not in refusals
|
| 5374 |
+
and _history_reopened(con, table, cohort)}
|
| 5375 |
+
if reopened_tables is not None:
|
| 5376 |
+
reopened_tables.update(reopened)
|
| 5377 |
missing = [table for table in TABLES
|
| 5378 |
if enabled.get(amazon_schema.DATABASE_PREFIX + table)
|
| 5379 |
and not read_watermark(con, table)
|
|
|
|
| 5381 |
owed = bool(load and load["state"] != "done" and missing)
|
| 5382 |
partially_refused = {name[len(PREFIX):] for name in restored.get("refused_tables", ())}
|
| 5383 |
return (bool(partially_refused.intersection(missing))
|
| 5384 |
+
or owed or bool(repairs) or bool(reopened)
|
| 5385 |
or (bool(missing) and not any(read_watermark(con, table) for table in TABLES))
|
| 5386 |
or any(table in missing for table in added))
|
| 5387 |
finally:
|
|
|
|
| 5405 |
job["tables"]["inventory"] = inventory_table_status(con)
|
| 5406 |
if job and isinstance(job.get("remaining"), int) and state["state"] != "done":
|
| 5407 |
remaining = job["remaining"]
|
| 5408 |
+
enabled_tables = [table for table in TABLES if enabled.get(amazon_schema.DATABASE_PREFIX + table)]
|
| 5409 |
+
load = _read_state(con, entity_key(FIRST_LOAD_END_KEY))
|
| 5410 |
+
try:
|
| 5411 |
+
cohort = _utc(load["detail"]) if load and load.get("detail") else datetime.now(timezone.utc)
|
| 5412 |
+
except (ValueError, TypeError, AttributeError, OverflowError):
|
| 5413 |
+
cohort = datetime.now(timezone.utc)
|
| 5414 |
out = {"state": state["state"], **first_load_payload(markers, remaining,
|
| 5415 |
int(values["etaSeconds"]), state=state["state"],
|
| 5416 |
+
tables=enabled_tables), "backfillDays": DEFAULT_BACKFILL_DAYS,
|
| 5417 |
+
"historyFloors": history_status(con, enabled_tables, cohort),
|
| 5418 |
"nextDue": _iso(_first_load_due(con)) if _first_load_due(con) else None}
|
| 5419 |
if job:
|
| 5420 |
out["job"] = job
|
|
|
|
| 5616 |
return task
|
| 5617 |
|
| 5618 |
|
| 5619 |
+
def _task_landed(con, table, w0, w1, batch, end=None, *, reopened=False):
|
| 5620 |
+
"""The mirror's own evidence that an earlier pass landed this window: watermark, floor or month marker.
|
| 5621 |
+
|
| 5622 |
+
A reopened table's watermark proves only its recent data, so its floor decides.
|
| 5623 |
+
"""
|
| 5624 |
if table == SQP_TABLE and w0 is not None:
|
| 5625 |
return _sqp_done(con, w0, w1, batch)
|
| 5626 |
+
if _after_cohort(table, w0, end):
|
| 5627 |
+
return _cohort_month_done(con, table, w0)
|
| 5628 |
+
if (not reopened and read_watermark(con, table)
|
| 5629 |
+
and (_read_state(con, entity_key(table)) or {}).get("state") == "done"):
|
| 5630 |
return True
|
| 5631 |
floor = read_backfill_floor(con, table)
|
| 5632 |
return w1 is not None and floor is not None and w1 > floor
|
|
|
|
| 5639 |
before a first pass froze one: the job then runs one bounded pass first.
|
| 5640 |
"""
|
| 5641 |
entries = account_entries(tenant_key, rt)
|
| 5642 |
+
completed, repairs, reopened = set(), set(), set()
|
| 5643 |
+
owed = needs_first_load(tenant_key, rt=rt, completed_tables=completed, repair_tables=repairs,
|
| 5644 |
+
reopened_tables=reopened)
|
| 5645 |
con, _path = _connect_tenant_mirror(tenant_key, rt)
|
| 5646 |
try:
|
| 5647 |
load = _read_state(con, entity_key(FIRST_LOAD_END_KEY))
|
|
|
|
| 5656 |
wanted = [t for t in tables if t not in completed or t in repairs] if single else tables
|
| 5657 |
sqp_asins = _sqp_catalogue(con) if SQP_TABLE in wanted else ()
|
| 5658 |
tasks, fields_by = [], {}
|
| 5659 |
+
starts = history_starts(con, wanted, end)
|
| 5660 |
for entry in entries:
|
| 5661 |
cid = entry["connection_id"]
|
| 5662 |
fields = fields_by[cid] = _sync_credential(tenant_key, rt, connection_id=cid)
|
| 5663 |
+
items = first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end, sqp_asins=sqp_asins,
|
| 5664 |
+
starts=starts)
|
| 5665 |
+
for item in _sqp_after_cohort(items, end, sqp_asins):
|
| 5666 |
table, w0, w1 = item[:3]
|
| 5667 |
batch = item[3] if len(item) == 4 else None
|
| 5668 |
+
if single and table != SQP_TABLE and not _after_cohort(table, w0, end):
|
| 5669 |
+
# The same clip sync applies, so a window straddling the landed floor is fetched below it only.
|
| 5670 |
+
owed_window = _owed_window(table, w0, w1,
|
| 5671 |
+
None if table in repairs else read_backfill_floor(con, table))
|
| 5672 |
+
w0, w1 = owed_window or (w0, w1)
|
| 5673 |
# Account-scoped floors live in each account's workspace; only one account reads them here.
|
| 5674 |
+
landed = single and _task_landed(con, table, w0, w1, batch, end, reopened=table in reopened)
|
| 5675 |
task = first_load_task(cid, fields, table, w0, w1, batch, landed=landed)
|
| 5676 |
# One sync writes every account, so a prefetch serves a single-account tenant only.
|
| 5677 |
task["prefetch"] = task["prefetch"] and single
|
|
|
|
| 6477 |
if not waiting_on and now() >= request_until:
|
| 6478 |
break
|
| 6479 |
sleep(poll_seconds if waiting_on else max(0.0, min(poll_seconds, next_token - now())))
|
| 6480 |
+
record["otherTablesPasses"] = 0
|
| 6481 |
+
while now() < request_until:
|
| 6482 |
+
# After the job's own reports. Each pass publishes; another follows while request time remains,
|
| 6483 |
+
# sized to that time at Amazon's one report a minute, so SQP history moves at Amazon's rate.
|
| 6484 |
+
budget = max(FIRST_LOAD_REPORTS_PER_PASS,
|
| 6485 |
+
min(FIRST_LOAD_DELEGATE_REPORTS, int((request_until - now()) // 60)))
|
| 6486 |
try:
|
| 6487 |
+
result = delegate(own_tables, budget)
|
| 6488 |
record["otherTablesPass"] = "done"
|
| 6489 |
except Exception as exc: # noqa: BLE001
|
| 6490 |
record["otherTablesPass"] = f"failed ({type(exc).__name__})"
|
| 6491 |
log(f"Amazon first-load pass for the other tables failed ({type(exc).__name__}); "
|
| 6492 |
"the next generation runs it again.")
|
| 6493 |
+
break
|
| 6494 |
+
record["otherTablesPasses"] += 1
|
| 6495 |
+
result = result if isinstance(result, dict) else {}
|
| 6496 |
+
if result.get("state") != "partial" or not result.get("landedTasks"):
|
| 6497 |
+
break
|
| 6498 |
try:
|
| 6499 |
after = plan_fn()
|
| 6500 |
except Exception: # noqa: BLE001
|
|
|
|
| 6538 |
pass
|
| 6539 |
|
| 6540 |
|
| 6541 |
+
# ----------------------------------------------------------------------------- history floors
|
| 6542 |
+
#: Amazon's own words for "this period is older than I keep". SQP's text also covers an unprocessed
|
| 6543 |
+
#: recent month, so `_close_history_floor` accepts it only below newer landed history.
|
| 6544 |
+
_AMAZON_FLOOR_TEXT = re.compile(
|
| 6545 |
+
r"outside the available data range"
|
| 6546 |
+
r"|start\s*date[^.]{0,80}?(?:cannot be before|must be no more than|is too old)", re.I)
|
| 6547 |
+
#: A refused period ending inside this many days is treated as unfinished, never as a floor.
|
| 6548 |
+
_FLOOR_MIN_AGE = timedelta(days=45)
|
| 6549 |
+
|
| 6550 |
+
|
| 6551 |
+
def _dated_table(table):
|
| 6552 |
+
"""A table whose first load plans dated history windows."""
|
| 6553 |
+
spec = _client().REPORT_TYPES.get(table)
|
| 6554 |
+
return bool(spec) and table != "settlements" and not spec.get("snapshot")
|
| 6555 |
+
|
| 6556 |
+
|
| 6557 |
+
def _history_grain(spec):
|
| 6558 |
+
return "month" if spec.get("monthly") else "week" if spec.get("weekly") else "day"
|
| 6559 |
+
|
| 6560 |
+
|
| 6561 |
+
def _grain_floor(value, grain):
|
| 6562 |
+
day = _utc(value).replace(hour=0, minute=0, second=0, microsecond=0)
|
| 6563 |
+
return day.replace(day=1) if grain == "month" else _week_start(day) if grain == "week" else day
|
| 6564 |
+
|
| 6565 |
+
|
| 6566 |
+
def _grain_ceil(value, grain):
|
| 6567 |
+
value = _utc(value)
|
| 6568 |
+
low = _grain_floor(value, grain)
|
| 6569 |
+
if low == value:
|
| 6570 |
+
return low
|
| 6571 |
+
if grain == "month":
|
| 6572 |
+
return (low.replace(day=28) + timedelta(days=4)).replace(day=1)
|
| 6573 |
+
return low + timedelta(days=7 if grain == "week" else 1)
|
| 6574 |
+
|
| 6575 |
+
|
| 6576 |
+
def _date_utc(text):
|
| 6577 |
+
return datetime.fromisoformat(str(text)[:10]).replace(tzinfo=timezone.utc)
|
| 6578 |
+
|
| 6579 |
+
|
| 6580 |
+
def _months_before(value, months):
|
| 6581 |
+
index = value.year * 12 + value.month - 1 - months
|
| 6582 |
+
return value.replace(year=index // 12, month=index % 12 + 1)
|
| 6583 |
+
|
| 6584 |
+
|
| 6585 |
+
def read_seller_first_month(con):
|
| 6586 |
+
"""-> (first month with orders or None, marker detail or None). A measured seller without orders has a detail."""
|
| 6587 |
+
state = _read_state(con, entity_key(SELLER_FIRST_MONTH_KEY))
|
| 6588 |
+
try:
|
| 6589 |
+
detail = json.loads(state["detail"]) if state and state.get("detail") else None
|
| 6590 |
+
if not isinstance(detail, dict):
|
| 6591 |
+
return None, None
|
| 6592 |
+
return (_date_utc(detail["month"]) if detail.get("month") else None), detail
|
| 6593 |
+
except (ValueError, TypeError, KeyError):
|
| 6594 |
+
return None, None
|
| 6595 |
+
|
| 6596 |
+
|
| 6597 |
+
def measure_seller_first_month(con, fields, end, *, log=print):
|
| 6598 |
+
"""Walk getOrderMetrics back in 24-month windows from `end`; the earliest month with orders wins.
|
| 6599 |
+
|
| 6600 |
+
Stops after two consecutive windows without orders, or at SELLER_FLOOR_EARLIEST. Records the marker.
|
| 6601 |
+
"""
|
| 6602 |
+
cx = _client()
|
| 6603 |
+
w1, empty, found = _grain_floor(end, "month"), 0, None
|
| 6604 |
+
while w1 > SELLER_FLOOR_EARLIEST and empty < 2:
|
| 6605 |
+
w0 = max(SELLER_FLOOR_EARLIEST, _months_before(w1, cx.ORDER_MONTHS_MAX))
|
| 6606 |
+
months = [month for month, count in cx.order_months(fields, w0, w1) if count > 0]
|
| 6607 |
+
if months:
|
| 6608 |
+
found, empty = min(months), 0
|
| 6609 |
+
else:
|
| 6610 |
+
empty += 1
|
| 6611 |
+
w1 = w0
|
| 6612 |
+
detail = {"month": found, "source": "getOrderMetrics", "measured_at": _iso(datetime.now(timezone.utc))}
|
| 6613 |
+
_record_state(con, entity_key(SELLER_FIRST_MONTH_KEY), "done", detail=json.dumps(detail))
|
| 6614 |
+
con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key(HISTORY_FLOOR_CAUSE_KEY)])
|
| 6615 |
+
log(f"Amazon seller first order month: {found or 'none found'}")
|
| 6616 |
+
return _date_utc(found) if found else None
|
| 6617 |
+
|
| 6618 |
+
|
| 6619 |
+
def _ensure_seller_first_month(con, fields, end, *, log=print):
|
| 6620 |
+
"""Measure once per seller; a failure records its cause and retries on a later UTC day, never every pass."""
|
| 6621 |
+
if os.environ.get("AIOS_AMAZON_SELLER_FLOOR_PROBE") == "0" or not fields:
|
| 6622 |
+
return
|
| 6623 |
+
if _read_state(con, entity_key(SELLER_FIRST_MONTH_KEY)):
|
| 6624 |
+
return
|
| 6625 |
+
cause = _read_state(con, entity_key(HISTORY_FLOOR_CAUSE_KEY))
|
| 6626 |
+
if cause and str(cause.get("updated_at") or "")[:10] == datetime.now(timezone.utc).date().isoformat():
|
| 6627 |
+
return
|
| 6628 |
+
if _client().prefetch_active():
|
| 6629 |
+
return # A job writer pass serves only prefetched documents; the delegate pass measures.
|
| 6630 |
+
try:
|
| 6631 |
+
measure_seller_first_month(con, fields, end, log=log)
|
| 6632 |
+
except Exception as exc: # noqa: BLE001
|
| 6633 |
+
reason = (f"The seller's first order month could not be read from Amazon ({type(exc).__name__}). "
|
| 6634 |
+
f"History reaches the {DEFAULT_BACKFILL_DAYS} day fallback; the reading retries tomorrow.")
|
| 6635 |
+
_record_state(con, entity_key(HISTORY_FLOOR_CAUSE_KEY), "failed",
|
| 6636 |
+
detail=json.dumps({"cause": reason, "error": str(exc)[:120]}))
|
| 6637 |
+
log(reason)
|
| 6638 |
+
|
| 6639 |
+
|
| 6640 |
+
def _seller_floor(con, end):
|
| 6641 |
+
"""-> (floor, cause). The measured first month, else end minus DEFAULT_BACKFILL_DAYS with its cause."""
|
| 6642 |
+
month, detail = read_seller_first_month(con)
|
| 6643 |
+
if month is not None:
|
| 6644 |
+
return month, None
|
| 6645 |
+
fallback = _utc(end) - timedelta(days=DEFAULT_BACKFILL_DAYS)
|
| 6646 |
+
if detail is not None:
|
| 6647 |
+
return fallback, (f"Amazon reports no orders for this seller, so history reaches the "
|
| 6648 |
+
f"{DEFAULT_BACKFILL_DAYS} day fallback.")
|
| 6649 |
+
failed = _read_state(con, entity_key(HISTORY_FLOOR_CAUSE_KEY))
|
| 6650 |
+
try:
|
| 6651 |
+
reason = json.loads(failed["detail"]).get("cause") if failed and failed.get("detail") else None
|
| 6652 |
+
except (ValueError, TypeError, AttributeError):
|
| 6653 |
+
reason = None
|
| 6654 |
+
return fallback, reason or (f"The seller's first order month is not measured yet, so history reaches the "
|
| 6655 |
+
f"{DEFAULT_BACKFILL_DAYS} day fallback.")
|
| 6656 |
+
|
| 6657 |
+
|
| 6658 |
+
def _history_limit_key(table):
|
| 6659 |
+
"""Amazon's observed floor for one table; `limit_` family, so a table reset clears it too."""
|
| 6660 |
+
return entity_key(f"limit_{table}_history")
|
| 6661 |
+
|
| 6662 |
+
|
| 6663 |
+
def _observed_history_limit(con, table):
|
| 6664 |
+
"""-> (floor, reason) Amazon showed by refusing an older window, or None."""
|
| 6665 |
+
state = _read_state(con, _history_limit_key(table))
|
| 6666 |
+
try:
|
| 6667 |
+
detail = json.loads(state["detail"]) if state and state.get("detail") else {}
|
| 6668 |
+
if detail.get("floor"):
|
| 6669 |
+
return _utc(detail["floor"]), str(detail.get("reason") or "")
|
| 6670 |
+
except (ValueError, TypeError, AttributeError):
|
| 6671 |
+
pass
|
| 6672 |
+
legacy = _read_state(con, entity_key("limit_" + table))
|
| 6673 |
+
try:
|
| 6674 |
+
reason = json.loads(legacy["detail"]).get("reason", "") if legacy and legacy.get("detail") else ""
|
| 6675 |
+
except (ValueError, TypeError, AttributeError):
|
| 6676 |
+
reason = ""
|
| 6677 |
+
floor = read_backfill_floor(con, table)
|
| 6678 |
+
if isinstance(reason, str) and reason.startswith("Amazon keeps this report back to ") and floor is not None:
|
| 6679 |
+
return floor, reason
|
| 6680 |
+
return None
|
| 6681 |
+
|
| 6682 |
+
|
| 6683 |
+
def history_bound(con, table, end, *, now=None):
|
| 6684 |
+
"""-> (floor, cause): the oldest history start for `table`, and why when Amazon or the fallback sets it.
|
| 6685 |
+
|
| 6686 |
+
max(seller floor, the table's `history` rule, Amazon's observed limit), on the table's window grain.
|
| 6687 |
+
A `from` rule or `seller_floor: False` replaces the seller floor (brand-level data predates it).
|
| 6688 |
+
`end` is the frozen cohort end (seller floor); `now` is the clock a rolling Amazon rule moves with.
|
| 6689 |
+
"""
|
| 6690 |
+
spec = _client().REPORT_TYPES.get(table) or {}
|
| 6691 |
+
grain = _history_grain(spec)
|
| 6692 |
+
now = _utc(now) if now is not None else datetime.now(timezone.utc)
|
| 6693 |
+
rule = spec.get("history") or {}
|
| 6694 |
+
if rule.get("from") or rule.get("seller_floor") is False:
|
| 6695 |
+
# Brand-level data (Brand Analytics) predates the seller's first order, so the seller month
|
| 6696 |
+
# does not bound it: a `from` rule is the floor, else history walks back until Amazon refuses.
|
| 6697 |
+
floor = _grain_floor(BRAND_FLOOR_EARLIEST, grain)
|
| 6698 |
+
cause = ("This data is brand level, so history walks back until Amazon refuses an older period or "
|
| 6699 |
+
f"returns {BRAND_EMPTY_STOP} empty periods in a row."
|
| 6700 |
+
if not rule.get("from") else None)
|
| 6701 |
+
else:
|
| 6702 |
+
floor, cause = _seller_floor(con, end)
|
| 6703 |
+
if cause is None:
|
| 6704 |
+
floor = _grain_floor(floor, grain)
|
| 6705 |
+
bound = None
|
| 6706 |
+
if rule.get("rolling_days"):
|
| 6707 |
+
cutoff = now - timedelta(days=float(rule["rolling_days"])) + ROLLING_FLOOR_MARGIN
|
| 6708 |
+
# An end edge admits the oldest window whose end is inside the rule; a start edge its start.
|
| 6709 |
+
bound = _grain_floor(cutoff, grain) if rule.get("edge") == "end" else _grain_ceil(cutoff, grain)
|
| 6710 |
+
rule_cause = (f"Amazon serves this data for the last {int(rule['rolling_days'])} days only, "
|
| 6711 |
+
f"so history starts {bound.date().isoformat()}.")
|
| 6712 |
+
elif rule.get("from"):
|
| 6713 |
+
bound = _grain_ceil(_date_utc(rule["from"]), grain)
|
| 6714 |
+
rule_cause = f"Amazon serves this data back to {bound.date().isoformat()} only."
|
| 6715 |
+
if bound is not None and bound > floor:
|
| 6716 |
+
floor, cause = bound, rule_cause
|
| 6717 |
+
observed = _observed_history_limit(con, table)
|
| 6718 |
+
if observed and observed[0] > floor:
|
| 6719 |
+
floor, cause = observed[0], observed[1] or (
|
| 6720 |
+
f"Amazon keeps this data back to {observed[0].date().isoformat()}; older history is not available.")
|
| 6721 |
+
return floor, cause
|
| 6722 |
+
|
| 6723 |
+
|
| 6724 |
+
def history_floor(con, table, end, *, now=None):
|
| 6725 |
+
"""The oldest history start `first_load_plan` and `needs_first_load` use for `table`."""
|
| 6726 |
+
return history_bound(con, table, end, now=now)[0]
|
| 6727 |
+
|
| 6728 |
+
|
| 6729 |
+
def history_starts(con, tables, end, *, now=None):
|
| 6730 |
+
"""{table: history_floor} for each dated table; the one dict every first-load planner passes."""
|
| 6731 |
+
return {table: history_floor(con, table, end, now=now) for table in tables if _dated_table(table)}
|
| 6732 |
+
|
| 6733 |
+
|
| 6734 |
+
def history_status(con, tables, end):
|
| 6735 |
+
"""{table: {historyFloor, cause?}} for status, beside `backfillDays`. Screen safe, no vendor text."""
|
| 6736 |
+
out = {}
|
| 6737 |
+
for table in tables:
|
| 6738 |
+
if not _dated_table(table):
|
| 6739 |
+
continue
|
| 6740 |
+
floor, cause = history_bound(con, table, end)
|
| 6741 |
+
row = {"historyFloor": _iso(floor)}
|
| 6742 |
+
if cause:
|
| 6743 |
+
row["cause"] = " ".join(cause.replace("\u2014", ". ").replace("\u2013", " to ").split())[:300]
|
| 6744 |
+
out[table] = row
|
| 6745 |
+
return out
|
| 6746 |
+
|
| 6747 |
+
|
| 6748 |
+
def _history_reopened(con, table, cohort):
|
| 6749 |
+
"""A loaded table whose history floor moved earlier than its landed floor: only its history is owed."""
|
| 6750 |
+
if not _dated_table(table):
|
| 6751 |
+
return False
|
| 6752 |
+
state = _read_state(con, entity_key(table)) or {}
|
| 6753 |
+
if state.get("state") != "done" or read_watermark(con, table) is None:
|
| 6754 |
+
return False
|
| 6755 |
+
bound = history_floor(con, table, cohort)
|
| 6756 |
+
if table == SQP_TABLE:
|
| 6757 |
+
return bool(_sqp_catalogue(con)) and any(
|
| 6758 |
+
not _sqp_month(con, w0, w1)["batches"] for w0, w1 in _month_windows(bound, cohort))
|
| 6759 |
+
floor = read_backfill_floor(con, table)
|
| 6760 |
+
return floor is not None and floor > bound
|
| 6761 |
+
|
| 6762 |
+
|
| 6763 |
+
def _brand_level(table):
|
| 6764 |
+
return (_client().REPORT_TYPES.get(table) or {}).get("history", {}).get("seller_floor") is False
|
| 6765 |
+
|
| 6766 |
+
|
| 6767 |
+
def _note_history_window(con, table, w0, count):
|
| 6768 |
+
"""Count a brand-level table's landed-empty windows older than its oldest window with rows.
|
| 6769 |
+
|
| 6770 |
+
-> the stop reason once BRAND_EMPTY_STOP empty windows follow it; the floor is then that oldest window.
|
| 6771 |
+
"""
|
| 6772 |
+
if w0 is None or not _brand_level(table):
|
| 6773 |
+
return None
|
| 6774 |
+
key = entity_key(f"limit_{table}_empty")
|
| 6775 |
+
state = _read_state(con, key)
|
| 6776 |
+
try:
|
| 6777 |
+
run = json.loads(state["detail"]) if state and state.get("detail") else {}
|
| 6778 |
+
except (ValueError, TypeError):
|
| 6779 |
+
run = {}
|
| 6780 |
+
rows_at = _utc(run["oldestRows"]) if run.get("oldestRows") else None
|
| 6781 |
+
if count:
|
| 6782 |
+
if rows_at is None or w0 < rows_at:
|
| 6783 |
+
run = {"oldestRows": _iso(w0), "empty": 0}
|
| 6784 |
+
elif rows_at is not None and w0 < rows_at:
|
| 6785 |
+
run["empty"] = int(run.get("empty") or 0) + 1
|
| 6786 |
+
_record_state(con, key, "done", detail=json.dumps(run))
|
| 6787 |
+
if rows_at is None or count or run.get("empty", 0) < BRAND_EMPTY_STOP:
|
| 6788 |
+
return None
|
| 6789 |
+
reason = (f"Amazon keeps this report back to {rows_at.date().isoformat()}; it returned no data for "
|
| 6790 |
+
f"{BRAND_EMPTY_STOP} periods in a row before that, so older history is not requested.")
|
| 6791 |
+
_record_state(con, _history_limit_key(table), "done", detail=json.dumps({
|
| 6792 |
+
"floor": _iso(rows_at), "reason": reason, "amazon": f"{BRAND_EMPTY_STOP} consecutive DONE reports with no rows",
|
| 6793 |
+
"window": [_iso(w0), _iso(rows_at)]}))
|
| 6794 |
+
return reason
|
| 6795 |
+
|
| 6796 |
+
|
| 6797 |
+
def _owed_window(table, w0, w1, floor):
|
| 6798 |
+
"""A history window still owed below the landed floor: whole, clipped at the floor, or None when landed.
|
| 6799 |
+
|
| 6800 |
+
Clipping keeps history contiguous when a moved floor shifts the day-window boundaries.
|
| 6801 |
+
"""
|
| 6802 |
+
if w1 is None or floor is None or w1 <= floor:
|
| 6803 |
+
return w0, w1
|
| 6804 |
+
if (w0 is not None and w0 < floor and table != SQP_TABLE
|
| 6805 |
+
and not _periodic(_client().REPORT_TYPES.get(table))):
|
| 6806 |
+
return w0, floor
|
| 6807 |
+
return None
|
| 6808 |
+
|
| 6809 |
+
|
| 6810 |
+
def sync_window(table, mark, *, now, since=None, backfill_days=None, intraday=False, floor=None):
|
| 6811 |
+
"""Pure request decision for one table: start, end, report, incremental and cause.
|
| 6812 |
+
|
| 6813 |
+
`floor` is the table's history floor for a first full load (`history_floor`).
|
| 6814 |
+
"""
|
| 6815 |
end = _utc(now)
|
| 6816 |
days = DEFAULT_BACKFILL_DAYS if backfill_days is None else backfill_days
|
| 6817 |
try:
|
|
|
|
| 6824 |
elif mark is not None:
|
| 6825 |
start, incremental = mark, True
|
| 6826 |
cause = f"incremental window since watermark {_iso(mark)}"
|
| 6827 |
+
elif floor is not None:
|
| 6828 |
+
start, incremental = _utc(floor), False
|
| 6829 |
+
cause = f"first full load: history from {start.date().isoformat()}"
|
| 6830 |
else:
|
| 6831 |
start, incremental = end - timedelta(days=days), False
|
| 6832 |
cause = f"first full load: {days} days of backfill"
|
|
|
|
| 6857 |
if incremental or since is not None:
|
| 6858 |
start = _week_start(start)
|
| 6859 |
cause += "; completed Sunday to Saturday weeks only"
|
| 6860 |
+
elif floor is not None:
|
| 6861 |
+
start = min(_week_start(floor), end - timedelta(days=7))
|
| 6862 |
+
cause = f"first load: completed Sunday to Saturday weeks back to {start.date().isoformat()}"
|
| 6863 |
else:
|
| 6864 |
start = end - timedelta(days=7)
|
| 6865 |
cause = "first load: the latest completed Sunday to Saturday week only (R18)"
|
|
|
|
| 7056 |
return result
|
| 7057 |
|
| 7058 |
|
| 7059 |
+
def _newer_history_landed(con, table, end):
|
| 7060 |
+
"""A window of `table` newer than `end` has landed, so a refusal at `end` is a floor, not an outage."""
|
| 7061 |
+
if table == SQP_TABLE:
|
| 7062 |
+
return any(any(not batch.get("refused") for batch in _sqp_month(con, w0, w1)["batches"].values())
|
| 7063 |
+
for w0, w1 in _month_windows(end, datetime.now(timezone.utc)))
|
| 7064 |
+
floor = read_backfill_floor(con, table)
|
| 7065 |
+
return floor is not None and end <= floor
|
| 7066 |
+
|
| 7067 |
+
|
| 7068 |
def _close_history_floor(con, table, exc, start, end):
|
| 7069 |
+
"""Amazon's observed history floor: an old window it refuses below newer landed history.
|
| 7070 |
+
|
| 7071 |
+
Counts Amazon's own words (out of range, start too old) on any dated table, and an unexplained
|
| 7072 |
+
old FATAL with no error document. Records the floor with Amazon's words; no older window is planned.
|
| 7073 |
+
"""
|
| 7074 |
+
if not _dated_table(table) or start is None or end is None:
|
| 7075 |
+
return None
|
| 7076 |
+
now = datetime.now(timezone.utc)
|
| 7077 |
+
status = getattr(exc, "status", None)
|
| 7078 |
+
words = str(getattr(exc, "vendor_reason", "") or exc)
|
| 7079 |
floor = read_backfill_floor(con, table)
|
| 7080 |
+
said = (bool(_AMAZON_FLOOR_TEXT.search(words)) and (status in ("FATAL", 400) or " answered 400" in str(exc))
|
| 7081 |
+
and end <= now - _FLOOR_MIN_AGE and _newer_history_landed(con, table, end))
|
| 7082 |
+
unexplained = (table != SQP_TABLE and status == "FATAL" and not getattr(exc, "has_error_document", True)
|
| 7083 |
+
and floor is not None and end <= floor and start < now - timedelta(days=365)
|
| 7084 |
+
and bool(_read_state(con, entity_key("reconcile_" + table))))
|
| 7085 |
+
if not (said or unexplained):
|
| 7086 |
return None
|
| 7087 |
+
stop = end if said else floor
|
| 7088 |
+
reason = (f"Amazon keeps this report back to {stop.date().isoformat()}; older history is not available. "
|
| 7089 |
"Keep the connection enabled for newer data; ask Seller Support for older history.")
|
| 7090 |
+
amazon = " ".join(words.replace("\u2014", "; ").replace("\u2013", " to ").split())[:300]
|
| 7091 |
+
_record_state(con, _history_limit_key(table), "done", detail=json.dumps({
|
| 7092 |
+
"floor": _iso(stop), "reason": reason, "amazon": amazon, "window": [_iso(start), _iso(end)]}))
|
| 7093 |
+
if table != SQP_TABLE:
|
| 7094 |
+
# SQP's limit_ key is its progress document; its floor lives in the history marker only.
|
| 7095 |
+
_record_state(con, entity_key("limit_" + table), "done", detail=json.dumps({"reason": reason}))
|
| 7096 |
return reason
|
| 7097 |
|
| 7098 |
|
|
|
|
| 9095 |
raise RuntimeError("Amazon deletion is outstanding; sync cannot repopulate purged data")
|
| 9096 |
load = _read_state(con, entity_key(FIRST_LOAD_END_KEY))
|
| 9097 |
# Table completion evidence also admits empty histories behind a stale aggregate marker.
|
| 9098 |
+
completed, repairs, invalid_proofs, reopened = set(), set(), set(), set()
|
| 9099 |
needs_first_load(tenant_key, rt=rt, completed_tables=completed, repair_tables=repairs,
|
| 9100 |
+
invalid_proofs=invalid_proofs, reopened_tables=reopened)
|
| 9101 |
prior_refusals = {row["table"]: row for row in _refused_reports(con)}
|
| 9102 |
exhausted = {table for table in want if prior_refusals.get(table, {}).get("state") == "failed"}
|
| 9103 |
|
|
|
|
| 9131 |
for key in ("problems", "audit_problems"):
|
| 9132 |
report[key].extend(refreshed.get(key) or [])
|
| 9133 |
con, _path = _connect_tenant_mirror(tenant_key, rt)
|
| 9134 |
+
# A reopened table refreshed above like any loaded table; its history continues in this pass.
|
| 9135 |
+
want = [table for table in want if table not in loaded or table in reopened]
|
| 9136 |
due = _restore_first_load_wait(tenant_key, con)
|
| 9137 |
if due and due > end and not _client().prefetch_active():
|
| 9138 |
progress = first_load_status(tenant_key, rt)
|
|
|
|
| 9305 |
"monthsDone": sum(r["done"] >= r["total"] for r in rows), "monthsTotal": len(rows),
|
| 9306 |
"etaSeconds": int((total - done + weekly) * REPORT_SUSTAINED_SECONDS)} if rows and batches else None
|
| 9307 |
|
| 9308 |
+
try:
|
| 9309 |
+
measure_end = _utc(load["detail"]) if first_load and load and load.get("detail") else end
|
| 9310 |
+
except (ValueError, TypeError, AttributeError, OverflowError):
|
| 9311 |
+
measure_end = end
|
| 9312 |
+
# H2: one getOrderMetrics walk per seller sets every table's history floor (or records why not).
|
| 9313 |
+
_ensure_seller_first_month(con, fields, measure_end, log=log)
|
| 9314 |
if first_load:
|
| 9315 |
# Freeze the cohort end across passes. Recomputing it would move every window boundary.
|
| 9316 |
end = _utc(load["detail"]) if load else _utc(_iso(end))
|
| 9317 |
_record_state(con, entity_key(FIRST_LOAD_END_KEY), "running", detail=_iso(end))
|
| 9318 |
+
# One floor per table, shared with needs_first_load and the job planner, so the load converges.
|
| 9319 |
+
starts = history_starts(con, pending, end)
|
| 9320 |
+
plans = {table: sync_window(table, None, now=end, floor=starts.get(table)) for table in pending}
|
| 9321 |
undocumented = repairs.intersection(pending)
|
| 9322 |
floors = {table: None if table in undocumented else read_backfill_floor(con, table) for table in pending}
|
| 9323 |
wanted = [table for table in pending if (table not in completed or table in exhausted) and (not read_watermark(con, table)
|
| 9324 |
+
or table in undocumented or table in reopened or read_watermark(con, table) < plans[table]["end"])]
|
| 9325 |
+
planned = first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end, sqp_asins=sqp_asins,
|
| 9326 |
+
starts=starts)
|
| 9327 |
+
for entry in _sqp_after_cohort(planned, end, sqp_asins):
|
| 9328 |
task = entry if len(entry) == 4 else (*entry, None)
|
| 9329 |
table, w0, w1, batch = task
|
| 9330 |
if table == SQP_TABLE:
|
| 9331 |
if not _sqp_done(con, w0, w1, batch):
|
| 9332 |
work.append(task)
|
| 9333 |
+
elif _after_cohort(table, w0, end):
|
| 9334 |
+
# A month after the frozen end has its own marker: the backfill floor only moves backwards.
|
| 9335 |
+
if not _cohort_month_done(con, table, w0):
|
| 9336 |
+
work.append(task)
|
| 9337 |
+
else:
|
| 9338 |
+
owed_window = _owed_window(table, w0, w1, floors[table])
|
| 9339 |
+
if owed_window is not None:
|
| 9340 |
+
work.append((table, *owed_window, batch))
|
| 9341 |
# A failed enrichment after the final report retries completion without fetching again.
|
| 9342 |
for table in wanted:
|
| 9343 |
if not any(task[0] == table for task in work):
|
|
|
|
| 9429 |
if not sqp_asins else sqp_limit["partialMonthReason"] if not sqp_limit["periods"] else "")
|
| 9430 |
_record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
|
| 9431 |
|
| 9432 |
+
history_cut = {} # table -> Amazon's floor observed this pass; older tasks are already counted landed.
|
| 9433 |
for index, (table, task_start, task_end, task_batch) in enumerate(work):
|
| 9434 |
if (first_load or table == SQP_TABLE) and table in failed:
|
| 9435 |
continue
|
| 9436 |
+
if history_cut.get(table) is not None and task_end is not None and task_end <= history_cut[table]:
|
| 9437 |
+
continue
|
| 9438 |
if ((first_load or table == SQP_TABLE) and isinstance(report_budget, int) and documents >= max(0, report_budget)
|
| 9439 |
and (task_end is not None or table == "settlements"
|
| 9440 |
or table in PERIOD_TABLES or table in LEGACY_SNAPSHOT_TABLES)):
|
|
|
|
| 9661 |
elif table in PERIOD_TABLES or table in LEGACY_SNAPSHOT_TABLES:
|
| 9662 |
operation = cx.REPORT_TYPES[table].get("api") or "report_document"
|
| 9663 |
spec = cx.REPORT_TYPES[table]
|
| 9664 |
+
after = first_load and _after_cohort(table, task_start, end)
|
| 9665 |
if first_load:
|
| 9666 |
+
# A month after the cohort end lies past the frozen plan end, so it keeps its own bounds.
|
| 9667 |
+
margin_windows = [(task_start, task_end if after else min(task_end, plan["end"]))] if task_end else (
|
| 9668 |
[(None, None)] if spec.get("snapshot") else [])
|
| 9669 |
elif spec.get("snapshot"):
|
| 9670 |
margin_windows = [(None, None)]
|
|
|
|
| 9723 |
cleared = True
|
| 9724 |
if first_load and table in GROUP_A_TABLES:
|
| 9725 |
_record_state(con, entity_key("sync_tick_" + table), "done")
|
| 9726 |
+
if after:
|
| 9727 |
+
_record_state(con, _cohort_month_key(table, w0), "done", count)
|
| 9728 |
+
elif first_load and w0 is not None:
|
| 9729 |
record_backfill_floor(con, table, w0)
|
| 9730 |
floors[table] = w0
|
| 9731 |
+
stop = _note_history_window(con, table, w0, count)
|
| 9732 |
+
if stop:
|
| 9733 |
+
history_cut[table] = w0
|
| 9734 |
+
landed.update(task for task in work if task[0] == table and task[2] is not None
|
| 9735 |
+
and task[2] <= w0)
|
| 9736 |
+
log(stop)
|
| 9737 |
elif table != "settlements":
|
| 9738 |
period_spec = cx.REPORT_TYPES[table]
|
| 9739 |
windows = (([(task_start, task_end)] if task_end is not None else []) if first_load
|
|
|
|
| 9874 |
if reached:
|
| 9875 |
record_backfill_floor(con, reached, w0)
|
| 9876 |
floors[reached] = w0
|
| 9877 |
+
stop = _note_history_window(con, table, w0, len(rows) if cols else 0)
|
| 9878 |
+
if stop:
|
| 9879 |
+
history_cut[table] = w0
|
| 9880 |
+
landed.update(task for task in work if task[0] == table and task[2] is not None
|
| 9881 |
+
and task[2] <= w0)
|
| 9882 |
+
log(stop)
|
| 9883 |
|
| 9884 |
if (first_load and table != "settlements" and table != SQP_TABLE
|
| 9885 |
and not cx.REPORT_TYPES[table].get("snapshot")
|
|
|
|
| 9966 |
(_read_state(con, entity_key("waiting_" + table)) or {}).get("detail") or "{}")
|
| 9967 |
elif period_repair_next is not None and landed_window:
|
| 9968 |
report["tables"][table]["nextWeek"] = _iso(period_repair_next)
|
| 9969 |
+
nothing_new = _landed_nothing(table, landed_window, landed)
|
| 9970 |
+
_record_state(con, key, "done", total_rows, NOTHING_NEW_DETAIL if nothing_new else None)
|
| 9971 |
+
prior_tick = _read_state(con, entity_key("sync_tick_" + table)) if nothing_new and first_load else None
|
| 9972 |
+
if not (prior_tick and prior_tick.get("state") == "done"):
|
| 9973 |
+
# A first-load pass that requested nothing keeps the last landing's tick, so it never reads as fresh.
|
| 9974 |
+
_record_state(con, entity_key("sync_tick_" + table), "done", total_rows,
|
| 9975 |
+
NOTHING_NEW_DETAIL if nothing_new else None)
|
| 9976 |
con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("resume_" + table)])
|
| 9977 |
if weekly and landed_window:
|
| 9978 |
# A landed week closes any earlier refusal of this table.
|
|
|
|
| 9987 |
mark = min(mark, waited_start)
|
| 9988 |
has_window = (cx.REPORT_TYPES[table].get("snapshot") or table in ("settlements", SQP_TABLE)
|
| 9989 |
or landed_window or (first_load and floors[table] is not None))
|
| 9990 |
+
if first_load and mark is not None and has_window:
|
| 9991 |
+
# Months landed after the cohort end advance the watermark, so a refresh never asks for them again.
|
| 9992 |
+
for w0, w1 in _month_windows(end, datetime.now(timezone.utc)):
|
| 9993 |
+
if not (_after_cohort(table, w0, end) and _cohort_month_done(con, table, w0)):
|
| 9994 |
+
break
|
| 9995 |
+
mark = max(mark, w1)
|
| 9996 |
+
if (first_load and mark is not None and waited_start is None
|
| 9997 |
+
and read_watermark(con, table) is not None):
|
| 9998 |
+
# A reopened history pass never rewinds the recent data the scheduled refresh landed.
|
| 9999 |
+
mark = max(mark, read_watermark(con, table))
|
| 10000 |
if mark is not None and has_window:
|
| 10001 |
record_watermark(con, table, mark, total_rows)
|
| 10002 |
if not first_load and not resync:
|
|
|
|
| 10039 |
break
|
| 10040 |
except Exception as e: # noqa: BLE001
|
| 10041 |
limit_reason = _close_history_floor(con, table, e, task_start, task_end) if first_load else None
|
| 10042 |
+
if limit_reason and table == SQP_TABLE:
|
| 10043 |
+
# Only months at or below Amazon's floor stop; newer months and the weekly batches continue.
|
| 10044 |
+
history_cut[table] = task_end
|
| 10045 |
+
landed.update(task for task in work if task[0] == table and task[2] is not None
|
| 10046 |
+
and task[2] <= task_end)
|
| 10047 |
+
if sqp_limit is not None:
|
| 10048 |
+
sqp_limit["errors"] = []
|
| 10049 |
+
_record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
|
| 10050 |
+
report["tables"][table] = {"state": "deferred", "current": True, "reason": limit_reason}
|
| 10051 |
+
log(limit_reason)
|
| 10052 |
+
continue
|
| 10053 |
if limit_reason:
|
| 10054 |
landed.update(task for task in work if task[0] == table)
|
| 10055 |
handled.add(table)
|
| 10056 |
total = con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0]
|
| 10057 |
_record_state(con, key, "done", total)
|
| 10058 |
+
record_watermark(con, table, max(plan["end"], read_watermark(con, table) or plan["end"]), total)
|
| 10059 |
report["tables"][table] = {"rows": total, "state": "done", "reason": limit_reason}
|
| 10060 |
log(limit_reason)
|
| 10061 |
continue
|
|
|
|
| 10138 |
"landedBatches": landed_batches}
|
| 10139 |
sqp_progress()
|
| 10140 |
_record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
|
| 10141 |
+
for table in pending if first_load else ():
|
| 10142 |
+
# Landed months publish with their markers, as SQP batches do; the months still owed run next pass.
|
| 10143 |
+
months = sum(task[0] == table and task[2] is not None for task in landed)
|
| 10144 |
+
row = report["tables"].get(table)
|
| 10145 |
+
if (table == SQP_TABLE or not cx.REPORT_TYPES[table].get("monthly") or not months
|
| 10146 |
+
or (row is not None and not (isinstance(row, dict)
|
| 10147 |
+
and row.get("state") in {"deferred", "throttled", "pending"}))):
|
| 10148 |
+
continue
|
| 10149 |
+
row = row or {"reason": f"{months} month{'' if months == 1 else 's'} landed. The remaining months continue next pass."}
|
| 10150 |
+
report["tables"][table] = {**row, "state": "deferred", "current": True, "landedMonths": months}
|
| 10151 |
if first_load:
|
| 10152 |
+
refusals ={row["table"]: row for row in _refused_reports(con)}
|
| 10153 |
not_offered = {table for table, row in refusals.items() if row["state"] == "not_offered"}
|
| 10154 |
retrying = {table for table, row in refusals.items() if row["state"] == "retrying"}
|
| 10155 |
landed.update(task for task in work if task[0] in not_offered)
|
|
|
|
| 10181 |
# refresh floors must not re-queue itself every 30 seconds; the daily pass resumes it.
|
| 10182 |
report["landedTasks"] = len(landed)
|
| 10183 |
markers = {row[0]: {"state": row[1], "detail": row[3]} for row in _sync_state_rows(con)}
|
| 10184 |
+
enabled_tables = [table for table in TABLES if enablement["databases"].get(amazon_schema.DATABASE_PREFIX + table)]
|
| 10185 |
progress = {"state": state, **first_load_payload(markers, remaining,
|
| 10186 |
max(first_load_eta(remaining, burst=0), floor_eta), state=state,
|
| 10187 |
+
tables=enabled_tables), "backfillDays": DEFAULT_BACKFILL_DAYS,
|
| 10188 |
+
"historyFloors": history_status(con, enabled_tables, end)}
|
| 10189 |
due = (datetime.now(timezone.utc) + timedelta(seconds=report["nextDueSeconds"])) if (
|
| 10190 |
state == "partial" and not landed and report.get("nextDueSeconds")) else None
|
| 10191 |
_record_state(con, entity_key(FIRST_LOAD_DUE_KEY), "pending" if due else "done",
|
|
|
|
| 12073 |
patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_: (con.cursor(), None)), \
|
| 12074 |
patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \
|
| 12075 |
patch.object(module, "_sync_event"), patch.object(cx, "report_document", side_effect=document), \
|
| 12076 |
+
patch.dict(cx.REPORT_TYPES, {SQP_TABLE: {k: v for k, v in cx.REPORT_TYPES[SQP_TABLE].items()
|
| 12077 |
+
if k != "history"}}), \
|
| 12078 |
patch.dict(cx.REPORT_TYPES[SQP_TABLE], {"periods": ("MONTH",)}):
|
| 12079 |
+
# The fixture's months sit on the 70-day fallback, so the measured SQP `from` rule is patched out.
|
| 12080 |
def run(**kwargs):
|
| 12081 |
result = sync(Runtime.key, tables=[SQP_TABLE], rt=Runtime(), log=lambda *_: None, **kwargs)
|
| 12082 |
problems.extend(result["problems"])
|
|
|
|
| 13007 |
return 1 if failures else 0
|
| 13008 |
|
| 13009 |
|
| 13010 |
+
def _without_history_rules(tables):
|
| 13011 |
+
"""Specs for a fixture that predates the measured `history` rules: same tables, rules removed."""
|
| 13012 |
+
specs = _client().REPORT_TYPES
|
| 13013 |
+
return {table: {k: v for k, v in specs[table].items() if k != "history"} for table in tables}
|
| 13014 |
+
|
| 13015 |
+
|
| 13016 |
+
def monthly_after_cohort_selftest(log=print):
|
| 13017 |
+
"""H5: monthly tables plan, land and publish the months closed after a frozen cohort end. No network."""
|
| 13018 |
+
from unittest.mock import patch
|
| 13019 |
+
import duckdb
|
| 13020 |
+
cx, module = _client(), sys.modules[__name__]
|
| 13021 |
+
fails, checks = [], [0]
|
| 13022 |
+
|
| 13023 |
+
def ck(name, cond, detail=""):
|
| 13024 |
+
checks[0] += 1
|
| 13025 |
+
if cond:
|
| 13026 |
+
log(f" ok {name}")
|
| 13027 |
+
return
|
| 13028 |
+
log(f" FAIL {name} {str(detail).encode('ascii', 'backslashreplace').decode('ascii')}")
|
| 13029 |
+
fails.append(name)
|
| 13030 |
+
|
| 13031 |
+
tables = ["search_catalog_performance", "market_basket", "repeat_purchase"]
|
| 13032 |
+
end, t0 = _utc("2026-09-26T00:00:00Z"), _utc("2026-11-14T00:00:30Z")
|
| 13033 |
+
clock, holder, calls, bounds = [t0], {}, [], []
|
| 13034 |
+
by_report = {cx.REPORT_TYPES[t]["report"]: t for t in tables}
|
| 13035 |
+
|
| 13036 |
+
class Clock(datetime):
|
| 13037 |
+
@classmethod
|
| 13038 |
+
def now(cls, tz=None):
|
| 13039 |
+
return clock[0].astimezone(tz)
|
| 13040 |
+
|
| 13041 |
+
class Runtime(_FixtureMirrorRuntime):
|
| 13042 |
+
key = "h5-fixture"
|
| 13043 |
+
|
| 13044 |
+
def get(self, *_args, **_kwargs):
|
| 13045 |
+
return None
|
| 13046 |
+
|
| 13047 |
+
def fresh(floor="2026-08-01T00:00:00Z"):
|
| 13048 |
+
con = holder["con"] = duckdb.connect(":memory:")
|
| 13049 |
+
con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, "
|
| 13050 |
+
"cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
|
| 13051 |
+
_record_state(con, entity_key(FIRST_LOAD_END_KEY), "running", detail=_iso(end))
|
| 13052 |
+
for table in tables:
|
| 13053 |
+
record_backfill_floor(con, table, _utc(floor))
|
| 13054 |
+
calls.clear()
|
| 13055 |
+
return con
|
| 13056 |
+
|
| 13057 |
+
def document(_fields, report_type, **kwargs):
|
| 13058 |
+
calls.append((by_report[report_type], kwargs["start"][:7]))
|
| 13059 |
+
bounds.append((kwargs["start"], kwargs["end"]))
|
| 13060 |
+
rows = [{"asin": f"B0H5{len(calls):06d}", "startDate": kwargs["start"][:10], "endDate": kwargs["end"][:10]}]
|
| 13061 |
+
return json.dumps({"dataByAsin": rows}), {"reportDocumentId": f"h5-{len(calls)}"}
|
| 13062 |
+
|
| 13063 |
+
def run(at):
|
| 13064 |
+
clock[0] = at
|
| 13065 |
+
return _sync_candidate(Runtime.key, tables=list(tables), rt=Runtime(), log=lambda *_: None,
|
| 13066 |
+
scheduled=True, first_load=True, report_budget=10)
|
| 13067 |
+
|
| 13068 |
+
def asked(start=0):
|
| 13069 |
+
return sorted({month for _t, month in calls[start:]})
|
| 13070 |
+
|
| 13071 |
+
def months(con, table):
|
| 13072 |
+
return sorted(str(r[0])[:7] for r in con.execute(
|
| 13073 |
+
f'SELECT DISTINCT "_period_start" FROM {table_name(table)}').fetchall())
|
| 13074 |
+
|
| 13075 |
+
day = timedelta(days=1, minutes=1)
|
| 13076 |
+
try:
|
| 13077 |
+
with patch.object(cx, "access_token", side_effect=AssertionError("fixture attempted LWA")), \
|
| 13078 |
+
patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("fixture attempted network")), \
|
| 13079 |
+
patch.object(module, "datetime", Clock), patch.object(module, "DEFAULT_BACKFILL_DAYS", 70), \
|
| 13080 |
+
patch.object(module, "_SCHEDULE_OFFSET_READER", lambda: 0), \
|
| 13081 |
+
patch.object(module, "credential", return_value={"fixture": True}), \
|
| 13082 |
+
patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_: (holder["con"].cursor(), None)), \
|
| 13083 |
+
patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \
|
| 13084 |
+
patch.object(module, "_sync_event"), patch.object(cx, "report_document", side_effect=document), \
|
| 13085 |
+
patch.object(cx, "settlement_reports", return_value=[]), \
|
| 13086 |
+
patch.dict(cx.REPORT_TYPES, _without_history_rules(tables)):
|
| 13087 |
+
# The plan for each table holds 2026-10 then 2026-09 before its own 2026-08 task.
|
| 13088 |
+
bare = first_load_plan(tables, end - timedelta(days=70), end)
|
| 13089 |
+
spliced = _sqp_after_cohort(bare, end, ())
|
| 13090 |
+
order = {t: [w0.strftime("%Y-%m") for name, w0, *_ in spliced if name == t] for t in tables}
|
| 13091 |
+
ck("H5 a frozen cohort plans 2026-10 and 2026-09 first for each Brand Analytics month table",
|
| 13092 |
+
all(order[t][:3] == ["2026-10", "2026-09", "2026-08"] for t in tables)
|
| 13093 |
+
and not any(w0.strftime("%Y-%m") in ("2026-09", "2026-10") for _n, w0, *_ in bare), order)
|
| 13094 |
+
|
| 13095 |
+
# Scheduled first-load passes land 2026-10, then 2026-09 after Amazon's daily floor.
|
| 13096 |
+
con = fresh()
|
| 13097 |
+
with patch.object(module, "_sqp_after_cohort", side_effect=lambda tasks, *_: list(tasks)):
|
| 13098 |
+
run(t0)
|
| 13099 |
+
red = asked()
|
| 13100 |
+
con.close()
|
| 13101 |
+
con = fresh()
|
| 13102 |
+
first = run(t0)
|
| 13103 |
+
one, mark = asked(), len(calls)
|
| 13104 |
+
run(t0 + day)
|
| 13105 |
+
two = asked(mark)
|
| 13106 |
+
ck("H5 scheduled first-load passes land 2026-09 and 2026-10 rows for search catalog performance",
|
| 13107 |
+
red == ["2026-07"] and one == ["2026-10"] and two == ["2026-09"]
|
| 13108 |
+
and months(con, "search_catalog_performance") == ["2026-09", "2026-10"]
|
| 13109 |
+
and bool(bounds) and all(a[:7] == b[:7] and a < b for a, b in bounds)
|
| 13110 |
+
and all(_cohort_month_done(con, t, _utc(m + "-01T00:00:00Z")) for t in tables for m in ("2026-09", "2026-10"))
|
| 13111 |
+
and all(read_backfill_floor(con, t) == _utc("2026-08-01T00:00:00Z") for t in tables),
|
| 13112 |
+
(red, one, two, months(con, "search_catalog_performance")))
|
| 13113 |
+
|
| 13114 |
+
# A landed month publishes; the same chunk without the landed mark publishes nothing.
|
| 13115 |
+
waiting = {table_name("orders")}
|
| 13116 |
+
row = first["tables"]["search_catalog_performance"]
|
| 13117 |
+
unmarked = {**first, "tables": {t: {k: v for k, v in r.items() if k not in ("current", "landedMonths")}
|
| 13118 |
+
for t, r in first["tables"].items()}}
|
| 13119 |
+
holds = _MIRROR_STORE._publication_holds(first, waiting)
|
| 13120 |
+
ck("H5 a first-load chunk that landed months publishes them, never holds them at the old rows",
|
| 13121 |
+
first.get("state") == "partial" and row.get("current") is True and row.get("landedMonths") == 1
|
| 13122 |
+
and holds is not None and not set(tables) & set(holds["held"])
|
| 13123 |
+
and _MIRROR_STORE._publication_holds(unmarked, waiting) is None, (row, holds))
|
| 13124 |
+
|
| 13125 |
+
# The next pass takes the owed history month only, then the table closes past 2026-10.
|
| 13126 |
+
before = len(calls)
|
| 13127 |
+
third = run(t0 + 2 * day)
|
| 13128 |
+
later = asked(before)
|
| 13129 |
+
run(t0 + 3 * day)
|
| 13130 |
+
ck("H5 a later pass never asks Amazon again for a landed month after the cohort",
|
| 13131 |
+
later == ["2026-07"] and asked(before + 3) == []
|
| 13132 |
+
and all(read_watermark(con, t) == _utc("2026-11-01T00:00:00Z") for t in tables)
|
| 13133 |
+
and third["tables"]["search_catalog_performance"].get("state", "done") == "done",
|
| 13134 |
+
(later, asked(before + 3), [read_watermark(con, t) for t in tables]))
|
| 13135 |
+
con.close()
|
| 13136 |
+
con = fresh()
|
| 13137 |
+
run(t0)
|
| 13138 |
+
run(t0 + day)
|
| 13139 |
+
with patch.object(module, "_cohort_month_done", return_value=False):
|
| 13140 |
+
before = len(calls)
|
| 13141 |
+
run(t0 + 2 * day)
|
| 13142 |
+
ck("H5 red control: without the month marker the next pass asks for 2026-10 again",
|
| 13143 |
+
asked(before) == ["2026-10"], asked(before))
|
| 13144 |
+
con.close()
|
| 13145 |
+
|
| 13146 |
+
# A pass whose table landed no window keeps the last landing's tick and says nothing new landed.
|
| 13147 |
+
def zero(nothing=None):
|
| 13148 |
+
clock[0] = t0
|
| 13149 |
+
con = fresh("2026-07-01T00:00:00Z")
|
| 13150 |
+
for t in tables:
|
| 13151 |
+
for m in ("2026-09", "2026-10"):
|
| 13152 |
+
_record_state(con, _cohort_month_key(t, _utc(m + "-01T00:00:00Z")), "done", 1)
|
| 13153 |
+
_record_state(con, entity_key("sync_tick_" + t), "done", 1)
|
| 13154 |
+
stamp = _read_state(con, entity_key("sync_tick_" + tables[0]))["updated_at"]
|
| 13155 |
+
if nothing is None:
|
| 13156 |
+
run(t0 + 3 * day)
|
| 13157 |
+
else:
|
| 13158 |
+
with patch.object(module, "_landed_nothing", return_value=nothing):
|
| 13159 |
+
run(t0 + 3 * day)
|
| 13160 |
+
tick = _read_state(con, entity_key("sync_tick_" + tables[0]))
|
| 13161 |
+
state = _read_state(con, entity_key(tables[0]))
|
| 13162 |
+
con.close()
|
| 13163 |
+
return stamp, tick, state, list(calls)
|
| 13164 |
+
stamp, tick, state, seen = zero()
|
| 13165 |
+
_s, red_tick, red_state, _c = zero(False)
|
| 13166 |
+
ck("H5 a pass that landed no window never stamps a plain done",
|
| 13167 |
+
seen == [] and state["state"] == "done" and state["detail"] == NOTHING_NEW_DETAIL
|
| 13168 |
+
and tick["state"] == "done" and tick["updated_at"] == stamp
|
| 13169 |
+
and red_state["detail"] is None and red_tick["updated_at"] != stamp,
|
| 13170 |
+
(seen, state, tick, red_state, red_tick))
|
| 13171 |
+
except Exception as exc: # noqa: BLE001
|
| 13172 |
+
ck("H5 monthly after-cohort fixture ran", False, f"{type(exc).__name__}: {exc}")
|
| 13173 |
+
log(f" monthly after cohort: {checks[0] - len(fails)}/{checks[0]}")
|
| 13174 |
+
return 1 if fails else 0
|
| 13175 |
+
|
| 13176 |
+
|
| 13177 |
def selftest(log=print):
|
| 13178 |
"""Prove the parsers, the typing, the naming rule and validate's refusal. No network, no store.
|
| 13179 |
|
|
|
|
| 13182 |
"""
|
| 13183 |
fails = []
|
| 13184 |
checks = 0
|
| 13185 |
+
# Offline fixtures never measure a seller floor; the H2 legs in verify_amazon turn it on.
|
| 13186 |
+
os.environ.setdefault("AIOS_AMAZON_SELLER_FLOOR_PROBE", "0")
|
| 13187 |
|
| 13188 |
def ck(name, cond, detail=""):
|
| 13189 |
"""⛔ THE DETAIL IS ASCII-ESCAPED, AND THAT IS NOT COSMETIC.
|
|
|
|
| 13515 |
ck("NC-AZ35 a large synthetic report parses and writes in bounded memory", bounded, detail)
|
| 13516 |
ck("NC-AZ38 brand analytics weekly wait and two week history fixtures", weekly_wait_selftest(log) == 0)
|
| 13517 |
ck("NC-AZ39 catalog picture fill fixtures", catalog_picture_selftest(log) == 0)
|
| 13518 |
+
ck("NC-AZ40 monthly tables after a frozen cohort fixtures", monthly_after_cohort_selftest(log) == 0)
|
| 13519 |
log(f"\n{'GREEN' if not fails else 'RED'}: {checks - len(fails)}/{checks}")
|
| 13520 |
return 0 if not fails else 1
|
| 13521 |
|
|
|
|
| 13661 |
|
| 13662 |
def accounts(tenant_key, **options):
|
| 13663 |
return _sync_candidate(tenant_key, **options)
|
| 13664 |
+
# This fixture's history is the latest week only, so weekly history (H2) plans nothing older.
|
| 13665 |
+
one_week = {**cx.REPORT_TYPES["brand_analytics"], "history": {"from": "2026-10-18", "evidence": "fixture"}}
|
| 13666 |
+
return [patch.dict(cx.REPORT_TYPES, {"brand_analytics": one_week}),
|
| 13667 |
+
patch.object(module, "datetime", Clock), patch.object(module, "_sync_credential", return_value={}),
|
| 13668 |
patch.object(module, "account_entries", return_value=[{"connection_id": "c", "label": "c"}]),
|
| 13669 |
patch.object(module, "restore_mirror", return_value={"dirty": set(), "problems": []}),
|
| 13670 |
patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_: (state.con.cursor(), None)),
|
platform/harness/mirror_store.py
CHANGED
|
@@ -50,6 +50,8 @@ _NEWEST_READERS = {}
|
|
| 50 |
_PUBLICATION_CHECKS = {}
|
| 51 |
_PUBLISH_LISTENERS = []
|
| 52 |
_RESTORE_FOLDER_MAX_AGE = 60 * 60
|
|
|
|
|
|
|
| 53 |
|
| 54 |
|
| 55 |
@contextmanager
|
|
@@ -386,8 +388,16 @@ def tracked_sync(connector, runtime, pull, *args, _catch_up_complete=True, **kwa
|
|
| 386 |
state = _catch_up_source(rt, connector)
|
| 387 |
if not state["current"]:
|
| 388 |
state.update(pass_pending=True, attempt_at=time.time())
|
|
|
|
|
|
|
| 389 |
try:
|
| 390 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 391 |
except Exception as exc:
|
| 392 |
if _catch_up_complete:
|
| 393 |
with _CATCH_UP_LOCK:
|
|
@@ -1032,6 +1042,64 @@ class MirrorStore:
|
|
| 1032 |
self.cache[cache_key] = result
|
| 1033 |
return result
|
| 1034 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1035 |
|
| 1036 |
def _refused_may_publish(self, tenant_key, rt, con, cache):
|
| 1037 |
"""Keep a refused process from checkpointing until a new process restores the saved copy."""
|
|
|
|
| 50 |
_PUBLICATION_CHECKS = {}
|
| 51 |
_PUBLISH_LISTENERS = []
|
| 52 |
_RESTORE_FOLDER_MAX_AGE = 60 * 60
|
| 53 |
+
# (connector, tenant) -> count of tracked_sync pulls running in this process. A newer publication waits for 0.
|
| 54 |
+
_ACTIVE_PULLS = {}
|
| 55 |
|
| 56 |
|
| 57 |
@contextmanager
|
|
|
|
| 388 |
state = _catch_up_source(rt, connector)
|
| 389 |
if not state["current"]:
|
| 390 |
state.update(pass_pending=True, attempt_at=time.time())
|
| 391 |
+
active = (connector, _tenant_key(rt.key))
|
| 392 |
+
_ACTIVE_PULLS[active] = _ACTIVE_PULLS.get(active, 0) + 1
|
| 393 |
try:
|
| 394 |
+
try:
|
| 395 |
+
report = pull(*args, **kwargs)
|
| 396 |
+
finally:
|
| 397 |
+
with _CATCH_UP_LOCK:
|
| 398 |
+
_ACTIVE_PULLS[active] -= 1
|
| 399 |
+
if _ACTIVE_PULLS[active] <= 0:
|
| 400 |
+
_ACTIVE_PULLS.pop(active, None)
|
| 401 |
except Exception as exc:
|
| 402 |
if _catch_up_complete:
|
| 403 |
with _CATCH_UP_LOCK:
|
|
|
|
| 1042 |
self.cache[cache_key] = result
|
| 1043 |
return result
|
| 1044 |
|
| 1045 |
+
def install_newer_publication(self, tenant_key, rt, log=None):
|
| 1046 |
+
"""Install a publication another writer made after this process restored. True when installed.
|
| 1047 |
+
|
| 1048 |
+
Skips while this process holds unpublished tables, runs a sync of this connector, or already
|
| 1049 |
+
serves the stored snapshot. The web process waits for the worker's hand-off instead.
|
| 1050 |
+
"""
|
| 1051 |
+
log = print if log is None else log
|
| 1052 |
+
cache_key = self._mirror_cache_key(tenant_key, rt)
|
| 1053 |
+
if _web_waits_for_worker(self.connector):
|
| 1054 |
+
return False
|
| 1055 |
+
with self.lock:
|
| 1056 |
+
with _CATCH_UP_LOCK:
|
| 1057 |
+
if _ACTIVE_PULLS.get((self.connector, _tenant_key(tenant_key))):
|
| 1058 |
+
return False
|
| 1059 |
+
cached = self.cache.get(cache_key)
|
| 1060 |
+
if cached is not None and cached.get("dirty"):
|
| 1061 |
+
return False
|
| 1062 |
+
raw = self._download(rt, _MANIFEST_KEY)
|
| 1063 |
+
if raw is None:
|
| 1064 |
+
return False
|
| 1065 |
+
stored = json.loads(raw)
|
| 1066 |
+
if stored.get("tenant") != _tenant_key(tenant_key):
|
| 1067 |
+
raise ValueError("Mirror manifest belongs to another tenant")
|
| 1068 |
+
stored_id = stored.get("snapshot_id")
|
| 1069 |
+
if not stored_id:
|
| 1070 |
+
return False
|
| 1071 |
+
# A snapshot this process already refused is not downloaded again on every tick.
|
| 1072 |
+
if cached is not None and cached.get("newer_attempt") == stored_id:
|
| 1073 |
+
return False
|
| 1074 |
+
con, _path = self._connect(tenant_key, rt)
|
| 1075 |
+
try:
|
| 1076 |
+
local_id = datastore._publication(con, self.connector).get("snapshot_id")
|
| 1077 |
+
finally:
|
| 1078 |
+
con.close()
|
| 1079 |
+
if local_id == stored_id:
|
| 1080 |
+
return False
|
| 1081 |
+
# Deletion and retention checks still refuse the install, as they refuse a restore.
|
| 1082 |
+
self._before_restore(tenant_key, rt)
|
| 1083 |
+
previous = self.cache.pop(cache_key, None)
|
| 1084 |
+
try:
|
| 1085 |
+
result = self.restore_mirror(tenant_key, rt, log=log)
|
| 1086 |
+
except BaseException:
|
| 1087 |
+
if previous is not None:
|
| 1088 |
+
self.cache[cache_key] = previous
|
| 1089 |
+
raise
|
| 1090 |
+
con, _path = self._connect(tenant_key, rt)
|
| 1091 |
+
try:
|
| 1092 |
+
installed = datastore._publication(con, self.connector).get("snapshot_id")
|
| 1093 |
+
finally:
|
| 1094 |
+
con.close()
|
| 1095 |
+
if installed == stored_id:
|
| 1096 |
+
return True
|
| 1097 |
+
# A refused newer snapshot leaves this process serving and checkpointing as before.
|
| 1098 |
+
kept = previous if previous is not None and result.get("state") == "refused" else result
|
| 1099 |
+
if not result.get("retry"):
|
| 1100 |
+
kept["newer_attempt"] = stored_id
|
| 1101 |
+
self.cache[cache_key] = kept
|
| 1102 |
+
return False
|
| 1103 |
|
| 1104 |
def _refused_may_publish(self, tenant_key, rt, con, cache):
|
| 1105 |
"""Keep a refused process from checkpointing until a new process restores the saved copy."""
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
a316de715da32a0bac5e51bcf06514188b87ecfbe38015edf489a6cd7fa04652
|