fsanyoto commited on
Commit
9ffb41a
·
verified ·
1 Parent(s): 1f7b5d2

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "current": "8165ce5cac (8165ce5cac1ed)",
3
  "releases": [
4
  {
5
  "version": "v86",
 
1
  {
2
+ "current": "c3003c1 (c3003c11762e5)",
3
  "releases": [
4
  {
5
  "version": "v86",
VERSION CHANGED
@@ -1 +1 @@
1
- 8165ce5cac (8165ce5cac1ed)
 
1
+ c3003c1 (c3003c11762e5)
api/main.py CHANGED
@@ -1057,10 +1057,9 @@ def _publish_connector_status(rt, provider, outcome=None, *, detail="", connecti
1057
  holds = _az._MIRROR_STORE._publication_holds(report, ()) or {}
1058
  extra["notOffered"] = len(holds.get("standing") or ())
1059
  extra["notOfferedTables"] = sorted(holds.get("standing") or ())
1060
- extra["held"] = {name: {"reason": "Amazon does not offer this report to this seller."
1061
- if name in (holds.get("standing") or ())
1062
- else "This table did not finish. Its last published rows are kept."}
1063
- for name in holds.get("held") or {}}
1064
  previous = _cs.status(rt, provider, connection_id, fresh=True) or {}
1065
  stamps = dict((previous.get("pass") or {}).get("tableSync") or {})
1066
  for name, row in (report.get("tables") or {}).items():
 
1057
  holds = _az._MIRROR_STORE._publication_holds(report, ()) or {}
1058
  extra["notOffered"] = len(holds.get("standing") or ())
1059
  extra["notOfferedTables"] = sorted(holds.get("standing") or ())
1060
+ import routes_connectors as _held_copy
1061
+ # Each held table names its cause: refusal, throttle with its retry time, or the pass end.
1062
+ extra["held"] = _held_copy.amazon_held_entries(report, holds)
 
1063
  previous = _cs.status(rt, provider, connection_id, fresh=True) or {}
1064
  stamps = dict((previous.get("pass") or {}).get("tableSync") or {})
1065
  for name, row in (report.get("tables") or {}).items():
api/routes_connectors.py CHANGED
@@ -3148,6 +3148,78 @@ def _amazon_mirror_completion(rt, *, ads=False):
3148
  return None, False
3149
 
3150
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3151
  def _amazon_table_schedule(rt, connection_id=None, now=None):
3152
  """Read account cadence metadata without restoring a mirror or reading source rows."""
3153
  import duckdb
@@ -3181,6 +3253,36 @@ def _amazon_table_schedule(rt, connection_id=None, now=None):
3181
  if (state := markers.get(prefix + az.entity_key("sync_tick_" + table)))}
3182
  if any(not state["updated_at"] for state in states.values()):
3183
  raise ValueError("Amazon cadence metadata has an invalid timestamp")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3184
  # C1 owns every due time, including throttle retries and calendar periods.
3185
  for row in az.sync_schedule(states, now):
3186
  table = row["table"]
@@ -3189,12 +3291,13 @@ def _amazon_table_schedule(rt, connection_id=None, now=None):
3189
  landed = markers.get(prefix + az.entity_key(table)) or {}
3190
  completed = (tick if tick.get("state") == "done"
3191
  and landed.get("state") not in ("refused", "failed", "throttled") else watermark)
 
 
 
3192
  schedule.append({**row, "connectionId": cid, "account": entry["label"],
3193
  "lastSyncAt": (completed.get("updated_at")
3194
  if completed.get("state") == "done" else None),
3195
- "state": (landed.get("state")
3196
- if landed.get("state") in ("refused", "failed", "throttled")
3197
- else tick.get("state") or "never_run")})
3198
  return schedule
3199
 
3200
  try:
@@ -5202,6 +5305,15 @@ def _amazon_databases(rt, is_admin, spec=None):
5202
  if diagnostics:
5203
  raw_sync["diagnostics"] = diagnostics
5204
  is_enabled = enabled.get(key) is True
 
 
 
 
 
 
 
 
 
5205
  out.append({"key": key,
5206
  "label": str(declared.get("label") or LABELS_AMAZON.get(key) or key),
5207
  "defaultLabel": str(declared.get("label")
@@ -5212,7 +5324,7 @@ def _amazon_databases(rt, is_admin, spec=None):
5212
  # definition untouched: only this provider config document changes.
5213
  "enabled": is_enabled,
5214
  **({"sync": raw_sync} if is_admin else {}),
5215
- "syncHealth": _amazon_database_sync_health(raw_sync, is_enabled, database_key=key),
5216
  "fields": fields,
5217
  "excluded": excluded,
5218
  "sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]})
 
3148
  return None, False
3149
 
3150
 
3151
+ #: The second sentence of every held Amazon table's reason.
3152
+ AMAZON_HELD_KEPT = "Its last published rows are kept."
3153
+
3154
+
3155
+ def _amazon_cause_text(text, limit=300):
3156
+ """Screen copy with no em or en dash, cut at a word near `limit` characters."""
3157
+ text = " ".join(dash_norm.plain(text).split())
3158
+ if len(text) > limit:
3159
+ text = text[:limit].rsplit(" ", 1)[0].rstrip(",;:") + "..."
3160
+ return text
3161
+
3162
+
3163
+ def amazon_hold_cause(row=None, held="", *, table=""):
3164
+ """One plain cause for an Amazon table a pass did not finish, from its report row or held text."""
3165
+ from harness.amazon_store import plain_refusal_reason
3166
+ row = row if isinstance(row, dict) else {}
3167
+ held = str(held or "")
3168
+ state = str(row.get("state") or "")
3169
+ status = str(row.get("amazonStatus") or "")
3170
+ reason = str(row.get("reason") or "")
3171
+ # The hold vocabulary is `MirrorStore._publication_holds`; the refusal vocabulary is `plain_refusal_reason`.
3172
+ if held.startswith("held with "):
3173
+ partner = held[len("held with "):].split(",", 1)[0].strip().replace("_", " ")
3174
+ cause = f"Held with {partner}, which reads the same report."
3175
+ elif state == "refused" and status.upper() == "NOT_OFFERED":
3176
+ cause = "Amazon does not offer this report to this seller."
3177
+ elif state == "throttled" or (not state and held.startswith("throttled")):
3178
+ cause = "Amazon is limiting requests. The next try is scheduled."
3179
+ elif state == "pending" or (not state and held.startswith("pending")):
3180
+ cause = "Amazon is still preparing this report."
3181
+ elif state == "deferred" or (not row and (not held or held.startswith("not reached"))):
3182
+ cause = "This pass ended before this table's turn. The next pass continues it."
3183
+ elif state == "never_run":
3184
+ # A table with no tick is due only in the minute after its daily slot, which a busy pass misses.
3185
+ cause = "This table has never synced. Its daily start window is too short."
3186
+ elif state in ("attempted", "running"):
3187
+ cause = "The last pass started this table and did not finish it. The next pass continues it."
3188
+ elif state == "refused":
3189
+ plain = plain_refusal_reason(reason or held, status=status, table=table)
3190
+ cause = plain if plain.startswith(("Amazon refused", "Amazon does not offer")) else (
3191
+ "Amazon refused this report: " + plain)
3192
+ else:
3193
+ problem = held[len("a problem names this table: "):] if held.startswith("a problem names this table: ") else ""
3194
+ cause = plain_refusal_reason(reason or problem or held, status=status, table=table)
3195
+ return _amazon_cause_text(cause)
3196
+
3197
+
3198
+ def amazon_held_entries(report, holds, now=None):
3199
+ """The status document's per-table held entries: the cause, the kept rows sentence, and any retry time."""
3200
+ from datetime import timedelta
3201
+ report = report if isinstance(report, dict) else {}
3202
+ holds = holds if isinstance(holds, dict) else {}
3203
+ rows = report.get("tables") or {}
3204
+ standing = set(holds.get("standing") or ())
3205
+ now = now or datetime.now(timezone.utc)
3206
+ wait = report.get("nextDueSeconds")
3207
+ retry = (now + timedelta(seconds=wait)).isoformat().replace("+00:00", "Z") if (
3208
+ type(wait) in (int, float) and 0 <= wait < 10 ** 7) else None
3209
+ entries = {}
3210
+ for name, held in (holds.get("held") or {}).items():
3211
+ row = rows.get(name) if isinstance(rows.get(name), dict) else {}
3212
+ cause = ("Amazon does not offer this report to this seller." if name in standing
3213
+ else amazon_hold_cause(row, held, table=name))
3214
+ entry = {"reason": f"{cause} {AMAZON_HELD_KEPT}", "cause": cause}
3215
+ if row.get("state") == "throttled" or str(held).startswith("throttled"):
3216
+ entry["nextTryAt"] = row.get("nextDue") or retry
3217
+ elif row.get("nextDue"):
3218
+ entry["nextTryAt"] = row["nextDue"]
3219
+ entries[name] = entry
3220
+ return entries
3221
+
3222
+
3223
  def _amazon_table_schedule(rt, connection_id=None, now=None):
3224
  """Read account cadence metadata without restoring a mirror or reading source rows."""
3225
  import duckdb
 
3253
  if (state := markers.get(prefix + az.entity_key("sync_tick_" + table)))}
3254
  if any(not state["updated_at"] for state in states.values()):
3255
  raise ValueError("Amazon cadence metadata has an invalid timestamp")
3256
+ # A row's cause reads the saved refusal and SQP limit markers, never source rows.
3257
+ cause_keys = ([prefix + az.entity_key(family + table) for table in az.TABLES for family in ("", "refused_")]
3258
+ + [prefix + az.SQP_LIMIT_KEY])
3259
+ saved = dict(cursor.execute("SELECT entity, cursor_wd FROM _sync_state WHERE entity IN (SELECT unnest(?))",
3260
+ [cause_keys]).fetchall())
3261
+
3262
+ def saved_json(key):
3263
+ try:
3264
+ value = json.loads(saved.get(key) or "{}")
3265
+ except (ValueError, TypeError):
3266
+ return {}
3267
+ return value if isinstance(value, dict) else {}
3268
+
3269
+ def row_reason(table, state):
3270
+ # A mirror still restoring has no ticks at all, so never_run names no cause until one loads.
3271
+ if state == "done" or (state == "never_run" and not states):
3272
+ return None
3273
+ refusal = saved_json(prefix + az.entity_key("refused_" + table))
3274
+ parts = [amazon_hold_cause({"state": state, "amazonStatus": refusal.get("amazonStatus") or "",
3275
+ "reason": refusal.get("reason") or saved.get(prefix + az.entity_key(table)) or ""},
3276
+ table=table)]
3277
+ limit = saved_json(prefix + az.SQP_LIMIT_KEY) if table == az.SQP_TABLE else {}
3278
+ if limit and hasattr(az, "sqp_progress_sentence"):
3279
+ # The card carries backfill progress only; the full SQP limit sits in the table description.
3280
+ try:
3281
+ parts.append(_amazon_cause_text(az.sqp_progress_sentence(limit)))
3282
+ except (ValueError, TypeError, KeyError):
3283
+ pass
3284
+ return " ".join(part for part in parts if part) or None
3285
+
3286
  # C1 owns every due time, including throttle retries and calendar periods.
3287
  for row in az.sync_schedule(states, now):
3288
  table = row["table"]
 
3291
  landed = markers.get(prefix + az.entity_key(table)) or {}
3292
  completed = (tick if tick.get("state") == "done"
3293
  and landed.get("state") not in ("refused", "failed", "throttled") else watermark)
3294
+ state = (landed.get("state") if landed.get("state") in ("refused", "failed", "throttled")
3295
+ else tick.get("state") or "never_run")
3296
+ reason = row_reason(table, state)
3297
  schedule.append({**row, "connectionId": cid, "account": entry["label"],
3298
  "lastSyncAt": (completed.get("updated_at")
3299
  if completed.get("state") == "done" else None),
3300
+ "state": state, **({"reason": reason} if reason else {})})
 
 
3301
  return schedule
3302
 
3303
  try:
 
5305
  if diagnostics:
5306
  raw_sync["diagnostics"] = diagnostics
5307
  is_enabled = enabled.get(key) is True
5308
+ # Health reads the cause before a held cause replaces it, so a throttle never reads as a refusal.
5309
+ health = _amazon_database_sync_health(raw_sync, is_enabled, database_key=key)
5310
+ held_cause = str(((raw_sync or {}).get("held") or {}).get("cause") or "") if isinstance(
5311
+ (raw_sync or {}).get("held"), dict) else ""
5312
+ if held_cause:
5313
+ pending = "This table has not loaded yet. Sync it to request rows from Amazon."
5314
+ cause = str(raw_sync.get("unknownCause") or "")
5315
+ raw_sync["unknownCause"] = (cause.replace(pending, held_cause) if pending in cause
5316
+ else cause or held_cause)
5317
  out.append({"key": key,
5318
  "label": str(declared.get("label") or LABELS_AMAZON.get(key) or key),
5319
  "defaultLabel": str(declared.get("label")
 
5324
  # definition untouched: only this provider config document changes.
5325
  "enabled": is_enabled,
5326
  **({"sync": raw_sync} if is_admin else {}),
5327
+ "syncHealth": health,
5328
  "fields": fields,
5329
  "excluded": excluded,
5330
  "sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]})
api/sync_alerts.py CHANGED
@@ -29,8 +29,8 @@ AMAZON_NOT_OFFERED = "Amazon does not offer this report to this seller account"
29
  AMAZON_ROLE = re.compile(r"has not granted access|access to requested resource is denied", re.I)
30
  # A missing role is a standing account fact: it alerts once and never reminds.
31
  ONCE_CODES = frozenset({"role"})
32
- # A throttle alerts only when it repeats on this many passes in a row.
33
- REPEAT_CODES = {"throttled": 2}
34
 
35
 
36
  def utc_now():
@@ -65,8 +65,8 @@ _SCRUB = [
65
  r"client_secret|password|passwd|pwd|signature|sig)=)[^&\s\"']+"), r"\1[masked]"),
66
  (re.compile(r"(?i)\b((?:access_token|refresh_token|client_secret|password|secret|api_key)"
67
  r"[\"']?\s*[:=]\s*[\"']?)[^\s,;\"'}]+"), r"\1[masked]"),
68
- # A long run holding a digit and a letter is a key; plain snake_case table names are not.
69
- (re.compile(r"\b(?=[\w\-+/=]*\d)(?=[\w\-+/=]*[A-Za-z])[\w\-+/=]{32,}"), "[token]"),
70
  ]
71
 
72
 
@@ -229,8 +229,10 @@ def _due(alert, now):
229
  return None
230
 
231
 
232
- def _reconcile(document, tenant, connector, current, observed, now, claim, quiet=frozenset()):
233
- """Fold this pass into the document; return the plan of what this pass must send."""
 
 
234
  document = document if isinstance(document, dict) else {}
235
  alerts = document.setdefault("alerts", {})
236
  recovered = document.setdefault("recovered", {})
@@ -251,7 +253,7 @@ def _reconcile(document, tenant, connector, current, observed, now, claim, quiet
251
  if alert is None and alert_id in recovered:
252
  # Failing again before its RECOVERED went out: the owner never heard it cleared.
253
  alert = {k: v for k, v in recovered.pop(alert_id).items() if k != "cleared_at"}
254
- if alert is None:
255
  # The same key under a new code is the same failure: it keeps what the owner was told.
256
  prior = next((v for a, v in mine.items() if a not in current and v.get("key") == failure["key"]
257
  and v.get("sent_at")), None)
@@ -317,9 +319,11 @@ def _pending(document, tenant, connector, now):
317
 
318
  def _plan_of(document, tenant, connector, now, claim):
319
  lease = (document.get("leases") or {}).get(_scope(tenant, connector)) or {}
320
- if lease.get("claim") != claim:
321
  return None
322
- return _pending(document, tenant, connector, now)
 
 
323
 
324
 
325
  def _item(alert):
@@ -375,11 +379,13 @@ def _mark(document, tenant, connector, plan, now, claim, delivered):
375
 
376
  # ---- entry point ----------------------------------------------------------------------------
377
 
378
- def notify(tenant, connector, failures, *, observed=None, now=None, sender=None, store=None, quiet=None):
 
379
  """Record this pass's failures and send at most one Telegram line; never raises.
380
 
381
  `failures` is the full current list for the tenant and connector; [] means healthy.
382
  `observed` limits clearing to the keys this pass checked; None means the list is complete.
 
383
  """
384
  result = {"ok": False, "sent": False, "failed": [], "reminder": [], "recovered": [], "cause": None}
385
  try:
@@ -396,12 +402,12 @@ def notify(tenant, connector, failures, *, observed=None, now=None, sender=None,
396
  # Save first, so a delivery marker from a past pass is never re-sent.
397
  quiet = frozenset(str(key) for key in (quiet or ()))
398
  document = store.update(DOCUMENT, lambda doc: _reconcile(
399
- doc, tenant, connector, current, observed, now, claim, quiet))
400
  plan = _plan_of(document or {}, tenant, connector, now, claim)
401
  if plan is None or not any(plan.values()):
402
  result.update(ok=True, cause=None if plan is not None else "another pass is sending")
403
  return result
404
- text = compose(tenant, connector, plan, document)
405
  try:
406
  response = (sender or default_sender)(text)
407
  delivered, detail = (response if isinstance(response, tuple) and len(response) == 2
 
29
  AMAZON_ROLE = re.compile(r"has not granted access|access to requested resource is denied", re.I)
30
  # A missing role is a standing account fact: it alerts once and never reminds.
31
  ONCE_CODES = frozenset({"role"})
32
+ # A throttle or a held table alerts only when it repeats on this many passes in a row.
33
+ REPEAT_CODES = {"throttled": 2, "held": 2}
34
 
35
 
36
  def utc_now():
 
65
  r"client_secret|password|passwd|pwd|signature|sig)=)[^&\s\"']+"), r"\1[masked]"),
66
  (re.compile(r"(?i)\b((?:access_token|refresh_token|client_secret|password|secret|api_key)"
67
  r"[\"']?\s*[:=]\s*[\"']?)[^\s,;\"'}]+"), r"\1[masked]"),
68
+ # A long run holding a digit and a letter is a key; snake_case and hashed ut_ table names are not.
69
+ (re.compile(r"\b(?!ut_)(?=[\w\-+/=]*\d)(?=[\w\-+/=]*[A-Za-z])[\w\-+/=]{32,}"), "[token]"),
70
  ]
71
 
72
 
 
229
  return None
230
 
231
 
232
+ def _reconcile(document, tenant, connector, current, observed, now, claim, quiet=frozenset(), carry=True):
233
+ """Fold this pass into the document; return the plan of what this pass must send.
234
+
235
+ `carry=False` sends a key whose code changed as a new failure instead of keeping what was told."""
236
  document = document if isinstance(document, dict) else {}
237
  alerts = document.setdefault("alerts", {})
238
  recovered = document.setdefault("recovered", {})
 
253
  if alert is None and alert_id in recovered:
254
  # Failing again before its RECOVERED went out: the owner never heard it cleared.
255
  alert = {k: v for k, v in recovered.pop(alert_id).items() if k != "cleared_at"}
256
+ if alert is None and carry:
257
  # The same key under a new code is the same failure: it keeps what the owner was told.
258
  prior = next((v for a, v in mine.items() if a not in current and v.get("key") == failure["key"]
259
  and v.get("sent_at")), None)
 
319
 
320
  def _plan_of(document, tenant, connector, now, claim):
321
  lease = (document.get("leases") or {}).get(_scope(tenant, connector)) or {}
322
+ if lease.get("claim") not in (None, claim):
323
  return None
324
+ # No lease means nothing was pending at the save: an empty plan, not another pass sending.
325
+ return _pending(document, tenant, connector, now) if lease.get("claim") == claim else {
326
+ "failed": [], "reminder": [], "recovered": []}
327
 
328
 
329
  def _item(alert):
 
379
 
380
  # ---- entry point ----------------------------------------------------------------------------
381
 
382
+ def notify(tenant, connector, failures, *, observed=None, now=None, sender=None, store=None, quiet=None,
383
+ composer=None, carry=True):
384
  """Record this pass's failures and send at most one Telegram line; never raises.
385
 
386
  `failures` is the full current list for the tenant and connector; [] means healthy.
387
  `observed` limits clearing to the keys this pass checked; None means the list is complete.
388
+ `composer(tenant, connector, plan, document)` writes the message for a sender other than Telegram.
389
  """
390
  result = {"ok": False, "sent": False, "failed": [], "reminder": [], "recovered": [], "cause": None}
391
  try:
 
402
  # Save first, so a delivery marker from a past pass is never re-sent.
403
  quiet = frozenset(str(key) for key in (quiet or ()))
404
  document = store.update(DOCUMENT, lambda doc: _reconcile(
405
+ doc, tenant, connector, current, observed, now, claim, quiet, carry))
406
  plan = _plan_of(document or {}, tenant, connector, now, claim)
407
  if plan is None or not any(plan.values()):
408
  result.update(ok=True, cause=None if plan is not None else "another pass is sending")
409
  return result
410
+ text = (composer or compose)(tenant, connector, plan, document)
411
  try:
412
  response = (sender or default_sender)(text)
413
  delivered, detail = (response if isinstance(response, tuple) and len(response) == 2
platform/harness/amazon_store.py CHANGED
@@ -79,6 +79,10 @@ TABLES = ("orders", "order_items", "listings", "settlements", "inventory",
79
  MARGIN_TABLES = ("economics", "transactions", "fba_fee_estimates", "referral_fees", "storage_fees")
80
  SQP_TABLE = "search_query_performance"
81
  SQP_LIMIT_KEY = "amzn_limit_search_query_performance"
 
 
 
 
82
  GROUP_A_TABLES = tuple(amazon_schema.GROUP_A_TABLES)
83
  TABLES += GROUP_A_TABLES
84
  DERIVED_TABLES = ("sku_keywords", "keyword_ranks")
@@ -613,6 +617,9 @@ def _interval_wait(table, state, now, floor_s=None):
613
  if state and state.get("state") == "throttled" and state.get("detail"):
614
  due = _utc(state["detail"])
615
  wait = (due - now).total_seconds()
 
 
 
616
  if wait > 0:
617
  if not phased:
618
  return wait, due, int(wait), False
@@ -678,6 +685,8 @@ FIRST_LOAD_RECENT_DAYS = MAX_WINDOW_DAYS
678
  #: about 2.7 h. A first load therefore runs in BOUNDED passes, or it would hold the resident
679
  #: thread for hours and the Odoo resync behind it would never run.
680
  FIRST_LOAD_REPORTS_PER_PASS = 15
 
 
681
  #: Rows one first-load pass writes before it takes no further document. A pacing bound, never a cap:
682
  #: the pass publishes and the next continues, so Brand Analytics months (3.7M rows and about 25
683
  #: minutes each) never hold the sync desk for hours (D-7081).
@@ -2513,8 +2522,27 @@ def sqp_limit_sentence(marker):
2513
  if any(type(n) is not int or n < 0 for n in numbers) or not all(numbers[:2]):
2514
  raise ValueError("Invalid saved Search Query Performance limits")
2515
  chars, maximum, count, batches = numbers
2516
- return (f"Amazon accepts at most {chars} characters of ASINs per Search Query Performance "
2517
- f"report, about {maximum} ASINs, so {count} ASINs are requested in {batches} reports per month.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2518
 
2519
 
2520
  def sqp_refusal_message(asin, reason):
@@ -2561,6 +2589,10 @@ class SQPBudgetDeferred(RuntimeError):
2561
  """A batch or its single retry remains durable for the next bounded pass."""
2562
 
2563
 
 
 
 
 
2564
  def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem):
2565
  """One completed period and batch using the shared report transport and pass budget.
2566
 
@@ -2701,10 +2733,13 @@ def refresh_sku_keywords(con):
2701
  raw_meta = (', "_connection_id"' if amazon_schema.CONNECTION_ID in raw_columns else
2702
  ', NULL::VARCHAR AS "_connection_id"') if scoped else ""
2703
  mapping_join = ' AND m."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else ""
2704
- query_join = ' AND q."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else ""
 
 
2705
  count_join = ' AND c."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else ""
2706
  grain_meta = ', "_connection_id"' if scoped else ""
2707
- matched_meta = ', l.store' if scoped else ""
 
2708
  output_meta = ', "_connection_id", store' if scoped else ""
2709
  backend_meta = ', l."_connection_id", l.store' if scoped else ""
2710
  backend_match = ' AND m."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else ""
@@ -2738,7 +2773,7 @@ def refresh_sku_keywords(con):
2738
  SELECT *, CAST(to_json([start_date, end_date, asin, search_query{grain_meta}]) AS VARCHAR) AS source_grain
2739
  FROM raw
2740
  ), matched AS (
2741
- SELECT l.sku, q.*, l.backend_keywords, c.mapped_sku_count{matched_meta}
2742
  FROM listings l JOIN mapping m ON m.sku=l.sku{mapping_join}
2743
  JOIN queries q ON q.asin=m.asin{query_join} JOIN mapped_counts c ON c.asin=m.asin{count_join}
2744
  )
@@ -2944,6 +2979,15 @@ def first_load_plan(tables, start, end, *, recent_days=None, size=MAX_WINDOW_DAY
2944
  return recent + history
2945
 
2946
 
 
 
 
 
 
 
 
 
 
2947
  def first_load_eta(report_count, *, stagger_seconds=None, burst=None, per_minute=None):
2948
  """Amazon createReport pacing floor in seconds; polling and other work can add time."""
2949
  from harness.amazon_client import REPORT_STAGGER_SECONDS, REPORT_BURST, REPORT_SUSTAINED_SECONDS
@@ -3874,9 +3918,10 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
3874
  """
3875
  rows = report.get("tables") or {}
3876
  problems = [str(problem) for key in ("problems", "audit_problems") for problem in report.get(key) or []]
 
3877
  if report.get("state") in {"partial", "failed"} and not any(
3878
- isinstance(row, dict) and row.get("state") != "deferred"
3879
- and _mirrors._table_sync_complete("amazon", row) for row in rows.values()):
3880
  return None
3881
  unfinished = {name[len(PREFIX):] if name.startswith(PREFIX) else name for name in waiting}
3882
  unfinished |= {name for name, row in rows.items() if isinstance(row, dict) and row.get("state") != "disabled"
@@ -5506,8 +5551,8 @@ def first_load_job_plan(tenant_key, rt, *, log=print):
5506
  for entry in entries:
5507
  cid = entry["connection_id"]
5508
  fields = fields_by[cid] = _sync_credential(tenant_key, rt, connection_id=cid)
5509
- for item in first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end,
5510
- sqp_asins=sqp_asins):
5511
  table, w0, w1 = item[:3]
5512
  batch = item[3] if len(item) == 4 else None
5513
  # Account-scoped floors live in each account's workspace; only one account reads them here.
@@ -6317,15 +6362,24 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon
6317
  if not waiting_on and now() >= request_until:
6318
  break
6319
  sleep(poll_seconds if waiting_on else max(0.0, min(poll_seconds, next_token - now())))
6320
- if now() < request_until:
6321
- # After the job's own reports, so its serial report waits never hold the parallel requests.
 
 
 
 
6322
  try:
6323
- delegate(own_tables, FIRST_LOAD_REPORTS_PER_PASS)
6324
  record["otherTablesPass"] = "done"
6325
  except Exception as exc: # noqa: BLE001
6326
  record["otherTablesPass"] = f"failed ({type(exc).__name__})"
6327
  log(f"Amazon first-load pass for the other tables failed ({type(exc).__name__}); "
6328
  "the next generation runs it again.")
 
 
 
 
 
6329
  try:
6330
  after = plan_fn()
6331
  except Exception: # noqa: BLE001
@@ -6539,7 +6593,9 @@ def _save_throttled_resume(con, table, exc, plan, completed, *, first_load, sinc
6539
  "completed": completed, "first_load": first_load,
6540
  "since": _iso(_utc(since)) if since is not None else None}
6541
  _record_state(con, entity_key("resume_" + table), "throttled", detail=json.dumps(continuation))
6542
- due = datetime.now(timezone.utc) + timedelta(seconds=max(60.0, float(exc.retry_after or 60)))
 
 
6543
  _record_state(con, entity_key("sync_tick_" + table), "throttled", detail=_iso(due))
6544
  return max(0, (due - datetime.now(timezone.utc)).total_seconds())
6545
 
@@ -8679,6 +8735,10 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
8679
  continue
8680
  report["tables"][table] = {"state": "deferred", "reason":
8681
  "The first load is continuing in bounded passes on the resident sync thread."}
 
 
 
 
8682
  want = [table for table in want if (read_watermark(con, table)
8683
  and (not refresh_loaded or table not in repairs))
8684
  or table in completed
@@ -8787,15 +8847,45 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
8787
  report["problems"].append(message)
8788
  log(message)
8789
 
 
 
8790
  def sqp_request(*args, **kwargs):
8791
- nonlocal documents
8792
  if isinstance(report_budget, int) and documents >= max(0, report_budget):
8793
  raise SQPBudgetDeferred("Search Query Performance report budget reached; remaining batches continue next pass.")
 
 
 
8794
  documents += 1
 
8795
  _sync_event("amazon.access", tenant_key, SQP_TABLE,
8796
  {"state": "access", "operation": "report_document"}, rt, report, log)
8797
  return _report_document(con, SQP_TABLE, *args, **kwargs, max_wait=max_wait, log=log)
8798
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8799
  if first_load:
8800
  # Freeze the cohort end across passes. Recomputing it would move every window boundary.
8801
  end = _utc(load["detail"]) if load else _utc(_iso(end))
@@ -8805,8 +8895,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
8805
  floors = {table: None if table in undocumented else read_backfill_floor(con, table) for table in pending}
8806
  wanted = [table for table in pending if (table not in completed or table in exhausted) and (not read_watermark(con, table)
8807
  or table in undocumented or read_watermark(con, table) < plans[table]["end"])]
8808
- for entry in first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end,
8809
- sqp_asins=sqp_asins):
8810
  task = entry if len(entry) == 4 else (*entry, None)
8811
  table, w0, w1, batch = task
8812
  if table == SQP_TABLE:
@@ -8849,15 +8939,14 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
8849
  plan = plans[table] = sync_window(table, mark, now=end, since=since,
8850
  intraday=_lookback_intraday(con, table, end, slot_pass=slot_pass))
8851
  if table == SQP_TABLE:
8852
- months = _month_windows(plan["start"], plan["end"])
8853
- if not plan["incremental"]:
8854
- months.reverse()
8855
  tasks = [(table, w0, w1, batch) for w0, w1 in months for batch in batches
8856
  if since is not None or not _sqp_done(con, w0, w1, batch)]
8857
  weekly_tasks = [task for task in _sqp_week_tasks(
8858
  _utc(since) if since is not None else end - timedelta(days=7), end, batches)
8859
  if since is not None or not _sqp_done(con, task[1], task[2], task[3])]
8860
- work.extend([*weekly_tasks, *(tasks or [(table, None, None, None)])])
8861
  else:
8862
  work.append((table, None, None, None))
8863
 
@@ -8894,7 +8983,13 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
8894
  if old_limit.get("weeklyStart") == sqp_limit["weeklyStart"]:
8895
  sqp_limit["weeklyCause"] = old_limit.get("weeklyCause", "")
8896
  plan = plans.get(SQP_TABLE) or sync_window(SQP_TABLE, read_watermark(con, SQP_TABLE), now=end, since=since)
8897
- sqp_limit["periods"] = [w0.strftime("%Y-%m") for w0, _w1 in _month_windows(plan["start"], plan["end"])]
 
 
 
 
 
 
8898
  sqp_limit["deferredReason"] = (
8899
  "Search Query Performance is waiting for ASINs in the listings or pricing mirror. Sync the catalogue first."
8900
  if not sqp_asins else sqp_limit["partialMonthReason"] if not sqp_limit["periods"] else "")
@@ -8927,6 +9022,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
8927
  con, task_start, task_end)["batches"][task_batch].get("refused", False)
8928
  spend_rows(count)
8929
  landed.add((table, task_start, task_end, task_batch))
 
 
 
8930
  except SQPBudgetDeferred as exc:
8931
  report["tables"][table] = {"state": "deferred", "reason": str(exc)}
8932
  weekly_deferred = True
@@ -9121,6 +9219,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
9121
  written += _sync_sqp_batch(con, fields, task_start, task_end, task_batch,
9122
  sqp_limit, sqp_request, sqp_problem)
9123
  sqp_limit["errors"] = []
 
9124
  _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
9125
  elif table in PERIOD_TABLES or table in LEGACY_SNAPSHOT_TABLES:
9126
  operation = cx.REPORT_TYPES[table].get("api") or "report_document"
@@ -9337,7 +9436,10 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
9337
  continue
9338
  if first_load or table == SQP_TABLE:
9339
  landed.add((table, task_start, task_end, task_batch))
9340
- if any(task[0] == table for task in work[index + 1:]):
 
 
 
9341
  continue
9342
 
9343
  if table == "pricing":
@@ -9460,6 +9562,10 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
9460
  {"state": "done", "rows": sibling_total, "operations_ok": True},
9461
  rt, report, log)
9462
  log(f" {sibling:<16} {sibling_total:>7,} rows {sibling_columns:>3} cols {sibling_prov}")
 
 
 
 
9463
  except SQPBudgetDeferred as exc:
9464
  report["tables"][table] = {"state": "deferred", "reason": str(exc)}
9465
  break
@@ -9545,6 +9651,13 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
9545
  reason = f"Search Query Performance has {remaining_sqp} month and batch tasks remaining; continuing next pass."
9546
  sqp_limit["deferredReason"] = reason
9547
  report["tables"][SQP_TABLE] = {"state": "deferred", "reason": reason}
 
 
 
 
 
 
 
9548
  _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
9549
  if first_load:
9550
  refusals = {row["table"]: row for row in _refused_reports(con)}
@@ -11613,9 +11726,189 @@ def sqp_selftest(log=print, check=None):
11613
  and json.loads(_read_state(con, SQP_LIMIT_KEY)["detail"])["errors"] == ["Amazon cannot generate this report"])
11614
  finally:
11615
  con.close()
 
11616
  return 1 if failures else 0
11617
 
11618
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11619
  def group_a_selftest(log=print, check=None):
11620
  """S66-T42 red-first fixture controls; memory mirrors and no credentials or network."""
11621
  from unittest.mock import patch
 
79
  MARGIN_TABLES = ("economics", "transactions", "fba_fee_estimates", "referral_fees", "storage_fees")
80
  SQP_TABLE = "search_query_performance"
81
  SQP_LIMIT_KEY = "amzn_limit_search_query_performance"
82
+ #: A scheduled pass spends at most Amazon's createReport burst on SQP documents.
83
+ SQP_PASS_REPORTS = 15
84
+ #: An SQP backlog resumes this long after its pass stops: 15 reports per 15 minutes is Amazon's sustained rate.
85
+ SQP_RESUME_SECONDS = 900
86
  GROUP_A_TABLES = tuple(amazon_schema.GROUP_A_TABLES)
87
  TABLES += GROUP_A_TABLES
88
  DERIVED_TABLES = ("sku_keywords", "keyword_ranks")
 
617
  if state and state.get("state") == "throttled" and state.get("detail"):
618
  due = _utc(state["detail"])
619
  wait = (due - now).total_seconds()
620
+ if table == SQP_TABLE:
621
+ # An SQP backlog resumes at its own deadline, never on the daily phase.
622
+ return (wait, due, int(wait), False) if wait > 0 else (0, None, seconds, False)
623
  if wait > 0:
624
  if not phased:
625
  return wait, due, int(wait), False
 
685
  #: about 2.7 h. A first load therefore runs in BOUNDED passes, or it would hold the resident
686
  #: thread for hours and the Odoo resync behind it would never run.
687
  FIRST_LOAD_REPORTS_PER_PASS = 15
688
+ #: A generation's repeated other-tables pass: about an hour of reports at Amazon's rate, then it publishes.
689
+ FIRST_LOAD_DELEGATE_REPORTS = 60
690
  #: Rows one first-load pass writes before it takes no further document. A pacing bound, never a cap:
691
  #: the pass publishes and the next continues, so Brand Analytics months (3.7M rows and about 25
692
  #: minutes each) never hold the sync desk for hours (D-7081).
 
2522
  if any(type(n) is not int or n < 0 for n in numbers) or not all(numbers[:2]):
2523
  raise ValueError("Invalid saved Search Query Performance limits")
2524
  chars, maximum, count, batches = numbers
2525
+ sentence = (f"Amazon accepts at most {chars} characters of ASINs per Search Query Performance "
2526
+ f"report, about {maximum} ASINs, so {count} ASINs are requested in {batches} reports per month.")
2527
+ progress = sqp_progress_sentence(marker)
2528
+ return f"{sentence} {progress}" if progress else sentence
2529
+
2530
+
2531
+ def sqp_progress_sentence(detail):
2532
+ """Batches done of total and the time left, from saved SQP progress; "" without progress."""
2533
+ progress = detail.get("progress")
2534
+ if not progress:
2535
+ return ""
2536
+ steps = [progress.get(k) for k in ("monthsDone", "monthsTotal", "batchesDone", "batchesTotal", "etaSeconds")]
2537
+ if any(type(n) is not int or n < 0 for n in steps):
2538
+ raise ValueError("Invalid saved Search Query Performance progress")
2539
+ months_done, months, done, total, eta = steps
2540
+ sentence = f"{months_done:,} of {months:,} months complete, {done:,} of {total:,} batches."
2541
+ if eta:
2542
+ hours, minutes = round(eta / 3600), max(1, round(eta / 60))
2543
+ left = (f"{hours:,} hours" if hours >= 2 else "1 minute" if minutes == 1 else f"{minutes} minutes")
2544
+ sentence += f" About {left} left at Amazon's rate of one report a minute."
2545
+ return sentence
2546
 
2547
 
2548
  def sqp_refusal_message(asin, reason):
 
2589
  """A batch or its single retry remains durable for the next bounded pass."""
2590
 
2591
 
2592
+ class SQPPassDeferred(SQPBudgetDeferred):
2593
+ """A scheduled pass spent `SQP_PASS_REPORTS`; only SQP stops, other tables keep running."""
2594
+
2595
+
2596
  def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem):
2597
  """One completed period and batch using the shared report transport and pass budget.
2598
 
 
2733
  raw_meta = (', "_connection_id"' if amazon_schema.CONNECTION_ID in raw_columns else
2734
  ', NULL::VARCHAR AS "_connection_id"') if scoped else ""
2735
  mapping_join = ' AND m."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else ""
2736
+ # An SQP row saved before account tagging has no account; it belongs to a single-account tenant's listings.
2737
+ query_join = (' AND (q."_connection_id" IS NOT DISTINCT FROM l."_connection_id" OR (q."_connection_id" IS NULL'
2738
+ ' AND (SELECT COUNT(DISTINCT "_connection_id") FROM listings) = 1))') if scoped else ""
2739
  count_join = ' AND c."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else ""
2740
  grain_meta = ', "_connection_id"' if scoped else ""
2741
+ matched_meta = ', l."_connection_id", l.store' if scoped else ""
2742
+ query_columns = 'q.* EXCLUDE ("_connection_id")' if scoped else "q.*"
2743
  output_meta = ', "_connection_id", store' if scoped else ""
2744
  backend_meta = ', l."_connection_id", l.store' if scoped else ""
2745
  backend_match = ' AND m."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else ""
 
2773
  SELECT *, CAST(to_json([start_date, end_date, asin, search_query{grain_meta}]) AS VARCHAR) AS source_grain
2774
  FROM raw
2775
  ), matched AS (
2776
+ SELECT l.sku, {query_columns}, l.backend_keywords, c.mapped_sku_count{matched_meta}
2777
  FROM listings l JOIN mapping m ON m.sku=l.sku{mapping_join}
2778
  JOIN queries q ON q.asin=m.asin{query_join} JOIN mapped_counts c ON c.asin=m.asin{count_join}
2779
  )
 
2979
  return recent + history
2980
 
2981
 
2982
+ def _sqp_after_cohort(tasks, end, asins):
2983
+ """Insert SQP months that closed after the frozen cohort end, newest first, before older SQP tasks."""
2984
+ spec = _client().REPORT_TYPES[SQP_TABLE]
2985
+ later = [(SQP_TABLE, w0, w1, batch) for w0, w1 in reversed(_month_windows(end, datetime.now(timezone.utc)))
2986
+ for batch in sqp_batches(asins, spec["asin_option_max_chars"])]
2987
+ at = next((i for i, task in enumerate(tasks) if task[0] == SQP_TABLE), len(tasks))
2988
+ return [*tasks[:at], *later, *tasks[at:]]
2989
+
2990
+
2991
  def first_load_eta(report_count, *, stagger_seconds=None, burst=None, per_minute=None):
2992
  """Amazon createReport pacing floor in seconds; polling and other work can add time."""
2993
  from harness.amazon_client import REPORT_STAGGER_SECONDS, REPORT_BURST, REPORT_SUSTAINED_SECONDS
 
3918
  """
3919
  rows = report.get("tables") or {}
3920
  problems = [str(problem) for key in ("problems", "audit_problems") for problem in report.get(key) or []]
3921
+ # SQP batches that landed count as finished work, so a chunk of only SQP progress still publishes.
3922
  if report.get("state") in {"partial", "failed"} and not any(
3923
+ isinstance(row, dict) and (row.get("state") != "deferred" or name == SQP_TABLE)
3924
+ and _mirrors._table_sync_complete("amazon", row) for name, row in rows.items()):
3925
  return None
3926
  unfinished = {name[len(PREFIX):] if name.startswith(PREFIX) else name for name in waiting}
3927
  unfinished |= {name for name, row in rows.items() if isinstance(row, dict) and row.get("state") != "disabled"
 
5551
  for entry in entries:
5552
  cid = entry["connection_id"]
5553
  fields = fields_by[cid] = _sync_credential(tenant_key, rt, connection_id=cid)
5554
+ items = first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end, sqp_asins=sqp_asins)
5555
+ for item in _sqp_after_cohort(items, end, sqp_asins) if SQP_TABLE in wanted else items:
5556
  table, w0, w1 = item[:3]
5557
  batch = item[3] if len(item) == 4 else None
5558
  # Account-scoped floors live in each account's workspace; only one account reads them here.
 
6362
  if not waiting_on and now() >= request_until:
6363
  break
6364
  sleep(poll_seconds if waiting_on else max(0.0, min(poll_seconds, next_token - now())))
6365
+ record["otherTablesPasses"] = 0
6366
+ while now() < request_until:
6367
+ # After the job's own reports. Each pass publishes; another follows while request time remains,
6368
+ # sized to that time at Amazon's one report a minute, so SQP history moves at Amazon's rate.
6369
+ budget = max(FIRST_LOAD_REPORTS_PER_PASS,
6370
+ min(FIRST_LOAD_DELEGATE_REPORTS, int((request_until - now()) // 60)))
6371
  try:
6372
+ result = delegate(own_tables, budget)
6373
  record["otherTablesPass"] = "done"
6374
  except Exception as exc: # noqa: BLE001
6375
  record["otherTablesPass"] = f"failed ({type(exc).__name__})"
6376
  log(f"Amazon first-load pass for the other tables failed ({type(exc).__name__}); "
6377
  "the next generation runs it again.")
6378
+ break
6379
+ record["otherTablesPasses"] += 1
6380
+ result = result if isinstance(result, dict) else {}
6381
+ if result.get("state") != "partial" or not result.get("landedTasks"):
6382
+ break
6383
  try:
6384
  after = plan_fn()
6385
  except Exception: # noqa: BLE001
 
6593
  "completed": completed, "first_load": first_load,
6594
  "since": _iso(_utc(since)) if since is not None else None}
6595
  _record_state(con, entity_key("resume_" + table), "throttled", detail=json.dumps(continuation))
6596
+ # A scheduled SQP backlog waits for Amazon's burst to refill, never less than SQP_RESUME_SECONDS.
6597
+ least = SQP_RESUME_SECONDS if table == SQP_TABLE and not first_load else 60.0
6598
+ due = datetime.now(timezone.utc) + timedelta(seconds=max(float(least), float(exc.retry_after or 60)))
6599
  _record_state(con, entity_key("sync_tick_" + table), "throttled", detail=_iso(due))
6600
  return max(0, (due - datetime.now(timezone.utc)).total_seconds())
6601
 
 
8735
  continue
8736
  report["tables"][table] = {"state": "deferred", "reason":
8737
  "The first load is continuing in bounded passes on the resident sync thread."}
8738
+ tick = _read_state(con, entity_key("sync_tick_" + table)) if table == SQP_TABLE else None
8739
+ if tick and tick.get("state") == "throttled":
8740
+ # The first load owns SQP now; a passed SQP deadline must not make it due every minute.
8741
+ _record_state(con, entity_key("sync_tick_" + table), "deferred", tick.get("rows") or 0, tick.get("detail"))
8742
  want = [table for table in want if (read_watermark(con, table)
8743
  and (not refresh_loaded or table not in repairs))
8744
  or table in completed
 
8847
  report["problems"].append(message)
8848
  log(message)
8849
 
8850
+ sqp_documents = 0
8851
+
8852
  def sqp_request(*args, **kwargs):
8853
+ nonlocal documents, sqp_documents
8854
  if isinstance(report_budget, int) and documents >= max(0, report_budget):
8855
  raise SQPBudgetDeferred("Search Query Performance report budget reached; remaining batches continue next pass.")
8856
+ if scheduled and not first_load and sqp_documents >= SQP_PASS_REPORTS:
8857
+ raise SQPPassDeferred(f"Search Query Performance requested {SQP_PASS_REPORTS} reports this pass; "
8858
+ f"remaining batches continue in {SQP_RESUME_SECONDS // 60} minutes.")
8859
  documents += 1
8860
+ sqp_documents += 1
8861
  _sync_event("amazon.access", tenant_key, SQP_TABLE,
8862
  {"state": "access", "operation": "report_document"}, rt, report, log)
8863
  return _report_document(con, SQP_TABLE, *args, **kwargs, max_wait=max_wait, log=log)
8864
 
8865
+ def pause_sqp(exc):
8866
+ """Stop SQP for this pass only, and leave its tick due after SQP_RESUME_SECONDS."""
8867
+ failed.add(SQP_TABLE)
8868
+ report["tables"][SQP_TABLE] = {"state": "deferred", "reason": str(exc)}
8869
+ due = datetime.now(timezone.utc) + timedelta(seconds=SQP_RESUME_SECONDS)
8870
+ _record_state(con, entity_key("sync_tick_" + SQP_TABLE), "throttled", detail=_iso(due))
8871
+ report["nextDueSeconds"] = min(report.get("nextDueSeconds") or SQP_RESUME_SECONDS, SQP_RESUME_SECONDS)
8872
+
8873
+ def sqp_progress():
8874
+ """Batches done of total per planned month, from the durable month markers."""
8875
+ rows = []
8876
+ for w0, w1 in sqp_months:
8877
+ marker = _sqp_month(con, w0, w1)
8878
+ rows.append({"month": w0.strftime("%Y-%m"), "total": len(batches),
8879
+ "done": sum(batch in marker["batches"] for batch in batches)})
8880
+ done, total = sum(r["done"] for r in rows), sum(r["total"] for r in rows)
8881
+ weekly = sum(not _sqp_done(con, w0, w1, batch) for table, w0, w1, batch in work
8882
+ if table == SQP_TABLE and w0 is not None and _sqp_period(w0, w1) == "WEEK")
8883
+ from harness.amazon_client import REPORT_SUSTAINED_SECONDS
8884
+ sqp_limit["progress"] = {
8885
+ "months": rows, "batchesDone": done, "batchesTotal": total,
8886
+ "monthsDone": sum(r["done"] >= r["total"] for r in rows), "monthsTotal": len(rows),
8887
+ "etaSeconds": int((total - done + weekly) * REPORT_SUSTAINED_SECONDS)} if rows and batches else None
8888
+
8889
  if first_load:
8890
  # Freeze the cohort end across passes. Recomputing it would move every window boundary.
8891
  end = _utc(load["detail"]) if load else _utc(_iso(end))
 
8895
  floors = {table: None if table in undocumented else read_backfill_floor(con, table) for table in pending}
8896
  wanted = [table for table in pending if (table not in completed or table in exhausted) and (not read_watermark(con, table)
8897
  or table in undocumented or read_watermark(con, table) < plans[table]["end"])]
8898
+ planned = first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end, sqp_asins=sqp_asins)
8899
+ for entry in _sqp_after_cohort(planned, end, sqp_asins) if SQP_TABLE in wanted else planned:
8900
  task = entry if len(entry) == 4 else (*entry, None)
8901
  table, w0, w1, batch = task
8902
  if table == SQP_TABLE:
 
8939
  plan = plans[table] = sync_window(table, mark, now=end, since=since,
8940
  intraday=_lookback_intraday(con, table, end, slot_pass=slot_pass))
8941
  if table == SQP_TABLE:
8942
+ # Closed months land newest first, before the latest week's batches.
8943
+ months = list(reversed(_month_windows(plan["start"], plan["end"])))
 
8944
  tasks = [(table, w0, w1, batch) for w0, w1 in months for batch in batches
8945
  if since is not None or not _sqp_done(con, w0, w1, batch)]
8946
  weekly_tasks = [task for task in _sqp_week_tasks(
8947
  _utc(since) if since is not None else end - timedelta(days=7), end, batches)
8948
  if since is not None or not _sqp_done(con, task[1], task[2], task[3])]
8949
+ work.extend([*tasks, *weekly_tasks] if tasks else [*weekly_tasks, (table, None, None, None)])
8950
  else:
8951
  work.append((table, None, None, None))
8952
 
 
8983
  if old_limit.get("weeklyStart") == sqp_limit["weeklyStart"]:
8984
  sqp_limit["weeklyCause"] = old_limit.get("weeklyCause", "")
8985
  plan = plans.get(SQP_TABLE) or sync_window(SQP_TABLE, read_watermark(con, SQP_TABLE), now=end, since=since)
8986
+ # A first load also owes the months that closed after its frozen cohort end.
8987
+ sqp_months = sorted({*_month_windows(plan["start"], plan["end"]),
8988
+ *(_month_windows(plan["end"], datetime.now(timezone.utc)) if first_load else ()),
8989
+ *((w0, w1) for name, w0, w1, _batch in work
8990
+ if name == SQP_TABLE and w0 is not None and _sqp_period(w0, w1) == "MONTH")})
8991
+ sqp_limit["periods"] = [w0.strftime("%Y-%m") for w0, _w1 in sqp_months]
8992
+ sqp_progress()
8993
  sqp_limit["deferredReason"] = (
8994
  "Search Query Performance is waiting for ASINs in the listings or pricing mirror. Sync the catalogue first."
8995
  if not sqp_asins else sqp_limit["partialMonthReason"] if not sqp_limit["periods"] else "")
 
9022
  con, task_start, task_end)["batches"][task_batch].get("refused", False)
9023
  spend_rows(count)
9024
  landed.add((table, task_start, task_end, task_batch))
9025
+ except SQPPassDeferred as exc:
9026
+ pause_sqp(exc)
9027
+ continue
9028
  except SQPBudgetDeferred as exc:
9029
  report["tables"][table] = {"state": "deferred", "reason": str(exc)}
9030
  weekly_deferred = True
 
9219
  written += _sync_sqp_batch(con, fields, task_start, task_end, task_batch,
9220
  sqp_limit, sqp_request, sqp_problem)
9221
  sqp_limit["errors"] = []
9222
+ sqp_progress()
9223
  _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
9224
  elif table in PERIOD_TABLES or table in LEGACY_SNAPSHOT_TABLES:
9225
  operation = cx.REPORT_TYPES[table].get("api") or "report_document"
 
9436
  continue
9437
  if first_load or table == SQP_TABLE:
9438
  landed.add((table, task_start, task_end, task_batch))
9439
+ # SQP weekly tasks never close the table, so the last month task does.
9440
+ if any(task[0] == table and not (table == SQP_TABLE and task[1] is not None
9441
+ and _sqp_period(task[1], task[2]) == "WEEK")
9442
+ for task in work[index + 1:]):
9443
  continue
9444
 
9445
  if table == "pricing":
 
9562
  {"state": "done", "rows": sibling_total, "operations_ok": True},
9563
  rt, report, log)
9564
  log(f" {sibling:<16} {sibling_total:>7,} rows {sibling_columns:>3} cols {sibling_prov}")
9565
+ except SQPPassDeferred as exc:
9566
+ # The SQP share is spent; the other tables in this pass still run.
9567
+ pause_sqp(exc)
9568
+ continue
9569
  except SQPBudgetDeferred as exc:
9570
  report["tables"][table] = {"state": "deferred", "reason": str(exc)}
9571
  break
 
9651
  reason = f"Search Query Performance has {remaining_sqp} month and batch tasks remaining; continuing next pass."
9652
  sqp_limit["deferredReason"] = reason
9653
  report["tables"][SQP_TABLE] = {"state": "deferred", "reason": reason}
9654
+ # Landed batches publish with their month markers; the backlog stays owed in the progress.
9655
+ landed_batches = sum(task[0] == SQP_TABLE and task[1] is not None for task in landed)
9656
+ row = report["tables"].get(SQP_TABLE)
9657
+ if landed_batches and isinstance(row, dict) and row.get("state") in {"deferred", "throttled", "pending"}:
9658
+ report["tables"][SQP_TABLE] = {**row, "state": "deferred", "current": True,
9659
+ "landedBatches": landed_batches}
9660
+ sqp_progress()
9661
  _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
9662
  if first_load:
9663
  refusals = {row["table"]: row for row in _refused_reports(con)}
 
11726
  and json.loads(_read_state(con, SQP_LIMIT_KEY)["detail"])["errors"] == ["Amazon cannot generate this report"])
11727
  finally:
11728
  con.close()
11729
+ _sqp_resume_legs(ck, Runtime, asins[:19])
11730
  return 1 if failures else 0
11731
 
11732
 
11733
+ def _sqp_resume_legs(ck, runtime, asins):
11734
+ """H1: a throttled or capped SQP backlog resumes at its own deadline. No network."""
11735
+ from unittest.mock import patch
11736
+ import duckdb
11737
+ cx, module = _client(), sys.modules[__name__]
11738
+ t0 = _utc("2026-11-14T00:00:30Z")
11739
+ clock, holder, calls = [t0], {}, []
11740
+
11741
+ class Clock(datetime):
11742
+ @classmethod
11743
+ def now(cls, tz=None):
11744
+ return clock[0].astimezone(tz)
11745
+
11746
+ def fresh(watermark=None):
11747
+ con = holder["con"] = duckdb.connect(":memory:")
11748
+ con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, "
11749
+ "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
11750
+ con.execute("CREATE TABLE amzn_listings (asin1 VARCHAR)")
11751
+ con.executemany("INSERT INTO amzn_listings VALUES (?)", [(a,) for a in asins])
11752
+ if watermark:
11753
+ record_watermark(con, SQP_TABLE, _utc(watermark))
11754
+ calls.clear()
11755
+ return con
11756
+
11757
+ throttle_at = [None]
11758
+
11759
+ def document(_fields, _report, **kwargs):
11760
+ calls.append((kwargs["start"][:7], kwargs["options"]["asin"]))
11761
+ if throttle_at[0] is not None and len(calls) - 1 == throttle_at[0]:
11762
+ raise cx.AmazonThrottled("Amazon is limiting requests", retry_after=5)
11763
+ raw = [{"asin": a, "startDate": kwargs["start"][:10], "endDate": kwargs["end"][:10],
11764
+ "searchQueryData": {"searchQuery": "q"}, "purchaseData": {"asinPurchaseCount": 1}}
11765
+ for a in kwargs["options"]["asin"].split()]
11766
+ return json.dumps({"dataByAsin": raw}), {"reportDocumentId": f"h1-{len(calls)}"}
11767
+
11768
+ def run(**kwargs):
11769
+ return _sync_candidate(runtime.key, tables=[SQP_TABLE], rt=runtime(), log=lambda *_: None, **kwargs)
11770
+
11771
+ tick = lambda con: _read_state(con, entity_key("sync_tick_" + SQP_TABLE))
11772
+ month = lambda w0: (w0, (w0.replace(day=28) + timedelta(days=4)).replace(day=1))
11773
+ sep, octo = month(_utc("2026-09-01T00:00:00Z")), month(_utc("2026-10-01T00:00:00Z"))
11774
+ batches = sqp_batches(asins, cx.REPORT_TYPES[SQP_TABLE]["asin_option_max_chars"])
11775
+ con = None
11776
+ try:
11777
+ with patch.object(cx, "access_token", side_effect=AssertionError("fixture attempted LWA")), \
11778
+ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("fixture attempted network")), \
11779
+ patch.object(module, "datetime", Clock), patch.object(module, "DEFAULT_BACKFILL_DAYS", 70), \
11780
+ patch.object(module, "_SCHEDULE_OFFSET_READER", lambda: 0), \
11781
+ patch.object(module, "credential", return_value={"fixture": True}), \
11782
+ patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_: (holder["con"].cursor(), None)), \
11783
+ patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \
11784
+ patch.object(module, "_sync_event"), patch.object(cx, "report_document", side_effect=document), \
11785
+ patch.object(cx, "settlement_reports", return_value=[]) as settlements, \
11786
+ patch.dict(cx.REPORT_TYPES[SQP_TABLE], {"periods": ("MONTH",)}):
11787
+ # A pass throttled at batch k keeps batches < k and resumes at k after the deadline.
11788
+ con = fresh("2026-09-01T00:00:00Z")
11789
+ throttle_at[0] = 2
11790
+ first = run(scheduled=True)
11791
+ saved = tick(con)
11792
+ kept = [_sqp_done(con, *octo, b) for b in batches] + [_sqp_done(con, *sep, batches[0])]
11793
+ holds = _MIRROR_STORE._publication_holds(first, ())
11794
+ row = first["tables"][SQP_TABLE]
11795
+ throttle_at[0] = None
11796
+ floor = _cadence_seconds(SQP_TABLE, saved)
11797
+ waits = [_interval_wait(SQP_TABLE, saved, t0 + timedelta(seconds=s), floor_s=floor)[0] for s in (60, 901)]
11798
+ daily = _interval_wait("storage_fees", saved, t0 + timedelta(seconds=901),
11799
+ floor_s=_cadence_seconds("storage_fees", saved))[0]
11800
+ ck("SQP throttled tick follows its own deadline while another table keeps its daily phase",
11801
+ saved["state"] == "throttled" and _utc(saved["detail"]) == t0 + timedelta(seconds=SQP_RESUME_SECONDS)
11802
+ and 0 < waits[0] <= SQP_RESUME_SECONDS and waits[1] == 0 and daily > SQP_RESUME_SECONDS)
11803
+ before = list(calls)
11804
+ clock[0] = t0 + timedelta(seconds=901)
11805
+ second = run(scheduled=True)
11806
+ resumed = calls[len(before):]
11807
+ ck("SQP throttled scheduled pass resumes at the throttled batch on the next pass, never re-requesting done batches",
11808
+ bool(row.get("reason")) and kept == [True, True, False]
11809
+ and [m for m, _a in before] == ["2026-10", "2026-10", "2026-09"]
11810
+ and resumed == [("2026-09", batches[0]), ("2026-09", batches[1])]
11811
+ and tick(con)["state"] == "done" and read_watermark(con, SQP_TABLE) == _utc("2026-11-01T00:00:00Z")
11812
+ and second["tables"][SQP_TABLE].get("state", "done") == "done")
11813
+ con.close()
11814
+
11815
+ # A pass that landed SQP batches publishes them; a pass that landed none holds SQP.
11816
+ clock[0] = t0
11817
+ con = fresh("2026-09-01T00:00:00Z")
11818
+ throttle_at[0] = 0
11819
+ nothing = run(scheduled=True)
11820
+ throttle_at[0] = None
11821
+ empty_holds = _MIRROR_STORE._publication_holds(nothing, ())
11822
+ ck("SQP landed batches publish with their month markers; a pass that landed none stays held",
11823
+ row.get("state") == "deferred" and row.get("current") is True and row.get("landedBatches") == 2
11824
+ and _mirrors._table_sync_complete("amazon", row) and (holds is None or SQP_TABLE not in holds["held"])
11825
+ and nothing["tables"][SQP_TABLE]["state"] == "throttled"
11826
+ and empty_holds is not None and SQP_TABLE in empty_holds["held"])
11827
+ con.close()
11828
+
11829
+ # A scheduled pass during an unfinished first load leaves SQP on the daily phase, not due every minute.
11830
+ t1 = t0 + timedelta(minutes=30)
11831
+ clock[0] = t1
11832
+ con = fresh()
11833
+ _record_state(con, entity_key(FIRST_LOAD_END_KEY), "running", detail="2026-09-26T00:00:00Z")
11834
+ _record_state(con, entity_key("sync_tick_" + SQP_TABLE), "throttled", detail=_iso(t1 - timedelta(seconds=10)))
11835
+ stale = tick(con)
11836
+ run(scheduled=True)
11837
+ later = t1 + timedelta(seconds=60)
11838
+ ck("SQP throttled tick deferred by an unfinished first load is not due a minute later",
11839
+ sync_plan({SQP_TABLE: stale}, [SQP_TABLE], now=later)["tables"] == [SQP_TABLE]
11840
+ and tick(con)["state"] == "deferred" and tick(con)["detail"] == stale["detail"]
11841
+ and sync_plan({SQP_TABLE: tick(con)}, [SQP_TABLE], now=later)["tables"] == []
11842
+ and _interval_wait(SQP_TABLE, tick(con), later, floor_s=_cadence_seconds(SQP_TABLE, tick(con)))[0] > 0
11843
+ and not calls)
11844
+ con.close()
11845
+ clock[0] = t0
11846
+
11847
+ # A scheduled pass spends at most SQP_PASS_REPORTS SQP documents; other due tables still run.
11848
+ clock[0] = t0
11849
+ with patch.object(module, "SQP_PASS_REPORTS", 10 ** 6):
11850
+ con = fresh("2026-01-01T00:00:00Z")
11851
+ run(scheduled=True)
11852
+ unbounded = len(calls)
11853
+ con.close()
11854
+ con = fresh("2026-01-01T00:00:00Z")
11855
+ settlements.reset_mock()
11856
+ capped = run(scheduled=True)
11857
+ saved = tick(con)
11858
+ ck("SQP scheduled pass stops at SQP_PASS_REPORTS documents and other due tables still run",
11859
+ unbounded == 20 and len(calls) == SQP_PASS_REPORTS
11860
+ and capped["tables"][SQP_TABLE]["state"] == "deferred" and settlements.call_count == 1
11861
+ and capped["tables"]["settlements"]["state"] == "empty"
11862
+ and saved["state"] == "throttled"
11863
+ and _utc(saved["detail"]) == t0 + timedelta(seconds=SQP_RESUME_SECONDS))
11864
+ limit = json.loads(_read_state(con, SQP_LIMIT_KEY)["detail"])
11865
+ progress = limit.get("progress") or {}
11866
+ sentence = sqp_limit_sentence(limit)
11867
+ bare = sqp_limit_sentence({k: v for k, v in limit.items() if k != "progress"})
11868
+ ck("SQP progress sentence names months and batches done of total, without a dash",
11869
+ "of 20 batches" not in bare and progress.get("batchesDone") == 15
11870
+ and progress.get("batchesTotal") == 20 and progress.get("monthsTotal") == 10
11871
+ and progress.get("etaSeconds") == 300 and progress["months"][2] == {"month": "2026-03", "total": 2, "done": 1}
11872
+ and "7 of 10 months complete, 15 of 20 batches." in sentence and "About 5 minutes left" in sentence
11873
+ and len(sentence) - len(bare) < 300 and not any(d in sentence for d in ("\u2013", "\u2014"))
11874
+ and sqp_progress_sentence({}) == "" and sentence.endswith(" " + sqp_progress_sentence(limit))
11875
+ and len(sqp_progress_sentence(limit)) < 140)
11876
+ con.close()
11877
+
11878
+ # A first load with a cohort frozen at 2026-09-26 plans 2026-10 and 2026-09 first.
11879
+ chunk = {}
11880
+
11881
+ def frozen(spliced=True):
11882
+ con = fresh()
11883
+ _record_state(con, entity_key(FIRST_LOAD_END_KEY), "running", detail="2026-09-26T00:00:00Z")
11884
+ if spliced:
11885
+ chunk.update(run(first_load=True, report_budget=4))
11886
+ else:
11887
+ with patch.object(module, "_sqp_after_cohort", side_effect=lambda tasks, *_: tasks):
11888
+ run(first_load=True, report_budget=4)
11889
+ con.close()
11890
+ return [m for m, _a in calls]
11891
+ end = _utc("2026-09-26T00:00:00Z")
11892
+ planned = _sqp_after_cohort(first_load_plan([SQP_TABLE], end - timedelta(days=70), end,
11893
+ sqp_asins=asins), end, asins)
11894
+ red = frozen(spliced=False)
11895
+ ck("SQP first load with a frozen cohort plans the months closed since, newest first",
11896
+ red[:1] == ["2026-08"] and frozen() == ["2026-10", "2026-10", "2026-09", "2026-09"]
11897
+ and [t[1].strftime("%Y-%m") for t in planned[:4]] == ["2026-10", "2026-10", "2026-09", "2026-09"])
11898
+ # A first-load chunk whose only progress is SQP batches publishes them; other tables stay held.
11899
+ waiting = {table_name("orders")}
11900
+ unmarked = {**chunk, "tables": {**chunk["tables"], SQP_TABLE: {
11901
+ k: v for k, v in chunk["tables"][SQP_TABLE].items() if k != "current"}}}
11902
+ chunk_holds = _MIRROR_STORE._publication_holds(chunk, waiting)
11903
+ ck("SQP first-load chunk of only SQP batches publishes them, never nothing",
11904
+ chunk.get("state") == "partial" and _MIRROR_STORE._publication_holds(unmarked, waiting) is None
11905
+ and chunk_holds is not None and SQP_TABLE not in chunk_holds["held"] and "orders" in chunk_holds["held"])
11906
+ con = None
11907
+ finally:
11908
+ if con is not None:
11909
+ con.close()
11910
+
11911
+
11912
  def group_a_selftest(log=print, check=None):
11913
  """S66-T42 red-first fixture controls; memory mirrors and no credentials or network."""
11914
  from unittest.mock import patch
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- b3436af160188b7d0b91b7654aad92979aab79eb3488ced1a43912bae20410d7
 
1
+ 75b681000cae6da4d22bca0114740865caadcf6e1fe3986ad2032e6bd969ebb5
web/src/connectors/ConnectorsPage.tsx CHANGED
@@ -264,6 +264,7 @@ export interface AmazonTableSync {
264
  nextSyncAt: string;
265
  perDay: number;
266
  state: string;
 
267
  }
268
 
269
  export interface AmazonSyncScheduleReading {
@@ -296,7 +297,8 @@ export function parseAmazonSyncSchedule(body: Record<string, unknown>): AmazonSy
296
  const identity = JSON.stringify([row.connectionId, row.table]);
297
  if (identities.has(identity)) return unavailable;
298
  identities.add(identity);
299
- tables.push(row as unknown as AmazonTableSync);
 
300
  }
301
  const state = body.scheduleState as AmazonSyncScheduleReading["state"];
302
  if ((state === "ready") !== (tables.length > 0)) return unavailable;
@@ -341,7 +343,8 @@ export function AmazonSyncSchedule({ reading, problem = "", onRetry }: {
341
  <td className="conn-settings-database-success">
342
  {Date.parse(row.nextSyncAt) <= Date.now() + 60000 ? "Due now"
343
  : <time dateTime={row.nextSyncAt}>{amazonSyncStamp(row.nextSyncAt, reading.timezone)}</time>}
344
- {row.state === "throttled" ? <p role="status">Rate limited, retry scheduled</p>
 
345
  : ["failed", "refused"].includes(row.state) ? <p role="status">Sync needs attention</p> : null}
346
  </td>
347
  <td className="conn-settings-number">{row.perDay}</td>
 
264
  nextSyncAt: string;
265
  perDay: number;
266
  state: string;
267
+ reason?: string;
268
  }
269
 
270
  export interface AmazonSyncScheduleReading {
 
297
  const identity = JSON.stringify([row.connectionId, row.table]);
298
  if (identities.has(identity)) return unavailable;
299
  identities.add(identity);
300
+ const reason = typeof row.reason === "string" ? row.reason.trim() : "";
301
+ tables.push({ ...(row as unknown as AmazonTableSync), reason: reason || undefined });
302
  }
303
  const state = body.scheduleState as AmazonSyncScheduleReading["state"];
304
  if ((state === "ready") !== (tables.length > 0)) return unavailable;
 
343
  <td className="conn-settings-database-success">
344
  {Date.parse(row.nextSyncAt) <= Date.now() + 60000 ? "Due now"
345
  : <time dateTime={row.nextSyncAt}>{amazonSyncStamp(row.nextSyncAt, reading.timezone)}</time>}
346
+ {row.reason && row.state !== "done" ? <p role="status">{row.reason}</p>
347
+ : row.state === "throttled" ? <p role="status">Rate limited, retry scheduled</p>
348
  : ["failed", "refused"].includes(row.state) ? <p role="status">Sync needs attention</p> : null}
349
  </td>
350
  <td className="conn-settings-number">{row.perDay}</td>