Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/main.py +3 -4
- api/routes_connectors.py +116 -4
- api/sync_alerts.py +18 -12
- platform/harness/amazon_store.py +315 -22
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
- web/src/connectors/ConnectorsPage.tsx +5 -2
RELEASES.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v86",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "c3003c1 (c3003c11762e5)",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v86",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
c3003c1 (c3003c11762e5)
|
api/main.py
CHANGED
|
@@ -1057,10 +1057,9 @@ def _publish_connector_status(rt, provider, outcome=None, *, detail="", connecti
|
|
| 1057 |
holds = _az._MIRROR_STORE._publication_holds(report, ()) or {}
|
| 1058 |
extra["notOffered"] = len(holds.get("standing") or ())
|
| 1059 |
extra["notOfferedTables"] = sorted(holds.get("standing") or ())
|
| 1060 |
-
|
| 1061 |
-
|
| 1062 |
-
|
| 1063 |
-
for name in holds.get("held") or {}}
|
| 1064 |
previous = _cs.status(rt, provider, connection_id, fresh=True) or {}
|
| 1065 |
stamps = dict((previous.get("pass") or {}).get("tableSync") or {})
|
| 1066 |
for name, row in (report.get("tables") or {}).items():
|
|
|
|
| 1057 |
holds = _az._MIRROR_STORE._publication_holds(report, ()) or {}
|
| 1058 |
extra["notOffered"] = len(holds.get("standing") or ())
|
| 1059 |
extra["notOfferedTables"] = sorted(holds.get("standing") or ())
|
| 1060 |
+
import routes_connectors as _held_copy
|
| 1061 |
+
# Each held table names its cause: refusal, throttle with its retry time, or the pass end.
|
| 1062 |
+
extra["held"] = _held_copy.amazon_held_entries(report, holds)
|
|
|
|
| 1063 |
previous = _cs.status(rt, provider, connection_id, fresh=True) or {}
|
| 1064 |
stamps = dict((previous.get("pass") or {}).get("tableSync") or {})
|
| 1065 |
for name, row in (report.get("tables") or {}).items():
|
api/routes_connectors.py
CHANGED
|
@@ -3148,6 +3148,78 @@ def _amazon_mirror_completion(rt, *, ads=False):
|
|
| 3148 |
return None, False
|
| 3149 |
|
| 3150 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3151 |
def _amazon_table_schedule(rt, connection_id=None, now=None):
|
| 3152 |
"""Read account cadence metadata without restoring a mirror or reading source rows."""
|
| 3153 |
import duckdb
|
|
@@ -3181,6 +3253,36 @@ def _amazon_table_schedule(rt, connection_id=None, now=None):
|
|
| 3181 |
if (state := markers.get(prefix + az.entity_key("sync_tick_" + table)))}
|
| 3182 |
if any(not state["updated_at"] for state in states.values()):
|
| 3183 |
raise ValueError("Amazon cadence metadata has an invalid timestamp")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3184 |
# C1 owns every due time, including throttle retries and calendar periods.
|
| 3185 |
for row in az.sync_schedule(states, now):
|
| 3186 |
table = row["table"]
|
|
@@ -3189,12 +3291,13 @@ def _amazon_table_schedule(rt, connection_id=None, now=None):
|
|
| 3189 |
landed = markers.get(prefix + az.entity_key(table)) or {}
|
| 3190 |
completed = (tick if tick.get("state") == "done"
|
| 3191 |
and landed.get("state") not in ("refused", "failed", "throttled") else watermark)
|
|
|
|
|
|
|
|
|
|
| 3192 |
schedule.append({**row, "connectionId": cid, "account": entry["label"],
|
| 3193 |
"lastSyncAt": (completed.get("updated_at")
|
| 3194 |
if completed.get("state") == "done" else None),
|
| 3195 |
-
"state": (
|
| 3196 |
-
if landed.get("state") in ("refused", "failed", "throttled")
|
| 3197 |
-
else tick.get("state") or "never_run")})
|
| 3198 |
return schedule
|
| 3199 |
|
| 3200 |
try:
|
|
@@ -5202,6 +5305,15 @@ def _amazon_databases(rt, is_admin, spec=None):
|
|
| 5202 |
if diagnostics:
|
| 5203 |
raw_sync["diagnostics"] = diagnostics
|
| 5204 |
is_enabled = enabled.get(key) is True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5205 |
out.append({"key": key,
|
| 5206 |
"label": str(declared.get("label") or LABELS_AMAZON.get(key) or key),
|
| 5207 |
"defaultLabel": str(declared.get("label")
|
|
@@ -5212,7 +5324,7 @@ def _amazon_databases(rt, is_admin, spec=None):
|
|
| 5212 |
# definition untouched: only this provider config document changes.
|
| 5213 |
"enabled": is_enabled,
|
| 5214 |
**({"sync": raw_sync} if is_admin else {}),
|
| 5215 |
-
"syncHealth":
|
| 5216 |
"fields": fields,
|
| 5217 |
"excluded": excluded,
|
| 5218 |
"sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]})
|
|
|
|
| 3148 |
return None, False
|
| 3149 |
|
| 3150 |
|
| 3151 |
+
#: The second sentence of every held Amazon table's reason.
|
| 3152 |
+
AMAZON_HELD_KEPT = "Its last published rows are kept."
|
| 3153 |
+
|
| 3154 |
+
|
| 3155 |
+
def _amazon_cause_text(text, limit=300):
|
| 3156 |
+
"""Screen copy with no em or en dash, cut at a word near `limit` characters."""
|
| 3157 |
+
text = " ".join(dash_norm.plain(text).split())
|
| 3158 |
+
if len(text) > limit:
|
| 3159 |
+
text = text[:limit].rsplit(" ", 1)[0].rstrip(",;:") + "..."
|
| 3160 |
+
return text
|
| 3161 |
+
|
| 3162 |
+
|
| 3163 |
+
def amazon_hold_cause(row=None, held="", *, table=""):
|
| 3164 |
+
"""One plain cause for an Amazon table a pass did not finish, from its report row or held text."""
|
| 3165 |
+
from harness.amazon_store import plain_refusal_reason
|
| 3166 |
+
row = row if isinstance(row, dict) else {}
|
| 3167 |
+
held = str(held or "")
|
| 3168 |
+
state = str(row.get("state") or "")
|
| 3169 |
+
status = str(row.get("amazonStatus") or "")
|
| 3170 |
+
reason = str(row.get("reason") or "")
|
| 3171 |
+
# The hold vocabulary is `MirrorStore._publication_holds`; the refusal vocabulary is `plain_refusal_reason`.
|
| 3172 |
+
if held.startswith("held with "):
|
| 3173 |
+
partner = held[len("held with "):].split(",", 1)[0].strip().replace("_", " ")
|
| 3174 |
+
cause = f"Held with {partner}, which reads the same report."
|
| 3175 |
+
elif state == "refused" and status.upper() == "NOT_OFFERED":
|
| 3176 |
+
cause = "Amazon does not offer this report to this seller."
|
| 3177 |
+
elif state == "throttled" or (not state and held.startswith("throttled")):
|
| 3178 |
+
cause = "Amazon is limiting requests. The next try is scheduled."
|
| 3179 |
+
elif state == "pending" or (not state and held.startswith("pending")):
|
| 3180 |
+
cause = "Amazon is still preparing this report."
|
| 3181 |
+
elif state == "deferred" or (not row and (not held or held.startswith("not reached"))):
|
| 3182 |
+
cause = "This pass ended before this table's turn. The next pass continues it."
|
| 3183 |
+
elif state == "never_run":
|
| 3184 |
+
# A table with no tick is due only in the minute after its daily slot, which a busy pass misses.
|
| 3185 |
+
cause = "This table has never synced. Its daily start window is too short."
|
| 3186 |
+
elif state in ("attempted", "running"):
|
| 3187 |
+
cause = "The last pass started this table and did not finish it. The next pass continues it."
|
| 3188 |
+
elif state == "refused":
|
| 3189 |
+
plain = plain_refusal_reason(reason or held, status=status, table=table)
|
| 3190 |
+
cause = plain if plain.startswith(("Amazon refused", "Amazon does not offer")) else (
|
| 3191 |
+
"Amazon refused this report: " + plain)
|
| 3192 |
+
else:
|
| 3193 |
+
problem = held[len("a problem names this table: "):] if held.startswith("a problem names this table: ") else ""
|
| 3194 |
+
cause = plain_refusal_reason(reason or problem or held, status=status, table=table)
|
| 3195 |
+
return _amazon_cause_text(cause)
|
| 3196 |
+
|
| 3197 |
+
|
| 3198 |
+
def amazon_held_entries(report, holds, now=None):
|
| 3199 |
+
"""The status document's per-table held entries: the cause, the kept rows sentence, and any retry time."""
|
| 3200 |
+
from datetime import timedelta
|
| 3201 |
+
report = report if isinstance(report, dict) else {}
|
| 3202 |
+
holds = holds if isinstance(holds, dict) else {}
|
| 3203 |
+
rows = report.get("tables") or {}
|
| 3204 |
+
standing = set(holds.get("standing") or ())
|
| 3205 |
+
now = now or datetime.now(timezone.utc)
|
| 3206 |
+
wait = report.get("nextDueSeconds")
|
| 3207 |
+
retry = (now + timedelta(seconds=wait)).isoformat().replace("+00:00", "Z") if (
|
| 3208 |
+
type(wait) in (int, float) and 0 <= wait < 10 ** 7) else None
|
| 3209 |
+
entries = {}
|
| 3210 |
+
for name, held in (holds.get("held") or {}).items():
|
| 3211 |
+
row = rows.get(name) if isinstance(rows.get(name), dict) else {}
|
| 3212 |
+
cause = ("Amazon does not offer this report to this seller." if name in standing
|
| 3213 |
+
else amazon_hold_cause(row, held, table=name))
|
| 3214 |
+
entry = {"reason": f"{cause} {AMAZON_HELD_KEPT}", "cause": cause}
|
| 3215 |
+
if row.get("state") == "throttled" or str(held).startswith("throttled"):
|
| 3216 |
+
entry["nextTryAt"] = row.get("nextDue") or retry
|
| 3217 |
+
elif row.get("nextDue"):
|
| 3218 |
+
entry["nextTryAt"] = row["nextDue"]
|
| 3219 |
+
entries[name] = entry
|
| 3220 |
+
return entries
|
| 3221 |
+
|
| 3222 |
+
|
| 3223 |
def _amazon_table_schedule(rt, connection_id=None, now=None):
|
| 3224 |
"""Read account cadence metadata without restoring a mirror or reading source rows."""
|
| 3225 |
import duckdb
|
|
|
|
| 3253 |
if (state := markers.get(prefix + az.entity_key("sync_tick_" + table)))}
|
| 3254 |
if any(not state["updated_at"] for state in states.values()):
|
| 3255 |
raise ValueError("Amazon cadence metadata has an invalid timestamp")
|
| 3256 |
+
# A row's cause reads the saved refusal and SQP limit markers, never source rows.
|
| 3257 |
+
cause_keys = ([prefix + az.entity_key(family + table) for table in az.TABLES for family in ("", "refused_")]
|
| 3258 |
+
+ [prefix + az.SQP_LIMIT_KEY])
|
| 3259 |
+
saved = dict(cursor.execute("SELECT entity, cursor_wd FROM _sync_state WHERE entity IN (SELECT unnest(?))",
|
| 3260 |
+
[cause_keys]).fetchall())
|
| 3261 |
+
|
| 3262 |
+
def saved_json(key):
|
| 3263 |
+
try:
|
| 3264 |
+
value = json.loads(saved.get(key) or "{}")
|
| 3265 |
+
except (ValueError, TypeError):
|
| 3266 |
+
return {}
|
| 3267 |
+
return value if isinstance(value, dict) else {}
|
| 3268 |
+
|
| 3269 |
+
def row_reason(table, state):
|
| 3270 |
+
# A mirror still restoring has no ticks at all, so never_run names no cause until one loads.
|
| 3271 |
+
if state == "done" or (state == "never_run" and not states):
|
| 3272 |
+
return None
|
| 3273 |
+
refusal = saved_json(prefix + az.entity_key("refused_" + table))
|
| 3274 |
+
parts = [amazon_hold_cause({"state": state, "amazonStatus": refusal.get("amazonStatus") or "",
|
| 3275 |
+
"reason": refusal.get("reason") or saved.get(prefix + az.entity_key(table)) or ""},
|
| 3276 |
+
table=table)]
|
| 3277 |
+
limit = saved_json(prefix + az.SQP_LIMIT_KEY) if table == az.SQP_TABLE else {}
|
| 3278 |
+
if limit and hasattr(az, "sqp_progress_sentence"):
|
| 3279 |
+
# The card carries backfill progress only; the full SQP limit sits in the table description.
|
| 3280 |
+
try:
|
| 3281 |
+
parts.append(_amazon_cause_text(az.sqp_progress_sentence(limit)))
|
| 3282 |
+
except (ValueError, TypeError, KeyError):
|
| 3283 |
+
pass
|
| 3284 |
+
return " ".join(part for part in parts if part) or None
|
| 3285 |
+
|
| 3286 |
# C1 owns every due time, including throttle retries and calendar periods.
|
| 3287 |
for row in az.sync_schedule(states, now):
|
| 3288 |
table = row["table"]
|
|
|
|
| 3291 |
landed = markers.get(prefix + az.entity_key(table)) or {}
|
| 3292 |
completed = (tick if tick.get("state") == "done"
|
| 3293 |
and landed.get("state") not in ("refused", "failed", "throttled") else watermark)
|
| 3294 |
+
state = (landed.get("state") if landed.get("state") in ("refused", "failed", "throttled")
|
| 3295 |
+
else tick.get("state") or "never_run")
|
| 3296 |
+
reason = row_reason(table, state)
|
| 3297 |
schedule.append({**row, "connectionId": cid, "account": entry["label"],
|
| 3298 |
"lastSyncAt": (completed.get("updated_at")
|
| 3299 |
if completed.get("state") == "done" else None),
|
| 3300 |
+
"state": state, **({"reason": reason} if reason else {})})
|
|
|
|
|
|
|
| 3301 |
return schedule
|
| 3302 |
|
| 3303 |
try:
|
|
|
|
| 5305 |
if diagnostics:
|
| 5306 |
raw_sync["diagnostics"] = diagnostics
|
| 5307 |
is_enabled = enabled.get(key) is True
|
| 5308 |
+
# Health reads the cause before a held cause replaces it, so a throttle never reads as a refusal.
|
| 5309 |
+
health = _amazon_database_sync_health(raw_sync, is_enabled, database_key=key)
|
| 5310 |
+
held_cause = str(((raw_sync or {}).get("held") or {}).get("cause") or "") if isinstance(
|
| 5311 |
+
(raw_sync or {}).get("held"), dict) else ""
|
| 5312 |
+
if held_cause:
|
| 5313 |
+
pending = "This table has not loaded yet. Sync it to request rows from Amazon."
|
| 5314 |
+
cause = str(raw_sync.get("unknownCause") or "")
|
| 5315 |
+
raw_sync["unknownCause"] = (cause.replace(pending, held_cause) if pending in cause
|
| 5316 |
+
else cause or held_cause)
|
| 5317 |
out.append({"key": key,
|
| 5318 |
"label": str(declared.get("label") or LABELS_AMAZON.get(key) or key),
|
| 5319 |
"defaultLabel": str(declared.get("label")
|
|
|
|
| 5324 |
# definition untouched: only this provider config document changes.
|
| 5325 |
"enabled": is_enabled,
|
| 5326 |
**({"sync": raw_sync} if is_admin else {}),
|
| 5327 |
+
"syncHealth": health,
|
| 5328 |
"fields": fields,
|
| 5329 |
"excluded": excluded,
|
| 5330 |
"sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]})
|
api/sync_alerts.py
CHANGED
|
@@ -29,8 +29,8 @@ AMAZON_NOT_OFFERED = "Amazon does not offer this report to this seller account"
|
|
| 29 |
AMAZON_ROLE = re.compile(r"has not granted access|access to requested resource is denied", re.I)
|
| 30 |
# A missing role is a standing account fact: it alerts once and never reminds.
|
| 31 |
ONCE_CODES = frozenset({"role"})
|
| 32 |
-
# A throttle alerts only when it repeats on this many passes in a row.
|
| 33 |
-
REPEAT_CODES = {"throttled": 2}
|
| 34 |
|
| 35 |
|
| 36 |
def utc_now():
|
|
@@ -65,8 +65,8 @@ _SCRUB = [
|
|
| 65 |
r"client_secret|password|passwd|pwd|signature|sig)=)[^&\s\"']+"), r"\1[masked]"),
|
| 66 |
(re.compile(r"(?i)\b((?:access_token|refresh_token|client_secret|password|secret|api_key)"
|
| 67 |
r"[\"']?\s*[:=]\s*[\"']?)[^\s,;\"'}]+"), r"\1[masked]"),
|
| 68 |
-
# A long run holding a digit and a letter is a key;
|
| 69 |
-
(re.compile(r"\b(?=[\w\-+/=]*\d)(?=[\w\-+/=]*[A-Za-z])[\w\-+/=]{32,}"), "[token]"),
|
| 70 |
]
|
| 71 |
|
| 72 |
|
|
@@ -229,8 +229,10 @@ def _due(alert, now):
|
|
| 229 |
return None
|
| 230 |
|
| 231 |
|
| 232 |
-
def _reconcile(document, tenant, connector, current, observed, now, claim, quiet=frozenset()):
|
| 233 |
-
"""Fold this pass into the document; return the plan of what this pass must send.
|
|
|
|
|
|
|
| 234 |
document = document if isinstance(document, dict) else {}
|
| 235 |
alerts = document.setdefault("alerts", {})
|
| 236 |
recovered = document.setdefault("recovered", {})
|
|
@@ -251,7 +253,7 @@ def _reconcile(document, tenant, connector, current, observed, now, claim, quiet
|
|
| 251 |
if alert is None and alert_id in recovered:
|
| 252 |
# Failing again before its RECOVERED went out: the owner never heard it cleared.
|
| 253 |
alert = {k: v for k, v in recovered.pop(alert_id).items() if k != "cleared_at"}
|
| 254 |
-
if alert is None:
|
| 255 |
# The same key under a new code is the same failure: it keeps what the owner was told.
|
| 256 |
prior = next((v for a, v in mine.items() if a not in current and v.get("key") == failure["key"]
|
| 257 |
and v.get("sent_at")), None)
|
|
@@ -317,9 +319,11 @@ def _pending(document, tenant, connector, now):
|
|
| 317 |
|
| 318 |
def _plan_of(document, tenant, connector, now, claim):
|
| 319 |
lease = (document.get("leases") or {}).get(_scope(tenant, connector)) or {}
|
| 320 |
-
if lease.get("claim")
|
| 321 |
return None
|
| 322 |
-
|
|
|
|
|
|
|
| 323 |
|
| 324 |
|
| 325 |
def _item(alert):
|
|
@@ -375,11 +379,13 @@ def _mark(document, tenant, connector, plan, now, claim, delivered):
|
|
| 375 |
|
| 376 |
# ---- entry point ----------------------------------------------------------------------------
|
| 377 |
|
| 378 |
-
def notify(tenant, connector, failures, *, observed=None, now=None, sender=None, store=None, quiet=None
|
|
|
|
| 379 |
"""Record this pass's failures and send at most one Telegram line; never raises.
|
| 380 |
|
| 381 |
`failures` is the full current list for the tenant and connector; [] means healthy.
|
| 382 |
`observed` limits clearing to the keys this pass checked; None means the list is complete.
|
|
|
|
| 383 |
"""
|
| 384 |
result = {"ok": False, "sent": False, "failed": [], "reminder": [], "recovered": [], "cause": None}
|
| 385 |
try:
|
|
@@ -396,12 +402,12 @@ def notify(tenant, connector, failures, *, observed=None, now=None, sender=None,
|
|
| 396 |
# Save first, so a delivery marker from a past pass is never re-sent.
|
| 397 |
quiet = frozenset(str(key) for key in (quiet or ()))
|
| 398 |
document = store.update(DOCUMENT, lambda doc: _reconcile(
|
| 399 |
-
doc, tenant, connector, current, observed, now, claim, quiet))
|
| 400 |
plan = _plan_of(document or {}, tenant, connector, now, claim)
|
| 401 |
if plan is None or not any(plan.values()):
|
| 402 |
result.update(ok=True, cause=None if plan is not None else "another pass is sending")
|
| 403 |
return result
|
| 404 |
-
text = compose(tenant, connector, plan, document)
|
| 405 |
try:
|
| 406 |
response = (sender or default_sender)(text)
|
| 407 |
delivered, detail = (response if isinstance(response, tuple) and len(response) == 2
|
|
|
|
| 29 |
AMAZON_ROLE = re.compile(r"has not granted access|access to requested resource is denied", re.I)
|
| 30 |
# A missing role is a standing account fact: it alerts once and never reminds.
|
| 31 |
ONCE_CODES = frozenset({"role"})
|
| 32 |
+
# A throttle or a held table alerts only when it repeats on this many passes in a row.
|
| 33 |
+
REPEAT_CODES = {"throttled": 2, "held": 2}
|
| 34 |
|
| 35 |
|
| 36 |
def utc_now():
|
|
|
|
| 65 |
r"client_secret|password|passwd|pwd|signature|sig)=)[^&\s\"']+"), r"\1[masked]"),
|
| 66 |
(re.compile(r"(?i)\b((?:access_token|refresh_token|client_secret|password|secret|api_key)"
|
| 67 |
r"[\"']?\s*[:=]\s*[\"']?)[^\s,;\"'}]+"), r"\1[masked]"),
|
| 68 |
+
# A long run holding a digit and a letter is a key; snake_case and hashed ut_ table names are not.
|
| 69 |
+
(re.compile(r"\b(?!ut_)(?=[\w\-+/=]*\d)(?=[\w\-+/=]*[A-Za-z])[\w\-+/=]{32,}"), "[token]"),
|
| 70 |
]
|
| 71 |
|
| 72 |
|
|
|
|
| 229 |
return None
|
| 230 |
|
| 231 |
|
| 232 |
+
def _reconcile(document, tenant, connector, current, observed, now, claim, quiet=frozenset(), carry=True):
|
| 233 |
+
"""Fold this pass into the document; return the plan of what this pass must send.
|
| 234 |
+
|
| 235 |
+
`carry=False` sends a key whose code changed as a new failure instead of keeping what was told."""
|
| 236 |
document = document if isinstance(document, dict) else {}
|
| 237 |
alerts = document.setdefault("alerts", {})
|
| 238 |
recovered = document.setdefault("recovered", {})
|
|
|
|
| 253 |
if alert is None and alert_id in recovered:
|
| 254 |
# Failing again before its RECOVERED went out: the owner never heard it cleared.
|
| 255 |
alert = {k: v for k, v in recovered.pop(alert_id).items() if k != "cleared_at"}
|
| 256 |
+
if alert is None and carry:
|
| 257 |
# The same key under a new code is the same failure: it keeps what the owner was told.
|
| 258 |
prior = next((v for a, v in mine.items() if a not in current and v.get("key") == failure["key"]
|
| 259 |
and v.get("sent_at")), None)
|
|
|
|
| 319 |
|
| 320 |
def _plan_of(document, tenant, connector, now, claim):
|
| 321 |
lease = (document.get("leases") or {}).get(_scope(tenant, connector)) or {}
|
| 322 |
+
if lease.get("claim") not in (None, claim):
|
| 323 |
return None
|
| 324 |
+
# No lease means nothing was pending at the save: an empty plan, not another pass sending.
|
| 325 |
+
return _pending(document, tenant, connector, now) if lease.get("claim") == claim else {
|
| 326 |
+
"failed": [], "reminder": [], "recovered": []}
|
| 327 |
|
| 328 |
|
| 329 |
def _item(alert):
|
|
|
|
| 379 |
|
| 380 |
# ---- entry point ----------------------------------------------------------------------------
|
| 381 |
|
| 382 |
+
def notify(tenant, connector, failures, *, observed=None, now=None, sender=None, store=None, quiet=None,
|
| 383 |
+
composer=None, carry=True):
|
| 384 |
"""Record this pass's failures and send at most one Telegram line; never raises.
|
| 385 |
|
| 386 |
`failures` is the full current list for the tenant and connector; [] means healthy.
|
| 387 |
`observed` limits clearing to the keys this pass checked; None means the list is complete.
|
| 388 |
+
`composer(tenant, connector, plan, document)` writes the message for a sender other than Telegram.
|
| 389 |
"""
|
| 390 |
result = {"ok": False, "sent": False, "failed": [], "reminder": [], "recovered": [], "cause": None}
|
| 391 |
try:
|
|
|
|
| 402 |
# Save first, so a delivery marker from a past pass is never re-sent.
|
| 403 |
quiet = frozenset(str(key) for key in (quiet or ()))
|
| 404 |
document = store.update(DOCUMENT, lambda doc: _reconcile(
|
| 405 |
+
doc, tenant, connector, current, observed, now, claim, quiet, carry))
|
| 406 |
plan = _plan_of(document or {}, tenant, connector, now, claim)
|
| 407 |
if plan is None or not any(plan.values()):
|
| 408 |
result.update(ok=True, cause=None if plan is not None else "another pass is sending")
|
| 409 |
return result
|
| 410 |
+
text = (composer or compose)(tenant, connector, plan, document)
|
| 411 |
try:
|
| 412 |
response = (sender or default_sender)(text)
|
| 413 |
delivered, detail = (response if isinstance(response, tuple) and len(response) == 2
|
platform/harness/amazon_store.py
CHANGED
|
@@ -79,6 +79,10 @@ TABLES = ("orders", "order_items", "listings", "settlements", "inventory",
|
|
| 79 |
MARGIN_TABLES = ("economics", "transactions", "fba_fee_estimates", "referral_fees", "storage_fees")
|
| 80 |
SQP_TABLE = "search_query_performance"
|
| 81 |
SQP_LIMIT_KEY = "amzn_limit_search_query_performance"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
GROUP_A_TABLES = tuple(amazon_schema.GROUP_A_TABLES)
|
| 83 |
TABLES += GROUP_A_TABLES
|
| 84 |
DERIVED_TABLES = ("sku_keywords", "keyword_ranks")
|
|
@@ -613,6 +617,9 @@ def _interval_wait(table, state, now, floor_s=None):
|
|
| 613 |
if state and state.get("state") == "throttled" and state.get("detail"):
|
| 614 |
due = _utc(state["detail"])
|
| 615 |
wait = (due - now).total_seconds()
|
|
|
|
|
|
|
|
|
|
| 616 |
if wait > 0:
|
| 617 |
if not phased:
|
| 618 |
return wait, due, int(wait), False
|
|
@@ -678,6 +685,8 @@ FIRST_LOAD_RECENT_DAYS = MAX_WINDOW_DAYS
|
|
| 678 |
#: about 2.7 h. A first load therefore runs in BOUNDED passes, or it would hold the resident
|
| 679 |
#: thread for hours and the Odoo resync behind it would never run.
|
| 680 |
FIRST_LOAD_REPORTS_PER_PASS = 15
|
|
|
|
|
|
|
| 681 |
#: Rows one first-load pass writes before it takes no further document. A pacing bound, never a cap:
|
| 682 |
#: the pass publishes and the next continues, so Brand Analytics months (3.7M rows and about 25
|
| 683 |
#: minutes each) never hold the sync desk for hours (D-7081).
|
|
@@ -2513,8 +2522,27 @@ def sqp_limit_sentence(marker):
|
|
| 2513 |
if any(type(n) is not int or n < 0 for n in numbers) or not all(numbers[:2]):
|
| 2514 |
raise ValueError("Invalid saved Search Query Performance limits")
|
| 2515 |
chars, maximum, count, batches = numbers
|
| 2516 |
-
|
| 2517 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2518 |
|
| 2519 |
|
| 2520 |
def sqp_refusal_message(asin, reason):
|
|
@@ -2561,6 +2589,10 @@ class SQPBudgetDeferred(RuntimeError):
|
|
| 2561 |
"""A batch or its single retry remains durable for the next bounded pass."""
|
| 2562 |
|
| 2563 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2564 |
def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem):
|
| 2565 |
"""One completed period and batch using the shared report transport and pass budget.
|
| 2566 |
|
|
@@ -2701,10 +2733,13 @@ def refresh_sku_keywords(con):
|
|
| 2701 |
raw_meta = (', "_connection_id"' if amazon_schema.CONNECTION_ID in raw_columns else
|
| 2702 |
', NULL::VARCHAR AS "_connection_id"') if scoped else ""
|
| 2703 |
mapping_join = ' AND m."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else ""
|
| 2704 |
-
|
|
|
|
|
|
|
| 2705 |
count_join = ' AND c."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else ""
|
| 2706 |
grain_meta = ', "_connection_id"' if scoped else ""
|
| 2707 |
-
matched_meta = ', l.store' if scoped else ""
|
|
|
|
| 2708 |
output_meta = ', "_connection_id", store' if scoped else ""
|
| 2709 |
backend_meta = ', l."_connection_id", l.store' if scoped else ""
|
| 2710 |
backend_match = ' AND m."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else ""
|
|
@@ -2738,7 +2773,7 @@ def refresh_sku_keywords(con):
|
|
| 2738 |
SELECT *, CAST(to_json([start_date, end_date, asin, search_query{grain_meta}]) AS VARCHAR) AS source_grain
|
| 2739 |
FROM raw
|
| 2740 |
), matched AS (
|
| 2741 |
-
SELECT l.sku,
|
| 2742 |
FROM listings l JOIN mapping m ON m.sku=l.sku{mapping_join}
|
| 2743 |
JOIN queries q ON q.asin=m.asin{query_join} JOIN mapped_counts c ON c.asin=m.asin{count_join}
|
| 2744 |
)
|
|
@@ -2944,6 +2979,15 @@ def first_load_plan(tables, start, end, *, recent_days=None, size=MAX_WINDOW_DAY
|
|
| 2944 |
return recent + history
|
| 2945 |
|
| 2946 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2947 |
def first_load_eta(report_count, *, stagger_seconds=None, burst=None, per_minute=None):
|
| 2948 |
"""Amazon createReport pacing floor in seconds; polling and other work can add time."""
|
| 2949 |
from harness.amazon_client import REPORT_STAGGER_SECONDS, REPORT_BURST, REPORT_SUSTAINED_SECONDS
|
|
@@ -3874,9 +3918,10 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
|
|
| 3874 |
"""
|
| 3875 |
rows = report.get("tables") or {}
|
| 3876 |
problems = [str(problem) for key in ("problems", "audit_problems") for problem in report.get(key) or []]
|
|
|
|
| 3877 |
if report.get("state") in {"partial", "failed"} and not any(
|
| 3878 |
-
isinstance(row, dict) and row.get("state") != "deferred"
|
| 3879 |
-
and _mirrors._table_sync_complete("amazon", row) for row in rows.
|
| 3880 |
return None
|
| 3881 |
unfinished = {name[len(PREFIX):] if name.startswith(PREFIX) else name for name in waiting}
|
| 3882 |
unfinished |= {name for name, row in rows.items() if isinstance(row, dict) and row.get("state") != "disabled"
|
|
@@ -5506,8 +5551,8 @@ def first_load_job_plan(tenant_key, rt, *, log=print):
|
|
| 5506 |
for entry in entries:
|
| 5507 |
cid = entry["connection_id"]
|
| 5508 |
fields = fields_by[cid] = _sync_credential(tenant_key, rt, connection_id=cid)
|
| 5509 |
-
|
| 5510 |
-
|
| 5511 |
table, w0, w1 = item[:3]
|
| 5512 |
batch = item[3] if len(item) == 4 else None
|
| 5513 |
# Account-scoped floors live in each account's workspace; only one account reads them here.
|
|
@@ -6317,15 +6362,24 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon
|
|
| 6317 |
if not waiting_on and now() >= request_until:
|
| 6318 |
break
|
| 6319 |
sleep(poll_seconds if waiting_on else max(0.0, min(poll_seconds, next_token - now())))
|
| 6320 |
-
|
| 6321 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6322 |
try:
|
| 6323 |
-
delegate(own_tables,
|
| 6324 |
record["otherTablesPass"] = "done"
|
| 6325 |
except Exception as exc: # noqa: BLE001
|
| 6326 |
record["otherTablesPass"] = f"failed ({type(exc).__name__})"
|
| 6327 |
log(f"Amazon first-load pass for the other tables failed ({type(exc).__name__}); "
|
| 6328 |
"the next generation runs it again.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6329 |
try:
|
| 6330 |
after = plan_fn()
|
| 6331 |
except Exception: # noqa: BLE001
|
|
@@ -6539,7 +6593,9 @@ def _save_throttled_resume(con, table, exc, plan, completed, *, first_load, sinc
|
|
| 6539 |
"completed": completed, "first_load": first_load,
|
| 6540 |
"since": _iso(_utc(since)) if since is not None else None}
|
| 6541 |
_record_state(con, entity_key("resume_" + table), "throttled", detail=json.dumps(continuation))
|
| 6542 |
-
|
|
|
|
|
|
|
| 6543 |
_record_state(con, entity_key("sync_tick_" + table), "throttled", detail=_iso(due))
|
| 6544 |
return max(0, (due - datetime.now(timezone.utc)).total_seconds())
|
| 6545 |
|
|
@@ -8679,6 +8735,10 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8679 |
continue
|
| 8680 |
report["tables"][table] = {"state": "deferred", "reason":
|
| 8681 |
"The first load is continuing in bounded passes on the resident sync thread."}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8682 |
want = [table for table in want if (read_watermark(con, table)
|
| 8683 |
and (not refresh_loaded or table not in repairs))
|
| 8684 |
or table in completed
|
|
@@ -8787,15 +8847,45 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8787 |
report["problems"].append(message)
|
| 8788 |
log(message)
|
| 8789 |
|
|
|
|
|
|
|
| 8790 |
def sqp_request(*args, **kwargs):
|
| 8791 |
-
nonlocal documents
|
| 8792 |
if isinstance(report_budget, int) and documents >= max(0, report_budget):
|
| 8793 |
raise SQPBudgetDeferred("Search Query Performance report budget reached; remaining batches continue next pass.")
|
|
|
|
|
|
|
|
|
|
| 8794 |
documents += 1
|
|
|
|
| 8795 |
_sync_event("amazon.access", tenant_key, SQP_TABLE,
|
| 8796 |
{"state": "access", "operation": "report_document"}, rt, report, log)
|
| 8797 |
return _report_document(con, SQP_TABLE, *args, **kwargs, max_wait=max_wait, log=log)
|
| 8798 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8799 |
if first_load:
|
| 8800 |
# Freeze the cohort end across passes. Recomputing it would move every window boundary.
|
| 8801 |
end = _utc(load["detail"]) if load else _utc(_iso(end))
|
|
@@ -8805,8 +8895,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8805 |
floors = {table: None if table in undocumented else read_backfill_floor(con, table) for table in pending}
|
| 8806 |
wanted = [table for table in pending if (table not in completed or table in exhausted) and (not read_watermark(con, table)
|
| 8807 |
or table in undocumented or read_watermark(con, table) < plans[table]["end"])]
|
| 8808 |
-
|
| 8809 |
-
|
| 8810 |
task = entry if len(entry) == 4 else (*entry, None)
|
| 8811 |
table, w0, w1, batch = task
|
| 8812 |
if table == SQP_TABLE:
|
|
@@ -8849,15 +8939,14 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8849 |
plan = plans[table] = sync_window(table, mark, now=end, since=since,
|
| 8850 |
intraday=_lookback_intraday(con, table, end, slot_pass=slot_pass))
|
| 8851 |
if table == SQP_TABLE:
|
| 8852 |
-
months
|
| 8853 |
-
|
| 8854 |
-
months.reverse()
|
| 8855 |
tasks = [(table, w0, w1, batch) for w0, w1 in months for batch in batches
|
| 8856 |
if since is not None or not _sqp_done(con, w0, w1, batch)]
|
| 8857 |
weekly_tasks = [task for task in _sqp_week_tasks(
|
| 8858 |
_utc(since) if since is not None else end - timedelta(days=7), end, batches)
|
| 8859 |
if since is not None or not _sqp_done(con, task[1], task[2], task[3])]
|
| 8860 |
-
work.extend([*
|
| 8861 |
else:
|
| 8862 |
work.append((table, None, None, None))
|
| 8863 |
|
|
@@ -8894,7 +8983,13 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8894 |
if old_limit.get("weeklyStart") == sqp_limit["weeklyStart"]:
|
| 8895 |
sqp_limit["weeklyCause"] = old_limit.get("weeklyCause", "")
|
| 8896 |
plan = plans.get(SQP_TABLE) or sync_window(SQP_TABLE, read_watermark(con, SQP_TABLE), now=end, since=since)
|
| 8897 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8898 |
sqp_limit["deferredReason"] = (
|
| 8899 |
"Search Query Performance is waiting for ASINs in the listings or pricing mirror. Sync the catalogue first."
|
| 8900 |
if not sqp_asins else sqp_limit["partialMonthReason"] if not sqp_limit["periods"] else "")
|
|
@@ -8927,6 +9022,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 8927 |
con, task_start, task_end)["batches"][task_batch].get("refused", False)
|
| 8928 |
spend_rows(count)
|
| 8929 |
landed.add((table, task_start, task_end, task_batch))
|
|
|
|
|
|
|
|
|
|
| 8930 |
except SQPBudgetDeferred as exc:
|
| 8931 |
report["tables"][table] = {"state": "deferred", "reason": str(exc)}
|
| 8932 |
weekly_deferred = True
|
|
@@ -9121,6 +9219,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9121 |
written += _sync_sqp_batch(con, fields, task_start, task_end, task_batch,
|
| 9122 |
sqp_limit, sqp_request, sqp_problem)
|
| 9123 |
sqp_limit["errors"] = []
|
|
|
|
| 9124 |
_record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
|
| 9125 |
elif table in PERIOD_TABLES or table in LEGACY_SNAPSHOT_TABLES:
|
| 9126 |
operation = cx.REPORT_TYPES[table].get("api") or "report_document"
|
|
@@ -9337,7 +9436,10 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9337 |
continue
|
| 9338 |
if first_load or table == SQP_TABLE:
|
| 9339 |
landed.add((table, task_start, task_end, task_batch))
|
| 9340 |
-
|
|
|
|
|
|
|
|
|
|
| 9341 |
continue
|
| 9342 |
|
| 9343 |
if table == "pricing":
|
|
@@ -9460,6 +9562,10 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9460 |
{"state": "done", "rows": sibling_total, "operations_ok": True},
|
| 9461 |
rt, report, log)
|
| 9462 |
log(f" {sibling:<16} {sibling_total:>7,} rows {sibling_columns:>3} cols {sibling_prov}")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9463 |
except SQPBudgetDeferred as exc:
|
| 9464 |
report["tables"][table] = {"state": "deferred", "reason": str(exc)}
|
| 9465 |
break
|
|
@@ -9545,6 +9651,13 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9545 |
reason = f"Search Query Performance has {remaining_sqp} month and batch tasks remaining; continuing next pass."
|
| 9546 |
sqp_limit["deferredReason"] = reason
|
| 9547 |
report["tables"][SQP_TABLE] = {"state": "deferred", "reason": reason}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9548 |
_record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
|
| 9549 |
if first_load:
|
| 9550 |
refusals = {row["table"]: row for row in _refused_reports(con)}
|
|
@@ -11613,9 +11726,189 @@ def sqp_selftest(log=print, check=None):
|
|
| 11613 |
and json.loads(_read_state(con, SQP_LIMIT_KEY)["detail"])["errors"] == ["Amazon cannot generate this report"])
|
| 11614 |
finally:
|
| 11615 |
con.close()
|
|
|
|
| 11616 |
return 1 if failures else 0
|
| 11617 |
|
| 11618 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11619 |
def group_a_selftest(log=print, check=None):
|
| 11620 |
"""S66-T42 red-first fixture controls; memory mirrors and no credentials or network."""
|
| 11621 |
from unittest.mock import patch
|
|
|
|
| 79 |
MARGIN_TABLES = ("economics", "transactions", "fba_fee_estimates", "referral_fees", "storage_fees")
|
| 80 |
SQP_TABLE = "search_query_performance"
|
| 81 |
SQP_LIMIT_KEY = "amzn_limit_search_query_performance"
|
| 82 |
+
#: A scheduled pass spends at most Amazon's createReport burst on SQP documents.
|
| 83 |
+
SQP_PASS_REPORTS = 15
|
| 84 |
+
#: An SQP backlog resumes this long after its pass stops: 15 reports per 15 minutes is Amazon's sustained rate.
|
| 85 |
+
SQP_RESUME_SECONDS = 900
|
| 86 |
GROUP_A_TABLES = tuple(amazon_schema.GROUP_A_TABLES)
|
| 87 |
TABLES += GROUP_A_TABLES
|
| 88 |
DERIVED_TABLES = ("sku_keywords", "keyword_ranks")
|
|
|
|
| 617 |
if state and state.get("state") == "throttled" and state.get("detail"):
|
| 618 |
due = _utc(state["detail"])
|
| 619 |
wait = (due - now).total_seconds()
|
| 620 |
+
if table == SQP_TABLE:
|
| 621 |
+
# An SQP backlog resumes at its own deadline, never on the daily phase.
|
| 622 |
+
return (wait, due, int(wait), False) if wait > 0 else (0, None, seconds, False)
|
| 623 |
if wait > 0:
|
| 624 |
if not phased:
|
| 625 |
return wait, due, int(wait), False
|
|
|
|
| 685 |
#: about 2.7 h. A first load therefore runs in BOUNDED passes, or it would hold the resident
|
| 686 |
#: thread for hours and the Odoo resync behind it would never run.
|
| 687 |
FIRST_LOAD_REPORTS_PER_PASS = 15
|
| 688 |
+
#: A generation's repeated other-tables pass: about an hour of reports at Amazon's rate, then it publishes.
|
| 689 |
+
FIRST_LOAD_DELEGATE_REPORTS = 60
|
| 690 |
#: Rows one first-load pass writes before it takes no further document. A pacing bound, never a cap:
|
| 691 |
#: the pass publishes and the next continues, so Brand Analytics months (3.7M rows and about 25
|
| 692 |
#: minutes each) never hold the sync desk for hours (D-7081).
|
|
|
|
| 2522 |
if any(type(n) is not int or n < 0 for n in numbers) or not all(numbers[:2]):
|
| 2523 |
raise ValueError("Invalid saved Search Query Performance limits")
|
| 2524 |
chars, maximum, count, batches = numbers
|
| 2525 |
+
sentence = (f"Amazon accepts at most {chars} characters of ASINs per Search Query Performance "
|
| 2526 |
+
f"report, about {maximum} ASINs, so {count} ASINs are requested in {batches} reports per month.")
|
| 2527 |
+
progress = sqp_progress_sentence(marker)
|
| 2528 |
+
return f"{sentence} {progress}" if progress else sentence
|
| 2529 |
+
|
| 2530 |
+
|
| 2531 |
+
def sqp_progress_sentence(detail):
|
| 2532 |
+
"""Batches done of total and the time left, from saved SQP progress; "" without progress."""
|
| 2533 |
+
progress = detail.get("progress")
|
| 2534 |
+
if not progress:
|
| 2535 |
+
return ""
|
| 2536 |
+
steps = [progress.get(k) for k in ("monthsDone", "monthsTotal", "batchesDone", "batchesTotal", "etaSeconds")]
|
| 2537 |
+
if any(type(n) is not int or n < 0 for n in steps):
|
| 2538 |
+
raise ValueError("Invalid saved Search Query Performance progress")
|
| 2539 |
+
months_done, months, done, total, eta = steps
|
| 2540 |
+
sentence = f"{months_done:,} of {months:,} months complete, {done:,} of {total:,} batches."
|
| 2541 |
+
if eta:
|
| 2542 |
+
hours, minutes = round(eta / 3600), max(1, round(eta / 60))
|
| 2543 |
+
left = (f"{hours:,} hours" if hours >= 2 else "1 minute" if minutes == 1 else f"{minutes} minutes")
|
| 2544 |
+
sentence += f" About {left} left at Amazon's rate of one report a minute."
|
| 2545 |
+
return sentence
|
| 2546 |
|
| 2547 |
|
| 2548 |
def sqp_refusal_message(asin, reason):
|
|
|
|
| 2589 |
"""A batch or its single retry remains durable for the next bounded pass."""
|
| 2590 |
|
| 2591 |
|
| 2592 |
+
class SQPPassDeferred(SQPBudgetDeferred):
|
| 2593 |
+
"""A scheduled pass spent `SQP_PASS_REPORTS`; only SQP stops, other tables keep running."""
|
| 2594 |
+
|
| 2595 |
+
|
| 2596 |
def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem):
|
| 2597 |
"""One completed period and batch using the shared report transport and pass budget.
|
| 2598 |
|
|
|
|
| 2733 |
raw_meta = (', "_connection_id"' if amazon_schema.CONNECTION_ID in raw_columns else
|
| 2734 |
', NULL::VARCHAR AS "_connection_id"') if scoped else ""
|
| 2735 |
mapping_join = ' AND m."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else ""
|
| 2736 |
+
# An SQP row saved before account tagging has no account; it belongs to a single-account tenant's listings.
|
| 2737 |
+
query_join = (' AND (q."_connection_id" IS NOT DISTINCT FROM l."_connection_id" OR (q."_connection_id" IS NULL'
|
| 2738 |
+
' AND (SELECT COUNT(DISTINCT "_connection_id") FROM listings) = 1))') if scoped else ""
|
| 2739 |
count_join = ' AND c."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else ""
|
| 2740 |
grain_meta = ', "_connection_id"' if scoped else ""
|
| 2741 |
+
matched_meta = ', l."_connection_id", l.store' if scoped else ""
|
| 2742 |
+
query_columns = 'q.* EXCLUDE ("_connection_id")' if scoped else "q.*"
|
| 2743 |
output_meta = ', "_connection_id", store' if scoped else ""
|
| 2744 |
backend_meta = ', l."_connection_id", l.store' if scoped else ""
|
| 2745 |
backend_match = ' AND m."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else ""
|
|
|
|
| 2773 |
SELECT *, CAST(to_json([start_date, end_date, asin, search_query{grain_meta}]) AS VARCHAR) AS source_grain
|
| 2774 |
FROM raw
|
| 2775 |
), matched AS (
|
| 2776 |
+
SELECT l.sku, {query_columns}, l.backend_keywords, c.mapped_sku_count{matched_meta}
|
| 2777 |
FROM listings l JOIN mapping m ON m.sku=l.sku{mapping_join}
|
| 2778 |
JOIN queries q ON q.asin=m.asin{query_join} JOIN mapped_counts c ON c.asin=m.asin{count_join}
|
| 2779 |
)
|
|
|
|
| 2979 |
return recent + history
|
| 2980 |
|
| 2981 |
|
| 2982 |
+
def _sqp_after_cohort(tasks, end, asins):
|
| 2983 |
+
"""Insert SQP months that closed after the frozen cohort end, newest first, before older SQP tasks."""
|
| 2984 |
+
spec = _client().REPORT_TYPES[SQP_TABLE]
|
| 2985 |
+
later = [(SQP_TABLE, w0, w1, batch) for w0, w1 in reversed(_month_windows(end, datetime.now(timezone.utc)))
|
| 2986 |
+
for batch in sqp_batches(asins, spec["asin_option_max_chars"])]
|
| 2987 |
+
at = next((i for i, task in enumerate(tasks) if task[0] == SQP_TABLE), len(tasks))
|
| 2988 |
+
return [*tasks[:at], *later, *tasks[at:]]
|
| 2989 |
+
|
| 2990 |
+
|
| 2991 |
def first_load_eta(report_count, *, stagger_seconds=None, burst=None, per_minute=None):
|
| 2992 |
"""Amazon createReport pacing floor in seconds; polling and other work can add time."""
|
| 2993 |
from harness.amazon_client import REPORT_STAGGER_SECONDS, REPORT_BURST, REPORT_SUSTAINED_SECONDS
|
|
|
|
| 3918 |
"""
|
| 3919 |
rows = report.get("tables") or {}
|
| 3920 |
problems = [str(problem) for key in ("problems", "audit_problems") for problem in report.get(key) or []]
|
| 3921 |
+
# SQP batches that landed count as finished work, so a chunk of only SQP progress still publishes.
|
| 3922 |
if report.get("state") in {"partial", "failed"} and not any(
|
| 3923 |
+
isinstance(row, dict) and (row.get("state") != "deferred" or name == SQP_TABLE)
|
| 3924 |
+
and _mirrors._table_sync_complete("amazon", row) for name, row in rows.items()):
|
| 3925 |
return None
|
| 3926 |
unfinished = {name[len(PREFIX):] if name.startswith(PREFIX) else name for name in waiting}
|
| 3927 |
unfinished |= {name for name, row in rows.items() if isinstance(row, dict) and row.get("state") != "disabled"
|
|
|
|
| 5551 |
for entry in entries:
|
| 5552 |
cid = entry["connection_id"]
|
| 5553 |
fields = fields_by[cid] = _sync_credential(tenant_key, rt, connection_id=cid)
|
| 5554 |
+
items = first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end, sqp_asins=sqp_asins)
|
| 5555 |
+
for item in _sqp_after_cohort(items, end, sqp_asins) if SQP_TABLE in wanted else items:
|
| 5556 |
table, w0, w1 = item[:3]
|
| 5557 |
batch = item[3] if len(item) == 4 else None
|
| 5558 |
# Account-scoped floors live in each account's workspace; only one account reads them here.
|
|
|
|
| 6362 |
if not waiting_on and now() >= request_until:
|
| 6363 |
break
|
| 6364 |
sleep(poll_seconds if waiting_on else max(0.0, min(poll_seconds, next_token - now())))
|
| 6365 |
+
record["otherTablesPasses"] = 0
|
| 6366 |
+
while now() < request_until:
|
| 6367 |
+
# After the job's own reports. Each pass publishes; another follows while request time remains,
|
| 6368 |
+
# sized to that time at Amazon's one report a minute, so SQP history moves at Amazon's rate.
|
| 6369 |
+
budget = max(FIRST_LOAD_REPORTS_PER_PASS,
|
| 6370 |
+
min(FIRST_LOAD_DELEGATE_REPORTS, int((request_until - now()) // 60)))
|
| 6371 |
try:
|
| 6372 |
+
result = delegate(own_tables, budget)
|
| 6373 |
record["otherTablesPass"] = "done"
|
| 6374 |
except Exception as exc: # noqa: BLE001
|
| 6375 |
record["otherTablesPass"] = f"failed ({type(exc).__name__})"
|
| 6376 |
log(f"Amazon first-load pass for the other tables failed ({type(exc).__name__}); "
|
| 6377 |
"the next generation runs it again.")
|
| 6378 |
+
break
|
| 6379 |
+
record["otherTablesPasses"] += 1
|
| 6380 |
+
result = result if isinstance(result, dict) else {}
|
| 6381 |
+
if result.get("state") != "partial" or not result.get("landedTasks"):
|
| 6382 |
+
break
|
| 6383 |
try:
|
| 6384 |
after = plan_fn()
|
| 6385 |
except Exception: # noqa: BLE001
|
|
|
|
| 6593 |
"completed": completed, "first_load": first_load,
|
| 6594 |
"since": _iso(_utc(since)) if since is not None else None}
|
| 6595 |
_record_state(con, entity_key("resume_" + table), "throttled", detail=json.dumps(continuation))
|
| 6596 |
+
# A scheduled SQP backlog waits for Amazon's burst to refill, never less than SQP_RESUME_SECONDS.
|
| 6597 |
+
least = SQP_RESUME_SECONDS if table == SQP_TABLE and not first_load else 60.0
|
| 6598 |
+
due = datetime.now(timezone.utc) + timedelta(seconds=max(float(least), float(exc.retry_after or 60)))
|
| 6599 |
_record_state(con, entity_key("sync_tick_" + table), "throttled", detail=_iso(due))
|
| 6600 |
return max(0, (due - datetime.now(timezone.utc)).total_seconds())
|
| 6601 |
|
|
|
|
| 8735 |
continue
|
| 8736 |
report["tables"][table] = {"state": "deferred", "reason":
|
| 8737 |
"The first load is continuing in bounded passes on the resident sync thread."}
|
| 8738 |
+
tick = _read_state(con, entity_key("sync_tick_" + table)) if table == SQP_TABLE else None
|
| 8739 |
+
if tick and tick.get("state") == "throttled":
|
| 8740 |
+
# The first load owns SQP now; a passed SQP deadline must not make it due every minute.
|
| 8741 |
+
_record_state(con, entity_key("sync_tick_" + table), "deferred", tick.get("rows") or 0, tick.get("detail"))
|
| 8742 |
want = [table for table in want if (read_watermark(con, table)
|
| 8743 |
and (not refresh_loaded or table not in repairs))
|
| 8744 |
or table in completed
|
|
|
|
| 8847 |
report["problems"].append(message)
|
| 8848 |
log(message)
|
| 8849 |
|
| 8850 |
+
sqp_documents = 0
|
| 8851 |
+
|
| 8852 |
def sqp_request(*args, **kwargs):
|
| 8853 |
+
nonlocal documents, sqp_documents
|
| 8854 |
if isinstance(report_budget, int) and documents >= max(0, report_budget):
|
| 8855 |
raise SQPBudgetDeferred("Search Query Performance report budget reached; remaining batches continue next pass.")
|
| 8856 |
+
if scheduled and not first_load and sqp_documents >= SQP_PASS_REPORTS:
|
| 8857 |
+
raise SQPPassDeferred(f"Search Query Performance requested {SQP_PASS_REPORTS} reports this pass; "
|
| 8858 |
+
f"remaining batches continue in {SQP_RESUME_SECONDS // 60} minutes.")
|
| 8859 |
documents += 1
|
| 8860 |
+
sqp_documents += 1
|
| 8861 |
_sync_event("amazon.access", tenant_key, SQP_TABLE,
|
| 8862 |
{"state": "access", "operation": "report_document"}, rt, report, log)
|
| 8863 |
return _report_document(con, SQP_TABLE, *args, **kwargs, max_wait=max_wait, log=log)
|
| 8864 |
|
| 8865 |
+
def pause_sqp(exc):
|
| 8866 |
+
"""Stop SQP for this pass only, and leave its tick due after SQP_RESUME_SECONDS."""
|
| 8867 |
+
failed.add(SQP_TABLE)
|
| 8868 |
+
report["tables"][SQP_TABLE] = {"state": "deferred", "reason": str(exc)}
|
| 8869 |
+
due = datetime.now(timezone.utc) + timedelta(seconds=SQP_RESUME_SECONDS)
|
| 8870 |
+
_record_state(con, entity_key("sync_tick_" + SQP_TABLE), "throttled", detail=_iso(due))
|
| 8871 |
+
report["nextDueSeconds"] = min(report.get("nextDueSeconds") or SQP_RESUME_SECONDS, SQP_RESUME_SECONDS)
|
| 8872 |
+
|
| 8873 |
+
def sqp_progress():
|
| 8874 |
+
"""Batches done of total per planned month, from the durable month markers."""
|
| 8875 |
+
rows = []
|
| 8876 |
+
for w0, w1 in sqp_months:
|
| 8877 |
+
marker = _sqp_month(con, w0, w1)
|
| 8878 |
+
rows.append({"month": w0.strftime("%Y-%m"), "total": len(batches),
|
| 8879 |
+
"done": sum(batch in marker["batches"] for batch in batches)})
|
| 8880 |
+
done, total = sum(r["done"] for r in rows), sum(r["total"] for r in rows)
|
| 8881 |
+
weekly = sum(not _sqp_done(con, w0, w1, batch) for table, w0, w1, batch in work
|
| 8882 |
+
if table == SQP_TABLE and w0 is not None and _sqp_period(w0, w1) == "WEEK")
|
| 8883 |
+
from harness.amazon_client import REPORT_SUSTAINED_SECONDS
|
| 8884 |
+
sqp_limit["progress"] = {
|
| 8885 |
+
"months": rows, "batchesDone": done, "batchesTotal": total,
|
| 8886 |
+
"monthsDone": sum(r["done"] >= r["total"] for r in rows), "monthsTotal": len(rows),
|
| 8887 |
+
"etaSeconds": int((total - done + weekly) * REPORT_SUSTAINED_SECONDS)} if rows and batches else None
|
| 8888 |
+
|
| 8889 |
if first_load:
|
| 8890 |
# Freeze the cohort end across passes. Recomputing it would move every window boundary.
|
| 8891 |
end = _utc(load["detail"]) if load else _utc(_iso(end))
|
|
|
|
| 8895 |
floors = {table: None if table in undocumented else read_backfill_floor(con, table) for table in pending}
|
| 8896 |
wanted = [table for table in pending if (table not in completed or table in exhausted) and (not read_watermark(con, table)
|
| 8897 |
or table in undocumented or read_watermark(con, table) < plans[table]["end"])]
|
| 8898 |
+
planned = first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end, sqp_asins=sqp_asins)
|
| 8899 |
+
for entry in _sqp_after_cohort(planned, end, sqp_asins) if SQP_TABLE in wanted else planned:
|
| 8900 |
task = entry if len(entry) == 4 else (*entry, None)
|
| 8901 |
table, w0, w1, batch = task
|
| 8902 |
if table == SQP_TABLE:
|
|
|
|
| 8939 |
plan = plans[table] = sync_window(table, mark, now=end, since=since,
|
| 8940 |
intraday=_lookback_intraday(con, table, end, slot_pass=slot_pass))
|
| 8941 |
if table == SQP_TABLE:
|
| 8942 |
+
# Closed months land newest first, before the latest week's batches.
|
| 8943 |
+
months = list(reversed(_month_windows(plan["start"], plan["end"])))
|
|
|
|
| 8944 |
tasks = [(table, w0, w1, batch) for w0, w1 in months for batch in batches
|
| 8945 |
if since is not None or not _sqp_done(con, w0, w1, batch)]
|
| 8946 |
weekly_tasks = [task for task in _sqp_week_tasks(
|
| 8947 |
_utc(since) if since is not None else end - timedelta(days=7), end, batches)
|
| 8948 |
if since is not None or not _sqp_done(con, task[1], task[2], task[3])]
|
| 8949 |
+
work.extend([*tasks, *weekly_tasks] if tasks else [*weekly_tasks, (table, None, None, None)])
|
| 8950 |
else:
|
| 8951 |
work.append((table, None, None, None))
|
| 8952 |
|
|
|
|
| 8983 |
if old_limit.get("weeklyStart") == sqp_limit["weeklyStart"]:
|
| 8984 |
sqp_limit["weeklyCause"] = old_limit.get("weeklyCause", "")
|
| 8985 |
plan = plans.get(SQP_TABLE) or sync_window(SQP_TABLE, read_watermark(con, SQP_TABLE), now=end, since=since)
|
| 8986 |
+
# A first load also owes the months that closed after its frozen cohort end.
|
| 8987 |
+
sqp_months = sorted({*_month_windows(plan["start"], plan["end"]),
|
| 8988 |
+
*(_month_windows(plan["end"], datetime.now(timezone.utc)) if first_load else ()),
|
| 8989 |
+
*((w0, w1) for name, w0, w1, _batch in work
|
| 8990 |
+
if name == SQP_TABLE and w0 is not None and _sqp_period(w0, w1) == "MONTH")})
|
| 8991 |
+
sqp_limit["periods"] = [w0.strftime("%Y-%m") for w0, _w1 in sqp_months]
|
| 8992 |
+
sqp_progress()
|
| 8993 |
sqp_limit["deferredReason"] = (
|
| 8994 |
"Search Query Performance is waiting for ASINs in the listings or pricing mirror. Sync the catalogue first."
|
| 8995 |
if not sqp_asins else sqp_limit["partialMonthReason"] if not sqp_limit["periods"] else "")
|
|
|
|
| 9022 |
con, task_start, task_end)["batches"][task_batch].get("refused", False)
|
| 9023 |
spend_rows(count)
|
| 9024 |
landed.add((table, task_start, task_end, task_batch))
|
| 9025 |
+
except SQPPassDeferred as exc:
|
| 9026 |
+
pause_sqp(exc)
|
| 9027 |
+
continue
|
| 9028 |
except SQPBudgetDeferred as exc:
|
| 9029 |
report["tables"][table] = {"state": "deferred", "reason": str(exc)}
|
| 9030 |
weekly_deferred = True
|
|
|
|
| 9219 |
written += _sync_sqp_batch(con, fields, task_start, task_end, task_batch,
|
| 9220 |
sqp_limit, sqp_request, sqp_problem)
|
| 9221 |
sqp_limit["errors"] = []
|
| 9222 |
+
sqp_progress()
|
| 9223 |
_record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
|
| 9224 |
elif table in PERIOD_TABLES or table in LEGACY_SNAPSHOT_TABLES:
|
| 9225 |
operation = cx.REPORT_TYPES[table].get("api") or "report_document"
|
|
|
|
| 9436 |
continue
|
| 9437 |
if first_load or table == SQP_TABLE:
|
| 9438 |
landed.add((table, task_start, task_end, task_batch))
|
| 9439 |
+
# SQP weekly tasks never close the table, so the last month task does.
|
| 9440 |
+
if any(task[0] == table and not (table == SQP_TABLE and task[1] is not None
|
| 9441 |
+
and _sqp_period(task[1], task[2]) == "WEEK")
|
| 9442 |
+
for task in work[index + 1:]):
|
| 9443 |
continue
|
| 9444 |
|
| 9445 |
if table == "pricing":
|
|
|
|
| 9562 |
{"state": "done", "rows": sibling_total, "operations_ok": True},
|
| 9563 |
rt, report, log)
|
| 9564 |
log(f" {sibling:<16} {sibling_total:>7,} rows {sibling_columns:>3} cols {sibling_prov}")
|
| 9565 |
+
except SQPPassDeferred as exc:
|
| 9566 |
+
# The SQP share is spent; the other tables in this pass still run.
|
| 9567 |
+
pause_sqp(exc)
|
| 9568 |
+
continue
|
| 9569 |
except SQPBudgetDeferred as exc:
|
| 9570 |
report["tables"][table] = {"state": "deferred", "reason": str(exc)}
|
| 9571 |
break
|
|
|
|
| 9651 |
reason = f"Search Query Performance has {remaining_sqp} month and batch tasks remaining; continuing next pass."
|
| 9652 |
sqp_limit["deferredReason"] = reason
|
| 9653 |
report["tables"][SQP_TABLE] = {"state": "deferred", "reason": reason}
|
| 9654 |
+
# Landed batches publish with their month markers; the backlog stays owed in the progress.
|
| 9655 |
+
landed_batches = sum(task[0] == SQP_TABLE and task[1] is not None for task in landed)
|
| 9656 |
+
row = report["tables"].get(SQP_TABLE)
|
| 9657 |
+
if landed_batches and isinstance(row, dict) and row.get("state") in {"deferred", "throttled", "pending"}:
|
| 9658 |
+
report["tables"][SQP_TABLE] = {**row, "state": "deferred", "current": True,
|
| 9659 |
+
"landedBatches": landed_batches}
|
| 9660 |
+
sqp_progress()
|
| 9661 |
_record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
|
| 9662 |
if first_load:
|
| 9663 |
refusals = {row["table"]: row for row in _refused_reports(con)}
|
|
|
|
| 11726 |
and json.loads(_read_state(con, SQP_LIMIT_KEY)["detail"])["errors"] == ["Amazon cannot generate this report"])
|
| 11727 |
finally:
|
| 11728 |
con.close()
|
| 11729 |
+
_sqp_resume_legs(ck, Runtime, asins[:19])
|
| 11730 |
return 1 if failures else 0
|
| 11731 |
|
| 11732 |
|
| 11733 |
+
def _sqp_resume_legs(ck, runtime, asins):
|
| 11734 |
+
"""H1: a throttled or capped SQP backlog resumes at its own deadline. No network."""
|
| 11735 |
+
from unittest.mock import patch
|
| 11736 |
+
import duckdb
|
| 11737 |
+
cx, module = _client(), sys.modules[__name__]
|
| 11738 |
+
t0 = _utc("2026-11-14T00:00:30Z")
|
| 11739 |
+
clock, holder, calls = [t0], {}, []
|
| 11740 |
+
|
| 11741 |
+
class Clock(datetime):
|
| 11742 |
+
@classmethod
|
| 11743 |
+
def now(cls, tz=None):
|
| 11744 |
+
return clock[0].astimezone(tz)
|
| 11745 |
+
|
| 11746 |
+
def fresh(watermark=None):
|
| 11747 |
+
con = holder["con"] = duckdb.connect(":memory:")
|
| 11748 |
+
con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, "
|
| 11749 |
+
"cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
|
| 11750 |
+
con.execute("CREATE TABLE amzn_listings (asin1 VARCHAR)")
|
| 11751 |
+
con.executemany("INSERT INTO amzn_listings VALUES (?)", [(a,) for a in asins])
|
| 11752 |
+
if watermark:
|
| 11753 |
+
record_watermark(con, SQP_TABLE, _utc(watermark))
|
| 11754 |
+
calls.clear()
|
| 11755 |
+
return con
|
| 11756 |
+
|
| 11757 |
+
throttle_at = [None]
|
| 11758 |
+
|
| 11759 |
+
def document(_fields, _report, **kwargs):
|
| 11760 |
+
calls.append((kwargs["start"][:7], kwargs["options"]["asin"]))
|
| 11761 |
+
if throttle_at[0] is not None and len(calls) - 1 == throttle_at[0]:
|
| 11762 |
+
raise cx.AmazonThrottled("Amazon is limiting requests", retry_after=5)
|
| 11763 |
+
raw = [{"asin": a, "startDate": kwargs["start"][:10], "endDate": kwargs["end"][:10],
|
| 11764 |
+
"searchQueryData": {"searchQuery": "q"}, "purchaseData": {"asinPurchaseCount": 1}}
|
| 11765 |
+
for a in kwargs["options"]["asin"].split()]
|
| 11766 |
+
return json.dumps({"dataByAsin": raw}), {"reportDocumentId": f"h1-{len(calls)}"}
|
| 11767 |
+
|
| 11768 |
+
def run(**kwargs):
|
| 11769 |
+
return _sync_candidate(runtime.key, tables=[SQP_TABLE], rt=runtime(), log=lambda *_: None, **kwargs)
|
| 11770 |
+
|
| 11771 |
+
tick = lambda con: _read_state(con, entity_key("sync_tick_" + SQP_TABLE))
|
| 11772 |
+
month = lambda w0: (w0, (w0.replace(day=28) + timedelta(days=4)).replace(day=1))
|
| 11773 |
+
sep, octo = month(_utc("2026-09-01T00:00:00Z")), month(_utc("2026-10-01T00:00:00Z"))
|
| 11774 |
+
batches = sqp_batches(asins, cx.REPORT_TYPES[SQP_TABLE]["asin_option_max_chars"])
|
| 11775 |
+
con = None
|
| 11776 |
+
try:
|
| 11777 |
+
with patch.object(cx, "access_token", side_effect=AssertionError("fixture attempted LWA")), \
|
| 11778 |
+
patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("fixture attempted network")), \
|
| 11779 |
+
patch.object(module, "datetime", Clock), patch.object(module, "DEFAULT_BACKFILL_DAYS", 70), \
|
| 11780 |
+
patch.object(module, "_SCHEDULE_OFFSET_READER", lambda: 0), \
|
| 11781 |
+
patch.object(module, "credential", return_value={"fixture": True}), \
|
| 11782 |
+
patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_: (holder["con"].cursor(), None)), \
|
| 11783 |
+
patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \
|
| 11784 |
+
patch.object(module, "_sync_event"), patch.object(cx, "report_document", side_effect=document), \
|
| 11785 |
+
patch.object(cx, "settlement_reports", return_value=[]) as settlements, \
|
| 11786 |
+
patch.dict(cx.REPORT_TYPES[SQP_TABLE], {"periods": ("MONTH",)}):
|
| 11787 |
+
# A pass throttled at batch k keeps batches < k and resumes at k after the deadline.
|
| 11788 |
+
con = fresh("2026-09-01T00:00:00Z")
|
| 11789 |
+
throttle_at[0] = 2
|
| 11790 |
+
first = run(scheduled=True)
|
| 11791 |
+
saved = tick(con)
|
| 11792 |
+
kept = [_sqp_done(con, *octo, b) for b in batches] + [_sqp_done(con, *sep, batches[0])]
|
| 11793 |
+
holds = _MIRROR_STORE._publication_holds(first, ())
|
| 11794 |
+
row = first["tables"][SQP_TABLE]
|
| 11795 |
+
throttle_at[0] = None
|
| 11796 |
+
floor = _cadence_seconds(SQP_TABLE, saved)
|
| 11797 |
+
waits = [_interval_wait(SQP_TABLE, saved, t0 + timedelta(seconds=s), floor_s=floor)[0] for s in (60, 901)]
|
| 11798 |
+
daily = _interval_wait("storage_fees", saved, t0 + timedelta(seconds=901),
|
| 11799 |
+
floor_s=_cadence_seconds("storage_fees", saved))[0]
|
| 11800 |
+
ck("SQP throttled tick follows its own deadline while another table keeps its daily phase",
|
| 11801 |
+
saved["state"] == "throttled" and _utc(saved["detail"]) == t0 + timedelta(seconds=SQP_RESUME_SECONDS)
|
| 11802 |
+
and 0 < waits[0] <= SQP_RESUME_SECONDS and waits[1] == 0 and daily > SQP_RESUME_SECONDS)
|
| 11803 |
+
before = list(calls)
|
| 11804 |
+
clock[0] = t0 + timedelta(seconds=901)
|
| 11805 |
+
second = run(scheduled=True)
|
| 11806 |
+
resumed = calls[len(before):]
|
| 11807 |
+
ck("SQP throttled scheduled pass resumes at the throttled batch on the next pass, never re-requesting done batches",
|
| 11808 |
+
bool(row.get("reason")) and kept == [True, True, False]
|
| 11809 |
+
and [m for m, _a in before] == ["2026-10", "2026-10", "2026-09"]
|
| 11810 |
+
and resumed == [("2026-09", batches[0]), ("2026-09", batches[1])]
|
| 11811 |
+
and tick(con)["state"] == "done" and read_watermark(con, SQP_TABLE) == _utc("2026-11-01T00:00:00Z")
|
| 11812 |
+
and second["tables"][SQP_TABLE].get("state", "done") == "done")
|
| 11813 |
+
con.close()
|
| 11814 |
+
|
| 11815 |
+
# A pass that landed SQP batches publishes them; a pass that landed none holds SQP.
|
| 11816 |
+
clock[0] = t0
|
| 11817 |
+
con = fresh("2026-09-01T00:00:00Z")
|
| 11818 |
+
throttle_at[0] = 0
|
| 11819 |
+
nothing = run(scheduled=True)
|
| 11820 |
+
throttle_at[0] = None
|
| 11821 |
+
empty_holds = _MIRROR_STORE._publication_holds(nothing, ())
|
| 11822 |
+
ck("SQP landed batches publish with their month markers; a pass that landed none stays held",
|
| 11823 |
+
row.get("state") == "deferred" and row.get("current") is True and row.get("landedBatches") == 2
|
| 11824 |
+
and _mirrors._table_sync_complete("amazon", row) and (holds is None or SQP_TABLE not in holds["held"])
|
| 11825 |
+
and nothing["tables"][SQP_TABLE]["state"] == "throttled"
|
| 11826 |
+
and empty_holds is not None and SQP_TABLE in empty_holds["held"])
|
| 11827 |
+
con.close()
|
| 11828 |
+
|
| 11829 |
+
# A scheduled pass during an unfinished first load leaves SQP on the daily phase, not due every minute.
|
| 11830 |
+
t1 = t0 + timedelta(minutes=30)
|
| 11831 |
+
clock[0] = t1
|
| 11832 |
+
con = fresh()
|
| 11833 |
+
_record_state(con, entity_key(FIRST_LOAD_END_KEY), "running", detail="2026-09-26T00:00:00Z")
|
| 11834 |
+
_record_state(con, entity_key("sync_tick_" + SQP_TABLE), "throttled", detail=_iso(t1 - timedelta(seconds=10)))
|
| 11835 |
+
stale = tick(con)
|
| 11836 |
+
run(scheduled=True)
|
| 11837 |
+
later = t1 + timedelta(seconds=60)
|
| 11838 |
+
ck("SQP throttled tick deferred by an unfinished first load is not due a minute later",
|
| 11839 |
+
sync_plan({SQP_TABLE: stale}, [SQP_TABLE], now=later)["tables"] == [SQP_TABLE]
|
| 11840 |
+
and tick(con)["state"] == "deferred" and tick(con)["detail"] == stale["detail"]
|
| 11841 |
+
and sync_plan({SQP_TABLE: tick(con)}, [SQP_TABLE], now=later)["tables"] == []
|
| 11842 |
+
and _interval_wait(SQP_TABLE, tick(con), later, floor_s=_cadence_seconds(SQP_TABLE, tick(con)))[0] > 0
|
| 11843 |
+
and not calls)
|
| 11844 |
+
con.close()
|
| 11845 |
+
clock[0] = t0
|
| 11846 |
+
|
| 11847 |
+
# A scheduled pass spends at most SQP_PASS_REPORTS SQP documents; other due tables still run.
|
| 11848 |
+
clock[0] = t0
|
| 11849 |
+
with patch.object(module, "SQP_PASS_REPORTS", 10 ** 6):
|
| 11850 |
+
con = fresh("2026-01-01T00:00:00Z")
|
| 11851 |
+
run(scheduled=True)
|
| 11852 |
+
unbounded = len(calls)
|
| 11853 |
+
con.close()
|
| 11854 |
+
con = fresh("2026-01-01T00:00:00Z")
|
| 11855 |
+
settlements.reset_mock()
|
| 11856 |
+
capped = run(scheduled=True)
|
| 11857 |
+
saved = tick(con)
|
| 11858 |
+
ck("SQP scheduled pass stops at SQP_PASS_REPORTS documents and other due tables still run",
|
| 11859 |
+
unbounded == 20 and len(calls) == SQP_PASS_REPORTS
|
| 11860 |
+
and capped["tables"][SQP_TABLE]["state"] == "deferred" and settlements.call_count == 1
|
| 11861 |
+
and capped["tables"]["settlements"]["state"] == "empty"
|
| 11862 |
+
and saved["state"] == "throttled"
|
| 11863 |
+
and _utc(saved["detail"]) == t0 + timedelta(seconds=SQP_RESUME_SECONDS))
|
| 11864 |
+
limit = json.loads(_read_state(con, SQP_LIMIT_KEY)["detail"])
|
| 11865 |
+
progress = limit.get("progress") or {}
|
| 11866 |
+
sentence = sqp_limit_sentence(limit)
|
| 11867 |
+
bare = sqp_limit_sentence({k: v for k, v in limit.items() if k != "progress"})
|
| 11868 |
+
ck("SQP progress sentence names months and batches done of total, without a dash",
|
| 11869 |
+
"of 20 batches" not in bare and progress.get("batchesDone") == 15
|
| 11870 |
+
and progress.get("batchesTotal") == 20 and progress.get("monthsTotal") == 10
|
| 11871 |
+
and progress.get("etaSeconds") == 300 and progress["months"][2] == {"month": "2026-03", "total": 2, "done": 1}
|
| 11872 |
+
and "7 of 10 months complete, 15 of 20 batches." in sentence and "About 5 minutes left" in sentence
|
| 11873 |
+
and len(sentence) - len(bare) < 300 and not any(d in sentence for d in ("\u2013", "\u2014"))
|
| 11874 |
+
and sqp_progress_sentence({}) == "" and sentence.endswith(" " + sqp_progress_sentence(limit))
|
| 11875 |
+
and len(sqp_progress_sentence(limit)) < 140)
|
| 11876 |
+
con.close()
|
| 11877 |
+
|
| 11878 |
+
# A first load with a cohort frozen at 2026-09-26 plans 2026-10 and 2026-09 first.
|
| 11879 |
+
chunk = {}
|
| 11880 |
+
|
| 11881 |
+
def frozen(spliced=True):
|
| 11882 |
+
con = fresh()
|
| 11883 |
+
_record_state(con, entity_key(FIRST_LOAD_END_KEY), "running", detail="2026-09-26T00:00:00Z")
|
| 11884 |
+
if spliced:
|
| 11885 |
+
chunk.update(run(first_load=True, report_budget=4))
|
| 11886 |
+
else:
|
| 11887 |
+
with patch.object(module, "_sqp_after_cohort", side_effect=lambda tasks, *_: tasks):
|
| 11888 |
+
run(first_load=True, report_budget=4)
|
| 11889 |
+
con.close()
|
| 11890 |
+
return [m for m, _a in calls]
|
| 11891 |
+
end = _utc("2026-09-26T00:00:00Z")
|
| 11892 |
+
planned = _sqp_after_cohort(first_load_plan([SQP_TABLE], end - timedelta(days=70), end,
|
| 11893 |
+
sqp_asins=asins), end, asins)
|
| 11894 |
+
red = frozen(spliced=False)
|
| 11895 |
+
ck("SQP first load with a frozen cohort plans the months closed since, newest first",
|
| 11896 |
+
red[:1] == ["2026-08"] and frozen() == ["2026-10", "2026-10", "2026-09", "2026-09"]
|
| 11897 |
+
and [t[1].strftime("%Y-%m") for t in planned[:4]] == ["2026-10", "2026-10", "2026-09", "2026-09"])
|
| 11898 |
+
# A first-load chunk whose only progress is SQP batches publishes them; other tables stay held.
|
| 11899 |
+
waiting = {table_name("orders")}
|
| 11900 |
+
unmarked = {**chunk, "tables": {**chunk["tables"], SQP_TABLE: {
|
| 11901 |
+
k: v for k, v in chunk["tables"][SQP_TABLE].items() if k != "current"}}}
|
| 11902 |
+
chunk_holds = _MIRROR_STORE._publication_holds(chunk, waiting)
|
| 11903 |
+
ck("SQP first-load chunk of only SQP batches publishes them, never nothing",
|
| 11904 |
+
chunk.get("state") == "partial" and _MIRROR_STORE._publication_holds(unmarked, waiting) is None
|
| 11905 |
+
and chunk_holds is not None and SQP_TABLE not in chunk_holds["held"] and "orders" in chunk_holds["held"])
|
| 11906 |
+
con = None
|
| 11907 |
+
finally:
|
| 11908 |
+
if con is not None:
|
| 11909 |
+
con.close()
|
| 11910 |
+
|
| 11911 |
+
|
| 11912 |
def group_a_selftest(log=print, check=None):
|
| 11913 |
"""S66-T42 red-first fixture controls; memory mirrors and no credentials or network."""
|
| 11914 |
from unittest.mock import patch
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
75b681000cae6da4d22bca0114740865caadcf6e1fe3986ad2032e6bd969ebb5
|
web/src/connectors/ConnectorsPage.tsx
CHANGED
|
@@ -264,6 +264,7 @@ export interface AmazonTableSync {
|
|
| 264 |
nextSyncAt: string;
|
| 265 |
perDay: number;
|
| 266 |
state: string;
|
|
|
|
| 267 |
}
|
| 268 |
|
| 269 |
export interface AmazonSyncScheduleReading {
|
|
@@ -296,7 +297,8 @@ export function parseAmazonSyncSchedule(body: Record<string, unknown>): AmazonSy
|
|
| 296 |
const identity = JSON.stringify([row.connectionId, row.table]);
|
| 297 |
if (identities.has(identity)) return unavailable;
|
| 298 |
identities.add(identity);
|
| 299 |
-
|
|
|
|
| 300 |
}
|
| 301 |
const state = body.scheduleState as AmazonSyncScheduleReading["state"];
|
| 302 |
if ((state === "ready") !== (tables.length > 0)) return unavailable;
|
|
@@ -341,7 +343,8 @@ export function AmazonSyncSchedule({ reading, problem = "", onRetry }: {
|
|
| 341 |
<td className="conn-settings-database-success">
|
| 342 |
{Date.parse(row.nextSyncAt) <= Date.now() + 60000 ? "Due now"
|
| 343 |
: <time dateTime={row.nextSyncAt}>{amazonSyncStamp(row.nextSyncAt, reading.timezone)}</time>}
|
| 344 |
-
{row.state ==
|
|
|
|
| 345 |
: ["failed", "refused"].includes(row.state) ? <p role="status">Sync needs attention</p> : null}
|
| 346 |
</td>
|
| 347 |
<td className="conn-settings-number">{row.perDay}</td>
|
|
|
|
| 264 |
nextSyncAt: string;
|
| 265 |
perDay: number;
|
| 266 |
state: string;
|
| 267 |
+
reason?: string;
|
| 268 |
}
|
| 269 |
|
| 270 |
export interface AmazonSyncScheduleReading {
|
|
|
|
| 297 |
const identity = JSON.stringify([row.connectionId, row.table]);
|
| 298 |
if (identities.has(identity)) return unavailable;
|
| 299 |
identities.add(identity);
|
| 300 |
+
const reason = typeof row.reason === "string" ? row.reason.trim() : "";
|
| 301 |
+
tables.push({ ...(row as unknown as AmazonTableSync), reason: reason || undefined });
|
| 302 |
}
|
| 303 |
const state = body.scheduleState as AmazonSyncScheduleReading["state"];
|
| 304 |
if ((state === "ready") !== (tables.length > 0)) return unavailable;
|
|
|
|
| 343 |
<td className="conn-settings-database-success">
|
| 344 |
{Date.parse(row.nextSyncAt) <= Date.now() + 60000 ? "Due now"
|
| 345 |
: <time dateTime={row.nextSyncAt}>{amazonSyncStamp(row.nextSyncAt, reading.timezone)}</time>}
|
| 346 |
+
{row.reason && row.state !== "done" ? <p role="status">{row.reason}</p>
|
| 347 |
+
: row.state === "throttled" ? <p role="status">Rate limited, retry scheduled</p>
|
| 348 |
: ["failed", "refused"].includes(row.state) ? <p role="status">Sync needs attention</p> : null}
|
| 349 |
</td>
|
| 350 |
<td className="conn-settings-number">{row.perDay}</td>
|