runloopable / api /amazon_quotes_loop.py
fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
95fb175 verified
Raw History Blame Contribute Delete
9.06 kB
"""Price and rank every rival once a day per tenant in the sync worker, outside the sync pass (C7).
The loop holds the SP-API client and passes its reads into `core.amazon_rivals.run_quotes`, because
core imports no harness module. It never joins `amazon_store.AFTER_SYNC` and never runs as an HF job.
"""
from contextlib import contextmanager
import importlib
import logging
from pathlib import Path
import threading
import time
import uuid
JOB = "amazon-rival-quotes"
#: The loop asks each tenant whether its daily run is due this often.
POLL_SECONDS = 600
SNAPSHOT_TABLES = ("competitor_snapshots", "competitor_images")
#: Called as `fn(tenant, changed_asins)` after a run finishes, where `changed_asins` is a tuple of the
#: ASINs whose newest quote changed. Each import is lazy and a failure is one log line, never a failed run.
#: Command Center patches the competitor cells of the rows those ASINs move (S82-T89).
AFTER_QUOTES = ("modules.amazon_profit:on_quotes", "modules.competitor_watch:on_quotes")
_log = logging.getLogger(__name__)
_missing = False
_unavailable = set()
_thread = None
_stop = None
class SnapshotBlocked(RuntimeError):
"""A safe admission cause that can be recorded without exposing provider details."""
def _say(message):
print(f"[{JOB}] {message}", flush=True)
def readers(fields):
"""Read prices and complete facts with the tenant credential and the shared Amazon pacers."""
from harness import amazon_client as cx
from harness import amazon_competitor_facts as facts
def complete(asins):
catalog, offers = facts.read_facts(fields, asins), facts.read_offers(fields, asins)
return {asin: {**catalog[asin], **{key: offers[asin][key] for key in ("prime", "offers")},
"cause": " ".join(c for c in (catalog[asin]["cause"], offers[asin]["cause"]) if c) or None}
for asin in asins}
return {"offers": lambda asins: cx.item_offers(fields, asins),
# The pricing sync draws this same `competitive_summary` pacer in this worker process.
"summaries": lambda asins: cx.competitive_summaries(fields, asins),
"facts": complete}
@contextmanager
def _mirror(runtime):
"""A read cursor on this tenant's Amazon mirror, None when the file does not exist yet."""
from harness import runtime as R
if not Path(runtime.datastore_path).is_file():
yield None
return
cur = R.mirror_cursor(runtime, require_ready=False)
try:
yield cur
finally:
cur.close()
def _after_quotes(tenant, changed):
for target in AFTER_QUOTES:
try:
module_name, _, name = target.partition(":")
getattr(importlib.import_module(module_name), name)(tenant, changed)
_say(f"tenant={tenant} after-quotes callback {target} completed")
except Exception as exc: # noqa: BLE001
_say(f"tenant={tenant} after-quotes callback {target} failed ({type(exc).__name__}); "
"the run result is unchanged.")
def _snapshot_cause(runtime, *, check_lease=True):
from harness import amazon_store
enabled = amazon_store.read_database_enablement(runtime)
if not enabled["available"]:
return "Competitor database settings could not be read. Restore settings access and retry the run."
if not all(enabled["databases"].get("ut_amzn_" + table) for table in SNAPSHOT_TABLES):
return "A competitor database is disabled. Enable competitor snapshots and images in Settings, Connectors and retry."
if check_lease and amazon_store.first_load_job_live(runtime):
return "The Amazon mirror writer lease is held. Let the current writer finish and retry the run."
return None
def _publish_snapshots(tenant, runtime, result):
from harness import amazon_competitor_facts as facts, amazon_store, mirror_store
lock = mirror_store.for_connector("amazon").lock
if not lock.acquire(blocking=False):
raise SnapshotBlocked("The Amazon mirror writer is busy.")
holder = f"{JOB}:{result['id']}"
leased = False
try:
leased = amazon_store.acquire_amazon_lease(runtime, holder)
if not leased:
raise SnapshotBlocked("The Amazon mirror writer lease is held.")
cause = _snapshot_cause(runtime, check_lease=False)
if cause:
raise SnapshotBlocked(cause)
report = {"tables": {}, "problems": []}
with mirror_store.sync_mirror("amazon", tenant, runtime, report=report, expected_tables=SNAPSHOT_TABLES):
con, _path = amazon_store._connect_tenant_mirror(tenant, runtime)
try:
stored = facts.import_snapshots(con, result["snapshots"])
for table in SNAPSHOT_TABLES:
total = con.execute(f"SELECT count(*) FROM amzn_{table}").fetchone()[0]
report["tables"][table] = {"state": "done", "rows": total}
finally:
con.close()
if report["problems"]:
raise RuntimeError("Competitor mirror publication failed")
result["counts"].update(snapshots=stored["snapshots"], images=stored["images"])
finally:
try:
if leased:
amazon_store.release_amazon_lease(runtime, holder)
finally:
lock.release()
def _run_blocked(runtime, cause):
from core import amazon_rivals as rivals
run_id = uuid.uuid4().hex
rivals.start_run(runtime, run_id, asin_total=0)
return {"id": run_id, "progress": 0, "cause": cause,
"counts": {"facts": dict.fromkeys(rivals.FACT_COUNT_COLUMNS, 0)}, "changed": (), "snapshots": []}
def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep):
"""Run today's quotes for one tenant when due. None when it is not due or has no Amazon store."""
from core import amazon_rivals as rivals
if str(tenant) != str(runtime.key) or str(runtime.key) != str(runtime.tenant.key):
raise ValueError("Competitor quotes require the matching tenant runtime")
try:
if not rivals.due(runtime, now=now):
return None
except rivals.BackendUnavailable:
if tenant not in _unavailable:
_unavailable.add(tenant)
_say(f"tenant={tenant} has no PostgreSQL store, so no rival quotes run")
return None
if fields is None:
from harness import amazon_store
fields = amazon_store.credential(tenant, runtime)
if not fields:
return None
started = time.monotonic()
cause = _snapshot_cause(runtime)
result = (_run_blocked(runtime, cause) if cause else
rivals.run_quotes(runtime, lambda: _mirror(runtime), **readers(fields),
seller_id=str(fields.get("seller_id") or "").strip() or None, now=now,
log=lambda message: _say(f"tenant={tenant} {message}"), sleep=sleep))
if result["snapshots"]:
try:
_publish_snapshots(tenant, runtime, result)
except Exception as exc:
cause = (str(exc) if isinstance(exc, SnapshotBlocked) else
f"Competitor snapshots could not be published ({type(exc).__name__}).")
result["cause"] = (f"{cause} Check database settings and mirror access, let any writer finish, "
f"and retry within {rivals.RETRY_RUN_HOURS} hours.")
result["counts"]["elapsed_seconds"] = round(time.monotonic() - started, 3)
rivals.mark_run(runtime, result["id"], progress=result["progress"], finished=True,
counts=result["counts"], cause=result["cause"])
if result["cause"] is None:
_after_quotes(tenant, result["changed"])
return result
def run_once(*, for_each_tenant=None, now=None):
"""Admit one independent per-tenant job without waiting for its body."""
global _missing
if for_each_tenant is None:
try:
from tenant_loops import for_each_tenant
except ImportError:
if not _missing:
_log.warning("Rival quotes unavailable: tenant_loops is not installed.")
_missing = True
return {}
_missing = False
return for_each_tenant(lambda tenant, runtime: run_tenant(tenant, runtime, now=now),
name=JOB, logger=_log, kind="heavy")
def start():
"""Start the daily loop only when process_role admits its heavy coordinator (the sync worker)."""
global _thread, _stop
import process_role
if _thread is not None and _thread.is_alive():
return _thread
stop_event = threading.Event()
_stop = stop_event
def run():
while not stop_event.is_set():
run_once()
stop_event.wait(POLL_SECONDS)
# A literal name: verify_background_loops classifies a job by it.
_thread = process_role.start("amazon-rival-quotes", "heavy", run)
return _thread
def stop():
if _stop is not None:
_stop.set()