File size: 9,055 Bytes
b2aa446 3e19b1e b2aa446 3e19b1e b2aa446 3e19b1e b2aa446 3e19b1e b2aa446 3e19b1e b2aa446 3e19b1e b2aa446 3e19b1e b2aa446 3e19b1e b2aa446 3e19b1e b2aa446 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 | """Price and rank every rival once a day per tenant in the sync worker, outside the sync pass (C7).
The loop holds the SP-API client and passes its reads into `core.amazon_rivals.run_quotes`, because
core imports no harness module. It never joins `amazon_store.AFTER_SYNC` and never runs as an HF job.
"""
from contextlib import contextmanager
import importlib
import logging
from pathlib import Path
import threading
import time
import uuid
JOB = "amazon-rival-quotes"
#: The loop asks each tenant whether its daily run is due this often.
POLL_SECONDS = 600
SNAPSHOT_TABLES = ("competitor_snapshots", "competitor_images")
#: Called as `fn(tenant, changed_asins)` after a run finishes, where `changed_asins` is a tuple of the
#: ASINs whose newest quote changed. Each import is lazy and a failure is one log line, never a failed run.
#: Command Center patches the competitor cells of the rows those ASINs move (S82-T89).
AFTER_QUOTES = ("modules.amazon_profit:on_quotes", "modules.competitor_watch:on_quotes")
_log = logging.getLogger(__name__)
_missing = False
_unavailable = set()
_thread = None
_stop = None
class SnapshotBlocked(RuntimeError):
"""A safe admission cause that can be recorded without exposing provider details."""
def _say(message):
print(f"[{JOB}] {message}", flush=True)
def readers(fields):
"""Read prices and complete facts with the tenant credential and the shared Amazon pacers."""
from harness import amazon_client as cx
from harness import amazon_competitor_facts as facts
def complete(asins):
catalog, offers = facts.read_facts(fields, asins), facts.read_offers(fields, asins)
return {asin: {**catalog[asin], **{key: offers[asin][key] for key in ("prime", "offers")},
"cause": " ".join(c for c in (catalog[asin]["cause"], offers[asin]["cause"]) if c) or None}
for asin in asins}
return {"offers": lambda asins: cx.item_offers(fields, asins),
# The pricing sync draws this same `competitive_summary` pacer in this worker process.
"summaries": lambda asins: cx.competitive_summaries(fields, asins),
"facts": complete}
@contextmanager
def _mirror(runtime):
"""A read cursor on this tenant's Amazon mirror, None when the file does not exist yet."""
from harness import runtime as R
if not Path(runtime.datastore_path).is_file():
yield None
return
cur = R.mirror_cursor(runtime, require_ready=False)
try:
yield cur
finally:
cur.close()
def _after_quotes(tenant, changed):
for target in AFTER_QUOTES:
try:
module_name, _, name = target.partition(":")
getattr(importlib.import_module(module_name), name)(tenant, changed)
_say(f"tenant={tenant} after-quotes callback {target} completed")
except Exception as exc: # noqa: BLE001
_say(f"tenant={tenant} after-quotes callback {target} failed ({type(exc).__name__}); "
"the run result is unchanged.")
def _snapshot_cause(runtime, *, check_lease=True):
from harness import amazon_store
enabled = amazon_store.read_database_enablement(runtime)
if not enabled["available"]:
return "Competitor database settings could not be read. Restore settings access and retry the run."
if not all(enabled["databases"].get("ut_amzn_" + table) for table in SNAPSHOT_TABLES):
return "A competitor database is disabled. Enable competitor snapshots and images in Settings, Connectors and retry."
if check_lease and amazon_store.first_load_job_live(runtime):
return "The Amazon mirror writer lease is held. Let the current writer finish and retry the run."
return None
def _publish_snapshots(tenant, runtime, result):
from harness import amazon_competitor_facts as facts, amazon_store, mirror_store
lock = mirror_store.for_connector("amazon").lock
if not lock.acquire(blocking=False):
raise SnapshotBlocked("The Amazon mirror writer is busy.")
holder = f"{JOB}:{result['id']}"
leased = False
try:
leased = amazon_store.acquire_amazon_lease(runtime, holder)
if not leased:
raise SnapshotBlocked("The Amazon mirror writer lease is held.")
cause = _snapshot_cause(runtime, check_lease=False)
if cause:
raise SnapshotBlocked(cause)
report = {"tables": {}, "problems": []}
with mirror_store.sync_mirror("amazon", tenant, runtime, report=report, expected_tables=SNAPSHOT_TABLES):
con, _path = amazon_store._connect_tenant_mirror(tenant, runtime)
try:
stored = facts.import_snapshots(con, result["snapshots"])
for table in SNAPSHOT_TABLES:
total = con.execute(f"SELECT count(*) FROM amzn_{table}").fetchone()[0]
report["tables"][table] = {"state": "done", "rows": total}
finally:
con.close()
if report["problems"]:
raise RuntimeError("Competitor mirror publication failed")
result["counts"].update(snapshots=stored["snapshots"], images=stored["images"])
finally:
try:
if leased:
amazon_store.release_amazon_lease(runtime, holder)
finally:
lock.release()
def _run_blocked(runtime, cause):
from core import amazon_rivals as rivals
run_id = uuid.uuid4().hex
rivals.start_run(runtime, run_id, asin_total=0)
return {"id": run_id, "progress": 0, "cause": cause,
"counts": {"facts": dict.fromkeys(rivals.FACT_COUNT_COLUMNS, 0)}, "changed": (), "snapshots": []}
def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep):
"""Run today's quotes for one tenant when due. None when it is not due or has no Amazon store."""
from core import amazon_rivals as rivals
if str(tenant) != str(runtime.key) or str(runtime.key) != str(runtime.tenant.key):
raise ValueError("Competitor quotes require the matching tenant runtime")
try:
if not rivals.due(runtime, now=now):
return None
except rivals.BackendUnavailable:
if tenant not in _unavailable:
_unavailable.add(tenant)
_say(f"tenant={tenant} has no PostgreSQL store, so no rival quotes run")
return None
if fields is None:
from harness import amazon_store
fields = amazon_store.credential(tenant, runtime)
if not fields:
return None
started = time.monotonic()
cause = _snapshot_cause(runtime)
result = (_run_blocked(runtime, cause) if cause else
rivals.run_quotes(runtime, lambda: _mirror(runtime), **readers(fields),
seller_id=str(fields.get("seller_id") or "").strip() or None, now=now,
log=lambda message: _say(f"tenant={tenant} {message}"), sleep=sleep))
if result["snapshots"]:
try:
_publish_snapshots(tenant, runtime, result)
except Exception as exc:
cause = (str(exc) if isinstance(exc, SnapshotBlocked) else
f"Competitor snapshots could not be published ({type(exc).__name__}).")
result["cause"] = (f"{cause} Check database settings and mirror access, let any writer finish, "
f"and retry within {rivals.RETRY_RUN_HOURS} hours.")
result["counts"]["elapsed_seconds"] = round(time.monotonic() - started, 3)
rivals.mark_run(runtime, result["id"], progress=result["progress"], finished=True,
counts=result["counts"], cause=result["cause"])
if result["cause"] is None:
_after_quotes(tenant, result["changed"])
return result
def run_once(*, for_each_tenant=None, now=None):
"""Admit one independent per-tenant job without waiting for its body."""
global _missing
if for_each_tenant is None:
try:
from tenant_loops import for_each_tenant
except ImportError:
if not _missing:
_log.warning("Rival quotes unavailable: tenant_loops is not installed.")
_missing = True
return {}
_missing = False
return for_each_tenant(lambda tenant, runtime: run_tenant(tenant, runtime, now=now),
name=JOB, logger=_log, kind="heavy")
def start():
"""Start the daily loop only when process_role admits its heavy coordinator (the sync worker)."""
global _thread, _stop
import process_role
if _thread is not None and _thread.is_alive():
return _thread
stop_event = threading.Event()
_stop = stop_event
def run():
while not stop_event.is_set():
run_once()
stop_event.wait(POLL_SECONDS)
# A literal name: verify_background_loops classifies a job by it.
_thread = process_role.start("amazon-rival-quotes", "heavy", run)
return _thread
def stop():
if _stop is not None:
_stop.set()
|