Download api/amazon_quotes_loop.py from fsanyoto/loopable: direct link, hf CLI and curl.
- Browser
- Download file 9.06 kB
-
https://huggingface.co/spaces/fsanyoto/loopable/resolve/f694a2bb8305731dbfb36abe080ff76fe2b3a6c4/api/amazon_quotes_loop.py
- Command line
-
hf download hf://spaces/fsanyoto/loopable@f694a2bb8305731dbfb36abe080ff76fe2b3a6c4/api/amazon_quotes_loop.py
-
curl -L -o amazon_quotes_loop.py https://huggingface.co/spaces/fsanyoto/loopable/resolve/f694a2bb8305731dbfb36abe080ff76fe2b3a6c4/api/amazon_quotes_loop.py
9.06 kB
| """Price and rank every rival once a day per tenant in the sync worker, outside the sync pass (C7). | |
| The loop holds the SP-API client and passes its reads into `core.amazon_rivals.run_quotes`, because | |
| core imports no harness module. It never joins `amazon_store.AFTER_SYNC` and never runs as an HF job. | |
| """ | |
| from contextlib import contextmanager | |
| import importlib | |
| import logging | |
| from pathlib import Path | |
| import threading | |
| import time | |
| import uuid | |
| JOB = "amazon-rival-quotes" | |
| #: The loop asks each tenant whether its daily run is due this often. | |
| POLL_SECONDS = 600 | |
| SNAPSHOT_TABLES = ("competitor_snapshots", "competitor_images") | |
| #: Called as `fn(tenant, changed_asins)` after a run finishes, where `changed_asins` is a tuple of the | |
| #: ASINs whose newest quote changed. Each import is lazy and a failure is one log line, never a failed run. | |
| #: Command Center patches the competitor cells of the rows those ASINs move (S82-T89). | |
| AFTER_QUOTES = ("modules.amazon_profit:on_quotes", "modules.competitor_watch:on_quotes") | |
| _log = logging.getLogger(__name__) | |
| _missing = False | |
| _unavailable = set() | |
| _thread = None | |
| _stop = None | |
| class SnapshotBlocked(RuntimeError): | |
| """A safe admission cause that can be recorded without exposing provider details.""" | |
| def _say(message): | |
| print(f"[{JOB}] {message}", flush=True) | |
| def readers(fields): | |
| """Read prices and complete facts with the tenant credential and the shared Amazon pacers.""" | |
| from harness import amazon_client as cx | |
| from harness import amazon_competitor_facts as facts | |
| def complete(asins): | |
| catalog, offers = facts.read_facts(fields, asins), facts.read_offers(fields, asins) | |
| return {asin: {**catalog[asin], **{key: offers[asin][key] for key in ("prime", "offers")}, | |
| "cause": " ".join(c for c in (catalog[asin]["cause"], offers[asin]["cause"]) if c) or None} | |
| for asin in asins} | |
| return {"offers": lambda asins: cx.item_offers(fields, asins), | |
| # The pricing sync draws this same `competitive_summary` pacer in this worker process. | |
| "summaries": lambda asins: cx.competitive_summaries(fields, asins), | |
| "facts": complete} | |
| def _mirror(runtime): | |
| """A read cursor on this tenant's Amazon mirror, None when the file does not exist yet.""" | |
| from harness import runtime as R | |
| if not Path(runtime.datastore_path).is_file(): | |
| yield None | |
| return | |
| cur = R.mirror_cursor(runtime, require_ready=False) | |
| try: | |
| yield cur | |
| finally: | |
| cur.close() | |
| def _after_quotes(tenant, changed): | |
| for target in AFTER_QUOTES: | |
| try: | |
| module_name, _, name = target.partition(":") | |
| getattr(importlib.import_module(module_name), name)(tenant, changed) | |
| _say(f"tenant={tenant} after-quotes callback {target} completed") | |
| except Exception as exc: # noqa: BLE001 | |
| _say(f"tenant={tenant} after-quotes callback {target} failed ({type(exc).__name__}); " | |
| "the run result is unchanged.") | |
| def _snapshot_cause(runtime, *, check_lease=True): | |
| from harness import amazon_store | |
| enabled = amazon_store.read_database_enablement(runtime) | |
| if not enabled["available"]: | |
| return "Competitor database settings could not be read. Restore settings access and retry the run." | |
| if not all(enabled["databases"].get("ut_amzn_" + table) for table in SNAPSHOT_TABLES): | |
| return "A competitor database is disabled. Enable competitor snapshots and images in Settings, Connectors and retry." | |
| if check_lease and amazon_store.first_load_job_live(runtime): | |
| return "The Amazon mirror writer lease is held. Let the current writer finish and retry the run." | |
| return None | |
| def _publish_snapshots(tenant, runtime, result): | |
| from harness import amazon_competitor_facts as facts, amazon_store, mirror_store | |
| lock = mirror_store.for_connector("amazon").lock | |
| if not lock.acquire(blocking=False): | |
| raise SnapshotBlocked("The Amazon mirror writer is busy.") | |
| holder = f"{JOB}:{result['id']}" | |
| leased = False | |
| try: | |
| leased = amazon_store.acquire_amazon_lease(runtime, holder) | |
| if not leased: | |
| raise SnapshotBlocked("The Amazon mirror writer lease is held.") | |
| cause = _snapshot_cause(runtime, check_lease=False) | |
| if cause: | |
| raise SnapshotBlocked(cause) | |
| report = {"tables": {}, "problems": []} | |
| with mirror_store.sync_mirror("amazon", tenant, runtime, report=report, expected_tables=SNAPSHOT_TABLES): | |
| con, _path = amazon_store._connect_tenant_mirror(tenant, runtime) | |
| try: | |
| stored = facts.import_snapshots(con, result["snapshots"]) | |
| for table in SNAPSHOT_TABLES: | |
| total = con.execute(f"SELECT count(*) FROM amzn_{table}").fetchone()[0] | |
| report["tables"][table] = {"state": "done", "rows": total} | |
| finally: | |
| con.close() | |
| if report["problems"]: | |
| raise RuntimeError("Competitor mirror publication failed") | |
| result["counts"].update(snapshots=stored["snapshots"], images=stored["images"]) | |
| finally: | |
| try: | |
| if leased: | |
| amazon_store.release_amazon_lease(runtime, holder) | |
| finally: | |
| lock.release() | |
| def _run_blocked(runtime, cause): | |
| from core import amazon_rivals as rivals | |
| run_id = uuid.uuid4().hex | |
| rivals.start_run(runtime, run_id, asin_total=0) | |
| return {"id": run_id, "progress": 0, "cause": cause, | |
| "counts": {"facts": dict.fromkeys(rivals.FACT_COUNT_COLUMNS, 0)}, "changed": (), "snapshots": []} | |
| def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep): | |
| """Run today's quotes for one tenant when due. None when it is not due or has no Amazon store.""" | |
| from core import amazon_rivals as rivals | |
| if str(tenant) != str(runtime.key) or str(runtime.key) != str(runtime.tenant.key): | |
| raise ValueError("Competitor quotes require the matching tenant runtime") | |
| try: | |
| if not rivals.due(runtime, now=now): | |
| return None | |
| except rivals.BackendUnavailable: | |
| if tenant not in _unavailable: | |
| _unavailable.add(tenant) | |
| _say(f"tenant={tenant} has no PostgreSQL store, so no rival quotes run") | |
| return None | |
| if fields is None: | |
| from harness import amazon_store | |
| fields = amazon_store.credential(tenant, runtime) | |
| if not fields: | |
| return None | |
| started = time.monotonic() | |
| cause = _snapshot_cause(runtime) | |
| result = (_run_blocked(runtime, cause) if cause else | |
| rivals.run_quotes(runtime, lambda: _mirror(runtime), **readers(fields), | |
| seller_id=str(fields.get("seller_id") or "").strip() or None, now=now, | |
| log=lambda message: _say(f"tenant={tenant} {message}"), sleep=sleep)) | |
| if result["snapshots"]: | |
| try: | |
| _publish_snapshots(tenant, runtime, result) | |
| except Exception as exc: | |
| cause = (str(exc) if isinstance(exc, SnapshotBlocked) else | |
| f"Competitor snapshots could not be published ({type(exc).__name__}).") | |
| result["cause"] = (f"{cause} Check database settings and mirror access, let any writer finish, " | |
| f"and retry within {rivals.RETRY_RUN_HOURS} hours.") | |
| result["counts"]["elapsed_seconds"] = round(time.monotonic() - started, 3) | |
| rivals.mark_run(runtime, result["id"], progress=result["progress"], finished=True, | |
| counts=result["counts"], cause=result["cause"]) | |
| if result["cause"] is None: | |
| _after_quotes(tenant, result["changed"]) | |
| return result | |
| def run_once(*, for_each_tenant=None, now=None): | |
| """Admit one independent per-tenant job without waiting for its body.""" | |
| global _missing | |
| if for_each_tenant is None: | |
| try: | |
| from tenant_loops import for_each_tenant | |
| except ImportError: | |
| if not _missing: | |
| _log.warning("Rival quotes unavailable: tenant_loops is not installed.") | |
| _missing = True | |
| return {} | |
| _missing = False | |
| return for_each_tenant(lambda tenant, runtime: run_tenant(tenant, runtime, now=now), | |
| name=JOB, logger=_log, kind="heavy") | |
| def start(): | |
| """Start the daily loop only when process_role admits its heavy coordinator (the sync worker).""" | |
| global _thread, _stop | |
| import process_role | |
| if _thread is not None and _thread.is_alive(): | |
| return _thread | |
| stop_event = threading.Event() | |
| _stop = stop_event | |
| def run(): | |
| while not stop_event.is_set(): | |
| run_once() | |
| stop_event.wait(POLL_SECONDS) | |
| # A literal name: verify_background_loops classifies a job by it. | |
| _thread = process_role.start("amazon-rival-quotes", "heavy", run) | |
| return _thread | |
| def stop(): | |
| if _stop is not None: | |
| _stop.set() | |