File size: 9,055 Bytes
8133df9
 
 
 
 
 
 
 
 
 
 
95fb175
8133df9
 
 
 
95fb175
8133df9
 
 
 
 
 
 
 
 
 
 
 
 
95fb175
 
 
 
8133df9
 
 
 
 
95fb175
8133df9
95fb175
 
 
 
 
 
 
 
8133df9
 
 
95fb175
8133df9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
95fb175
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8133df9
 
 
95fb175
 
8133df9
 
 
 
 
 
 
 
 
 
 
 
 
95fb175
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8133df9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
"""Price and rank every rival once a day per tenant in the sync worker, outside the sync pass (C7).

The loop holds the SP-API client and passes its reads into `core.amazon_rivals.run_quotes`, because
core imports no harness module. It never joins `amazon_store.AFTER_SYNC` and never runs as an HF job.
"""
from contextlib import contextmanager
import importlib
import logging
from pathlib import Path
import threading
import time
import uuid

JOB = "amazon-rival-quotes"
#: The loop asks each tenant whether its daily run is due this often.
POLL_SECONDS = 600
SNAPSHOT_TABLES = ("competitor_snapshots", "competitor_images")

#: Called as `fn(tenant, changed_asins)` after a run finishes, where `changed_asins` is a tuple of the
#: ASINs whose newest quote changed. Each import is lazy and a failure is one log line, never a failed run.
#: Command Center patches the competitor cells of the rows those ASINs move (S82-T89).
AFTER_QUOTES = ("modules.amazon_profit:on_quotes", "modules.competitor_watch:on_quotes")

_log = logging.getLogger(__name__)
_missing = False
_unavailable = set()
_thread = None
_stop = None


class SnapshotBlocked(RuntimeError):
    """A safe admission cause that can be recorded without exposing provider details."""


def _say(message):
    print(f"[{JOB}] {message}", flush=True)


def readers(fields):
    """Read prices and complete facts with the tenant credential and the shared Amazon pacers."""
    from harness import amazon_client as cx
    from harness import amazon_competitor_facts as facts

    def complete(asins):
        catalog, offers = facts.read_facts(fields, asins), facts.read_offers(fields, asins)
        return {asin: {**catalog[asin], **{key: offers[asin][key] for key in ("prime", "offers")},
                       "cause": " ".join(c for c in (catalog[asin]["cause"], offers[asin]["cause"]) if c) or None}
                for asin in asins}

    return {"offers": lambda asins: cx.item_offers(fields, asins),
            # The pricing sync draws this same `competitive_summary` pacer in this worker process.
            "summaries": lambda asins: cx.competitive_summaries(fields, asins),
            "facts": complete}


@contextmanager
def _mirror(runtime):
    """A read cursor on this tenant's Amazon mirror, None when the file does not exist yet."""
    from harness import runtime as R
    if not Path(runtime.datastore_path).is_file():
        yield None
        return
    cur = R.mirror_cursor(runtime, require_ready=False)
    try:
        yield cur
    finally:
        cur.close()


def _after_quotes(tenant, changed):
    for target in AFTER_QUOTES:
        try:
            module_name, _, name = target.partition(":")
            getattr(importlib.import_module(module_name), name)(tenant, changed)
            _say(f"tenant={tenant} after-quotes callback {target} completed")
        except Exception as exc:                                       # noqa: BLE001
            _say(f"tenant={tenant} after-quotes callback {target} failed ({type(exc).__name__}); "
                 "the run result is unchanged.")


def _snapshot_cause(runtime, *, check_lease=True):
    from harness import amazon_store

    enabled = amazon_store.read_database_enablement(runtime)
    if not enabled["available"]:
        return "Competitor database settings could not be read. Restore settings access and retry the run."
    if not all(enabled["databases"].get("ut_amzn_" + table) for table in SNAPSHOT_TABLES):
        return "A competitor database is disabled. Enable competitor snapshots and images in Settings, Connectors and retry."
    if check_lease and amazon_store.first_load_job_live(runtime):
        return "The Amazon mirror writer lease is held. Let the current writer finish and retry the run."
    return None


def _publish_snapshots(tenant, runtime, result):
    from harness import amazon_competitor_facts as facts, amazon_store, mirror_store

    lock = mirror_store.for_connector("amazon").lock
    if not lock.acquire(blocking=False):
        raise SnapshotBlocked("The Amazon mirror writer is busy.")
    holder = f"{JOB}:{result['id']}"
    leased = False
    try:
        leased = amazon_store.acquire_amazon_lease(runtime, holder)
        if not leased:
            raise SnapshotBlocked("The Amazon mirror writer lease is held.")
        cause = _snapshot_cause(runtime, check_lease=False)
        if cause:
            raise SnapshotBlocked(cause)
        report = {"tables": {}, "problems": []}
        with mirror_store.sync_mirror("amazon", tenant, runtime, report=report, expected_tables=SNAPSHOT_TABLES):
            con, _path = amazon_store._connect_tenant_mirror(tenant, runtime)
            try:
                stored = facts.import_snapshots(con, result["snapshots"])
                for table in SNAPSHOT_TABLES:
                    total = con.execute(f"SELECT count(*) FROM amzn_{table}").fetchone()[0]
                    report["tables"][table] = {"state": "done", "rows": total}
            finally:
                con.close()
        if report["problems"]:
            raise RuntimeError("Competitor mirror publication failed")
        result["counts"].update(snapshots=stored["snapshots"], images=stored["images"])
    finally:
        try:
            if leased:
                amazon_store.release_amazon_lease(runtime, holder)
        finally:
            lock.release()


def _run_blocked(runtime, cause):
    from core import amazon_rivals as rivals

    run_id = uuid.uuid4().hex
    rivals.start_run(runtime, run_id, asin_total=0)
    return {"id": run_id, "progress": 0, "cause": cause,
            "counts": {"facts": dict.fromkeys(rivals.FACT_COUNT_COLUMNS, 0)}, "changed": (), "snapshots": []}


def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep):
    """Run today's quotes for one tenant when due. None when it is not due or has no Amazon store."""
    from core import amazon_rivals as rivals
    if str(tenant) != str(runtime.key) or str(runtime.key) != str(runtime.tenant.key):
        raise ValueError("Competitor quotes require the matching tenant runtime")
    try:
        if not rivals.due(runtime, now=now):
            return None
    except rivals.BackendUnavailable:
        if tenant not in _unavailable:
            _unavailable.add(tenant)
            _say(f"tenant={tenant} has no PostgreSQL store, so no rival quotes run")
        return None
    if fields is None:
        from harness import amazon_store
        fields = amazon_store.credential(tenant, runtime)
    if not fields:
        return None
    started = time.monotonic()
    cause = _snapshot_cause(runtime)
    result = (_run_blocked(runtime, cause) if cause else
              rivals.run_quotes(runtime, lambda: _mirror(runtime), **readers(fields),
                                seller_id=str(fields.get("seller_id") or "").strip() or None, now=now,
                                log=lambda message: _say(f"tenant={tenant} {message}"), sleep=sleep))
    if result["snapshots"]:
        try:
            _publish_snapshots(tenant, runtime, result)
        except Exception as exc:
            cause = (str(exc) if isinstance(exc, SnapshotBlocked) else
                     f"Competitor snapshots could not be published ({type(exc).__name__}).")
            result["cause"] = (f"{cause} Check database settings and mirror access, let any writer finish, "
                               f"and retry within {rivals.RETRY_RUN_HOURS} hours.")
    result["counts"]["elapsed_seconds"] = round(time.monotonic() - started, 3)
    rivals.mark_run(runtime, result["id"], progress=result["progress"], finished=True,
                    counts=result["counts"], cause=result["cause"])
    if result["cause"] is None:
        _after_quotes(tenant, result["changed"])
    return result


def run_once(*, for_each_tenant=None, now=None):
    """Admit one independent per-tenant job without waiting for its body."""
    global _missing
    if for_each_tenant is None:
        try:
            from tenant_loops import for_each_tenant
        except ImportError:
            if not _missing:
                _log.warning("Rival quotes unavailable: tenant_loops is not installed.")
                _missing = True
            return {}
    _missing = False
    return for_each_tenant(lambda tenant, runtime: run_tenant(tenant, runtime, now=now),
                           name=JOB, logger=_log, kind="heavy")


def start():
    """Start the daily loop only when process_role admits its heavy coordinator (the sync worker)."""
    global _thread, _stop
    import process_role
    if _thread is not None and _thread.is_alive():
        return _thread
    stop_event = threading.Event()
    _stop = stop_event

    def run():
        while not stop_event.is_set():
            run_once()
            stop_event.wait(POLL_SECONDS)

    # A literal name: verify_background_loops classifies a job by it.
    _thread = process_role.start("amazon-rival-quotes", "heavy", run)
    return _thread


def stop():
    if _stop is not None:
        _stop.set()