File size: 12,381 Bytes
81a8217
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
"""Apify rank transport with one sealed vendor door and a fixture-only default."""
from __future__ import annotations

import hashlib
import json
import os
import re
from datetime import datetime, timezone
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen

import providers


def _budget(budget):
    return providers.amazon_rank_budget(budget["query_count"], budget["max_results_per_query"],
                                       budget["max_search_pages"])


def apify_call(path, *, budget, allow_paid=False, payload=None, params=None):
    """Open only reviewed Apify rank URLs after spend, scope and explicit paid admission."""
    approved = _budget(budget)
    if allow_paid is not True:
        raise providers.ProviderRefusal("apify_paid_disabled", "Paid Apify requests are disabled. Use a recorded dataset fixture.")
    actor = providers.PROVIDERS["apify"].cap("amazon_keyword_rank").actor_id.replace("/", "~")
    start = path == f"acts/{actor}/runs"
    if not (start or re.fullmatch(r"actor-runs/[A-Za-z0-9]+", path)
            or re.fullmatch(r"datasets/[A-Za-z0-9]+/items", path)) or start != (payload is not None):
        raise providers.ProviderRefusal("apify_scope", "Apify refused an unreviewed rank operation.")
    if start and (set(payload) != {"searchQueries", "maxSearchPages", "maxResultsPerQuery"}
                  or not isinstance(payload["searchQueries"], list)
                  or len(payload["searchQueries"]) != approved["query_count"]
                  or payload["maxSearchPages"] != approved["max_search_pages"]
                  or payload["maxResultsPerQuery"] != approved["max_results_per_query"]):
        raise providers.ProviderRefusal("apify_scope", "Apify rank input differs from its spend estimate.")
    key = (os.environ.get("AIOS_APIFY_KEY") or "").strip()
    if not key:
        raise providers.ProviderRefusal("apify_key", "Apify is not configured for Amazon keyword ranks.")
    query = dict(params or {})
    if start:
        query.update(maxTotalChargeUsd=approved["max_total_charge_usd"], memory=1024, waitForFinish=60)
    url = providers.APIFY_BASE + "/" + path + ("?" + urlencode(query) if query else "")
    request = Request(url, data=json.dumps(payload).encode() if start else None,
                      headers={"Authorization": "Bearer " + key, "Content-Type": "application/json"},
                      method="POST" if start else "GET")
    try:
        with urlopen(request, timeout=90) as response:
            return json.loads(response.read())
    except HTTPError as exc:
        reasons = {401: "credential rejected", 403: "actor access refused", 402: "insufficient credit",
                   429: "rate limit", 404: "actor, run or dataset not found"}
        raise providers.ProviderRefusal("apify_http", f"Apify Amazon ranks refused: {reasons.get(exc.code, 'vendor HTTP failure')} (HTTP {exc.code}).") from None
    except (URLError, TimeoutError, ValueError):
        raise providers.ProviderRefusal("apify_transport", "Apify Amazon ranks returned an unavailable or invalid response. Check existing Apify runs before retrying a start.") from None


def read_amazon_rank_run(run_id, *, budget, allow_paid=False):
    """Read one existing run and all its dataset pages without starting another run."""
    budget = _budget(budget)
    if not re.fullmatch(r"[A-Za-z0-9]+", str(run_id)):
        raise providers.ProviderRefusal("apify_run", "Apify rank run ID is invalid.")
    run = apify_call(f"actor-runs/{run_id}", budget=budget, allow_paid=allow_paid).get("data", {})
    status = run.get("status")
    if status != "SUCCEEDED":
        allowed = {"READY", "RUNNING", "ABORTING", "ABORTED", "FAILED", "TIMING-OUT", "TIMED-OUT"}
        status = status if status in allowed else "unknown status"
        raise providers.ProviderRefusal("apify_run_state", f"Apify rank run {run_id} is {status}. Resume this run without starting a second paid run.")
    dataset_id = run.get("defaultDatasetId", "")
    if not re.fullmatch(r"[A-Za-z0-9]+", str(dataset_id)):
        raise providers.ProviderRefusal("apify_dataset", "Apify rank run returned no valid dataset ID.")
    items, offset = [], 0
    while True:
        page = apify_call(f"datasets/{dataset_id}/items", budget=budget, allow_paid=allow_paid,
                          params={"format": "json", "clean": "false", "offset": offset, "limit": 1000})
        if not isinstance(page, list):
            raise providers.ProviderRefusal("apify_dataset", "Apify rank dataset returned an invalid page.")
        items.extend(page)
        if len(page) < 1000:
            break
        offset += len(page)
    return {"actor_id": budget["actor_id"], "dataset_id": dataset_id, "run_id": str(run_id),
            "observed_at": run.get("finishedAt"), "items": items,
            "cost_usd": run.get("usageTotalUsd"), "budget": budget, "fixture": False}


def amazon_keyword_rank_dataset(queries, *, max_results_per_query, max_search_pages=1,
                                recorded=None, allow_paid=False):
    """Route one rank capture by capability, or consume its recorded dataset without vendor I/O."""
    if not isinstance(queries, (tuple, list)) or any(not isinstance(q, str) or not q.strip() for q in queries):
        raise providers.ProviderRefusal("apify_rank_scope", "Amazon rank search queries must be nonempty text.")
    queries = list(dict.fromkeys(q.strip() for q in queries))
    budget = providers.amazon_rank_budget(len(queries), max_results_per_query, max_search_pages)
    if recorded is not None:
        if not isinstance(recorded, dict) or recorded.get("actor_id") != budget["actor_id"]:
            raise providers.ProviderRefusal("apify_actor", "Recorded Amazon ranks must name the reviewed Apify actor.")
        if not isinstance(recorded.get("items"), list):
            raise providers.ProviderRefusal("apify_dataset", "Recorded Amazon ranks must contain dataset items.")
        return {**recorded, "budget": budget, "fixture": recorded.get("fixture", True)}
    if allow_paid is not True:
        raise providers.ProviderRefusal("apify_paid_disabled", "Paid Apify requests are disabled. Use a recorded dataset fixture.")
    if not any(p.key == "apify" for p in providers.chain("amazon_keyword_rank")):
        raise providers.ProviderRefusal("apify_provider", "No configured provider supports Amazon keyword ranks.")
    result = apify_call("acts/" + budget["actor_id"].replace("/", "~") + "/runs",
                        budget=budget, allow_paid=allow_paid,
                        payload={"searchQueries": queries, "maxSearchPages": max_search_pages,
                                 "maxResultsPerQuery": max_results_per_query})
    run_id = (result.get("data") or {}).get("id")
    return read_amazon_rank_run(run_id, budget=budget, allow_paid=allow_paid)


def amazon_rank_rows(dataset):
    """Normalize recorded placements while preserving raw records and separate rank meanings."""
    if dataset.get("actor_id") != providers.APIFY_AMAZON_ACTOR or not dataset.get("dataset_id"):
        raise ValueError("Amazon ranks require the reviewed actor and dataset ID")
    try:
        observed = datetime.fromisoformat(dataset["observed_at"].replace("Z", "+00:00"))
        if observed.tzinfo is None:
            raise ValueError()
        date = observed.astimezone(timezone.utc).date().isoformat()
    except (KeyError, ValueError, TypeError, AttributeError):
        raise ValueError("Amazon ranks require a timezone-aware observation date") from None

    def rank(value, field, optional=True):
        if value is None and optional:
            return None
        if type(value) is not int or value < 1:
            raise ValueError("Amazon rank " + field + " must be a positive integer")
        return value

    rows, positions = [], set()
    for item in dataset["items"]:
        if not isinstance(item, dict):
            raise ValueError("Amazon rank dataset row must be a record")
        keyword, asin = item.get("keyword"), item.get("asin")
        if not isinstance(keyword, str) or not keyword.strip() or not isinstance(asin, str) or not asin.strip():
            raise ValueError("Amazon rank dataset row lacks keyword or ASIN")
        sponsored = item.get("isSponsored")
        if type(sponsored) is not bool:
            raise ValueError("Amazon rank dataset row lacks a boolean isSponsored")
        position = rank(item.get("globalPosition"), "globalPosition", optional=False)
        key = (keyword.strip().casefold(), position)
        if key in positions:
            raise ValueError("Amazon rank dataset repeats a keyword placement")
        positions.add(key)
        organic = rank(item.get("organicRank"), "organicRank")
        paid = rank(item.get("sponsoredRank"), "sponsoredRank")
        if sponsored and organic is not None or not sponsored and paid is not None:
            raise ValueError("Amazon organic and sponsored rank semantics conflict")
        identity = json.dumps([dataset["dataset_id"], keyword.strip(), asin.strip(), position], separators=(",", ":"))
        rows.append({"observation_id": hashlib.sha256(identity.encode()).hexdigest(),
                     "dataset_id": str(dataset["dataset_id"]), "keyword": keyword.strip(), "asin": asin.strip(),
                     "date": date, "global_position": position, "organic_rank": organic,
                     "sponsored_rank": paid, "is_sponsored": sponsored,
                     "sponsored_rank_basis": "vendor sponsoredRank" if paid is not None else None,
                     "source_json": json.dumps(item, ensure_ascii=False, separators=(",", ":"))})
    counts = {}
    for row in sorted(rows, key=lambda row: (row["keyword"].casefold(), row["global_position"])):
        if row["is_sponsored"]:
            key = row["keyword"].casefold()
            counts[key] = counts.get(key, 0) + 1
            if row["sponsored_rank"] is None:
                row["sponsored_rank"] = counts[key]
                row["sponsored_rank_basis"] = "Ordered sponsored placements in the captured dataset, sorted by globalPosition."
    return rows


def import_amazon_keyword_ranks(con, dataset):
    """Import recorded rank rows into the supplied tenant mirror through its sole writer."""
    from harness import amazon_store
    return amazon_store.import_keyword_ranks(con, amazon_rank_rows(dataset), dataset["dataset_id"])


def sync_amazon_keyword_ranks(tenant_key, rt, queries, *, max_results_per_query,
                             max_search_pages=1, recorded=None, allow_paid=False):
    """Publish one explicitly requested rank dataset through the tenant's existing mirror boundary."""
    from harness import amazon_store, mirror_store

    if rt is None or str(rt.key) != str(tenant_key):
        raise providers.ProviderRefusal("apify_tenant", "Amazon keyword ranks require the matching tenant runtime.")
    enabled = amazon_store.read_database_enablement(rt)
    if not enabled["available"] or not enabled["databases"].get("ut_amzn_keyword_ranks"):
        raise providers.ProviderRefusal("apify_database", "Amazon keyword rank database is disabled or its enablement is unavailable.")
    dataset = amazon_keyword_rank_dataset(queries, max_results_per_query=max_results_per_query,
                                          max_search_pages=max_search_pages, recorded=recorded,
                                          allow_paid=allow_paid)
    rows = amazon_rank_rows(dataset)
    report = {"tables": {}, "problems": []}
    with mirror_store.sync_mirror("amazon", tenant_key, rt, report=report,
                                  expected_tables=("apify_rank_observations",)):
        con, _path = amazon_store._connect_tenant_mirror(tenant_key, rt)
        result = amazon_store.import_keyword_ranks(con, rows, dataset["dataset_id"])
        report["tables"]["apify_rank_observations"] = {"state": "done", "rows": len(rows)}
    if report["problems"]:
        raise providers.ProviderRefusal("apify_publication", "Amazon keyword rank publication failed. Retry importing the saved dataset without starting another paid run.")
    return {**result, "budget": dataset["budget"], "cost_usd": dataset.get("cost_usd"),
            "fixture": dataset["fixture"], "dataset_id": dataset["dataset_id"], "run_id": dataset.get("run_id")}