File size: 12,381 Bytes
81a8217 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 | """Apify rank transport with one sealed vendor door and a fixture-only default."""
from __future__ import annotations
import hashlib
import json
import os
import re
from datetime import datetime, timezone
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
import providers
def _budget(budget):
return providers.amazon_rank_budget(budget["query_count"], budget["max_results_per_query"],
budget["max_search_pages"])
def apify_call(path, *, budget, allow_paid=False, payload=None, params=None):
"""Open only reviewed Apify rank URLs after spend, scope and explicit paid admission."""
approved = _budget(budget)
if allow_paid is not True:
raise providers.ProviderRefusal("apify_paid_disabled", "Paid Apify requests are disabled. Use a recorded dataset fixture.")
actor = providers.PROVIDERS["apify"].cap("amazon_keyword_rank").actor_id.replace("/", "~")
start = path == f"acts/{actor}/runs"
if not (start or re.fullmatch(r"actor-runs/[A-Za-z0-9]+", path)
or re.fullmatch(r"datasets/[A-Za-z0-9]+/items", path)) or start != (payload is not None):
raise providers.ProviderRefusal("apify_scope", "Apify refused an unreviewed rank operation.")
if start and (set(payload) != {"searchQueries", "maxSearchPages", "maxResultsPerQuery"}
or not isinstance(payload["searchQueries"], list)
or len(payload["searchQueries"]) != approved["query_count"]
or payload["maxSearchPages"] != approved["max_search_pages"]
or payload["maxResultsPerQuery"] != approved["max_results_per_query"]):
raise providers.ProviderRefusal("apify_scope", "Apify rank input differs from its spend estimate.")
key = (os.environ.get("AIOS_APIFY_KEY") or "").strip()
if not key:
raise providers.ProviderRefusal("apify_key", "Apify is not configured for Amazon keyword ranks.")
query = dict(params or {})
if start:
query.update(maxTotalChargeUsd=approved["max_total_charge_usd"], memory=1024, waitForFinish=60)
url = providers.APIFY_BASE + "/" + path + ("?" + urlencode(query) if query else "")
request = Request(url, data=json.dumps(payload).encode() if start else None,
headers={"Authorization": "Bearer " + key, "Content-Type": "application/json"},
method="POST" if start else "GET")
try:
with urlopen(request, timeout=90) as response:
return json.loads(response.read())
except HTTPError as exc:
reasons = {401: "credential rejected", 403: "actor access refused", 402: "insufficient credit",
429: "rate limit", 404: "actor, run or dataset not found"}
raise providers.ProviderRefusal("apify_http", f"Apify Amazon ranks refused: {reasons.get(exc.code, 'vendor HTTP failure')} (HTTP {exc.code}).") from None
except (URLError, TimeoutError, ValueError):
raise providers.ProviderRefusal("apify_transport", "Apify Amazon ranks returned an unavailable or invalid response. Check existing Apify runs before retrying a start.") from None
def read_amazon_rank_run(run_id, *, budget, allow_paid=False):
"""Read one existing run and all its dataset pages without starting another run."""
budget = _budget(budget)
if not re.fullmatch(r"[A-Za-z0-9]+", str(run_id)):
raise providers.ProviderRefusal("apify_run", "Apify rank run ID is invalid.")
run = apify_call(f"actor-runs/{run_id}", budget=budget, allow_paid=allow_paid).get("data", {})
status = run.get("status")
if status != "SUCCEEDED":
allowed = {"READY", "RUNNING", "ABORTING", "ABORTED", "FAILED", "TIMING-OUT", "TIMED-OUT"}
status = status if status in allowed else "unknown status"
raise providers.ProviderRefusal("apify_run_state", f"Apify rank run {run_id} is {status}. Resume this run without starting a second paid run.")
dataset_id = run.get("defaultDatasetId", "")
if not re.fullmatch(r"[A-Za-z0-9]+", str(dataset_id)):
raise providers.ProviderRefusal("apify_dataset", "Apify rank run returned no valid dataset ID.")
items, offset = [], 0
while True:
page = apify_call(f"datasets/{dataset_id}/items", budget=budget, allow_paid=allow_paid,
params={"format": "json", "clean": "false", "offset": offset, "limit": 1000})
if not isinstance(page, list):
raise providers.ProviderRefusal("apify_dataset", "Apify rank dataset returned an invalid page.")
items.extend(page)
if len(page) < 1000:
break
offset += len(page)
return {"actor_id": budget["actor_id"], "dataset_id": dataset_id, "run_id": str(run_id),
"observed_at": run.get("finishedAt"), "items": items,
"cost_usd": run.get("usageTotalUsd"), "budget": budget, "fixture": False}
def amazon_keyword_rank_dataset(queries, *, max_results_per_query, max_search_pages=1,
recorded=None, allow_paid=False):
"""Route one rank capture by capability, or consume its recorded dataset without vendor I/O."""
if not isinstance(queries, (tuple, list)) or any(not isinstance(q, str) or not q.strip() for q in queries):
raise providers.ProviderRefusal("apify_rank_scope", "Amazon rank search queries must be nonempty text.")
queries = list(dict.fromkeys(q.strip() for q in queries))
budget = providers.amazon_rank_budget(len(queries), max_results_per_query, max_search_pages)
if recorded is not None:
if not isinstance(recorded, dict) or recorded.get("actor_id") != budget["actor_id"]:
raise providers.ProviderRefusal("apify_actor", "Recorded Amazon ranks must name the reviewed Apify actor.")
if not isinstance(recorded.get("items"), list):
raise providers.ProviderRefusal("apify_dataset", "Recorded Amazon ranks must contain dataset items.")
return {**recorded, "budget": budget, "fixture": recorded.get("fixture", True)}
if allow_paid is not True:
raise providers.ProviderRefusal("apify_paid_disabled", "Paid Apify requests are disabled. Use a recorded dataset fixture.")
if not any(p.key == "apify" for p in providers.chain("amazon_keyword_rank")):
raise providers.ProviderRefusal("apify_provider", "No configured provider supports Amazon keyword ranks.")
result = apify_call("acts/" + budget["actor_id"].replace("/", "~") + "/runs",
budget=budget, allow_paid=allow_paid,
payload={"searchQueries": queries, "maxSearchPages": max_search_pages,
"maxResultsPerQuery": max_results_per_query})
run_id = (result.get("data") or {}).get("id")
return read_amazon_rank_run(run_id, budget=budget, allow_paid=allow_paid)
def amazon_rank_rows(dataset):
"""Normalize recorded placements while preserving raw records and separate rank meanings."""
if dataset.get("actor_id") != providers.APIFY_AMAZON_ACTOR or not dataset.get("dataset_id"):
raise ValueError("Amazon ranks require the reviewed actor and dataset ID")
try:
observed = datetime.fromisoformat(dataset["observed_at"].replace("Z", "+00:00"))
if observed.tzinfo is None:
raise ValueError()
date = observed.astimezone(timezone.utc).date().isoformat()
except (KeyError, ValueError, TypeError, AttributeError):
raise ValueError("Amazon ranks require a timezone-aware observation date") from None
def rank(value, field, optional=True):
if value is None and optional:
return None
if type(value) is not int or value < 1:
raise ValueError("Amazon rank " + field + " must be a positive integer")
return value
rows, positions = [], set()
for item in dataset["items"]:
if not isinstance(item, dict):
raise ValueError("Amazon rank dataset row must be a record")
keyword, asin = item.get("keyword"), item.get("asin")
if not isinstance(keyword, str) or not keyword.strip() or not isinstance(asin, str) or not asin.strip():
raise ValueError("Amazon rank dataset row lacks keyword or ASIN")
sponsored = item.get("isSponsored")
if type(sponsored) is not bool:
raise ValueError("Amazon rank dataset row lacks a boolean isSponsored")
position = rank(item.get("globalPosition"), "globalPosition", optional=False)
key = (keyword.strip().casefold(), position)
if key in positions:
raise ValueError("Amazon rank dataset repeats a keyword placement")
positions.add(key)
organic = rank(item.get("organicRank"), "organicRank")
paid = rank(item.get("sponsoredRank"), "sponsoredRank")
if sponsored and organic is not None or not sponsored and paid is not None:
raise ValueError("Amazon organic and sponsored rank semantics conflict")
identity = json.dumps([dataset["dataset_id"], keyword.strip(), asin.strip(), position], separators=(",", ":"))
rows.append({"observation_id": hashlib.sha256(identity.encode()).hexdigest(),
"dataset_id": str(dataset["dataset_id"]), "keyword": keyword.strip(), "asin": asin.strip(),
"date": date, "global_position": position, "organic_rank": organic,
"sponsored_rank": paid, "is_sponsored": sponsored,
"sponsored_rank_basis": "vendor sponsoredRank" if paid is not None else None,
"source_json": json.dumps(item, ensure_ascii=False, separators=(",", ":"))})
counts = {}
for row in sorted(rows, key=lambda row: (row["keyword"].casefold(), row["global_position"])):
if row["is_sponsored"]:
key = row["keyword"].casefold()
counts[key] = counts.get(key, 0) + 1
if row["sponsored_rank"] is None:
row["sponsored_rank"] = counts[key]
row["sponsored_rank_basis"] = "Ordered sponsored placements in the captured dataset, sorted by globalPosition."
return rows
def import_amazon_keyword_ranks(con, dataset):
"""Import recorded rank rows into the supplied tenant mirror through its sole writer."""
from harness import amazon_store
return amazon_store.import_keyword_ranks(con, amazon_rank_rows(dataset), dataset["dataset_id"])
def sync_amazon_keyword_ranks(tenant_key, rt, queries, *, max_results_per_query,
max_search_pages=1, recorded=None, allow_paid=False):
"""Publish one explicitly requested rank dataset through the tenant's existing mirror boundary."""
from harness import amazon_store, mirror_store
if rt is None or str(rt.key) != str(tenant_key):
raise providers.ProviderRefusal("apify_tenant", "Amazon keyword ranks require the matching tenant runtime.")
enabled = amazon_store.read_database_enablement(rt)
if not enabled["available"] or not enabled["databases"].get("ut_amzn_keyword_ranks"):
raise providers.ProviderRefusal("apify_database", "Amazon keyword rank database is disabled or its enablement is unavailable.")
dataset = amazon_keyword_rank_dataset(queries, max_results_per_query=max_results_per_query,
max_search_pages=max_search_pages, recorded=recorded,
allow_paid=allow_paid)
rows = amazon_rank_rows(dataset)
report = {"tables": {}, "problems": []}
with mirror_store.sync_mirror("amazon", tenant_key, rt, report=report,
expected_tables=("apify_rank_observations",)):
con, _path = amazon_store._connect_tenant_mirror(tenant_key, rt)
result = amazon_store.import_keyword_ranks(con, rows, dataset["dataset_id"])
report["tables"]["apify_rank_observations"] = {"state": "done", "rows": len(rows)}
if report["problems"]:
raise providers.ProviderRefusal("apify_publication", "Amazon keyword rank publication failed. Retry importing the saved dataset without starting another paid run.")
return {**result, "budget": dataset["budget"], "cost_usd": dataset.get("cost_usd"),
"fixture": dataset["fixture"], "dataset_id": dataset["dataset_id"], "run_id": dataset.get("run_id")}
|