Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Download api/connectors_apify.py from fsanyoto/runloopable: direct link, hf CLI and curl.
- Browser
- Download file 12.4 kB
-
https://huggingface.co/spaces/fsanyoto/runloopable/resolve/main/api/connectors_apify.py
- Command line
-
hf download hf://spaces/fsanyoto/runloopable/api/connectors_apify.py
-
curl -L -o connectors_apify.py https://huggingface.co/spaces/fsanyoto/runloopable/resolve/main/api/connectors_apify.py
12.4 kB
| """Apify rank transport with one sealed vendor door and a fixture-only default.""" | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import os | |
| import re | |
| from datetime import datetime, timezone | |
| from urllib.error import HTTPError, URLError | |
| from urllib.parse import urlencode | |
| from urllib.request import Request, urlopen | |
| import providers | |
| def _budget(budget): | |
| return providers.amazon_rank_budget(budget["query_count"], budget["max_results_per_query"], | |
| budget["max_search_pages"]) | |
| def apify_call(path, *, budget, allow_paid=False, payload=None, params=None): | |
| """Open only reviewed Apify rank URLs after spend, scope and explicit paid admission.""" | |
| approved = _budget(budget) | |
| if allow_paid is not True: | |
| raise providers.ProviderRefusal("apify_paid_disabled", "Paid Apify requests are disabled. Use a recorded dataset fixture.") | |
| actor = providers.PROVIDERS["apify"].cap("amazon_keyword_rank").actor_id.replace("/", "~") | |
| start = path == f"acts/{actor}/runs" | |
| if not (start or re.fullmatch(r"actor-runs/[A-Za-z0-9]+", path) | |
| or re.fullmatch(r"datasets/[A-Za-z0-9]+/items", path)) or start != (payload is not None): | |
| raise providers.ProviderRefusal("apify_scope", "Apify refused an unreviewed rank operation.") | |
| if start and (set(payload) != {"searchQueries", "maxSearchPages", "maxResultsPerQuery"} | |
| or not isinstance(payload["searchQueries"], list) | |
| or len(payload["searchQueries"]) != approved["query_count"] | |
| or payload["maxSearchPages"] != approved["max_search_pages"] | |
| or payload["maxResultsPerQuery"] != approved["max_results_per_query"]): | |
| raise providers.ProviderRefusal("apify_scope", "Apify rank input differs from its spend estimate.") | |
| key = (os.environ.get("AIOS_APIFY_KEY") or "").strip() | |
| if not key: | |
| raise providers.ProviderRefusal("apify_key", "Apify is not configured for Amazon keyword ranks.") | |
| query = dict(params or {}) | |
| if start: | |
| query.update(maxTotalChargeUsd=approved["max_total_charge_usd"], memory=1024, waitForFinish=60) | |
| url = providers.APIFY_BASE + "/" + path + ("?" + urlencode(query) if query else "") | |
| request = Request(url, data=json.dumps(payload).encode() if start else None, | |
| headers={"Authorization": "Bearer " + key, "Content-Type": "application/json"}, | |
| method="POST" if start else "GET") | |
| try: | |
| with urlopen(request, timeout=90) as response: | |
| return json.loads(response.read()) | |
| except HTTPError as exc: | |
| reasons = {401: "credential rejected", 403: "actor access refused", 402: "insufficient credit", | |
| 429: "rate limit", 404: "actor, run or dataset not found"} | |
| raise providers.ProviderRefusal("apify_http", f"Apify Amazon ranks refused: {reasons.get(exc.code, 'vendor HTTP failure')} (HTTP {exc.code}).") from None | |
| except (URLError, TimeoutError, ValueError): | |
| raise providers.ProviderRefusal("apify_transport", "Apify Amazon ranks returned an unavailable or invalid response. Check existing Apify runs before retrying a start.") from None | |
| def read_amazon_rank_run(run_id, *, budget, allow_paid=False): | |
| """Read one existing run and all its dataset pages without starting another run.""" | |
| budget = _budget(budget) | |
| if not re.fullmatch(r"[A-Za-z0-9]+", str(run_id)): | |
| raise providers.ProviderRefusal("apify_run", "Apify rank run ID is invalid.") | |
| run = apify_call(f"actor-runs/{run_id}", budget=budget, allow_paid=allow_paid).get("data", {}) | |
| status = run.get("status") | |
| if status != "SUCCEEDED": | |
| allowed = {"READY", "RUNNING", "ABORTING", "ABORTED", "FAILED", "TIMING-OUT", "TIMED-OUT"} | |
| status = status if status in allowed else "unknown status" | |
| raise providers.ProviderRefusal("apify_run_state", f"Apify rank run {run_id} is {status}. Resume this run without starting a second paid run.") | |
| dataset_id = run.get("defaultDatasetId", "") | |
| if not re.fullmatch(r"[A-Za-z0-9]+", str(dataset_id)): | |
| raise providers.ProviderRefusal("apify_dataset", "Apify rank run returned no valid dataset ID.") | |
| items, offset = [], 0 | |
| while True: | |
| page = apify_call(f"datasets/{dataset_id}/items", budget=budget, allow_paid=allow_paid, | |
| params={"format": "json", "clean": "false", "offset": offset, "limit": 1000}) | |
| if not isinstance(page, list): | |
| raise providers.ProviderRefusal("apify_dataset", "Apify rank dataset returned an invalid page.") | |
| items.extend(page) | |
| if len(page) < 1000: | |
| break | |
| offset += len(page) | |
| return {"actor_id": budget["actor_id"], "dataset_id": dataset_id, "run_id": str(run_id), | |
| "observed_at": run.get("finishedAt"), "items": items, | |
| "cost_usd": run.get("usageTotalUsd"), "budget": budget, "fixture": False} | |
| def amazon_keyword_rank_dataset(queries, *, max_results_per_query, max_search_pages=1, | |
| recorded=None, allow_paid=False): | |
| """Route one rank capture by capability, or consume its recorded dataset without vendor I/O.""" | |
| if not isinstance(queries, (tuple, list)) or any(not isinstance(q, str) or not q.strip() for q in queries): | |
| raise providers.ProviderRefusal("apify_rank_scope", "Amazon rank search queries must be nonempty text.") | |
| queries = list(dict.fromkeys(q.strip() for q in queries)) | |
| budget = providers.amazon_rank_budget(len(queries), max_results_per_query, max_search_pages) | |
| if recorded is not None: | |
| if not isinstance(recorded, dict) or recorded.get("actor_id") != budget["actor_id"]: | |
| raise providers.ProviderRefusal("apify_actor", "Recorded Amazon ranks must name the reviewed Apify actor.") | |
| if not isinstance(recorded.get("items"), list): | |
| raise providers.ProviderRefusal("apify_dataset", "Recorded Amazon ranks must contain dataset items.") | |
| return {**recorded, "budget": budget, "fixture": recorded.get("fixture", True)} | |
| if allow_paid is not True: | |
| raise providers.ProviderRefusal("apify_paid_disabled", "Paid Apify requests are disabled. Use a recorded dataset fixture.") | |
| if not any(p.key == "apify" for p in providers.chain("amazon_keyword_rank")): | |
| raise providers.ProviderRefusal("apify_provider", "No configured provider supports Amazon keyword ranks.") | |
| result = apify_call("acts/" + budget["actor_id"].replace("/", "~") + "/runs", | |
| budget=budget, allow_paid=allow_paid, | |
| payload={"searchQueries": queries, "maxSearchPages": max_search_pages, | |
| "maxResultsPerQuery": max_results_per_query}) | |
| run_id = (result.get("data") or {}).get("id") | |
| return read_amazon_rank_run(run_id, budget=budget, allow_paid=allow_paid) | |
| def amazon_rank_rows(dataset): | |
| """Normalize recorded placements while preserving raw records and separate rank meanings.""" | |
| if dataset.get("actor_id") != providers.APIFY_AMAZON_ACTOR or not dataset.get("dataset_id"): | |
| raise ValueError("Amazon ranks require the reviewed actor and dataset ID") | |
| try: | |
| observed = datetime.fromisoformat(dataset["observed_at"].replace("Z", "+00:00")) | |
| if observed.tzinfo is None: | |
| raise ValueError() | |
| date = observed.astimezone(timezone.utc).date().isoformat() | |
| except (KeyError, ValueError, TypeError, AttributeError): | |
| raise ValueError("Amazon ranks require a timezone-aware observation date") from None | |
| def rank(value, field, optional=True): | |
| if value is None and optional: | |
| return None | |
| if type(value) is not int or value < 1: | |
| raise ValueError("Amazon rank " + field + " must be a positive integer") | |
| return value | |
| rows, positions = [], set() | |
| for item in dataset["items"]: | |
| if not isinstance(item, dict): | |
| raise ValueError("Amazon rank dataset row must be a record") | |
| keyword, asin = item.get("keyword"), item.get("asin") | |
| if not isinstance(keyword, str) or not keyword.strip() or not isinstance(asin, str) or not asin.strip(): | |
| raise ValueError("Amazon rank dataset row lacks keyword or ASIN") | |
| sponsored = item.get("isSponsored") | |
| if type(sponsored) is not bool: | |
| raise ValueError("Amazon rank dataset row lacks a boolean isSponsored") | |
| position = rank(item.get("globalPosition"), "globalPosition", optional=False) | |
| key = (keyword.strip().casefold(), position) | |
| if key in positions: | |
| raise ValueError("Amazon rank dataset repeats a keyword placement") | |
| positions.add(key) | |
| organic = rank(item.get("organicRank"), "organicRank") | |
| paid = rank(item.get("sponsoredRank"), "sponsoredRank") | |
| if sponsored and organic is not None or not sponsored and paid is not None: | |
| raise ValueError("Amazon organic and sponsored rank semantics conflict") | |
| identity = json.dumps([dataset["dataset_id"], keyword.strip(), asin.strip(), position], separators=(",", ":")) | |
| rows.append({"observation_id": hashlib.sha256(identity.encode()).hexdigest(), | |
| "dataset_id": str(dataset["dataset_id"]), "keyword": keyword.strip(), "asin": asin.strip(), | |
| "date": date, "global_position": position, "organic_rank": organic, | |
| "sponsored_rank": paid, "is_sponsored": sponsored, | |
| "sponsored_rank_basis": "vendor sponsoredRank" if paid is not None else None, | |
| "source_json": json.dumps(item, ensure_ascii=False, separators=(",", ":"))}) | |
| counts = {} | |
| for row in sorted(rows, key=lambda row: (row["keyword"].casefold(), row["global_position"])): | |
| if row["is_sponsored"]: | |
| key = row["keyword"].casefold() | |
| counts[key] = counts.get(key, 0) + 1 | |
| if row["sponsored_rank"] is None: | |
| row["sponsored_rank"] = counts[key] | |
| row["sponsored_rank_basis"] = "Ordered sponsored placements in the captured dataset, sorted by globalPosition." | |
| return rows | |
| def import_amazon_keyword_ranks(con, dataset): | |
| """Import recorded rank rows into the supplied tenant mirror through its sole writer.""" | |
| from harness import amazon_store | |
| return amazon_store.import_keyword_ranks(con, amazon_rank_rows(dataset), dataset["dataset_id"]) | |
| def sync_amazon_keyword_ranks(tenant_key, rt, queries, *, max_results_per_query, | |
| max_search_pages=1, recorded=None, allow_paid=False): | |
| """Publish one explicitly requested rank dataset through the tenant's existing mirror boundary.""" | |
| from harness import amazon_store, mirror_store | |
| if rt is None or str(rt.key) != str(tenant_key): | |
| raise providers.ProviderRefusal("apify_tenant", "Amazon keyword ranks require the matching tenant runtime.") | |
| enabled = amazon_store.read_database_enablement(rt) | |
| if not enabled["available"] or not enabled["databases"].get("ut_amzn_keyword_ranks"): | |
| raise providers.ProviderRefusal("apify_database", "Amazon keyword rank database is disabled or its enablement is unavailable.") | |
| dataset = amazon_keyword_rank_dataset(queries, max_results_per_query=max_results_per_query, | |
| max_search_pages=max_search_pages, recorded=recorded, | |
| allow_paid=allow_paid) | |
| rows = amazon_rank_rows(dataset) | |
| report = {"tables": {}, "problems": []} | |
| with mirror_store.sync_mirror("amazon", tenant_key, rt, report=report, | |
| expected_tables=("apify_rank_observations",)): | |
| con, _path = amazon_store._connect_tenant_mirror(tenant_key, rt) | |
| result = amazon_store.import_keyword_ranks(con, rows, dataset["dataset_id"]) | |
| report["tables"]["apify_rank_observations"] = {"state": "done", "rows": len(rows)} | |
| if report["problems"]: | |
| raise providers.ProviderRefusal("apify_publication", "Amazon keyword rank publication failed. Retry importing the saved dataset without starting another paid run.") | |
| return {**result, "budget": dataset["budget"], "cost_usd": dataset.get("cost_usd"), | |
| "fixture": dataset["fixture"], "dataset_id": dataset["dataset_id"], "run_id": dataset.get("run_id")} | |