Spaces:
Sleeping
Sleeping
Download tool.py from bugrahans/amazon-scraper-tool: direct link, hf CLI and curl.
- Browser
- Download file 9.37 kB
-
https://huggingface.co/spaces/bugrahans/amazon-scraper-tool/resolve/main/tool.py
- Command line
-
hf download hf://spaces/bugrahans/amazon-scraper-tool/tool.py
-
curl -L -o tool.py https://huggingface.co/spaces/bugrahans/amazon-scraper-tool/resolve/main/tool.py
9.37 kB
| from __future__ import annotations | |
| import json | |
| import os | |
| import re | |
| import urllib.parse | |
| from typing import Any, Dict, Optional | |
| import requests | |
| from smolagents.tools import Tool | |
| SCRAPE_DO_SIGNUP_URL = ( | |
| "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=amazon-scraper" | |
| ) | |
| API_BASE = "https://api.scrape.do/plugin/amazon" | |
| # geocode -> human label, shown in the UI dropdown. | |
| MARKETPLACES: Dict[str, str] = { | |
| "us": "United States (amazon.com)", | |
| "gb": "United Kingdom (amazon.co.uk)", | |
| "de": "Germany (amazon.de)", | |
| "fr": "France (amazon.fr)", | |
| "it": "Italy (amazon.it)", | |
| "es": "Spain (amazon.es)", | |
| "ca": "Canada (amazon.ca)", | |
| "jp": "Japan (amazon.co.jp)", | |
| "in": "India (amazon.in)", | |
| "au": "Australia (amazon.com.au)", | |
| "br": "Brazil (amazon.com.br)", | |
| "mx": "Mexico (amazon.com.mx)", | |
| "nl": "Netherlands (amazon.nl)", | |
| "pl": "Poland (amazon.pl)", | |
| "se": "Sweden (amazon.se)", | |
| "tr": "Turkey (amazon.com.tr)", | |
| "ae": "UAE (amazon.ae)", | |
| "sa": "Saudi Arabia (amazon.sa)", | |
| "sg": "Singapore (amazon.sg)", | |
| "eg": "Egypt (amazon.eg)", | |
| "be": "Belgium (amazon.com.be)", | |
| } | |
| # Marketplaces where Amazon localises delivery by postal code. Everything else | |
| # takes a country-level location instead, via countryName. | |
| ZIP_LEVEL_MARKETS = {"us", "gb", "uk", "de", "fr", "it", "es", "ca", "jp", "in"} | |
| COUNTRY_NAMES: Dict[str, str] = { | |
| "tr": "Turkey", | |
| "br": "Brazil", | |
| "au": "Australia", | |
| "mx": "Mexico", | |
| "nl": "Netherlands", | |
| "pl": "Poland", | |
| "se": "Sweden", | |
| "ae": "United Arab Emirates", | |
| "sa": "Saudi Arabia", | |
| "sg": "Singapore", | |
| "eg": "Egypt", | |
| "be": "Belgium", | |
| } | |
| _ASIN_PATTERN = re.compile(r"(?:/dp/|/gp/product/|/gp/aw/d/|/product/|[?&]asin=)([A-Z0-9]{10})", re.I) | |
| _BARE_ASIN = re.compile(r"^[A-Z0-9]{10}$", re.I) | |
| def extract_asin(value: str) -> Optional[str]: | |
| """Accept a full Amazon URL or a bare ASIN and return the ASIN.""" | |
| if not value: | |
| return None | |
| value = value.strip() | |
| if _BARE_ASIN.match(value): | |
| return value.upper() | |
| match = _ASIN_PATTERN.search(value) | |
| if match: | |
| return match.group(1).upper() | |
| return None | |
| class ScrapeDoAmazonTool(Tool): | |
| name = "amazon_product_scraper" | |
| description = """ | |
| Scrape structured product data from any Amazon marketplace: product detail | |
| pages (title, brand, price, rating, images, rankings, technical specs), | |
| all seller offers for an ASIN, or keyword search results with organic and | |
| sponsored positions. Returns clean JSON, no HTML parsing required. | |
| Powered by Scrape.do's Amazon Scraper API, which handles AWS WAF, proxy | |
| rotation, sessions and CAPTCHAs automatically across 21 marketplaces. | |
| Use this when you need Amazon pricing, catalogue or competitive data. | |
| """ | |
| output_type = "string" | |
| def __init__(self, api_token: Optional[str] = None) -> None: | |
| # Token can be passed explicitly (e.g. by a Gradio UI) or read from the | |
| # SCRAPE_DO_API_TOKEN environment variable for programmatic use. | |
| self._api_token = api_token | |
| self.inputs = { | |
| "mode": { | |
| "type": "string", | |
| "description": ( | |
| "What to scrape: 'product' for a product detail page, " | |
| "'offers' for every seller offer on an ASIN, or 'search' " | |
| "for keyword/category search results." | |
| ), | |
| }, | |
| "query": { | |
| "type": "string", | |
| "description": ( | |
| "An Amazon product URL or ASIN for 'product' and 'offers'. " | |
| "A search keyword for 'search'." | |
| ), | |
| }, | |
| "geocode": { | |
| "type": "string", | |
| "description": ( | |
| "Marketplace country code: 'us', 'gb', 'de', 'jp', 'tr'… " | |
| "Defaults to 'us'." | |
| ), | |
| "nullable": True, | |
| }, | |
| "zipcode": { | |
| "type": "string", | |
| "description": ( | |
| "Postal code for ZIP-level marketplaces (us, gb, de, fr, it, " | |
| "es, ca, jp, in) to localise price, stock and delivery. " | |
| "Ignored elsewhere, where a country-level location is used." | |
| ), | |
| "nullable": True, | |
| }, | |
| "page": { | |
| "type": "integer", | |
| "description": "Search results page number. Only used when mode='search'.", | |
| "nullable": True, | |
| }, | |
| "language": { | |
| "type": "string", | |
| "description": ( | |
| "ISO 639-1 language code in uppercase, e.g. 'EN', 'DE', 'FR'. " | |
| "Must be supported by the target marketplace." | |
| ), | |
| "nullable": True, | |
| }, | |
| } | |
| super().__init__() | |
| # ------------------------------------------------------------------ token | |
| def _resolve_token(self) -> Optional[str]: | |
| if self._api_token: | |
| return self._api_token.strip() | |
| env_token = os.getenv("SCRAPE_DO_API_TOKEN") | |
| return env_token.strip() if env_token else None | |
| # ---------------------------------------------------------------- request | |
| def request( | |
| self, | |
| mode: str, | |
| query: str, | |
| geocode: str = "us", | |
| zipcode: Optional[str] = None, | |
| page: Optional[int] = None, | |
| language: Optional[str] = None, | |
| ) -> Dict[str, Any]: | |
| """Call the Amazon Scraper API and return a parsed dict.""" | |
| token = self._resolve_token() | |
| if not token: | |
| return { | |
| "status": "error", | |
| "errorMessage": "Missing Scrape.do API token.", | |
| "hint": f"Get a free token at {SCRAPE_DO_SIGNUP_URL}", | |
| } | |
| if not query or not query.strip(): | |
| return {"status": "error", "errorMessage": "No ASIN, URL or keyword provided."} | |
| geocode = (geocode or "us").strip().lower() | |
| params: Dict[str, Any] = {"token": token, "geocode": geocode} | |
| if mode in ("product", "offers"): | |
| asin = extract_asin(query) | |
| if not asin: | |
| return { | |
| "status": "error", | |
| "errorMessage": ( | |
| "Could not read an ASIN from that input. Paste a full " | |
| "Amazon product URL (…/dp/B0BLRJ4R8F) or the 10-character " | |
| "ASIN on its own." | |
| ), | |
| } | |
| params["asin"] = asin | |
| endpoint = f"{API_BASE}/pdp" if mode == "product" else f"{API_BASE}/offer-listing" | |
| elif mode == "search": | |
| params["keyword"] = query.strip() | |
| params["page"] = int(page or 1) | |
| endpoint = f"{API_BASE}/search" | |
| else: | |
| return {"status": "error", "errorMessage": f"Unknown mode: {mode}"} | |
| # Location: ZIP for ZIP-level marketplaces, country name for the rest. | |
| if geocode in ZIP_LEVEL_MARKETS: | |
| if zipcode and zipcode.strip(): | |
| params["zipcode"] = zipcode.strip() | |
| elif geocode in COUNTRY_NAMES: | |
| # Passing the marketplace's own country is a no-op, so only send it | |
| # when the caller explicitly asked for a different delivery country. | |
| if zipcode and zipcode.strip(): | |
| params["countryName"] = zipcode.strip() | |
| if language and language.strip(): | |
| params["language"] = language.strip().upper() | |
| url = endpoint + "?" + urllib.parse.urlencode(params, quote_via=urllib.parse.quote) | |
| try: | |
| response = requests.get(url, timeout=90) | |
| except requests.exceptions.RequestException as exc: | |
| return {"status": "error", "errorMessage": f"Request failed: {exc}"} | |
| try: | |
| data = response.json() | |
| except ValueError: | |
| return { | |
| "status": "error", | |
| "errorMessage": f"Unexpected non-JSON response (HTTP {response.status_code}).", | |
| "details": response.text[:500], | |
| } | |
| if response.status_code == 401 or "token" in str(data.get("error", "")).lower(): | |
| data.setdefault( | |
| "hint", f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" | |
| ) | |
| if isinstance(data, dict) and "status" not in data: | |
| data["status"] = "success" if response.ok else "error" | |
| if isinstance(data, dict) and not response.ok and not data.get("errorMessage"): | |
| data["errorMessage"] = data.get("message") or f"HTTP {response.status_code}" | |
| return data | |
| # ---------------------------------------------------------------- forward | |
| def forward( | |
| self, | |
| mode: str, | |
| query: str, | |
| geocode: Optional[str] = None, | |
| zipcode: Optional[str] = None, | |
| page: Optional[int] = None, | |
| language: Optional[str] = None, | |
| ) -> str: | |
| result = self.request( | |
| mode=mode, | |
| query=query, | |
| geocode=geocode or "us", | |
| zipcode=zipcode, | |
| page=page, | |
| language=language, | |
| ) | |
| return json.dumps(result, ensure_ascii=False, indent=2) | |