Spaces:
Sleeping
Sleeping
Download tool.py from bugrahans/walmart-scraper: direct link, hf CLI and curl.
- Browser
- Download file 17.3 kB
-
https://huggingface.co/spaces/bugrahans/walmart-scraper/resolve/main/tool.py
- Command line
-
hf download hf://spaces/bugrahans/walmart-scraper/tool.py
-
curl -L -o tool.py https://huggingface.co/spaces/bugrahans/walmart-scraper/resolve/main/tool.py
17.3 kB
| from __future__ import annotations | |
| import json | |
| import os | |
| import re | |
| import time | |
| import urllib.parse | |
| from concurrent.futures import ThreadPoolExecutor | |
| from typing import Any, Dict, List, Optional, Tuple | |
| import requests | |
| from smolagents.tools import Tool | |
| SCRAPE_DO_SIGNUP_URL = ( | |
| "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=walmart-scraper" | |
| ) | |
| SEARCH_ENDPOINT = "https://api.scrape.do/plugin/walmart/search" | |
| PRODUCT_ENDPOINT = "https://api.scrape.do/plugin/walmart/product" | |
| # From the Scrape.do docs: 10 credits per successful call, failed calls are free. | |
| CREDITS_PER_CALL = 10 | |
| # Secaucus, NJ. The store used in Scrape.do's Walmart scraping guide. | |
| DEFAULT_LOCATION = "07094" | |
| # Secaucus NJ, Deer Park IL, Dallas TX, Los Angeles CA. | |
| DEFAULT_COMPARE_LOCATIONS = ["07094", "60074", "75201", "90012"] | |
| MAX_COMPARE_LOCATIONS = 5 | |
| # ------------------------------------------------------------------ helpers | |
| def parse_location(value: Optional[str]) -> Tuple[Optional[Dict[str, str]], Optional[str]]: | |
| """Read a 5-digit ZIP code or a Walmart store number. | |
| Returns ({"zipcode": ...} or {"store": ...}, None) or (None, error message). | |
| """ | |
| text = (value or "").strip() or DEFAULT_LOCATION | |
| cleaned = text.lower().replace("#", "").replace("no.", "").strip() | |
| match = re.fullmatch(r"store\s*:?\s*(\d{1,6})", cleaned) | |
| if match: | |
| return {"store": match.group(1)}, None | |
| if re.fullmatch(r"\d{5}", cleaned): | |
| return {"zipcode": cleaned}, None | |
| if re.fullmatch(r"\d{5}-\d{4}", cleaned): | |
| return {"zipcode": cleaned[:5]}, None | |
| if re.fullmatch(r"\d{1,4}", cleaned): | |
| return {"store": cleaned}, None | |
| return None, f"“{text}” isn't a 5-digit US ZIP code or a Walmart store number." | |
| def split_locations(value: Optional[str]) -> List[str]: | |
| parts = [p.strip() for p in re.split(r"[,;\n]+", value or "") if p.strip()] | |
| seen: List[str] = [] | |
| for part in parts: | |
| if part not in seen: | |
| seen.append(part) | |
| return seen | |
| def parse_item_id(text: Optional[str]) -> Optional[str]: | |
| """Pull Walmart's numeric item id out of a product link, or accept a bare id.""" | |
| decoded = urllib.parse.unquote(text or "").strip() | |
| match = re.search(r"/ip/(?:[^/?#]+/)*(\d{4,})", decoded) | |
| if match: | |
| return match.group(1) | |
| match = re.search(r"[?&](?:itemId|item_id|usItemId|item)=(\d{4,})", decoded) | |
| if match: | |
| return match.group(1) | |
| if re.fullmatch(r"\d{4,15}", decoded): | |
| return decoded | |
| return None | |
| def _is_url(text: str) -> bool: | |
| return text.lower().startswith(("http://", "https://", "www.", "walmart.com")) | |
| def describe_store(store: Optional[Dict[str, Any]]) -> str: | |
| if not isinstance(store, dict) or not store: | |
| return "" | |
| place = ", ".join(p for p in [store.get("city"), store.get("state")] if p) | |
| label = f"Walmart #{store.get('store_id')}" if store.get("store_id") else "Walmart" | |
| if place: | |
| label += f", {place}" | |
| if store.get("postal_code"): | |
| label += f" {store['postal_code']}" | |
| return label | |
| # --------------------------------------------------------------------- tool | |
| class ScrapeDoWalmartTool(Tool): | |
| name = "walmart_scraper" | |
| description = """ | |
| Scrape Walmart.com. Walmart prices and stock depend on the store, so every | |
| call is tied to a US ZIP code or a Walmart store number. Three modes: | |
| - 'search': products for a keyword at one store, with price, list price, | |
| rating, review count, seller, stock status, pickup/delivery and item id. | |
| - 'product': full details for one product: price, discount, stock, seller, | |
| images, highlights, specifications, UPC/GTIN, model number, description. | |
| - 'compare': the same product's price and stock at several stores at once. | |
| For 'product' and 'compare', pass a Walmart product link, an item id, or a | |
| product name. Powered by Scrape.do's Walmart API, which handles proxies, | |
| bot protection and store selection. | |
| """ | |
| output_type = "string" | |
| def __init__(self, api_token: Optional[str] = None) -> None: | |
| self._api_token = api_token | |
| self.inputs = { | |
| "mode": { | |
| "type": "string", | |
| "description": "'search' for a keyword, 'product' for one product's details, 'compare' for one product's price across stores.", | |
| }, | |
| "query": { | |
| "type": "string", | |
| "description": ( | |
| "For 'search': the keyword, e.g. 'whole milk'. For 'product' and 'compare': a Walmart " | |
| "product link, the numeric item id, or a product name." | |
| ), | |
| }, | |
| "location": { | |
| "type": "string", | |
| "description": ( | |
| "A 5-digit US ZIP code or a Walmart store number. For 'compare', a comma-separated list " | |
| "of up to 5. Defaults to ZIP 07094, or four cities across the US for 'compare'." | |
| ), | |
| "nullable": True, | |
| }, | |
| } | |
| super().__init__() | |
| # ------------------------------------------------------------ plumbing | |
| def _resolve_token(self) -> Optional[str]: | |
| if self._api_token: | |
| return self._api_token.strip() | |
| env_token = os.getenv("SCRAPE_DO_API_TOKEN") | |
| return env_token.strip() if env_token else None | |
| def _missing_token() -> Dict[str, Any]: | |
| return { | |
| "status": "error", | |
| "errorMessage": "Missing Scrape.do API token.", | |
| "hint": f"Get a free token at {SCRAPE_DO_SIGNUP_URL}", | |
| } | |
| def _as_error(data: Dict[str, Any]) -> Dict[str, Any]: | |
| result = {"status": "error", "errorMessage": data.get("_error", "Unknown error")} | |
| if data.get("_auth"): | |
| result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" | |
| return result | |
| def _get(self, url: str, params: Dict[str, Any], retries: int = 1) -> Dict[str, Any]: | |
| """GET a Scrape.do endpoint. Retries once on 5xx, which Scrape.do doesn't charge for.""" | |
| token = str(params.get("token") or "") | |
| def redact(text: str) -> str: | |
| return text.replace(token, "***") if token else text | |
| full_url = url + "?" + urllib.parse.urlencode(params, quote_via=urllib.parse.quote_plus) | |
| last: Dict[str, Any] = {"_error": "Request failed."} | |
| for attempt in range(retries + 1): | |
| try: | |
| response = requests.get(full_url, timeout=120) | |
| except requests.exceptions.RequestException as exc: | |
| last = {"_error": redact(f"Request failed: {exc}")} | |
| if attempt < retries: | |
| time.sleep(1.5) | |
| continue | |
| return last | |
| if response.status_code == 401: | |
| return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True} | |
| try: | |
| data = response.json() | |
| except ValueError: | |
| data = None | |
| if response.ok and isinstance(data, dict): | |
| return data | |
| message = "" | |
| if isinstance(data, dict): | |
| parts = [str(data.get(key)).strip().rstrip(".") for key in ("error", "message") if data.get(key)] | |
| message = ": ".join(parts) | |
| if not message: | |
| message = (response.text or "").strip()[:200] or "no details" | |
| if response.status_code == 502: | |
| text = ( | |
| "Walmart didn't return a valid page for this store. Failed calls aren't charged, " | |
| f"so try again or use another ZIP code. ({message})" | |
| ) | |
| else: | |
| text = f"Scrape.do returned HTTP {response.status_code}: {message}" | |
| last = {"_error": redact(text), "_status": response.status_code} | |
| if response.status_code in (500, 502, 503, 504) and attempt < retries: | |
| time.sleep(1.5) | |
| continue | |
| return last | |
| return last | |
| def _resolve_item( | |
| self, token: str, ref: str, location: Dict[str, str] | |
| ) -> Dict[str, Any]: | |
| """Item id from a link, an id, or (one extra search) a product name.""" | |
| text = (ref or "").strip() | |
| if not text: | |
| return {"error": "Paste a Walmart product link, an item id, or a product name."} | |
| item_id = parse_item_id(text) | |
| if item_id: | |
| return {"item_id": item_id, "credits": 0, "note": None} | |
| if _is_url(text): | |
| if "walmart." not in text.lower(): | |
| return {"error": "That isn't a Walmart link. Paste a walmart.com product link."} | |
| return { | |
| "error": "Couldn't find a product id in that link. Open the product page on Walmart " | |
| "(the address contains /ip/) and copy that link, or use the Search tab." | |
| } | |
| data = self._get(SEARCH_ENDPOINT, dict({"token": token, "q": text}, **location)) | |
| if "_error" in data: | |
| return {"error": data["_error"], "auth": data.get("_auth")} | |
| results = [r for r in data.get("results") or [] if r.get("item_id")] | |
| if not results: | |
| return {"error": f"Walmart found no product for “{text}”."} | |
| best = next((r for r in results if not r.get("sponsored")), results[0]) | |
| note = ( | |
| f"Searched Walmart for “{text}” and used the first match: {best.get('name')} " | |
| f"({CREDITS_PER_CALL} credits)." | |
| ) | |
| return {"item_id": str(best["item_id"]), "credits": CREDITS_PER_CALL, "note": note} | |
| # --------------------------------------------------------------- search | |
| def search(self, query: str, location: Optional[str] = None) -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| if not query or not query.strip(): | |
| return {"status": "error", "errorMessage": "Type what you're looking for, e.g. 'whole milk'."} | |
| loc, error = parse_location(location) | |
| if error: | |
| return {"status": "error", "errorMessage": error} | |
| data = self._get(SEARCH_ENDPOINT, dict({"token": token, "q": query.strip()}, **loc)) | |
| if "_error" in data: | |
| return self._as_error(data) | |
| results = [r for r in data.get("results") or [] if isinstance(r, dict)] | |
| return { | |
| "status": "success", | |
| "query": query.strip(), | |
| "store": data.get("store") or {}, | |
| "total_results": data.get("total_results"), | |
| "total_pages": data.get("total_pages"), | |
| "results": results, | |
| "credits_used": CREDITS_PER_CALL, | |
| } | |
| # -------------------------------------------------------------- product | |
| def product(self, ref: str, location: Optional[str] = None) -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| loc, error = parse_location(location) | |
| if error: | |
| return {"status": "error", "errorMessage": error} | |
| resolved = self._resolve_item(token, ref, loc) | |
| if "error" in resolved: | |
| result = {"status": "error", "errorMessage": resolved["error"]} | |
| if resolved.get("auth"): | |
| result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" | |
| return result | |
| data = self._get(PRODUCT_ENDPOINT, dict({"token": token, "item": resolved["item_id"]}, **loc)) | |
| if "_error" in data: | |
| return self._as_error(data) | |
| product = data.get("product") | |
| if not isinstance(product, dict) or not product: | |
| return {"status": "error", "errorMessage": "Walmart returned no details for this product."} | |
| result: Dict[str, Any] = { | |
| "status": "success", | |
| "store": data.get("store") or {}, | |
| "product": product, | |
| "credits_used": resolved["credits"] + CREDITS_PER_CALL, | |
| } | |
| if resolved.get("note"): | |
| result["note"] = resolved["note"] | |
| return result | |
| # -------------------------------------------------------------- compare | |
| def compare(self, ref: str, locations: Optional[str] = None) -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| raw = split_locations(locations) or list(DEFAULT_COMPARE_LOCATIONS) | |
| warnings: List[str] = [] | |
| if len(raw) > MAX_COMPARE_LOCATIONS: | |
| warnings.append(f"Only the first {MAX_COMPARE_LOCATIONS} locations were checked.") | |
| raw = raw[:MAX_COMPARE_LOCATIONS] | |
| parsed: List[Tuple[str, Dict[str, str]]] = [] | |
| for item in raw: | |
| loc, error = parse_location(item) | |
| if error: | |
| warnings.append(error) | |
| else: | |
| parsed.append((item, loc)) | |
| if not parsed: | |
| return {"status": "error", "errorMessage": "Enter at least one 5-digit ZIP code or store number."} | |
| resolved = self._resolve_item(token, ref, parsed[0][1]) | |
| if "error" in resolved: | |
| result = {"status": "error", "errorMessage": resolved["error"]} | |
| if resolved.get("auth"): | |
| result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" | |
| return result | |
| item_id = resolved["item_id"] | |
| def fetch(entry: Tuple[str, Dict[str, str]]) -> Dict[str, Any]: | |
| label, loc = entry | |
| data = self._get(PRODUCT_ENDPOINT, dict({"token": token, "item": item_id}, **loc)) | |
| return {"label": label, "data": data} | |
| # Three at a time keeps free-plan accounts inside their concurrency limit. | |
| with ThreadPoolExecutor(max_workers=3) as pool: | |
| fetched = list(pool.map(fetch, parsed)) | |
| credits = resolved["credits"] | |
| rows: List[Dict[str, Any]] = [] | |
| product_info: Dict[str, Any] = {} | |
| auth_failed = False | |
| for entry in fetched: | |
| data = entry["data"] | |
| if "_error" in data: | |
| auth_failed = auth_failed or bool(data.get("_auth")) | |
| rows.append({"location": entry["label"], "error": data["_error"]}) | |
| continue | |
| credits += CREDITS_PER_CALL | |
| product = data.get("product") or {} | |
| store = data.get("store") or {} | |
| if product and not product_info: | |
| product_info = { | |
| key: product.get(key) | |
| for key in ("item_id", "name", "brand", "url", "rating", "reviews_count", "category") | |
| } | |
| images = product.get("images") or [] | |
| product_info["image"] = images[0] if images else None | |
| rows.append( | |
| { | |
| "location": entry["label"], | |
| "store_id": store.get("store_id"), | |
| "city": store.get("city"), | |
| "state": store.get("state"), | |
| "postal_code": store.get("postal_code"), | |
| "price": product.get("price"), | |
| "list_price": product.get("list_price"), | |
| "currency": product.get("currency"), | |
| "availability": product.get("availability"), | |
| "fulfillment": product.get("fulfillment") or [], | |
| "seller": product.get("seller"), | |
| } | |
| ) | |
| if all("error" in row for row in rows): | |
| result = {"status": "error", "errorMessage": rows[0]["error"]} | |
| if auth_failed: | |
| result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" | |
| return result | |
| priced = [r for r in rows if isinstance(r.get("price"), (int, float))] | |
| result: Dict[str, Any] = { | |
| "status": "success", | |
| "item_id": item_id, | |
| "product": product_info, | |
| "stores": rows, | |
| "credits_used": credits, | |
| } | |
| if priced: | |
| cheapest = min(priced, key=lambda r: r["price"]) | |
| dearest = max(priced, key=lambda r: r["price"]) | |
| result["cheapest"] = cheapest | |
| result["most_expensive"] = dearest | |
| result["price_gap"] = round(dearest["price"] - cheapest["price"], 2) | |
| if resolved.get("note"): | |
| result["note"] = resolved["note"] | |
| if warnings: | |
| result["warnings"] = warnings | |
| return result | |
| # -------------------------------------------------------------- forward | |
| def forward(self, mode: str, query: str, location: Optional[str] = None) -> str: | |
| mode_key = (mode or "").strip().lower() | |
| if mode_key in ("search", "find"): | |
| result = self.search(query, location) | |
| elif mode_key in ("product", "details", "item"): | |
| result = self.product(query, location) | |
| elif mode_key in ("compare", "prices", "price_compare"): | |
| result = self.compare(query, location) | |
| else: | |
| result = {"status": "error", "errorMessage": "mode must be 'search', 'product' or 'compare'."} | |
| return json.dumps(result, ensure_ascii=False, indent=2) | |