Spaces:
Sleeping
Sleeping
Download tool.py from bugrahans/blinkit-scraper: direct link, hf CLI and curl.
- Browser
- Download file 18.3 kB
-
https://huggingface.co/spaces/bugrahans/blinkit-scraper/resolve/main/tool.py
- Command line
-
hf download hf://spaces/bugrahans/blinkit-scraper/tool.py
-
curl -L -o tool.py https://huggingface.co/spaces/bugrahans/blinkit-scraper/resolve/main/tool.py
18.3 kB
| from __future__ import annotations | |
| import json | |
| import os | |
| import re | |
| import time | |
| from typing import Any, Dict, List, Optional, Tuple | |
| from urllib.parse import parse_qs, urlparse | |
| import requests | |
| from smolagents.tools import Tool | |
| SCRAPE_DO_SIGNUP_URL = ( | |
| "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=blinkit-scraper" | |
| ) | |
| API_URL = "https://api.scrape.do/" | |
| BLINKIT = "https://blinkit.com" | |
| # Blinkit sits behind Cloudflare; through Scrape.do every request uses residential proxies in | |
| # India (super=true, geoCode=in) and costs 10 credits. | |
| CREDITS_PER_REQUEST = 10 | |
| MAX_PAGES = 10 | |
| CARD_TYPE = "product_card_snippet_type_2" | |
| DEFAULT_CATEGORY_URL = "https://blinkit.com/cn/chips-crisps/cid/1237/940" | |
| DEFAULT_PRODUCT_URL = "https://blinkit.com/prn/unibic-kerala-banana-chips-classic-salted/prid/594125" | |
| CITIES = { | |
| "Gurugram": (28.413333, 77.072833), | |
| "Delhi": (28.6315, 77.2167), | |
| "Mumbai": (19.0760, 72.8777), | |
| "Bengaluru": (12.9716, 77.5946), | |
| "Hyderabad": (17.3850, 78.4867), | |
| "Pune": (18.5204, 73.8567), | |
| "Kolkata": (22.5726, 88.3639), | |
| "Chennai": (13.0827, 80.2707), | |
| } | |
| _STATE_RE = re.compile(r"window\.grofers\.PRELOADED_STATE\s*=\s*") | |
| _decoder = json.JSONDecoder() | |
| # ------------------------------------------------------------------ parsing | |
| def extract_state(html: str) -> Dict[str, Any]: | |
| match = _STATE_RE.search(html or "") | |
| if not match: | |
| return {} | |
| try: | |
| data, _ = _decoder.raw_decode(html, match.end()) | |
| except json.JSONDecodeError: | |
| return {} | |
| return data if isinstance(data, dict) else {} | |
| def _text(node: Any) -> Optional[str]: | |
| if isinstance(node, dict): | |
| value = node.get("text") | |
| return value.strip() if isinstance(value, str) else None | |
| return node.strip() if isinstance(node, str) else None | |
| def _num(value: Any) -> Optional[float]: | |
| if isinstance(value, bool) or value is None: | |
| return None | |
| if isinstance(value, (int, float)): | |
| return value | |
| match = re.search(r"\d[\d,]*(?:\.\d+)?", str(value)) | |
| if not match: | |
| return None | |
| number = float(match.group(0).replace(",", "")) | |
| return int(number) if number.is_integer() else number | |
| def _int(value: Any) -> Optional[int]: | |
| try: | |
| return int(value) | |
| except (TypeError, ValueError): | |
| return None | |
| def _pct(price: Any, mrp: Any) -> int: | |
| if isinstance(price, (int, float)) and isinstance(mrp, (int, float)) and mrp > 0 and price <= mrp: | |
| return int((mrp - price) * 100 // mrp) | |
| return 0 | |
| _MULT = {"k": 1_000, "lac": 100_000, "lakh": 100_000, "l": 100_000, "cr": 10_000_000} | |
| def parse_count(text: Any) -> Optional[int]: | |
| match = re.search(r"(\d[\d,]*(?:\.\d+)?)\s*([a-zA-Z]*)", str(text or "")) | |
| if not match: | |
| return None | |
| return int(round(float(match.group(1).replace(",", "")) * _MULT.get(match.group(2).lower(), 1))) | |
| def product_url(product_id: Any, name: Any = None) -> str: | |
| slug = re.sub(r"[^a-z0-9]+", "-", str(name or "").lower()).strip("-") or "product" | |
| return f"{BLINKIT}/prn/{slug}/prid/{product_id}" if product_id else "" | |
| def classify_url(text: Optional[str]) -> Optional[str]: | |
| raw = (text or "").strip() | |
| if not raw: | |
| return None | |
| if not raw.lower().startswith(("http://", "https://")): | |
| raw = "https://" + raw | |
| parsed = urlparse(raw) | |
| if not parsed.netloc.lower().endswith("blinkit.com"): | |
| return None | |
| if re.search(r"/prid/\d+", parsed.path): | |
| return "product" | |
| if re.search(r"/cid/\d+", parsed.path): | |
| return "category" | |
| return None | |
| def clean_url(text: str) -> str: | |
| raw = text.strip() | |
| return raw if raw.lower().startswith(("http://", "https://")) else "https://" + raw | |
| def parse_card(data: Dict[str, Any]) -> Optional[Dict[str, Any]]: | |
| if not isinstance(data, dict): | |
| return None | |
| cart = ((data.get("atc_action") or {}).get("add_to_cart") or {}).get("cart_item") or {} | |
| product_id = _int(data.get("product_id") or (data.get("identity") or {}).get("id") or cart.get("product_id")) | |
| if not product_id: | |
| return None | |
| name = _text(data.get("display_name")) or _text(data.get("name")) or cart.get("product_name") | |
| price = cart.get("price") if isinstance(cart.get("price"), (int, float)) else _num(_text(data.get("normal_price"))) | |
| mrp = cart.get("mrp") if isinstance(cart.get("mrp"), (int, float)) else _num(_text(data.get("mrp"))) | |
| if mrp is None: | |
| mrp = price | |
| inventory = data.get("inventory", cart.get("inventory")) | |
| inventory = inventory if isinstance(inventory, int) else 0 | |
| sold_out = bool(data.get("is_sold_out")) or data.get("product_state") == "out_of_stock" or inventory <= 0 | |
| bar = (data.get("rating") or {}).get("bar") or {} | |
| rating = bar.get("value") | |
| return { | |
| "product_id": product_id, | |
| "name": name, | |
| "brand": _text(data.get("brand_name")) or cart.get("brand"), | |
| "pack_size": _text(data.get("variant")) or cart.get("unit"), | |
| "price_inr": price, | |
| "mrp_inr": mrp, | |
| "discount_pct": _pct(price, mrp), | |
| "in_stock": not sold_out, | |
| "inventory": max(inventory, 0), | |
| "rating": round(rating, 2) if isinstance(rating, (int, float)) else None, | |
| "rating_count": parse_count(_text(bar.get("title"))) if bar else None, | |
| "store_id": _int(data.get("merchant_id") or cart.get("merchant_id")), | |
| "image": (data.get("image") or {}).get("url") or cart.get("image_url"), | |
| "url": product_url(product_id, name), | |
| } | |
| def parse_cards(snippets: List[Any]) -> List[Dict[str, Any]]: | |
| rows = [] | |
| for snippet in snippets or []: | |
| if isinstance(snippet, dict) and snippet.get("widget_type") == CARD_TYPE: | |
| row = parse_card(snippet.get("data")) | |
| if row: | |
| rows.append(row) | |
| return rows | |
| def _total(next_url: Optional[str]) -> Optional[int]: | |
| if not next_url: | |
| return None | |
| values = parse_qs(urlparse(next_url).query).get("total_pagination_items") | |
| return _int(values[0]) if values else None | |
| _ATTRS = { | |
| "shelf_life": ("Shelf Life",), | |
| "fssai_license": ("FSSAI License", "FSSAI License Number"), | |
| "country_of_origin": ("Country of Origin",), | |
| "pack_size": ("Unit", "Net Weight"), | |
| "ingredients": ("Ingredients",), | |
| "manufacturer": ("Manufacturer Details", "Manufacturer's Name and Address", "Manufacturer Name"), | |
| "return_policy": ("Return Policy",), | |
| } | |
| def parse_product(html: str) -> Optional[Dict[str, Any]]: | |
| state = extract_state(html) | |
| bff = ((((state.get("ui") or {}).get("pdp") or {}).get("bffPdp") or {}).get("bffData")) or {} | |
| seo = ((((bff.get("tracking") or {}).get("le_meta") or {}).get("custom_data")) or {}).get("seo") or {} | |
| product_id = _int(seo.get("id")) | |
| if not product_id: | |
| return None | |
| attributes: Dict[str, str] = {} | |
| for item in seo.get("attributes") or []: | |
| name, value = item.get("name"), item.get("value") | |
| if name and value not in (None, "") and name not in attributes: | |
| attributes[name] = str(value).strip() | |
| picked = {k: next((attributes[n] for n in names if n in attributes), None) for k, names in _ATTRS.items()} | |
| store_id = None | |
| for snippet in bff.get("snippets") or []: | |
| if snippet.get("widget_type") == "product_atc_strip": | |
| found = re.search(r'"merchant_id":\s*(\d+)', json.dumps(snippet.get("data") or {})) | |
| store_id = int(found.group(1)) if found else None | |
| break | |
| price, mrp = seo.get("price") or None, seo.get("mrp") or None | |
| inventory = seo.get("inventory") if isinstance(seo.get("inventory"), int) else 0 | |
| return { | |
| "product_id": product_id, | |
| "name": seo.get("product_name"), | |
| "brand": seo.get("brand"), | |
| "price_inr": price, | |
| "mrp_inr": mrp, | |
| "discount_pct": _pct(price, mrp) if price else 0, | |
| "in_stock": bool(price) and inventory > 0, | |
| "inventory": max(inventory, 0), | |
| **picked, | |
| "store_id": store_id, | |
| "images": [u for u in seo.get("images") or [] if isinstance(u, str)][:6], | |
| "attributes": attributes, | |
| "url": product_url(product_id, seo.get("product_name")), | |
| } | |
| # --------------------------------------------------------------------- tool | |
| class ScrapeDoBlinkitTool(Tool): | |
| name = "blinkit_scraper" | |
| description = """ | |
| Scrape Blinkit (blinkit.com), India's 10-minute grocery app, for one delivery location. Two modes: | |
| - 'category': products of a category URL (https://blinkit.com/cn/<name>/cid/<l0>/<l1>) with name, | |
| brand, pack size, price and MRP in rupees, discount, in-stock flag, units in stock, rating and URL, | |
| for the dark store that serves the location. Page 1 has 30 products, every further page 15. | |
| - 'product': one product URL (.../prn/<name>/prid/<id>): price, MRP, stock, FSSAI licence, | |
| shelf life, country of origin, ingredients, manufacturer and the full attribute list. | |
| Location: a city (Gurugram, Delhi, Mumbai, Bengaluru, Hyderabad, Pune, Kolkata, Chennai) or | |
| "lat,lon". Powered by Scrape.do (residential proxies in India, 10 credits per request). | |
| """ | |
| output_type = "string" | |
| def __init__(self, api_token: Optional[str] = None) -> None: | |
| self._api_token = api_token | |
| self.inputs = { | |
| "mode": {"type": "string", "description": "'category' for a category page, 'product' for one product."}, | |
| "url": {"type": "string", "description": "A Blinkit category URL or product URL."}, | |
| "location": { | |
| "type": "string", | |
| "description": "City name (e.g. 'Mumbai') or 'lat,lon'. Default Gurugram, Blinkit's own default.", | |
| "nullable": True, | |
| }, | |
| "max_pages": { | |
| "type": "integer", | |
| "description": "For 'category': pages to read (30 products, then 15 per page). Default 1, maximum 10.", | |
| "nullable": True, | |
| }, | |
| } | |
| super().__init__() | |
| # ------------------------------------------------------------ plumbing | |
| def _resolve_token(self) -> Optional[str]: | |
| if self._api_token: | |
| return self._api_token.strip() | |
| env_token = os.getenv("SCRAPE_DO_API_TOKEN") | |
| return env_token.strip() if env_token else None | |
| def resolve_location(location: Optional[str]) -> Tuple[float, float, str]: | |
| text = (location or "").strip() | |
| if not text: | |
| return (*CITIES["Gurugram"], "Gurugram") | |
| for city, coords in CITIES.items(): | |
| if city.lower() == text.lower(): | |
| return (*coords, city) | |
| match = re.fullmatch(r"\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*", text) | |
| if match: | |
| return float(match.group(1)), float(match.group(2)), text | |
| raise ValueError(f"Unknown location {text!r}. Use a city ({', '.join(CITIES)}) or 'lat,lon'.") | |
| def _missing_token() -> Dict[str, Any]: | |
| return {"status": "error", "errorMessage": "Missing Scrape.do API token.", | |
| "hint": f"Get a free token at {SCRAPE_DO_SIGNUP_URL}"} | |
| def _request(self, token: str, url: str, lat: float, lon: float, api: bool = False) -> Dict[str, Any]: | |
| """One Scrape.do call with two attempts. Returns {'text', 'credits'} or {'_error', ...}.""" | |
| params = {"token": token, "url": url, "super": "true", "geoCode": "in"} | |
| headers = None | |
| if api: | |
| params["extraHeaders"] = "true" | |
| headers = {"sd-lat": str(lat), "sd-lon": str(lon), "sd-app_client": "consumer_web", | |
| "sd-Content-Type": "application/json", "sd-Accept": "application/json"} | |
| else: | |
| params["setCookies"] = f"gr_1_lat={lat}; gr_1_lon={lon}" | |
| last: Dict[str, Any] = {"_error": "Request failed."} | |
| credits = 0 | |
| for attempt in (1, 2): | |
| started = time.time() | |
| print(f"[fetch] attempt {attempt}: {'POST ' if api else ''}{url[:110]}", flush=True) | |
| try: | |
| response = requests.request("POST" if api else "GET", API_URL, params=params, headers=headers, | |
| data="{}" if api else None, timeout=(10, 90)) | |
| except requests.exceptions.Timeout: | |
| last = {"_error": "Scrape.do didn't answer within 90 seconds. Try again in a minute."} | |
| continue | |
| except requests.exceptions.RequestException as exc: | |
| last = {"_error": f"Request failed: {str(exc).replace(token, '***')[:200]}"} | |
| continue | |
| body = response.text or "" | |
| from_target = "scrape.do-initial-status-code" in {k.lower() for k in response.headers} | |
| try: | |
| credits += int(response.headers.get("scrape.do-request-cost") or 0) | |
| except ValueError: | |
| pass | |
| print(f"[fetch] HTTP {response.status_code}, {len(body)} chars, {time.time() - started:.1f}s", flush=True) | |
| if response.status_code == 401 and not from_target: | |
| return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True} | |
| if response.status_code == 200 and body: | |
| if api: | |
| try: | |
| ok = isinstance(json.loads(body), dict) | |
| except ValueError: | |
| ok = False | |
| else: | |
| ok = bool(extract_state(body)) | |
| if ok: | |
| return {"text": body, "credits": credits} | |
| last = {"_error": "Blinkit returned a page without its data. Try again."} | |
| continue | |
| last = {"_error": f"Blinkit didn't return the page (HTTP {response.status_code})."} | |
| if response.status_code not in (429, 500, 502, 503, 504): | |
| break | |
| last["credits"] = credits | |
| return last | |
| def _as_error(data: Dict[str, Any]) -> Dict[str, Any]: | |
| result = {"status": "error", "errorMessage": data.get("_error", "Unknown error")} | |
| if data.get("_auth"): | |
| result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" | |
| return result | |
| # ------------------------------------------------------------- category | |
| def category(self, url: str, location: Optional[str] = None, max_pages: Optional[int] = 1) -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| if classify_url(url) != "category": | |
| return {"status": "error", "errorMessage": "Paste a Blinkit category URL, e.g. https://blinkit.com/cn/chips-crisps/cid/1237/940"} | |
| try: | |
| lat, lon, label = self.resolve_location(location) | |
| except ValueError as exc: | |
| return {"status": "error", "errorMessage": str(exc)} | |
| start = clean_url(url).split("#")[0] | |
| pages_wanted = max(1, min(int(max_pages or 1), MAX_PAGES)) | |
| page = self._request(token, start, lat, lon) | |
| if "_error" in page: | |
| return self._as_error(page) | |
| credits = page["credits"] | |
| state = extract_state(page["text"]) | |
| feed = (((state.get("ui") or {}).get("plpContainer") or {}).get("feedData")) or {} | |
| next_url = (feed.get("pagination") or {}).get("next_url") | |
| total = _total(next_url) | |
| products: Dict[int, Dict[str, Any]] = {r["product_id"]: r for r in parse_cards(feed.get("snippets") or [])} | |
| pages_read = 1 | |
| while next_url and pages_read < pages_wanted: | |
| res = self._request(token, BLINKIT + next_url if next_url.startswith("/") else next_url, lat, lon, api=True) | |
| credits += res.get("credits", 0) | |
| if "_error" in res: | |
| break | |
| response = json.loads(res["text"]).get("response") or {} | |
| rows = parse_cards(response.get("snippets") or []) | |
| pages_read += 1 | |
| for row in rows: | |
| products.setdefault(row["product_id"], row) | |
| next_url = (response.get("pagination") or {}).get("next_url") | |
| if not rows: | |
| break | |
| rows = list(products.values()) | |
| return {"status": "success", "url": start, "location": label, "latitude": lat, "longitude": lon, | |
| "store_id": next((r["store_id"] for r in rows if r.get("store_id")), None), | |
| "total_count": total, "pages_read": pages_read, "has_more": bool(next_url), | |
| "products": rows, "credits_used": credits} | |
| # -------------------------------------------------------------- product | |
| def product(self, url: str, location: Optional[str] = None) -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| if classify_url(url) != "product": | |
| return {"status": "error", "errorMessage": "Paste a Blinkit product URL (…/prn/<name>/prid/<id>)."} | |
| try: | |
| lat, lon, label = self.resolve_location(location) | |
| except ValueError as exc: | |
| return {"status": "error", "errorMessage": str(exc)} | |
| product_id = re.search(r"/prid/(\d+)", url).group(1) | |
| data = self._request(token, f"{BLINKIT}/prn/product/prid/{product_id}", lat, lon) | |
| if "_error" in data: | |
| return self._as_error(data) | |
| product = parse_product(data["text"]) | |
| if not product: | |
| return {"status": "error", "errorMessage": "That page had no product data. Check the link."} | |
| return {"status": "success", "location": label, "product": product, "credits_used": data.get("credits", CREDITS_PER_REQUEST)} | |
| # -------------------------------------------------------------- forward | |
| def forward(self, mode: str, url: str, location: Optional[str] = None, max_pages: Optional[int] = None) -> str: | |
| key = (mode or "").strip().lower() | |
| if key in ("category", "listing", "categories"): | |
| result = self.category(url, location, max_pages or 1) | |
| elif key in ("product", "details", "item"): | |
| result = self.product(url, location) | |
| else: | |
| result = {"status": "error", "errorMessage": "mode must be 'category' or 'product'."} | |
| return json.dumps(result, ensure_ascii=False, indent=2) | |