from __future__ import annotations import json import os import re import time from typing import Any, Dict, List, Optional, Tuple from urllib.parse import parse_qs, urlparse import requests from smolagents.tools import Tool SCRAPE_DO_SIGNUP_URL = ( "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=blinkit-scraper" ) API_URL = "https://api.scrape.do/" BLINKIT = "https://blinkit.com" # Blinkit sits behind Cloudflare; through Scrape.do every request uses residential proxies in # India (super=true, geoCode=in) and costs 10 credits. CREDITS_PER_REQUEST = 10 MAX_PAGES = 10 CARD_TYPE = "product_card_snippet_type_2" DEFAULT_CATEGORY_URL = "https://blinkit.com/cn/chips-crisps/cid/1237/940" DEFAULT_PRODUCT_URL = "https://blinkit.com/prn/unibic-kerala-banana-chips-classic-salted/prid/594125" CITIES = { "Gurugram": (28.413333, 77.072833), "Delhi": (28.6315, 77.2167), "Mumbai": (19.0760, 72.8777), "Bengaluru": (12.9716, 77.5946), "Hyderabad": (17.3850, 78.4867), "Pune": (18.5204, 73.8567), "Kolkata": (22.5726, 88.3639), "Chennai": (13.0827, 80.2707), } _STATE_RE = re.compile(r"window\.grofers\.PRELOADED_STATE\s*=\s*") _decoder = json.JSONDecoder() # ------------------------------------------------------------------ parsing def extract_state(html: str) -> Dict[str, Any]: match = _STATE_RE.search(html or "") if not match: return {} try: data, _ = _decoder.raw_decode(html, match.end()) except json.JSONDecodeError: return {} return data if isinstance(data, dict) else {} def _text(node: Any) -> Optional[str]: if isinstance(node, dict): value = node.get("text") return value.strip() if isinstance(value, str) else None return node.strip() if isinstance(node, str) else None def _num(value: Any) -> Optional[float]: if isinstance(value, bool) or value is None: return None if isinstance(value, (int, float)): return value match = re.search(r"\d[\d,]*(?:\.\d+)?", str(value)) if not match: return None number = float(match.group(0).replace(",", "")) return int(number) if number.is_integer() else number def _int(value: Any) -> Optional[int]: try: return int(value) except (TypeError, ValueError): return None def _pct(price: Any, mrp: Any) -> int: if isinstance(price, (int, float)) and isinstance(mrp, (int, float)) and mrp > 0 and price <= mrp: return int((mrp - price) * 100 // mrp) return 0 _MULT = {"k": 1_000, "lac": 100_000, "lakh": 100_000, "l": 100_000, "cr": 10_000_000} def parse_count(text: Any) -> Optional[int]: match = re.search(r"(\d[\d,]*(?:\.\d+)?)\s*([a-zA-Z]*)", str(text or "")) if not match: return None return int(round(float(match.group(1).replace(",", "")) * _MULT.get(match.group(2).lower(), 1))) def product_url(product_id: Any, name: Any = None) -> str: slug = re.sub(r"[^a-z0-9]+", "-", str(name or "").lower()).strip("-") or "product" return f"{BLINKIT}/prn/{slug}/prid/{product_id}" if product_id else "" def classify_url(text: Optional[str]) -> Optional[str]: raw = (text or "").strip() if not raw: return None if not raw.lower().startswith(("http://", "https://")): raw = "https://" + raw parsed = urlparse(raw) if not parsed.netloc.lower().endswith("blinkit.com"): return None if re.search(r"/prid/\d+", parsed.path): return "product" if re.search(r"/cid/\d+", parsed.path): return "category" return None def clean_url(text: str) -> str: raw = text.strip() return raw if raw.lower().startswith(("http://", "https://")) else "https://" + raw def parse_card(data: Dict[str, Any]) -> Optional[Dict[str, Any]]: if not isinstance(data, dict): return None cart = ((data.get("atc_action") or {}).get("add_to_cart") or {}).get("cart_item") or {} product_id = _int(data.get("product_id") or (data.get("identity") or {}).get("id") or cart.get("product_id")) if not product_id: return None name = _text(data.get("display_name")) or _text(data.get("name")) or cart.get("product_name") price = cart.get("price") if isinstance(cart.get("price"), (int, float)) else _num(_text(data.get("normal_price"))) mrp = cart.get("mrp") if isinstance(cart.get("mrp"), (int, float)) else _num(_text(data.get("mrp"))) if mrp is None: mrp = price inventory = data.get("inventory", cart.get("inventory")) inventory = inventory if isinstance(inventory, int) else 0 sold_out = bool(data.get("is_sold_out")) or data.get("product_state") == "out_of_stock" or inventory <= 0 bar = (data.get("rating") or {}).get("bar") or {} rating = bar.get("value") return { "product_id": product_id, "name": name, "brand": _text(data.get("brand_name")) or cart.get("brand"), "pack_size": _text(data.get("variant")) or cart.get("unit"), "price_inr": price, "mrp_inr": mrp, "discount_pct": _pct(price, mrp), "in_stock": not sold_out, "inventory": max(inventory, 0), "rating": round(rating, 2) if isinstance(rating, (int, float)) else None, "rating_count": parse_count(_text(bar.get("title"))) if bar else None, "store_id": _int(data.get("merchant_id") or cart.get("merchant_id")), "image": (data.get("image") or {}).get("url") or cart.get("image_url"), "url": product_url(product_id, name), } def parse_cards(snippets: List[Any]) -> List[Dict[str, Any]]: rows = [] for snippet in snippets or []: if isinstance(snippet, dict) and snippet.get("widget_type") == CARD_TYPE: row = parse_card(snippet.get("data")) if row: rows.append(row) return rows def _total(next_url: Optional[str]) -> Optional[int]: if not next_url: return None values = parse_qs(urlparse(next_url).query).get("total_pagination_items") return _int(values[0]) if values else None _ATTRS = { "shelf_life": ("Shelf Life",), "fssai_license": ("FSSAI License", "FSSAI License Number"), "country_of_origin": ("Country of Origin",), "pack_size": ("Unit", "Net Weight"), "ingredients": ("Ingredients",), "manufacturer": ("Manufacturer Details", "Manufacturer's Name and Address", "Manufacturer Name"), "return_policy": ("Return Policy",), } def parse_product(html: str) -> Optional[Dict[str, Any]]: state = extract_state(html) bff = ((((state.get("ui") or {}).get("pdp") or {}).get("bffPdp") or {}).get("bffData")) or {} seo = ((((bff.get("tracking") or {}).get("le_meta") or {}).get("custom_data")) or {}).get("seo") or {} product_id = _int(seo.get("id")) if not product_id: return None attributes: Dict[str, str] = {} for item in seo.get("attributes") or []: name, value = item.get("name"), item.get("value") if name and value not in (None, "") and name not in attributes: attributes[name] = str(value).strip() picked = {k: next((attributes[n] for n in names if n in attributes), None) for k, names in _ATTRS.items()} store_id = None for snippet in bff.get("snippets") or []: if snippet.get("widget_type") == "product_atc_strip": found = re.search(r'"merchant_id":\s*(\d+)', json.dumps(snippet.get("data") or {})) store_id = int(found.group(1)) if found else None break price, mrp = seo.get("price") or None, seo.get("mrp") or None inventory = seo.get("inventory") if isinstance(seo.get("inventory"), int) else 0 return { "product_id": product_id, "name": seo.get("product_name"), "brand": seo.get("brand"), "price_inr": price, "mrp_inr": mrp, "discount_pct": _pct(price, mrp) if price else 0, "in_stock": bool(price) and inventory > 0, "inventory": max(inventory, 0), **picked, "store_id": store_id, "images": [u for u in seo.get("images") or [] if isinstance(u, str)][:6], "attributes": attributes, "url": product_url(product_id, seo.get("product_name")), } # --------------------------------------------------------------------- tool class ScrapeDoBlinkitTool(Tool): name = "blinkit_scraper" description = """ Scrape Blinkit (blinkit.com), India's 10-minute grocery app, for one delivery location. Two modes: - 'category': products of a category URL (https://blinkit.com/cn//cid//) with name, brand, pack size, price and MRP in rupees, discount, in-stock flag, units in stock, rating and URL, for the dark store that serves the location. Page 1 has 30 products, every further page 15. - 'product': one product URL (.../prn//prid/): price, MRP, stock, FSSAI licence, shelf life, country of origin, ingredients, manufacturer and the full attribute list. Location: a city (Gurugram, Delhi, Mumbai, Bengaluru, Hyderabad, Pune, Kolkata, Chennai) or "lat,lon". Powered by Scrape.do (residential proxies in India, 10 credits per request). """ output_type = "string" def __init__(self, api_token: Optional[str] = None) -> None: self._api_token = api_token self.inputs = { "mode": {"type": "string", "description": "'category' for a category page, 'product' for one product."}, "url": {"type": "string", "description": "A Blinkit category URL or product URL."}, "location": { "type": "string", "description": "City name (e.g. 'Mumbai') or 'lat,lon'. Default Gurugram, Blinkit's own default.", "nullable": True, }, "max_pages": { "type": "integer", "description": "For 'category': pages to read (30 products, then 15 per page). Default 1, maximum 10.", "nullable": True, }, } super().__init__() # ------------------------------------------------------------ plumbing def _resolve_token(self) -> Optional[str]: if self._api_token: return self._api_token.strip() env_token = os.getenv("SCRAPE_DO_API_TOKEN") return env_token.strip() if env_token else None @staticmethod def resolve_location(location: Optional[str]) -> Tuple[float, float, str]: text = (location or "").strip() if not text: return (*CITIES["Gurugram"], "Gurugram") for city, coords in CITIES.items(): if city.lower() == text.lower(): return (*coords, city) match = re.fullmatch(r"\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*", text) if match: return float(match.group(1)), float(match.group(2)), text raise ValueError(f"Unknown location {text!r}. Use a city ({', '.join(CITIES)}) or 'lat,lon'.") @staticmethod def _missing_token() -> Dict[str, Any]: return {"status": "error", "errorMessage": "Missing Scrape.do API token.", "hint": f"Get a free token at {SCRAPE_DO_SIGNUP_URL}"} def _request(self, token: str, url: str, lat: float, lon: float, api: bool = False) -> Dict[str, Any]: """One Scrape.do call with two attempts. Returns {'text', 'credits'} or {'_error', ...}.""" params = {"token": token, "url": url, "super": "true", "geoCode": "in"} headers = None if api: params["extraHeaders"] = "true" headers = {"sd-lat": str(lat), "sd-lon": str(lon), "sd-app_client": "consumer_web", "sd-Content-Type": "application/json", "sd-Accept": "application/json"} else: params["setCookies"] = f"gr_1_lat={lat}; gr_1_lon={lon}" last: Dict[str, Any] = {"_error": "Request failed."} credits = 0 for attempt in (1, 2): started = time.time() print(f"[fetch] attempt {attempt}: {'POST ' if api else ''}{url[:110]}", flush=True) try: response = requests.request("POST" if api else "GET", API_URL, params=params, headers=headers, data="{}" if api else None, timeout=(10, 90)) except requests.exceptions.Timeout: last = {"_error": "Scrape.do didn't answer within 90 seconds. Try again in a minute."} continue except requests.exceptions.RequestException as exc: last = {"_error": f"Request failed: {str(exc).replace(token, '***')[:200]}"} continue body = response.text or "" from_target = "scrape.do-initial-status-code" in {k.lower() for k in response.headers} try: credits += int(response.headers.get("scrape.do-request-cost") or 0) except ValueError: pass print(f"[fetch] HTTP {response.status_code}, {len(body)} chars, {time.time() - started:.1f}s", flush=True) if response.status_code == 401 and not from_target: return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True} if response.status_code == 200 and body: if api: try: ok = isinstance(json.loads(body), dict) except ValueError: ok = False else: ok = bool(extract_state(body)) if ok: return {"text": body, "credits": credits} last = {"_error": "Blinkit returned a page without its data. Try again."} continue last = {"_error": f"Blinkit didn't return the page (HTTP {response.status_code})."} if response.status_code not in (429, 500, 502, 503, 504): break last["credits"] = credits return last @staticmethod def _as_error(data: Dict[str, Any]) -> Dict[str, Any]: result = {"status": "error", "errorMessage": data.get("_error", "Unknown error")} if data.get("_auth"): result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" return result # ------------------------------------------------------------- category def category(self, url: str, location: Optional[str] = None, max_pages: Optional[int] = 1) -> Dict[str, Any]: token = self._resolve_token() if not token: return self._missing_token() if classify_url(url) != "category": return {"status": "error", "errorMessage": "Paste a Blinkit category URL, e.g. https://blinkit.com/cn/chips-crisps/cid/1237/940"} try: lat, lon, label = self.resolve_location(location) except ValueError as exc: return {"status": "error", "errorMessage": str(exc)} start = clean_url(url).split("#")[0] pages_wanted = max(1, min(int(max_pages or 1), MAX_PAGES)) page = self._request(token, start, lat, lon) if "_error" in page: return self._as_error(page) credits = page["credits"] state = extract_state(page["text"]) feed = (((state.get("ui") or {}).get("plpContainer") or {}).get("feedData")) or {} next_url = (feed.get("pagination") or {}).get("next_url") total = _total(next_url) products: Dict[int, Dict[str, Any]] = {r["product_id"]: r for r in parse_cards(feed.get("snippets") or [])} pages_read = 1 while next_url and pages_read < pages_wanted: res = self._request(token, BLINKIT + next_url if next_url.startswith("/") else next_url, lat, lon, api=True) credits += res.get("credits", 0) if "_error" in res: break response = json.loads(res["text"]).get("response") or {} rows = parse_cards(response.get("snippets") or []) pages_read += 1 for row in rows: products.setdefault(row["product_id"], row) next_url = (response.get("pagination") or {}).get("next_url") if not rows: break rows = list(products.values()) return {"status": "success", "url": start, "location": label, "latitude": lat, "longitude": lon, "store_id": next((r["store_id"] for r in rows if r.get("store_id")), None), "total_count": total, "pages_read": pages_read, "has_more": bool(next_url), "products": rows, "credits_used": credits} # -------------------------------------------------------------- product def product(self, url: str, location: Optional[str] = None) -> Dict[str, Any]: token = self._resolve_token() if not token: return self._missing_token() if classify_url(url) != "product": return {"status": "error", "errorMessage": "Paste a Blinkit product URL (…/prn//prid/)."} try: lat, lon, label = self.resolve_location(location) except ValueError as exc: return {"status": "error", "errorMessage": str(exc)} product_id = re.search(r"/prid/(\d+)", url).group(1) data = self._request(token, f"{BLINKIT}/prn/product/prid/{product_id}", lat, lon) if "_error" in data: return self._as_error(data) product = parse_product(data["text"]) if not product: return {"status": "error", "errorMessage": "That page had no product data. Check the link."} return {"status": "success", "location": label, "product": product, "credits_used": data.get("credits", CREDITS_PER_REQUEST)} # -------------------------------------------------------------- forward def forward(self, mode: str, url: str, location: Optional[str] = None, max_pages: Optional[int] = None) -> str: key = (mode or "").strip().lower() if key in ("category", "listing", "categories"): result = self.category(url, location, max_pages or 1) elif key in ("product", "details", "item"): result = self.product(url, location) else: result = {"status": "error", "errorMessage": "mode must be 'category' or 'product'."} return json.dumps(result, ensure_ascii=False, indent=2)