blinkit-scraper / tool.py
bugrahans's picture
Upload 4 files
2b5d098 verified
Raw History Blame Contribute Delete
18.3 kB
from __future__ import annotations
import json
import os
import re
import time
from typing import Any, Dict, List, Optional, Tuple
from urllib.parse import parse_qs, urlparse
import requests
from smolagents.tools import Tool
SCRAPE_DO_SIGNUP_URL = (
"https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=blinkit-scraper"
)
API_URL = "https://api.scrape.do/"
BLINKIT = "https://blinkit.com"
# Blinkit sits behind Cloudflare; through Scrape.do every request uses residential proxies in
# India (super=true, geoCode=in) and costs 10 credits.
CREDITS_PER_REQUEST = 10
MAX_PAGES = 10
CARD_TYPE = "product_card_snippet_type_2"
DEFAULT_CATEGORY_URL = "https://blinkit.com/cn/chips-crisps/cid/1237/940"
DEFAULT_PRODUCT_URL = "https://blinkit.com/prn/unibic-kerala-banana-chips-classic-salted/prid/594125"
CITIES = {
"Gurugram": (28.413333, 77.072833),
"Delhi": (28.6315, 77.2167),
"Mumbai": (19.0760, 72.8777),
"Bengaluru": (12.9716, 77.5946),
"Hyderabad": (17.3850, 78.4867),
"Pune": (18.5204, 73.8567),
"Kolkata": (22.5726, 88.3639),
"Chennai": (13.0827, 80.2707),
}
_STATE_RE = re.compile(r"window\.grofers\.PRELOADED_STATE\s*=\s*")
_decoder = json.JSONDecoder()
# ------------------------------------------------------------------ parsing
def extract_state(html: str) -> Dict[str, Any]:
match = _STATE_RE.search(html or "")
if not match:
return {}
try:
data, _ = _decoder.raw_decode(html, match.end())
except json.JSONDecodeError:
return {}
return data if isinstance(data, dict) else {}
def _text(node: Any) -> Optional[str]:
if isinstance(node, dict):
value = node.get("text")
return value.strip() if isinstance(value, str) else None
return node.strip() if isinstance(node, str) else None
def _num(value: Any) -> Optional[float]:
if isinstance(value, bool) or value is None:
return None
if isinstance(value, (int, float)):
return value
match = re.search(r"\d[\d,]*(?:\.\d+)?", str(value))
if not match:
return None
number = float(match.group(0).replace(",", ""))
return int(number) if number.is_integer() else number
def _int(value: Any) -> Optional[int]:
try:
return int(value)
except (TypeError, ValueError):
return None
def _pct(price: Any, mrp: Any) -> int:
if isinstance(price, (int, float)) and isinstance(mrp, (int, float)) and mrp > 0 and price <= mrp:
return int((mrp - price) * 100 // mrp)
return 0
_MULT = {"k": 1_000, "lac": 100_000, "lakh": 100_000, "l": 100_000, "cr": 10_000_000}
def parse_count(text: Any) -> Optional[int]:
match = re.search(r"(\d[\d,]*(?:\.\d+)?)\s*([a-zA-Z]*)", str(text or ""))
if not match:
return None
return int(round(float(match.group(1).replace(",", "")) * _MULT.get(match.group(2).lower(), 1)))
def product_url(product_id: Any, name: Any = None) -> str:
slug = re.sub(r"[^a-z0-9]+", "-", str(name or "").lower()).strip("-") or "product"
return f"{BLINKIT}/prn/{slug}/prid/{product_id}" if product_id else ""
def classify_url(text: Optional[str]) -> Optional[str]:
raw = (text or "").strip()
if not raw:
return None
if not raw.lower().startswith(("http://", "https://")):
raw = "https://" + raw
parsed = urlparse(raw)
if not parsed.netloc.lower().endswith("blinkit.com"):
return None
if re.search(r"/prid/\d+", parsed.path):
return "product"
if re.search(r"/cid/\d+", parsed.path):
return "category"
return None
def clean_url(text: str) -> str:
raw = text.strip()
return raw if raw.lower().startswith(("http://", "https://")) else "https://" + raw
def parse_card(data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
if not isinstance(data, dict):
return None
cart = ((data.get("atc_action") or {}).get("add_to_cart") or {}).get("cart_item") or {}
product_id = _int(data.get("product_id") or (data.get("identity") or {}).get("id") or cart.get("product_id"))
if not product_id:
return None
name = _text(data.get("display_name")) or _text(data.get("name")) or cart.get("product_name")
price = cart.get("price") if isinstance(cart.get("price"), (int, float)) else _num(_text(data.get("normal_price")))
mrp = cart.get("mrp") if isinstance(cart.get("mrp"), (int, float)) else _num(_text(data.get("mrp")))
if mrp is None:
mrp = price
inventory = data.get("inventory", cart.get("inventory"))
inventory = inventory if isinstance(inventory, int) else 0
sold_out = bool(data.get("is_sold_out")) or data.get("product_state") == "out_of_stock" or inventory <= 0
bar = (data.get("rating") or {}).get("bar") or {}
rating = bar.get("value")
return {
"product_id": product_id,
"name": name,
"brand": _text(data.get("brand_name")) or cart.get("brand"),
"pack_size": _text(data.get("variant")) or cart.get("unit"),
"price_inr": price,
"mrp_inr": mrp,
"discount_pct": _pct(price, mrp),
"in_stock": not sold_out,
"inventory": max(inventory, 0),
"rating": round(rating, 2) if isinstance(rating, (int, float)) else None,
"rating_count": parse_count(_text(bar.get("title"))) if bar else None,
"store_id": _int(data.get("merchant_id") or cart.get("merchant_id")),
"image": (data.get("image") or {}).get("url") or cart.get("image_url"),
"url": product_url(product_id, name),
}
def parse_cards(snippets: List[Any]) -> List[Dict[str, Any]]:
rows = []
for snippet in snippets or []:
if isinstance(snippet, dict) and snippet.get("widget_type") == CARD_TYPE:
row = parse_card(snippet.get("data"))
if row:
rows.append(row)
return rows
def _total(next_url: Optional[str]) -> Optional[int]:
if not next_url:
return None
values = parse_qs(urlparse(next_url).query).get("total_pagination_items")
return _int(values[0]) if values else None
_ATTRS = {
"shelf_life": ("Shelf Life",),
"fssai_license": ("FSSAI License", "FSSAI License Number"),
"country_of_origin": ("Country of Origin",),
"pack_size": ("Unit", "Net Weight"),
"ingredients": ("Ingredients",),
"manufacturer": ("Manufacturer Details", "Manufacturer's Name and Address", "Manufacturer Name"),
"return_policy": ("Return Policy",),
}
def parse_product(html: str) -> Optional[Dict[str, Any]]:
state = extract_state(html)
bff = ((((state.get("ui") or {}).get("pdp") or {}).get("bffPdp") or {}).get("bffData")) or {}
seo = ((((bff.get("tracking") or {}).get("le_meta") or {}).get("custom_data")) or {}).get("seo") or {}
product_id = _int(seo.get("id"))
if not product_id:
return None
attributes: Dict[str, str] = {}
for item in seo.get("attributes") or []:
name, value = item.get("name"), item.get("value")
if name and value not in (None, "") and name not in attributes:
attributes[name] = str(value).strip()
picked = {k: next((attributes[n] for n in names if n in attributes), None) for k, names in _ATTRS.items()}
store_id = None
for snippet in bff.get("snippets") or []:
if snippet.get("widget_type") == "product_atc_strip":
found = re.search(r'"merchant_id":\s*(\d+)', json.dumps(snippet.get("data") or {}))
store_id = int(found.group(1)) if found else None
break
price, mrp = seo.get("price") or None, seo.get("mrp") or None
inventory = seo.get("inventory") if isinstance(seo.get("inventory"), int) else 0
return {
"product_id": product_id,
"name": seo.get("product_name"),
"brand": seo.get("brand"),
"price_inr": price,
"mrp_inr": mrp,
"discount_pct": _pct(price, mrp) if price else 0,
"in_stock": bool(price) and inventory > 0,
"inventory": max(inventory, 0),
**picked,
"store_id": store_id,
"images": [u for u in seo.get("images") or [] if isinstance(u, str)][:6],
"attributes": attributes,
"url": product_url(product_id, seo.get("product_name")),
}
# --------------------------------------------------------------------- tool
class ScrapeDoBlinkitTool(Tool):
name = "blinkit_scraper"
description = """
Scrape Blinkit (blinkit.com), India's 10-minute grocery app, for one delivery location. Two modes:
- 'category': products of a category URL (https://blinkit.com/cn/<name>/cid/<l0>/<l1>) with name,
brand, pack size, price and MRP in rupees, discount, in-stock flag, units in stock, rating and URL,
for the dark store that serves the location. Page 1 has 30 products, every further page 15.
- 'product': one product URL (.../prn/<name>/prid/<id>): price, MRP, stock, FSSAI licence,
shelf life, country of origin, ingredients, manufacturer and the full attribute list.
Location: a city (Gurugram, Delhi, Mumbai, Bengaluru, Hyderabad, Pune, Kolkata, Chennai) or
"lat,lon". Powered by Scrape.do (residential proxies in India, 10 credits per request).
"""
output_type = "string"
def __init__(self, api_token: Optional[str] = None) -> None:
self._api_token = api_token
self.inputs = {
"mode": {"type": "string", "description": "'category' for a category page, 'product' for one product."},
"url": {"type": "string", "description": "A Blinkit category URL or product URL."},
"location": {
"type": "string",
"description": "City name (e.g. 'Mumbai') or 'lat,lon'. Default Gurugram, Blinkit's own default.",
"nullable": True,
},
"max_pages": {
"type": "integer",
"description": "For 'category': pages to read (30 products, then 15 per page). Default 1, maximum 10.",
"nullable": True,
},
}
super().__init__()
# ------------------------------------------------------------ plumbing
def _resolve_token(self) -> Optional[str]:
if self._api_token:
return self._api_token.strip()
env_token = os.getenv("SCRAPE_DO_API_TOKEN")
return env_token.strip() if env_token else None
@staticmethod
def resolve_location(location: Optional[str]) -> Tuple[float, float, str]:
text = (location or "").strip()
if not text:
return (*CITIES["Gurugram"], "Gurugram")
for city, coords in CITIES.items():
if city.lower() == text.lower():
return (*coords, city)
match = re.fullmatch(r"\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*", text)
if match:
return float(match.group(1)), float(match.group(2)), text
raise ValueError(f"Unknown location {text!r}. Use a city ({', '.join(CITIES)}) or 'lat,lon'.")
@staticmethod
def _missing_token() -> Dict[str, Any]:
return {"status": "error", "errorMessage": "Missing Scrape.do API token.",
"hint": f"Get a free token at {SCRAPE_DO_SIGNUP_URL}"}
def _request(self, token: str, url: str, lat: float, lon: float, api: bool = False) -> Dict[str, Any]:
"""One Scrape.do call with two attempts. Returns {'text', 'credits'} or {'_error', ...}."""
params = {"token": token, "url": url, "super": "true", "geoCode": "in"}
headers = None
if api:
params["extraHeaders"] = "true"
headers = {"sd-lat": str(lat), "sd-lon": str(lon), "sd-app_client": "consumer_web",
"sd-Content-Type": "application/json", "sd-Accept": "application/json"}
else:
params["setCookies"] = f"gr_1_lat={lat}; gr_1_lon={lon}"
last: Dict[str, Any] = {"_error": "Request failed."}
credits = 0
for attempt in (1, 2):
started = time.time()
print(f"[fetch] attempt {attempt}: {'POST ' if api else ''}{url[:110]}", flush=True)
try:
response = requests.request("POST" if api else "GET", API_URL, params=params, headers=headers,
data="{}" if api else None, timeout=(10, 90))
except requests.exceptions.Timeout:
last = {"_error": "Scrape.do didn't answer within 90 seconds. Try again in a minute."}
continue
except requests.exceptions.RequestException as exc:
last = {"_error": f"Request failed: {str(exc).replace(token, '***')[:200]}"}
continue
body = response.text or ""
from_target = "scrape.do-initial-status-code" in {k.lower() for k in response.headers}
try:
credits += int(response.headers.get("scrape.do-request-cost") or 0)
except ValueError:
pass
print(f"[fetch] HTTP {response.status_code}, {len(body)} chars, {time.time() - started:.1f}s", flush=True)
if response.status_code == 401 and not from_target:
return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True}
if response.status_code == 200 and body:
if api:
try:
ok = isinstance(json.loads(body), dict)
except ValueError:
ok = False
else:
ok = bool(extract_state(body))
if ok:
return {"text": body, "credits": credits}
last = {"_error": "Blinkit returned a page without its data. Try again."}
continue
last = {"_error": f"Blinkit didn't return the page (HTTP {response.status_code})."}
if response.status_code not in (429, 500, 502, 503, 504):
break
last["credits"] = credits
return last
@staticmethod
def _as_error(data: Dict[str, Any]) -> Dict[str, Any]:
result = {"status": "error", "errorMessage": data.get("_error", "Unknown error")}
if data.get("_auth"):
result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}"
return result
# ------------------------------------------------------------- category
def category(self, url: str, location: Optional[str] = None, max_pages: Optional[int] = 1) -> Dict[str, Any]:
token = self._resolve_token()
if not token:
return self._missing_token()
if classify_url(url) != "category":
return {"status": "error", "errorMessage": "Paste a Blinkit category URL, e.g. https://blinkit.com/cn/chips-crisps/cid/1237/940"}
try:
lat, lon, label = self.resolve_location(location)
except ValueError as exc:
return {"status": "error", "errorMessage": str(exc)}
start = clean_url(url).split("#")[0]
pages_wanted = max(1, min(int(max_pages or 1), MAX_PAGES))
page = self._request(token, start, lat, lon)
if "_error" in page:
return self._as_error(page)
credits = page["credits"]
state = extract_state(page["text"])
feed = (((state.get("ui") or {}).get("plpContainer") or {}).get("feedData")) or {}
next_url = (feed.get("pagination") or {}).get("next_url")
total = _total(next_url)
products: Dict[int, Dict[str, Any]] = {r["product_id"]: r for r in parse_cards(feed.get("snippets") or [])}
pages_read = 1
while next_url and pages_read < pages_wanted:
res = self._request(token, BLINKIT + next_url if next_url.startswith("/") else next_url, lat, lon, api=True)
credits += res.get("credits", 0)
if "_error" in res:
break
response = json.loads(res["text"]).get("response") or {}
rows = parse_cards(response.get("snippets") or [])
pages_read += 1
for row in rows:
products.setdefault(row["product_id"], row)
next_url = (response.get("pagination") or {}).get("next_url")
if not rows:
break
rows = list(products.values())
return {"status": "success", "url": start, "location": label, "latitude": lat, "longitude": lon,
"store_id": next((r["store_id"] for r in rows if r.get("store_id")), None),
"total_count": total, "pages_read": pages_read, "has_more": bool(next_url),
"products": rows, "credits_used": credits}
# -------------------------------------------------------------- product
def product(self, url: str, location: Optional[str] = None) -> Dict[str, Any]:
token = self._resolve_token()
if not token:
return self._missing_token()
if classify_url(url) != "product":
return {"status": "error", "errorMessage": "Paste a Blinkit product URL (…/prn/<name>/prid/<id>)."}
try:
lat, lon, label = self.resolve_location(location)
except ValueError as exc:
return {"status": "error", "errorMessage": str(exc)}
product_id = re.search(r"/prid/(\d+)", url).group(1)
data = self._request(token, f"{BLINKIT}/prn/product/prid/{product_id}", lat, lon)
if "_error" in data:
return self._as_error(data)
product = parse_product(data["text"])
if not product:
return {"status": "error", "errorMessage": "That page had no product data. Check the link."}
return {"status": "success", "location": label, "product": product, "credits_used": data.get("credits", CREDITS_PER_REQUEST)}
# -------------------------------------------------------------- forward
def forward(self, mode: str, url: str, location: Optional[str] = None, max_pages: Optional[int] = None) -> str:
key = (mode or "").strip().lower()
if key in ("category", "listing", "categories"):
result = self.category(url, location, max_pages or 1)
elif key in ("product", "details", "item"):
result = self.product(url, location)
else:
result = {"status": "error", "errorMessage": "mode must be 'category' or 'product'."}
return json.dumps(result, ensure_ascii=False, indent=2)