zepto-scraper / tool.py
bugrahans's picture
Always use residential proxies in India (super=true, geoCode=in)
cfc4175 verified
Raw History Blame Contribute Delete
17.1 kB
from __future__ import annotations
import json
import os
import re
import time
import urllib.parse
from typing import Any, Dict, List, Optional
import requests
from smolagents.tools import Tool
SCRAPE_DO_SIGNUP_URL = (
"https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=zepto-scraper"
)
API_URL = "https://api.scrape.do/"
ZEPTO = "https://www.zepto.com"
# Every page goes through Scrape.do residential proxies in India (super=true&geoCode=in).
CREDITS_PER_PAGE = 10
DEFAULT_SUBCATEGORY_URL = (
"https://www.zepto.com/cn/munchies/chips-crisps/cid/d2c2a144-43cd-43e5-b308-92628fa68596"
"/scid/df4f5100-c02f-4906-83b8-ddb744081a7a"
)
DEFAULT_PRODUCT_URL = (
"https://www.zepto.com/pn/bingo-original-style-chilli-sprinkled-flat-cut-spicy-potato-chips-pack-for-snacks"
"/pvid/cc0c8026-9b71-4bdd-bc88-b55d08e50185"
)
MAX_SUBCATEGORIES = 20
UUID = r"[0-9a-f-]{36}"
PUSH_RE = re.compile(r'self\.__next_f\.push\(\[1,"((?:[^"\\]|\\.)*)"\]\)')
# Card keys are sorted alphabetically. In-stock cards start with availableQuantity;
# out-of-stock cards drop that key and start with baseProductId.
ITEM_START_RE = re.compile(r'\{"(?:availableQuantity|baseProductId)":')
PRODUCT_URL_RE = re.compile(r"https://www\.zepto\.com/pn/[a-z0-9-]+/pvid/" + UUID)
STORE_RE = re.compile(r'"storeServiceableResponse":\{[^}]*?"storeId":"(' + UUID + ')"')
LD_RE = re.compile(r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>', re.S)
SUBCATEGORY_PATH_RE = re.compile(r"^/cn/[a-z0-9-]+/[a-z0-9-]+/cid/" + UUID + "/scid/" + UUID + "/?$")
PRODUCT_PATH_RE = re.compile(r"^/pn/[a-z0-9-]+/pvid/" + UUID + "/?$")
_decoder = json.JSONDecoder()
# ------------------------------------------------------------------ parsing
def classify_url(text: Optional[str]) -> Optional[str]:
"""'subcategory', 'product' or None."""
raw = (text or "").strip()
if not raw:
return None
if not raw.lower().startswith(("http://", "https://")):
raw = "https://" + raw
parsed = urllib.parse.urlparse(raw)
if not parsed.netloc.lower().endswith("zepto.com"):
return None
if SUBCATEGORY_PATH_RE.match(parsed.path):
return "subcategory"
if PRODUCT_PATH_RE.match(parsed.path):
return "product"
return None
def clean_url(text: str) -> str:
raw = text.strip()
if not raw.lower().startswith(("http://", "https://")):
raw = "https://" + raw
parsed = urllib.parse.urlparse(raw)
return f"{ZEPTO}{parsed.path.rstrip('/')}"
def decode_rsc(html: str) -> str:
"""Join Zepto's Next.js stream chunks back into one text.
Each chunk is a JavaScript string literal; JSON strings use the same escaping.
"""
out = []
for chunk in PUSH_RE.findall(html or ""):
try:
out.append(json.loads('"' + chunk + '"'))
except json.JSONDecodeError:
continue
return "".join(out)
def rupees(paise: Any) -> Optional[float]:
"""Zepto stores every price in paise: 1900 means Rs 19."""
if isinstance(paise, bool) or not isinstance(paise, (int, float)):
return None
return round(paise / 100, 2)
def extract_items(flat: str) -> List[Dict[str, Any]]:
"""Every product card in a decoded stream, in stock and out of stock."""
items: Dict[str, Dict[str, Any]] = {}
for match in ITEM_START_RE.finditer(flat):
try:
obj, _ = _decoder.raw_decode(flat, match.start())
except json.JSONDecodeError:
continue
variant = obj.get("productVariant") or {}
# Look-alike objects elsewhere in the stream have no product block
if variant.get("id") and obj.get("product"):
items.setdefault(variant["id"], obj)
return list(items.values())
def parse_listing(html: str) -> Dict[str, Any]:
flat = decode_rsc(html)
store = STORE_RE.search(flat)
urls = {link.rsplit("/", 1)[1]: link for link in PRODUCT_URL_RE.findall(flat)}
rows = []
for item in extract_items(flat):
product = item.get("product") or {}
variant = item.get("productVariant") or {}
rating = variant.get("ratingSummary") or {}
rows.append(
{
"variant_id": variant.get("id"),
"name": product.get("name"),
"brand": product.get("brand"),
"pack_size": variant.get("formattedPacksize"),
"price_inr": rupees(item.get("discountedSellingPrice")),
"mrp_inr": rupees(item.get("mrp")),
"discount_pct": item.get("discountPercent", 0),
"in_stock": not item.get("outOfStock", False),
"available_qty": item.get("availableQuantity", 0),
"rating": rating.get("averageRating"),
"rating_count": rating.get("totalRatings"),
"subcategory": item.get("primarySubcategoryName"),
"store_id": store.group(1) if store else None,
"url": urls.get(variant.get("id")),
}
)
return {"products": rows, "end_of_page": '"endOfPage":true' in flat}
def subcategory_urls(html: str, start_url: str) -> List[str]:
"""The start subcategory plus its siblings, deduped on scid.
The same subcategory can sit under a second category slug that renders no
products, so only links under the start URL's own category slug are kept.
"""
start_path = urllib.parse.urlparse(start_url).path.rstrip("/")
cid = re.search(r"/cid/(" + UUID + ")", start_path)
if not cid:
return [start_url]
slug = start_path.split("/")[2]
sibling_re = re.compile(r"/cn/[a-z0-9-]+/[a-z0-9-]+/cid/" + cid.group(1) + r"/scid/" + UUID)
found: Dict[str, str] = {}
for path in [start_path] + sibling_re.findall(decode_rsc(html)):
if path.split("/")[2] == slug:
found.setdefault(path.rsplit("/", 1)[1], ZEPTO + path)
return list(found.values())
def product_jsonld(html: str) -> Dict[str, Any]:
for block in LD_RE.findall(html or ""):
try:
data = json.loads(block)
except json.JSONDecodeError:
continue
if isinstance(data, dict) and data.get("@type") == "Product":
return data
return {}
def variant_data(flat: str, variant_id: str) -> Dict[str, Any]:
"""productVariant carries the printed MRP and order limit for this exact pack size."""
for match in re.finditer(r'"productVariant":\{', flat):
try:
obj, _ = _decoder.raw_decode(flat, match.end() - 1)
except json.JSONDecodeError:
continue
if obj.get("id") == variant_id:
return obj
return {}
def parse_product(html: str, url: str) -> Optional[Dict[str, Any]]:
"""Catalog sheet for one product page. Price and stock are left out on purpose:
the product page caps the stock count and can lag the listing."""
ld = product_jsonld(html)
if not ld:
return None
variant_id = url.rstrip("/").rsplit("/", 1)[1]
variant = variant_data(decode_rsc(html), variant_id)
props = {
p.get("name"): p.get("value")
for p in ld.get("additionalProperty") or []
if isinstance(p, dict) and p.get("name")
}
rating = ld.get("aggregateRating") or {}
brand = ld.get("brand")
images = ld.get("image") or []
if isinstance(images, str):
images = [images]
return {
"variant_id": variant_id,
"name": ld.get("name"),
"brand": brand.get("name") if isinstance(brand, dict) else brand,
"category": ld.get("category"),
"pack_size": ld.get("size"),
"mrp_inr": rupees(variant.get("mrp")),
"max_per_order": variant.get("maxAllowedQuantity"),
"rating": rating.get("ratingValue"),
"rating_count": rating.get("ratingCount"),
"shelf_life": props.get("Shelf life"),
"country_of_origin": props.get("Country of origin"),
"seller": props.get("Seller"),
"fssai_licence": props.get("FSSAI licence number"),
"images": [i for i in images if isinstance(i, str)],
"description": ld.get("description"),
"url": url,
}
# --------------------------------------------------------------------- tool
class ScrapeDoZeptoTool(Tool):
name = "zepto_scraper"
description = """
Scrape Zepto (zepto.com), India's quick-commerce grocery app. Two modes:
- 'category': every product in a Zepto subcategory, and optionally its sibling
subcategories, with price and MRP in rupees, discount, in-stock flag, units in
stock, rating, rating count and the dark store the prices belong to.
Out-of-stock products are included. Zepto shows at most 32 products per subcategory.
- 'product': the catalog sheet for one product page: brand, category path, pack
size, MRP, order limit, seller, shelf life, FSSAI licence number, country of
origin, images and description.
Pass a Zepto subcategory URL (/cn/.../cid/.../scid/...) or product URL (/pn/.../pvid/...).
Powered by Scrape.do, which gets past the AWS WAF challenge Zepto shows to scripts.
10 Scrape.do credits per page.
"""
output_type = "string"
def __init__(self, api_token: Optional[str] = None) -> None:
self._api_token = api_token
self.inputs = {
"mode": {
"type": "string",
"description": "'category' for a subcategory's product listings, 'product' for one product's details.",
},
"url": {
"type": "string",
"description": (
"For 'category': a Zepto subcategory URL (zepto.com/cn/<category>/<subcategory>/cid/.../scid/...). "
"For 'product': a Zepto product URL (zepto.com/pn/<name>/pvid/...)."
),
},
"max_subcategories": {
"type": "integer",
"description": "For 'category': how many subcategories to read, the start one plus its siblings. 1 reads only the start page. Default 1, maximum 20.",
"nullable": True,
},
}
super().__init__()
# ------------------------------------------------------------ plumbing
def _resolve_token(self) -> Optional[str]:
if self._api_token:
return self._api_token.strip()
env_token = os.getenv("SCRAPE_DO_API_TOKEN")
return env_token.strip() if env_token else None
@staticmethod
def _missing_token() -> Dict[str, Any]:
return {
"status": "error",
"errorMessage": "Missing Scrape.do API token.",
"hint": f"Get a free token at {SCRAPE_DO_SIGNUP_URL}",
}
# Zepto's WAF blocks Scrape.do's default rotation (502 ROTATION_FAILED after ~60 s), so every
# page goes straight through the residential pool in India: about 3 s, 10 credits.
PARAMS = {"super": "true", "geoCode": "in"}
def _fetch(self, token: str, url: str) -> Dict[str, Any]:
"""GET a Zepto page through Scrape.do. Anything other than HTTP 200 is a failure:
Zepto's AWS WAF answers scripts with an empty 202, which requests treats as success."""
last: Dict[str, Any] = {"_error": "Request failed."}
for attempt in (1, 2):
params = {"token": token, "url": url, **self.PARAMS}
started = time.time()
print(f"[fetch] attempt {attempt}: {url}", flush=True)
try:
response = requests.get(API_URL, params=params, timeout=(10, 90))
except requests.exceptions.Timeout:
print(f"[fetch] timeout after {time.time() - started:.1f}s", flush=True)
last = {"_error": "Scrape.do didn't answer within 90 seconds. Try again in a minute."}
continue
except requests.exceptions.RequestException as exc:
msg = str(exc).replace(token, "***")
print(f"[fetch] error after {time.time() - started:.1f}s: {msg[:300]}", flush=True)
last = {"_error": f"Request failed: {msg}"}
continue
body = response.text or ""
elapsed = time.time() - started
print(f"[fetch] HTTP {response.status_code}, {len(body)} chars, {elapsed:.1f}s", flush=True)
if response.status_code == 401:
return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True}
if response.status_code == 200 and body:
return {"html": body, "_credits": CREDITS_PER_PAGE}
if response.status_code != 200:
print(f"[fetch] body: {body[:300].replace(token, '***')}", flush=True)
last = {"_error": f"Zepto didn't return the page (HTTP {response.status_code}).",
"_status": response.status_code}
if response.status_code not in (200, 202, 403, 429, 500, 502, 503, 504):
return last
return last
@staticmethod
def _as_error(data: Dict[str, Any]) -> Dict[str, Any]:
result = {"status": "error", "errorMessage": data.get("_error", "Unknown error")}
if data.get("_auth"):
result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}"
return result
# -------------------------------------------------------------- category
def category(self, url: str, max_subcategories: Optional[int] = 1) -> Dict[str, Any]:
token = self._resolve_token()
if not token:
return self._missing_token()
if classify_url(url) != "subcategory":
return {
"status": "error",
"errorMessage": "Paste a Zepto subcategory URL: open a category on zepto.com, click a "
"subcategory in the left sidebar and copy the address (it contains /cid/ and /scid/).",
}
start = clean_url(url)
limit = max(1, min(int(max_subcategories or 1), MAX_SUBCATEGORIES))
first = self._fetch(token, start)
if "_error" in first:
return self._as_error(first)
pages = subcategory_urls(first["html"], start)
found_subcategories = len(pages)
pages = pages[:limit]
products: Dict[str, Dict[str, Any]] = {}
per_page: List[Dict[str, Any]] = []
credits = 0
for page_url in pages:
data = first if page_url == start else self._fetch(token, page_url)
if "_error" in data:
if data.get("_auth"):
return self._as_error(data)
per_page.append({"url": page_url, "error": data["_error"]})
continue
credits += data.get("_credits", CREDITS_PER_PAGE)
parsed = parse_listing(data["html"])
rows = parsed["products"]
per_page.append(
{
"url": page_url,
"subcategory": page_url.split("/")[5],
"products": len(rows),
"out_of_stock": sum(1 for r in rows if not r["in_stock"]),
}
)
for row in rows:
products.setdefault(row["variant_id"], row)
if not products and all("error" in p for p in per_page):
return {"status": "error", "errorMessage": per_page[0]["error"]}
return {
"status": "success",
"start_url": start,
"subcategories_found": found_subcategories,
"subcategories_read": per_page,
"products": list(products.values()),
"credits_used": credits,
}
# --------------------------------------------------------------- product
def product(self, url: str) -> Dict[str, Any]:
token = self._resolve_token()
if not token:
return self._missing_token()
if classify_url(url) != "product":
return {
"status": "error",
"errorMessage": "Paste a Zepto product URL (it contains /pn/ and /pvid/).",
}
page_url = clean_url(url)
data = self._fetch(token, page_url)
if "_error" in data:
return self._as_error(data)
product = parse_product(data["html"], page_url)
if not product:
return {"status": "error", "errorMessage": "That page had no product data. Check the link."}
return {"status": "success", "product": product, "credits_used": data.get("_credits", CREDITS_PER_PAGE)}
# --------------------------------------------------------------- forward
def forward(self, mode: str, url: str, max_subcategories: Optional[int] = None) -> str:
key = (mode or "").strip().lower()
if key in ("category", "listing", "listings", "subcategory"):
result = self.category(url, max_subcategories or 1)
elif key in ("product", "details", "item"):
result = self.product(url)
else:
result = {"status": "error", "errorMessage": "mode must be 'category' or 'product'."}
return json.dumps(result, ensure_ascii=False, indent=2)