from __future__ import annotations import json import os import re import time import urllib.parse from typing import Any, Dict, List, Optional import requests from smolagents.tools import Tool SCRAPE_DO_SIGNUP_URL = ( "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=zepto-scraper" ) API_URL = "https://api.scrape.do/" ZEPTO = "https://www.zepto.com" # Every page goes through Scrape.do residential proxies in India (super=true&geoCode=in). CREDITS_PER_PAGE = 10 DEFAULT_SUBCATEGORY_URL = ( "https://www.zepto.com/cn/munchies/chips-crisps/cid/d2c2a144-43cd-43e5-b308-92628fa68596" "/scid/df4f5100-c02f-4906-83b8-ddb744081a7a" ) DEFAULT_PRODUCT_URL = ( "https://www.zepto.com/pn/bingo-original-style-chilli-sprinkled-flat-cut-spicy-potato-chips-pack-for-snacks" "/pvid/cc0c8026-9b71-4bdd-bc88-b55d08e50185" ) MAX_SUBCATEGORIES = 20 UUID = r"[0-9a-f-]{36}" PUSH_RE = re.compile(r'self\.__next_f\.push\(\[1,"((?:[^"\\]|\\.)*)"\]\)') # Card keys are sorted alphabetically. In-stock cards start with availableQuantity; # out-of-stock cards drop that key and start with baseProductId. ITEM_START_RE = re.compile(r'\{"(?:availableQuantity|baseProductId)":') PRODUCT_URL_RE = re.compile(r"https://www\.zepto\.com/pn/[a-z0-9-]+/pvid/" + UUID) STORE_RE = re.compile(r'"storeServiceableResponse":\{[^}]*?"storeId":"(' + UUID + ')"') LD_RE = re.compile(r']*type="application/ld\+json"[^>]*>(.*?)', re.S) SUBCATEGORY_PATH_RE = re.compile(r"^/cn/[a-z0-9-]+/[a-z0-9-]+/cid/" + UUID + "/scid/" + UUID + "/?$") PRODUCT_PATH_RE = re.compile(r"^/pn/[a-z0-9-]+/pvid/" + UUID + "/?$") _decoder = json.JSONDecoder() # ------------------------------------------------------------------ parsing def classify_url(text: Optional[str]) -> Optional[str]: """'subcategory', 'product' or None.""" raw = (text or "").strip() if not raw: return None if not raw.lower().startswith(("http://", "https://")): raw = "https://" + raw parsed = urllib.parse.urlparse(raw) if not parsed.netloc.lower().endswith("zepto.com"): return None if SUBCATEGORY_PATH_RE.match(parsed.path): return "subcategory" if PRODUCT_PATH_RE.match(parsed.path): return "product" return None def clean_url(text: str) -> str: raw = text.strip() if not raw.lower().startswith(("http://", "https://")): raw = "https://" + raw parsed = urllib.parse.urlparse(raw) return f"{ZEPTO}{parsed.path.rstrip('/')}" def decode_rsc(html: str) -> str: """Join Zepto's Next.js stream chunks back into one text. Each chunk is a JavaScript string literal; JSON strings use the same escaping. """ out = [] for chunk in PUSH_RE.findall(html or ""): try: out.append(json.loads('"' + chunk + '"')) except json.JSONDecodeError: continue return "".join(out) def rupees(paise: Any) -> Optional[float]: """Zepto stores every price in paise: 1900 means Rs 19.""" if isinstance(paise, bool) or not isinstance(paise, (int, float)): return None return round(paise / 100, 2) def extract_items(flat: str) -> List[Dict[str, Any]]: """Every product card in a decoded stream, in stock and out of stock.""" items: Dict[str, Dict[str, Any]] = {} for match in ITEM_START_RE.finditer(flat): try: obj, _ = _decoder.raw_decode(flat, match.start()) except json.JSONDecodeError: continue variant = obj.get("productVariant") or {} # Look-alike objects elsewhere in the stream have no product block if variant.get("id") and obj.get("product"): items.setdefault(variant["id"], obj) return list(items.values()) def parse_listing(html: str) -> Dict[str, Any]: flat = decode_rsc(html) store = STORE_RE.search(flat) urls = {link.rsplit("/", 1)[1]: link for link in PRODUCT_URL_RE.findall(flat)} rows = [] for item in extract_items(flat): product = item.get("product") or {} variant = item.get("productVariant") or {} rating = variant.get("ratingSummary") or {} rows.append( { "variant_id": variant.get("id"), "name": product.get("name"), "brand": product.get("brand"), "pack_size": variant.get("formattedPacksize"), "price_inr": rupees(item.get("discountedSellingPrice")), "mrp_inr": rupees(item.get("mrp")), "discount_pct": item.get("discountPercent", 0), "in_stock": not item.get("outOfStock", False), "available_qty": item.get("availableQuantity", 0), "rating": rating.get("averageRating"), "rating_count": rating.get("totalRatings"), "subcategory": item.get("primarySubcategoryName"), "store_id": store.group(1) if store else None, "url": urls.get(variant.get("id")), } ) return {"products": rows, "end_of_page": '"endOfPage":true' in flat} def subcategory_urls(html: str, start_url: str) -> List[str]: """The start subcategory plus its siblings, deduped on scid. The same subcategory can sit under a second category slug that renders no products, so only links under the start URL's own category slug are kept. """ start_path = urllib.parse.urlparse(start_url).path.rstrip("/") cid = re.search(r"/cid/(" + UUID + ")", start_path) if not cid: return [start_url] slug = start_path.split("/")[2] sibling_re = re.compile(r"/cn/[a-z0-9-]+/[a-z0-9-]+/cid/" + cid.group(1) + r"/scid/" + UUID) found: Dict[str, str] = {} for path in [start_path] + sibling_re.findall(decode_rsc(html)): if path.split("/")[2] == slug: found.setdefault(path.rsplit("/", 1)[1], ZEPTO + path) return list(found.values()) def product_jsonld(html: str) -> Dict[str, Any]: for block in LD_RE.findall(html or ""): try: data = json.loads(block) except json.JSONDecodeError: continue if isinstance(data, dict) and data.get("@type") == "Product": return data return {} def variant_data(flat: str, variant_id: str) -> Dict[str, Any]: """productVariant carries the printed MRP and order limit for this exact pack size.""" for match in re.finditer(r'"productVariant":\{', flat): try: obj, _ = _decoder.raw_decode(flat, match.end() - 1) except json.JSONDecodeError: continue if obj.get("id") == variant_id: return obj return {} def parse_product(html: str, url: str) -> Optional[Dict[str, Any]]: """Catalog sheet for one product page. Price and stock are left out on purpose: the product page caps the stock count and can lag the listing.""" ld = product_jsonld(html) if not ld: return None variant_id = url.rstrip("/").rsplit("/", 1)[1] variant = variant_data(decode_rsc(html), variant_id) props = { p.get("name"): p.get("value") for p in ld.get("additionalProperty") or [] if isinstance(p, dict) and p.get("name") } rating = ld.get("aggregateRating") or {} brand = ld.get("brand") images = ld.get("image") or [] if isinstance(images, str): images = [images] return { "variant_id": variant_id, "name": ld.get("name"), "brand": brand.get("name") if isinstance(brand, dict) else brand, "category": ld.get("category"), "pack_size": ld.get("size"), "mrp_inr": rupees(variant.get("mrp")), "max_per_order": variant.get("maxAllowedQuantity"), "rating": rating.get("ratingValue"), "rating_count": rating.get("ratingCount"), "shelf_life": props.get("Shelf life"), "country_of_origin": props.get("Country of origin"), "seller": props.get("Seller"), "fssai_licence": props.get("FSSAI licence number"), "images": [i for i in images if isinstance(i, str)], "description": ld.get("description"), "url": url, } # --------------------------------------------------------------------- tool class ScrapeDoZeptoTool(Tool): name = "zepto_scraper" description = """ Scrape Zepto (zepto.com), India's quick-commerce grocery app. Two modes: - 'category': every product in a Zepto subcategory, and optionally its sibling subcategories, with price and MRP in rupees, discount, in-stock flag, units in stock, rating, rating count and the dark store the prices belong to. Out-of-stock products are included. Zepto shows at most 32 products per subcategory. - 'product': the catalog sheet for one product page: brand, category path, pack size, MRP, order limit, seller, shelf life, FSSAI licence number, country of origin, images and description. Pass a Zepto subcategory URL (/cn/.../cid/.../scid/...) or product URL (/pn/.../pvid/...). Powered by Scrape.do, which gets past the AWS WAF challenge Zepto shows to scripts. 10 Scrape.do credits per page. """ output_type = "string" def __init__(self, api_token: Optional[str] = None) -> None: self._api_token = api_token self.inputs = { "mode": { "type": "string", "description": "'category' for a subcategory's product listings, 'product' for one product's details.", }, "url": { "type": "string", "description": ( "For 'category': a Zepto subcategory URL (zepto.com/cn///cid/.../scid/...). " "For 'product': a Zepto product URL (zepto.com/pn//pvid/...)." ), }, "max_subcategories": { "type": "integer", "description": "For 'category': how many subcategories to read, the start one plus its siblings. 1 reads only the start page. Default 1, maximum 20.", "nullable": True, }, } super().__init__() # ------------------------------------------------------------ plumbing def _resolve_token(self) -> Optional[str]: if self._api_token: return self._api_token.strip() env_token = os.getenv("SCRAPE_DO_API_TOKEN") return env_token.strip() if env_token else None @staticmethod def _missing_token() -> Dict[str, Any]: return { "status": "error", "errorMessage": "Missing Scrape.do API token.", "hint": f"Get a free token at {SCRAPE_DO_SIGNUP_URL}", } # Zepto's WAF blocks Scrape.do's default rotation (502 ROTATION_FAILED after ~60 s), so every # page goes straight through the residential pool in India: about 3 s, 10 credits. PARAMS = {"super": "true", "geoCode": "in"} def _fetch(self, token: str, url: str) -> Dict[str, Any]: """GET a Zepto page through Scrape.do. Anything other than HTTP 200 is a failure: Zepto's AWS WAF answers scripts with an empty 202, which requests treats as success.""" last: Dict[str, Any] = {"_error": "Request failed."} for attempt in (1, 2): params = {"token": token, "url": url, **self.PARAMS} started = time.time() print(f"[fetch] attempt {attempt}: {url}", flush=True) try: response = requests.get(API_URL, params=params, timeout=(10, 90)) except requests.exceptions.Timeout: print(f"[fetch] timeout after {time.time() - started:.1f}s", flush=True) last = {"_error": "Scrape.do didn't answer within 90 seconds. Try again in a minute."} continue except requests.exceptions.RequestException as exc: msg = str(exc).replace(token, "***") print(f"[fetch] error after {time.time() - started:.1f}s: {msg[:300]}", flush=True) last = {"_error": f"Request failed: {msg}"} continue body = response.text or "" elapsed = time.time() - started print(f"[fetch] HTTP {response.status_code}, {len(body)} chars, {elapsed:.1f}s", flush=True) if response.status_code == 401: return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True} if response.status_code == 200 and body: return {"html": body, "_credits": CREDITS_PER_PAGE} if response.status_code != 200: print(f"[fetch] body: {body[:300].replace(token, '***')}", flush=True) last = {"_error": f"Zepto didn't return the page (HTTP {response.status_code}).", "_status": response.status_code} if response.status_code not in (200, 202, 403, 429, 500, 502, 503, 504): return last return last @staticmethod def _as_error(data: Dict[str, Any]) -> Dict[str, Any]: result = {"status": "error", "errorMessage": data.get("_error", "Unknown error")} if data.get("_auth"): result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" return result # -------------------------------------------------------------- category def category(self, url: str, max_subcategories: Optional[int] = 1) -> Dict[str, Any]: token = self._resolve_token() if not token: return self._missing_token() if classify_url(url) != "subcategory": return { "status": "error", "errorMessage": "Paste a Zepto subcategory URL: open a category on zepto.com, click a " "subcategory in the left sidebar and copy the address (it contains /cid/ and /scid/).", } start = clean_url(url) limit = max(1, min(int(max_subcategories or 1), MAX_SUBCATEGORIES)) first = self._fetch(token, start) if "_error" in first: return self._as_error(first) pages = subcategory_urls(first["html"], start) found_subcategories = len(pages) pages = pages[:limit] products: Dict[str, Dict[str, Any]] = {} per_page: List[Dict[str, Any]] = [] credits = 0 for page_url in pages: data = first if page_url == start else self._fetch(token, page_url) if "_error" in data: if data.get("_auth"): return self._as_error(data) per_page.append({"url": page_url, "error": data["_error"]}) continue credits += data.get("_credits", CREDITS_PER_PAGE) parsed = parse_listing(data["html"]) rows = parsed["products"] per_page.append( { "url": page_url, "subcategory": page_url.split("/")[5], "products": len(rows), "out_of_stock": sum(1 for r in rows if not r["in_stock"]), } ) for row in rows: products.setdefault(row["variant_id"], row) if not products and all("error" in p for p in per_page): return {"status": "error", "errorMessage": per_page[0]["error"]} return { "status": "success", "start_url": start, "subcategories_found": found_subcategories, "subcategories_read": per_page, "products": list(products.values()), "credits_used": credits, } # --------------------------------------------------------------- product def product(self, url: str) -> Dict[str, Any]: token = self._resolve_token() if not token: return self._missing_token() if classify_url(url) != "product": return { "status": "error", "errorMessage": "Paste a Zepto product URL (it contains /pn/ and /pvid/).", } page_url = clean_url(url) data = self._fetch(token, page_url) if "_error" in data: return self._as_error(data) product = parse_product(data["html"], page_url) if not product: return {"status": "error", "errorMessage": "That page had no product data. Check the link."} return {"status": "success", "product": product, "credits_used": data.get("_credits", CREDITS_PER_PAGE)} # --------------------------------------------------------------- forward def forward(self, mode: str, url: str, max_subcategories: Optional[int] = None) -> str: key = (mode or "").strip().lower() if key in ("category", "listing", "listings", "subcategory"): result = self.category(url, max_subcategories or 1) elif key in ("product", "details", "item"): result = self.product(url) else: result = {"status": "error", "errorMessage": "mode must be 'category' or 'product'."} return json.dumps(result, ensure_ascii=False, indent=2)