Spaces:
Sleeping
Sleeping
Download tool.py from bugrahans/zepto-scraper: direct link, hf CLI and curl.
- Browser
- Download file 17.1 kB
-
https://huggingface.co/spaces/bugrahans/zepto-scraper/resolve/main/tool.py
- Command line
-
hf download hf://spaces/bugrahans/zepto-scraper/tool.py
-
curl -L -o tool.py https://huggingface.co/spaces/bugrahans/zepto-scraper/resolve/main/tool.py
17.1 kB
| from __future__ import annotations | |
| import json | |
| import os | |
| import re | |
| import time | |
| import urllib.parse | |
| from typing import Any, Dict, List, Optional | |
| import requests | |
| from smolagents.tools import Tool | |
| SCRAPE_DO_SIGNUP_URL = ( | |
| "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=zepto-scraper" | |
| ) | |
| API_URL = "https://api.scrape.do/" | |
| ZEPTO = "https://www.zepto.com" | |
| # Every page goes through Scrape.do residential proxies in India (super=true&geoCode=in). | |
| CREDITS_PER_PAGE = 10 | |
| DEFAULT_SUBCATEGORY_URL = ( | |
| "https://www.zepto.com/cn/munchies/chips-crisps/cid/d2c2a144-43cd-43e5-b308-92628fa68596" | |
| "/scid/df4f5100-c02f-4906-83b8-ddb744081a7a" | |
| ) | |
| DEFAULT_PRODUCT_URL = ( | |
| "https://www.zepto.com/pn/bingo-original-style-chilli-sprinkled-flat-cut-spicy-potato-chips-pack-for-snacks" | |
| "/pvid/cc0c8026-9b71-4bdd-bc88-b55d08e50185" | |
| ) | |
| MAX_SUBCATEGORIES = 20 | |
| UUID = r"[0-9a-f-]{36}" | |
| PUSH_RE = re.compile(r'self\.__next_f\.push\(\[1,"((?:[^"\\]|\\.)*)"\]\)') | |
| # Card keys are sorted alphabetically. In-stock cards start with availableQuantity; | |
| # out-of-stock cards drop that key and start with baseProductId. | |
| ITEM_START_RE = re.compile(r'\{"(?:availableQuantity|baseProductId)":') | |
| PRODUCT_URL_RE = re.compile(r"https://www\.zepto\.com/pn/[a-z0-9-]+/pvid/" + UUID) | |
| STORE_RE = re.compile(r'"storeServiceableResponse":\{[^}]*?"storeId":"(' + UUID + ')"') | |
| LD_RE = re.compile(r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>', re.S) | |
| SUBCATEGORY_PATH_RE = re.compile(r"^/cn/[a-z0-9-]+/[a-z0-9-]+/cid/" + UUID + "/scid/" + UUID + "/?$") | |
| PRODUCT_PATH_RE = re.compile(r"^/pn/[a-z0-9-]+/pvid/" + UUID + "/?$") | |
| _decoder = json.JSONDecoder() | |
| # ------------------------------------------------------------------ parsing | |
| def classify_url(text: Optional[str]) -> Optional[str]: | |
| """'subcategory', 'product' or None.""" | |
| raw = (text or "").strip() | |
| if not raw: | |
| return None | |
| if not raw.lower().startswith(("http://", "https://")): | |
| raw = "https://" + raw | |
| parsed = urllib.parse.urlparse(raw) | |
| if not parsed.netloc.lower().endswith("zepto.com"): | |
| return None | |
| if SUBCATEGORY_PATH_RE.match(parsed.path): | |
| return "subcategory" | |
| if PRODUCT_PATH_RE.match(parsed.path): | |
| return "product" | |
| return None | |
| def clean_url(text: str) -> str: | |
| raw = text.strip() | |
| if not raw.lower().startswith(("http://", "https://")): | |
| raw = "https://" + raw | |
| parsed = urllib.parse.urlparse(raw) | |
| return f"{ZEPTO}{parsed.path.rstrip('/')}" | |
| def decode_rsc(html: str) -> str: | |
| """Join Zepto's Next.js stream chunks back into one text. | |
| Each chunk is a JavaScript string literal; JSON strings use the same escaping. | |
| """ | |
| out = [] | |
| for chunk in PUSH_RE.findall(html or ""): | |
| try: | |
| out.append(json.loads('"' + chunk + '"')) | |
| except json.JSONDecodeError: | |
| continue | |
| return "".join(out) | |
| def rupees(paise: Any) -> Optional[float]: | |
| """Zepto stores every price in paise: 1900 means Rs 19.""" | |
| if isinstance(paise, bool) or not isinstance(paise, (int, float)): | |
| return None | |
| return round(paise / 100, 2) | |
| def extract_items(flat: str) -> List[Dict[str, Any]]: | |
| """Every product card in a decoded stream, in stock and out of stock.""" | |
| items: Dict[str, Dict[str, Any]] = {} | |
| for match in ITEM_START_RE.finditer(flat): | |
| try: | |
| obj, _ = _decoder.raw_decode(flat, match.start()) | |
| except json.JSONDecodeError: | |
| continue | |
| variant = obj.get("productVariant") or {} | |
| # Look-alike objects elsewhere in the stream have no product block | |
| if variant.get("id") and obj.get("product"): | |
| items.setdefault(variant["id"], obj) | |
| return list(items.values()) | |
| def parse_listing(html: str) -> Dict[str, Any]: | |
| flat = decode_rsc(html) | |
| store = STORE_RE.search(flat) | |
| urls = {link.rsplit("/", 1)[1]: link for link in PRODUCT_URL_RE.findall(flat)} | |
| rows = [] | |
| for item in extract_items(flat): | |
| product = item.get("product") or {} | |
| variant = item.get("productVariant") or {} | |
| rating = variant.get("ratingSummary") or {} | |
| rows.append( | |
| { | |
| "variant_id": variant.get("id"), | |
| "name": product.get("name"), | |
| "brand": product.get("brand"), | |
| "pack_size": variant.get("formattedPacksize"), | |
| "price_inr": rupees(item.get("discountedSellingPrice")), | |
| "mrp_inr": rupees(item.get("mrp")), | |
| "discount_pct": item.get("discountPercent", 0), | |
| "in_stock": not item.get("outOfStock", False), | |
| "available_qty": item.get("availableQuantity", 0), | |
| "rating": rating.get("averageRating"), | |
| "rating_count": rating.get("totalRatings"), | |
| "subcategory": item.get("primarySubcategoryName"), | |
| "store_id": store.group(1) if store else None, | |
| "url": urls.get(variant.get("id")), | |
| } | |
| ) | |
| return {"products": rows, "end_of_page": '"endOfPage":true' in flat} | |
| def subcategory_urls(html: str, start_url: str) -> List[str]: | |
| """The start subcategory plus its siblings, deduped on scid. | |
| The same subcategory can sit under a second category slug that renders no | |
| products, so only links under the start URL's own category slug are kept. | |
| """ | |
| start_path = urllib.parse.urlparse(start_url).path.rstrip("/") | |
| cid = re.search(r"/cid/(" + UUID + ")", start_path) | |
| if not cid: | |
| return [start_url] | |
| slug = start_path.split("/")[2] | |
| sibling_re = re.compile(r"/cn/[a-z0-9-]+/[a-z0-9-]+/cid/" + cid.group(1) + r"/scid/" + UUID) | |
| found: Dict[str, str] = {} | |
| for path in [start_path] + sibling_re.findall(decode_rsc(html)): | |
| if path.split("/")[2] == slug: | |
| found.setdefault(path.rsplit("/", 1)[1], ZEPTO + path) | |
| return list(found.values()) | |
| def product_jsonld(html: str) -> Dict[str, Any]: | |
| for block in LD_RE.findall(html or ""): | |
| try: | |
| data = json.loads(block) | |
| except json.JSONDecodeError: | |
| continue | |
| if isinstance(data, dict) and data.get("@type") == "Product": | |
| return data | |
| return {} | |
| def variant_data(flat: str, variant_id: str) -> Dict[str, Any]: | |
| """productVariant carries the printed MRP and order limit for this exact pack size.""" | |
| for match in re.finditer(r'"productVariant":\{', flat): | |
| try: | |
| obj, _ = _decoder.raw_decode(flat, match.end() - 1) | |
| except json.JSONDecodeError: | |
| continue | |
| if obj.get("id") == variant_id: | |
| return obj | |
| return {} | |
| def parse_product(html: str, url: str) -> Optional[Dict[str, Any]]: | |
| """Catalog sheet for one product page. Price and stock are left out on purpose: | |
| the product page caps the stock count and can lag the listing.""" | |
| ld = product_jsonld(html) | |
| if not ld: | |
| return None | |
| variant_id = url.rstrip("/").rsplit("/", 1)[1] | |
| variant = variant_data(decode_rsc(html), variant_id) | |
| props = { | |
| p.get("name"): p.get("value") | |
| for p in ld.get("additionalProperty") or [] | |
| if isinstance(p, dict) and p.get("name") | |
| } | |
| rating = ld.get("aggregateRating") or {} | |
| brand = ld.get("brand") | |
| images = ld.get("image") or [] | |
| if isinstance(images, str): | |
| images = [images] | |
| return { | |
| "variant_id": variant_id, | |
| "name": ld.get("name"), | |
| "brand": brand.get("name") if isinstance(brand, dict) else brand, | |
| "category": ld.get("category"), | |
| "pack_size": ld.get("size"), | |
| "mrp_inr": rupees(variant.get("mrp")), | |
| "max_per_order": variant.get("maxAllowedQuantity"), | |
| "rating": rating.get("ratingValue"), | |
| "rating_count": rating.get("ratingCount"), | |
| "shelf_life": props.get("Shelf life"), | |
| "country_of_origin": props.get("Country of origin"), | |
| "seller": props.get("Seller"), | |
| "fssai_licence": props.get("FSSAI licence number"), | |
| "images": [i for i in images if isinstance(i, str)], | |
| "description": ld.get("description"), | |
| "url": url, | |
| } | |
| # --------------------------------------------------------------------- tool | |
| class ScrapeDoZeptoTool(Tool): | |
| name = "zepto_scraper" | |
| description = """ | |
| Scrape Zepto (zepto.com), India's quick-commerce grocery app. Two modes: | |
| - 'category': every product in a Zepto subcategory, and optionally its sibling | |
| subcategories, with price and MRP in rupees, discount, in-stock flag, units in | |
| stock, rating, rating count and the dark store the prices belong to. | |
| Out-of-stock products are included. Zepto shows at most 32 products per subcategory. | |
| - 'product': the catalog sheet for one product page: brand, category path, pack | |
| size, MRP, order limit, seller, shelf life, FSSAI licence number, country of | |
| origin, images and description. | |
| Pass a Zepto subcategory URL (/cn/.../cid/.../scid/...) or product URL (/pn/.../pvid/...). | |
| Powered by Scrape.do, which gets past the AWS WAF challenge Zepto shows to scripts. | |
| 10 Scrape.do credits per page. | |
| """ | |
| output_type = "string" | |
| def __init__(self, api_token: Optional[str] = None) -> None: | |
| self._api_token = api_token | |
| self.inputs = { | |
| "mode": { | |
| "type": "string", | |
| "description": "'category' for a subcategory's product listings, 'product' for one product's details.", | |
| }, | |
| "url": { | |
| "type": "string", | |
| "description": ( | |
| "For 'category': a Zepto subcategory URL (zepto.com/cn/<category>/<subcategory>/cid/.../scid/...). " | |
| "For 'product': a Zepto product URL (zepto.com/pn/<name>/pvid/...)." | |
| ), | |
| }, | |
| "max_subcategories": { | |
| "type": "integer", | |
| "description": "For 'category': how many subcategories to read, the start one plus its siblings. 1 reads only the start page. Default 1, maximum 20.", | |
| "nullable": True, | |
| }, | |
| } | |
| super().__init__() | |
| # ------------------------------------------------------------ plumbing | |
| def _resolve_token(self) -> Optional[str]: | |
| if self._api_token: | |
| return self._api_token.strip() | |
| env_token = os.getenv("SCRAPE_DO_API_TOKEN") | |
| return env_token.strip() if env_token else None | |
| def _missing_token() -> Dict[str, Any]: | |
| return { | |
| "status": "error", | |
| "errorMessage": "Missing Scrape.do API token.", | |
| "hint": f"Get a free token at {SCRAPE_DO_SIGNUP_URL}", | |
| } | |
| # Zepto's WAF blocks Scrape.do's default rotation (502 ROTATION_FAILED after ~60 s), so every | |
| # page goes straight through the residential pool in India: about 3 s, 10 credits. | |
| PARAMS = {"super": "true", "geoCode": "in"} | |
| def _fetch(self, token: str, url: str) -> Dict[str, Any]: | |
| """GET a Zepto page through Scrape.do. Anything other than HTTP 200 is a failure: | |
| Zepto's AWS WAF answers scripts with an empty 202, which requests treats as success.""" | |
| last: Dict[str, Any] = {"_error": "Request failed."} | |
| for attempt in (1, 2): | |
| params = {"token": token, "url": url, **self.PARAMS} | |
| started = time.time() | |
| print(f"[fetch] attempt {attempt}: {url}", flush=True) | |
| try: | |
| response = requests.get(API_URL, params=params, timeout=(10, 90)) | |
| except requests.exceptions.Timeout: | |
| print(f"[fetch] timeout after {time.time() - started:.1f}s", flush=True) | |
| last = {"_error": "Scrape.do didn't answer within 90 seconds. Try again in a minute."} | |
| continue | |
| except requests.exceptions.RequestException as exc: | |
| msg = str(exc).replace(token, "***") | |
| print(f"[fetch] error after {time.time() - started:.1f}s: {msg[:300]}", flush=True) | |
| last = {"_error": f"Request failed: {msg}"} | |
| continue | |
| body = response.text or "" | |
| elapsed = time.time() - started | |
| print(f"[fetch] HTTP {response.status_code}, {len(body)} chars, {elapsed:.1f}s", flush=True) | |
| if response.status_code == 401: | |
| return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True} | |
| if response.status_code == 200 and body: | |
| return {"html": body, "_credits": CREDITS_PER_PAGE} | |
| if response.status_code != 200: | |
| print(f"[fetch] body: {body[:300].replace(token, '***')}", flush=True) | |
| last = {"_error": f"Zepto didn't return the page (HTTP {response.status_code}).", | |
| "_status": response.status_code} | |
| if response.status_code not in (200, 202, 403, 429, 500, 502, 503, 504): | |
| return last | |
| return last | |
| def _as_error(data: Dict[str, Any]) -> Dict[str, Any]: | |
| result = {"status": "error", "errorMessage": data.get("_error", "Unknown error")} | |
| if data.get("_auth"): | |
| result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" | |
| return result | |
| # -------------------------------------------------------------- category | |
| def category(self, url: str, max_subcategories: Optional[int] = 1) -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| if classify_url(url) != "subcategory": | |
| return { | |
| "status": "error", | |
| "errorMessage": "Paste a Zepto subcategory URL: open a category on zepto.com, click a " | |
| "subcategory in the left sidebar and copy the address (it contains /cid/ and /scid/).", | |
| } | |
| start = clean_url(url) | |
| limit = max(1, min(int(max_subcategories or 1), MAX_SUBCATEGORIES)) | |
| first = self._fetch(token, start) | |
| if "_error" in first: | |
| return self._as_error(first) | |
| pages = subcategory_urls(first["html"], start) | |
| found_subcategories = len(pages) | |
| pages = pages[:limit] | |
| products: Dict[str, Dict[str, Any]] = {} | |
| per_page: List[Dict[str, Any]] = [] | |
| credits = 0 | |
| for page_url in pages: | |
| data = first if page_url == start else self._fetch(token, page_url) | |
| if "_error" in data: | |
| if data.get("_auth"): | |
| return self._as_error(data) | |
| per_page.append({"url": page_url, "error": data["_error"]}) | |
| continue | |
| credits += data.get("_credits", CREDITS_PER_PAGE) | |
| parsed = parse_listing(data["html"]) | |
| rows = parsed["products"] | |
| per_page.append( | |
| { | |
| "url": page_url, | |
| "subcategory": page_url.split("/")[5], | |
| "products": len(rows), | |
| "out_of_stock": sum(1 for r in rows if not r["in_stock"]), | |
| } | |
| ) | |
| for row in rows: | |
| products.setdefault(row["variant_id"], row) | |
| if not products and all("error" in p for p in per_page): | |
| return {"status": "error", "errorMessage": per_page[0]["error"]} | |
| return { | |
| "status": "success", | |
| "start_url": start, | |
| "subcategories_found": found_subcategories, | |
| "subcategories_read": per_page, | |
| "products": list(products.values()), | |
| "credits_used": credits, | |
| } | |
| # --------------------------------------------------------------- product | |
| def product(self, url: str) -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| if classify_url(url) != "product": | |
| return { | |
| "status": "error", | |
| "errorMessage": "Paste a Zepto product URL (it contains /pn/ and /pvid/).", | |
| } | |
| page_url = clean_url(url) | |
| data = self._fetch(token, page_url) | |
| if "_error" in data: | |
| return self._as_error(data) | |
| product = parse_product(data["html"], page_url) | |
| if not product: | |
| return {"status": "error", "errorMessage": "That page had no product data. Check the link."} | |
| return {"status": "success", "product": product, "credits_used": data.get("_credits", CREDITS_PER_PAGE)} | |
| # --------------------------------------------------------------- forward | |
| def forward(self, mode: str, url: str, max_subcategories: Optional[int] = None) -> str: | |
| key = (mode or "").strip().lower() | |
| if key in ("category", "listing", "listings", "subcategory"): | |
| result = self.category(url, max_subcategories or 1) | |
| elif key in ("product", "details", "item"): | |
| result = self.product(url) | |
| else: | |
| result = {"status": "error", "errorMessage": "mode must be 'category' or 'product'."} | |
| return json.dumps(result, ensure_ascii=False, indent=2) | |