File size: 18,346 Bytes
2b5d098
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
from __future__ import annotations

import json
import os
import re
import time
from typing import Any, Dict, List, Optional, Tuple
from urllib.parse import parse_qs, urlparse

import requests
from smolagents.tools import Tool

SCRAPE_DO_SIGNUP_URL = (
    "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=blinkit-scraper"
)
API_URL = "https://api.scrape.do/"
BLINKIT = "https://blinkit.com"
# Blinkit sits behind Cloudflare; through Scrape.do every request uses residential proxies in
# India (super=true, geoCode=in) and costs 10 credits.
CREDITS_PER_REQUEST = 10
MAX_PAGES = 10
CARD_TYPE = "product_card_snippet_type_2"
DEFAULT_CATEGORY_URL = "https://blinkit.com/cn/chips-crisps/cid/1237/940"
DEFAULT_PRODUCT_URL = "https://blinkit.com/prn/unibic-kerala-banana-chips-classic-salted/prid/594125"
CITIES = {
    "Gurugram": (28.413333, 77.072833),
    "Delhi": (28.6315, 77.2167),
    "Mumbai": (19.0760, 72.8777),
    "Bengaluru": (12.9716, 77.5946),
    "Hyderabad": (17.3850, 78.4867),
    "Pune": (18.5204, 73.8567),
    "Kolkata": (22.5726, 88.3639),
    "Chennai": (13.0827, 80.2707),
}

_STATE_RE = re.compile(r"window\.grofers\.PRELOADED_STATE\s*=\s*")
_decoder = json.JSONDecoder()

# ------------------------------------------------------------------ parsing


def extract_state(html: str) -> Dict[str, Any]:
    match = _STATE_RE.search(html or "")
    if not match:
        return {}
    try:
        data, _ = _decoder.raw_decode(html, match.end())
    except json.JSONDecodeError:
        return {}
    return data if isinstance(data, dict) else {}


def _text(node: Any) -> Optional[str]:
    if isinstance(node, dict):
        value = node.get("text")
        return value.strip() if isinstance(value, str) else None
    return node.strip() if isinstance(node, str) else None


def _num(value: Any) -> Optional[float]:
    if isinstance(value, bool) or value is None:
        return None
    if isinstance(value, (int, float)):
        return value
    match = re.search(r"\d[\d,]*(?:\.\d+)?", str(value))
    if not match:
        return None
    number = float(match.group(0).replace(",", ""))
    return int(number) if number.is_integer() else number


def _int(value: Any) -> Optional[int]:
    try:
        return int(value)
    except (TypeError, ValueError):
        return None


def _pct(price: Any, mrp: Any) -> int:
    if isinstance(price, (int, float)) and isinstance(mrp, (int, float)) and mrp > 0 and price <= mrp:
        return int((mrp - price) * 100 // mrp)
    return 0


_MULT = {"k": 1_000, "lac": 100_000, "lakh": 100_000, "l": 100_000, "cr": 10_000_000}


def parse_count(text: Any) -> Optional[int]:
    match = re.search(r"(\d[\d,]*(?:\.\d+)?)\s*([a-zA-Z]*)", str(text or ""))
    if not match:
        return None
    return int(round(float(match.group(1).replace(",", "")) * _MULT.get(match.group(2).lower(), 1)))


def product_url(product_id: Any, name: Any = None) -> str:
    slug = re.sub(r"[^a-z0-9]+", "-", str(name or "").lower()).strip("-") or "product"
    return f"{BLINKIT}/prn/{slug}/prid/{product_id}" if product_id else ""


def classify_url(text: Optional[str]) -> Optional[str]:
    raw = (text or "").strip()
    if not raw:
        return None
    if not raw.lower().startswith(("http://", "https://")):
        raw = "https://" + raw
    parsed = urlparse(raw)
    if not parsed.netloc.lower().endswith("blinkit.com"):
        return None
    if re.search(r"/prid/\d+", parsed.path):
        return "product"
    if re.search(r"/cid/\d+", parsed.path):
        return "category"
    return None


def clean_url(text: str) -> str:
    raw = text.strip()
    return raw if raw.lower().startswith(("http://", "https://")) else "https://" + raw


def parse_card(data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
    if not isinstance(data, dict):
        return None
    cart = ((data.get("atc_action") or {}).get("add_to_cart") or {}).get("cart_item") or {}
    product_id = _int(data.get("product_id") or (data.get("identity") or {}).get("id") or cart.get("product_id"))
    if not product_id:
        return None
    name = _text(data.get("display_name")) or _text(data.get("name")) or cart.get("product_name")
    price = cart.get("price") if isinstance(cart.get("price"), (int, float)) else _num(_text(data.get("normal_price")))
    mrp = cart.get("mrp") if isinstance(cart.get("mrp"), (int, float)) else _num(_text(data.get("mrp")))
    if mrp is None:
        mrp = price
    inventory = data.get("inventory", cart.get("inventory"))
    inventory = inventory if isinstance(inventory, int) else 0
    sold_out = bool(data.get("is_sold_out")) or data.get("product_state") == "out_of_stock" or inventory <= 0
    bar = (data.get("rating") or {}).get("bar") or {}
    rating = bar.get("value")
    return {
        "product_id": product_id,
        "name": name,
        "brand": _text(data.get("brand_name")) or cart.get("brand"),
        "pack_size": _text(data.get("variant")) or cart.get("unit"),
        "price_inr": price,
        "mrp_inr": mrp,
        "discount_pct": _pct(price, mrp),
        "in_stock": not sold_out,
        "inventory": max(inventory, 0),
        "rating": round(rating, 2) if isinstance(rating, (int, float)) else None,
        "rating_count": parse_count(_text(bar.get("title"))) if bar else None,
        "store_id": _int(data.get("merchant_id") or cart.get("merchant_id")),
        "image": (data.get("image") or {}).get("url") or cart.get("image_url"),
        "url": product_url(product_id, name),
    }


def parse_cards(snippets: List[Any]) -> List[Dict[str, Any]]:
    rows = []
    for snippet in snippets or []:
        if isinstance(snippet, dict) and snippet.get("widget_type") == CARD_TYPE:
            row = parse_card(snippet.get("data"))
            if row:
                rows.append(row)
    return rows


def _total(next_url: Optional[str]) -> Optional[int]:
    if not next_url:
        return None
    values = parse_qs(urlparse(next_url).query).get("total_pagination_items")
    return _int(values[0]) if values else None


_ATTRS = {
    "shelf_life": ("Shelf Life",),
    "fssai_license": ("FSSAI License", "FSSAI License Number"),
    "country_of_origin": ("Country of Origin",),
    "pack_size": ("Unit", "Net Weight"),
    "ingredients": ("Ingredients",),
    "manufacturer": ("Manufacturer Details", "Manufacturer's Name and Address", "Manufacturer Name"),
    "return_policy": ("Return Policy",),
}


def parse_product(html: str) -> Optional[Dict[str, Any]]:
    state = extract_state(html)
    bff = ((((state.get("ui") or {}).get("pdp") or {}).get("bffPdp") or {}).get("bffData")) or {}
    seo = ((((bff.get("tracking") or {}).get("le_meta") or {}).get("custom_data")) or {}).get("seo") or {}
    product_id = _int(seo.get("id"))
    if not product_id:
        return None
    attributes: Dict[str, str] = {}
    for item in seo.get("attributes") or []:
        name, value = item.get("name"), item.get("value")
        if name and value not in (None, "") and name not in attributes:
            attributes[name] = str(value).strip()
    picked = {k: next((attributes[n] for n in names if n in attributes), None) for k, names in _ATTRS.items()}
    store_id = None
    for snippet in bff.get("snippets") or []:
        if snippet.get("widget_type") == "product_atc_strip":
            found = re.search(r'"merchant_id":\s*(\d+)', json.dumps(snippet.get("data") or {}))
            store_id = int(found.group(1)) if found else None
            break
    price, mrp = seo.get("price") or None, seo.get("mrp") or None
    inventory = seo.get("inventory") if isinstance(seo.get("inventory"), int) else 0
    return {
        "product_id": product_id,
        "name": seo.get("product_name"),
        "brand": seo.get("brand"),
        "price_inr": price,
        "mrp_inr": mrp,
        "discount_pct": _pct(price, mrp) if price else 0,
        "in_stock": bool(price) and inventory > 0,
        "inventory": max(inventory, 0),
        **picked,
        "store_id": store_id,
        "images": [u for u in seo.get("images") or [] if isinstance(u, str)][:6],
        "attributes": attributes,
        "url": product_url(product_id, seo.get("product_name")),
    }

# --------------------------------------------------------------------- tool


class ScrapeDoBlinkitTool(Tool):
    name = "blinkit_scraper"
    description = """
    Scrape Blinkit (blinkit.com), India's 10-minute grocery app, for one delivery location. Two modes:
    - 'category': products of a category URL (https://blinkit.com/cn/<name>/cid/<l0>/<l1>) with name,
      brand, pack size, price and MRP in rupees, discount, in-stock flag, units in stock, rating and URL,
      for the dark store that serves the location. Page 1 has 30 products, every further page 15.
    - 'product': one product URL (.../prn/<name>/prid/<id>): price, MRP, stock, FSSAI licence,
      shelf life, country of origin, ingredients, manufacturer and the full attribute list.
    Location: a city (Gurugram, Delhi, Mumbai, Bengaluru, Hyderabad, Pune, Kolkata, Chennai) or
    "lat,lon". Powered by Scrape.do (residential proxies in India, 10 credits per request).
    """
    output_type = "string"

    def __init__(self, api_token: Optional[str] = None) -> None:
        self._api_token = api_token
        self.inputs = {
            "mode": {"type": "string", "description": "'category' for a category page, 'product' for one product."},
            "url": {"type": "string", "description": "A Blinkit category URL or product URL."},
            "location": {
                "type": "string",
                "description": "City name (e.g. 'Mumbai') or 'lat,lon'. Default Gurugram, Blinkit's own default.",
                "nullable": True,
            },
            "max_pages": {
                "type": "integer",
                "description": "For 'category': pages to read (30 products, then 15 per page). Default 1, maximum 10.",
                "nullable": True,
            },
        }
        super().__init__()

    # ------------------------------------------------------------ plumbing

    def _resolve_token(self) -> Optional[str]:
        if self._api_token:
            return self._api_token.strip()
        env_token = os.getenv("SCRAPE_DO_API_TOKEN")
        return env_token.strip() if env_token else None

    @staticmethod
    def resolve_location(location: Optional[str]) -> Tuple[float, float, str]:
        text = (location or "").strip()
        if not text:
            return (*CITIES["Gurugram"], "Gurugram")
        for city, coords in CITIES.items():
            if city.lower() == text.lower():
                return (*coords, city)
        match = re.fullmatch(r"\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*", text)
        if match:
            return float(match.group(1)), float(match.group(2)), text
        raise ValueError(f"Unknown location {text!r}. Use a city ({', '.join(CITIES)}) or 'lat,lon'.")

    @staticmethod
    def _missing_token() -> Dict[str, Any]:
        return {"status": "error", "errorMessage": "Missing Scrape.do API token.",
                "hint": f"Get a free token at {SCRAPE_DO_SIGNUP_URL}"}

    def _request(self, token: str, url: str, lat: float, lon: float, api: bool = False) -> Dict[str, Any]:
        """One Scrape.do call with two attempts. Returns {'text', 'credits'} or {'_error', ...}."""
        params = {"token": token, "url": url, "super": "true", "geoCode": "in"}
        headers = None
        if api:
            params["extraHeaders"] = "true"
            headers = {"sd-lat": str(lat), "sd-lon": str(lon), "sd-app_client": "consumer_web",
                       "sd-Content-Type": "application/json", "sd-Accept": "application/json"}
        else:
            params["setCookies"] = f"gr_1_lat={lat}; gr_1_lon={lon}"
        last: Dict[str, Any] = {"_error": "Request failed."}
        credits = 0
        for attempt in (1, 2):
            started = time.time()
            print(f"[fetch] attempt {attempt}: {'POST ' if api else ''}{url[:110]}", flush=True)
            try:
                response = requests.request("POST" if api else "GET", API_URL, params=params, headers=headers,
                                            data="{}" if api else None, timeout=(10, 90))
            except requests.exceptions.Timeout:
                last = {"_error": "Scrape.do didn't answer within 90 seconds. Try again in a minute."}
                continue
            except requests.exceptions.RequestException as exc:
                last = {"_error": f"Request failed: {str(exc).replace(token, '***')[:200]}"}
                continue
            body = response.text or ""
            from_target = "scrape.do-initial-status-code" in {k.lower() for k in response.headers}
            try:
                credits += int(response.headers.get("scrape.do-request-cost") or 0)
            except ValueError:
                pass
            print(f"[fetch] HTTP {response.status_code}, {len(body)} chars, {time.time() - started:.1f}s", flush=True)
            if response.status_code == 401 and not from_target:
                return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True}
            if response.status_code == 200 and body:
                if api:
                    try:
                        ok = isinstance(json.loads(body), dict)
                    except ValueError:
                        ok = False
                else:
                    ok = bool(extract_state(body))
                if ok:
                    return {"text": body, "credits": credits}
                last = {"_error": "Blinkit returned a page without its data. Try again."}
                continue
            last = {"_error": f"Blinkit didn't return the page (HTTP {response.status_code})."}
            if response.status_code not in (429, 500, 502, 503, 504):
                break
        last["credits"] = credits
        return last

    @staticmethod
    def _as_error(data: Dict[str, Any]) -> Dict[str, Any]:
        result = {"status": "error", "errorMessage": data.get("_error", "Unknown error")}
        if data.get("_auth"):
            result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}"
        return result

    # ------------------------------------------------------------- category

    def category(self, url: str, location: Optional[str] = None, max_pages: Optional[int] = 1) -> Dict[str, Any]:
        token = self._resolve_token()
        if not token:
            return self._missing_token()
        if classify_url(url) != "category":
            return {"status": "error", "errorMessage": "Paste a Blinkit category URL, e.g. https://blinkit.com/cn/chips-crisps/cid/1237/940"}
        try:
            lat, lon, label = self.resolve_location(location)
        except ValueError as exc:
            return {"status": "error", "errorMessage": str(exc)}
        start = clean_url(url).split("#")[0]
        pages_wanted = max(1, min(int(max_pages or 1), MAX_PAGES))
        page = self._request(token, start, lat, lon)
        if "_error" in page:
            return self._as_error(page)
        credits = page["credits"]
        state = extract_state(page["text"])
        feed = (((state.get("ui") or {}).get("plpContainer") or {}).get("feedData")) or {}
        next_url = (feed.get("pagination") or {}).get("next_url")
        total = _total(next_url)
        products: Dict[int, Dict[str, Any]] = {r["product_id"]: r for r in parse_cards(feed.get("snippets") or [])}
        pages_read = 1
        while next_url and pages_read < pages_wanted:
            res = self._request(token, BLINKIT + next_url if next_url.startswith("/") else next_url, lat, lon, api=True)
            credits += res.get("credits", 0)
            if "_error" in res:
                break
            response = json.loads(res["text"]).get("response") or {}
            rows = parse_cards(response.get("snippets") or [])
            pages_read += 1
            for row in rows:
                products.setdefault(row["product_id"], row)
            next_url = (response.get("pagination") or {}).get("next_url")
            if not rows:
                break
        rows = list(products.values())
        return {"status": "success", "url": start, "location": label, "latitude": lat, "longitude": lon,
                "store_id": next((r["store_id"] for r in rows if r.get("store_id")), None),
                "total_count": total, "pages_read": pages_read, "has_more": bool(next_url),
                "products": rows, "credits_used": credits}

    # -------------------------------------------------------------- product

    def product(self, url: str, location: Optional[str] = None) -> Dict[str, Any]:
        token = self._resolve_token()
        if not token:
            return self._missing_token()
        if classify_url(url) != "product":
            return {"status": "error", "errorMessage": "Paste a Blinkit product URL (…/prn/<name>/prid/<id>)."}
        try:
            lat, lon, label = self.resolve_location(location)
        except ValueError as exc:
            return {"status": "error", "errorMessage": str(exc)}
        product_id = re.search(r"/prid/(\d+)", url).group(1)
        data = self._request(token, f"{BLINKIT}/prn/product/prid/{product_id}", lat, lon)
        if "_error" in data:
            return self._as_error(data)
        product = parse_product(data["text"])
        if not product:
            return {"status": "error", "errorMessage": "That page had no product data. Check the link."}
        return {"status": "success", "location": label, "product": product, "credits_used": data.get("credits", CREDITS_PER_REQUEST)}

    # -------------------------------------------------------------- forward

    def forward(self, mode: str, url: str, location: Optional[str] = None, max_pages: Optional[int] = None) -> str:
        key = (mode or "").strip().lower()
        if key in ("category", "listing", "categories"):
            result = self.category(url, location, max_pages or 1)
        elif key in ("product", "details", "item"):
            result = self.product(url, location)
        else:
            result = {"status": "error", "errorMessage": "mode must be 'category' or 'product'."}
        return json.dumps(result, ensure_ascii=False, indent=2)