from __future__ import annotations import json import os import re import time from typing import Any, Dict, List, Optional import requests from smolagents.tools import Tool SCRAPE_DO_SIGNUP_URL = "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=google-maps-leads-scraper" API_BASE = "https://api.scrape.do/plugin/google/maps" # The Scrape.do Google Maps API returns parsed JSON; every successful call is 10 credits. CREDITS_PER_REQUEST = 10 PAGE_SIZE = 20 MAX_PAGES = 6 SORT_OPTIONS = ("newest", "most_relevant", "highest_rating", "lowest_rating") DAYS = ("monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday") # ------------------------------------------------------------------ helpers # (same mapping as `pip install google-maps-leads-scraper`) _SPACES = re.compile(r"[   ]") _LL_RE = re.compile(r"^@?\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)(?:\s*,\s*(\d+(?:\.\d+)?)z?)?\s*$") def clean(value: Any) -> str: return "" if value is None else _SPACES.sub(" ", str(value)).strip() def format_ll(value: str, zoom: int = 14) -> Optional[str]: match = _LL_RE.match(value or "") if not match: return None lat, lng, z = match.groups() if abs(float(lat)) > 90 or abs(float(lng)) > 180: return None return f"@{lat},{lng},{z or zoom}z" def format_hours(hours: Any) -> str: merged: Dict[str, str] = {} if isinstance(hours, dict): merged = {str(k).lower(): clean(v) for k, v in hours.items()} elif isinstance(hours, list): for item in hours: if isinstance(item, dict): merged.update({str(k).lower(): clean(v) for k, v in item.items()}) return "; ".join(f"{d[:3].title()}: {merged[d]}" for d in DAYS if d in merged) def _num(value: Any) -> Optional[float]: try: return float(str(value).replace(",", "")) if value not in (None, "") else None except ValueError: return None def normalize_place(item: Dict[str, Any]) -> Dict[str, Any]: coords = item.get("gps_coordinates") or {} raw_type = item.get("type") types = [clean(t) for t in (item.get("types") or (raw_type if isinstance(raw_type, list) else [])) if t] category = (clean(raw_type) if isinstance(raw_type, str) else "") or (types[0] if types else "") if category and category not in types: types.insert(0, category) place_id = clean(item.get("place_id")) cid = clean(item.get("data_cid")) rating = _num(item.get("rating")) reviews = _num(item.get("reviews")) attributes = {} for ext in item.get("extensions") or []: if isinstance(ext, dict): for k, v in ext.items(): if isinstance(v, list): attributes[str(k)] = [clean(x) for x in v] return { "name": clean(item.get("title")), "category": category, "categories": types, "rating": rating, "reviews": int(reviews) if reviews is not None else None, "phone": clean(item.get("phone")), "website": clean(item.get("website")), "address": clean(item.get("address")), "latitude": _num(coords.get("latitude")), "longitude": _num(coords.get("longitude")), "hours": format_hours(item.get("operating_hours") or (item.get("hours") if not isinstance(item.get("hours"), str) else None)), "open_state": clean(item.get("open_state")), "plus_code": clean(item.get("plus_code")), "rating_breakdown": {str(r.get("stars")): int(r.get("amount") or 0) for r in item.get("rating_summary") or [] if isinstance(r, dict)}, "attributes": attributes, "place_id": place_id, "data_id": clean(item.get("data_id")), "cid": cid, "maps_url": f"https://maps.google.com/?cid={cid}" if cid else (f"https://www.google.com/maps/place/?q=place_id:{place_id}" if place_id else ""), "thumbnail": clean(item.get("thumbnail")), } def normalize_review(item: Dict[str, Any]) -> Dict[str, Any]: user = item.get("user") or {} resp = item.get("response") or {} return { "rating": item.get("rating"), "date": clean(item.get("date")), "iso_date": clean(item.get("iso_date")), "text": clean(item.get("snippet") or (item.get("extracted_snippet") or {}).get("original")), "likes": item.get("likes") or 0, "author": clean(user.get("name")), "local_guide": bool(user.get("local_guide")), "owner_reply": clean(resp.get("snippet") or (resp.get("extracted_snippet") or {}).get("original")), "review_id": clean(item.get("review_id")), } def parse_place_ref(value: str) -> Dict[str, str]: value = (value or "").strip() if re.fullmatch(r"ChI[A-Za-z0-9_-]{10,}", value): return {"place_id": value} if re.fullmatch(r"\d{6,25}", value): return {"data_cid": value} m = re.fullmatch(r"(0x[0-9a-f]+):0x([0-9a-f]+)", value, re.I) if m: return {"data_id": value.lower(), "data_cid": str(int(m.group(2), 16))} m = re.search(r"[?&]cid=(\d+)", value) if m: return {"data_cid": m.group(1)} m = re.search(r"place_id[:=](ChI[A-Za-z0-9_-]+)", value) if m: return {"place_id": m.group(1)} m = re.search(r"!1s(0x[0-9a-f]+):0x([0-9a-f]+)", value, re.I) if m: return {"data_id": f"{m.group(1)}:0x{m.group(2)}".lower(), "data_cid": str(int(m.group(2), 16))} return {} # ------------------------------------------------------------------ tool class ScrapeDoGoogleMapsTool(Tool): name = "google_maps_scraper" description = """ Get Google Maps business data through the Scrape.do Google Maps API. Three modes: - 'search': a query like 'dentist' or 'coffee shop in Austin, TX', optionally centred on `location` ('lat,lng'). Returns up to 20 places per page (page 1-6): name, category, rating, review count, phone, website, address, coordinates, hours, place_id and data_id. - 'place': a place_id (ChIJ...), CID, feature id (0x..:0x..) or Google Maps place URL. Returns full details: hours, plus code, rating breakdown, attributes. - 'reviews': a place_id or data_id. Returns about 10 reviews per page with owner replies. Each call is 10 Scrape.do credits. """ output_type = "string" def __init__(self, api_token: Optional[str] = None) -> None: self._api_token = api_token self.inputs = { "mode": {"type": "string", "description": "'search', 'place' or 'reviews'."}, "query": {"type": "string", "description": "Search terms for 'search'; a place_id, CID, data_id or Maps URL for 'place' and 'reviews'."}, "location": {"type": "string", "description": "Optional 'lat,lng' centre for 'search', e.g. '30.2672,-97.7431'.", "nullable": True}, "page": {"type": "integer", "description": "Result page (1-6) for 'search', review page for 'reviews'. Default 1.", "nullable": True}, "sort_by": {"type": "string", "description": "For 'reviews': newest, most_relevant, highest_rating or lowest_rating.", "nullable": True}, } super().__init__() def _token(self) -> Optional[str]: token = (self._api_token or os.environ.get("SCRAPE_DO_API_TOKEN") or "").strip() return token or None @staticmethod def _missing_token() -> Dict[str, Any]: return {"status": "error", "errorMessage": "A Scrape.do API token is required.", "hint": f"Get a free one at {SCRAPE_DO_SIGNUP_URL}"} def _call(self, token: str, endpoint: str, params: Dict[str, Any]) -> Dict[str, Any]: query = {k: v for k, v in params.items() if v not in (None, "")} query["token"] = token last: Dict[str, Any] = {"_error": "No response."} for attempt in range(3): started = time.time() try: r = requests.get(f"{API_BASE}/{endpoint}", params=query, timeout=90) except requests.RequestException as exc: last = {"_error": f"Request failed: {exc.__class__.__name__}"} continue print(f"[{endpoint}] HTTP {r.status_code} in {time.time() - started:.1f}s", flush=True) if r.status_code == 401: return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True} if r.status_code == 200: try: return {"json": r.json()} except ValueError: last = {"_error": "The API returned a body that is not JSON."} continue last = {"_error": f"The Google Maps API returned HTTP {r.status_code}."} if r.status_code not in (429, 500, 502, 503, 504): break time.sleep(1 + attempt) return last @staticmethod def _as_error(res: Dict[str, Any]) -> Dict[str, Any]: out = {"status": "error", "errorMessage": res.get("_error", "Unknown error")} if res.get("_auth"): out["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" return out def search(self, query: str, location: Optional[str] = None, page: int = 1, gl: str = "us", hl: str = "en") -> Dict[str, Any]: token = self._token() if not token: return self._missing_token() if not (query or "").strip(): return {"status": "error", "errorMessage": "Type what to search for, e.g. 'dentist'."} ll = None if location and location.strip(): ll = format_ll(location) if not ll: return {"status": "error", "errorMessage": "Location must be 'lat,lng', e.g. 30.2672,-97.7431."} page = max(1, min(MAX_PAGES, int(page or 1))) res = self._call(token, "search", {"q": query.strip(), "ll": ll, "gl": gl, "hl": hl, "start": (page - 1) * PAGE_SIZE or None}) if "_error" in res: return self._as_error(res) results = res["json"].get("local_results") or [] if isinstance(results, dict): results = [results] places = [normalize_place(x) for x in results if isinstance(x, dict)] return {"status": "success", "query": query.strip(), "location": ll, "page": page, "results": places, "maps_url": (res["json"].get("search_metadata") or {}).get("google_maps_url", ""), "credits_used": CREDITS_PER_REQUEST} def place(self, ref: str, hl: str = "en") -> Dict[str, Any]: token = self._token() if not token: return self._missing_token() ids = parse_place_ref(ref) if not ids.get("place_id") and not ids.get("data_cid"): return {"status": "error", "errorMessage": "Paste a place ID (ChIJ...), a CID, a feature ID (0x...:0x...) or a Google Maps place URL."} params = {"place_id": ids.get("place_id"), "data_cid": None if ids.get("place_id") else ids.get("data_cid"), "hl": hl} res = self._call(token, "place", params) if "_error" in res: return self._as_error(res) item = res["json"].get("place_results") if not isinstance(item, dict): return {"status": "error", "errorMessage": "Google Maps has no place for that ID."} return {"status": "success", "place": normalize_place(item), "credits_used": CREDITS_PER_REQUEST} def reviews(self, ref: str, sort_by: str = "newest", pages: int = 1, hl: str = "en") -> Dict[str, Any]: token = self._token() if not token: return self._missing_token() ids = parse_place_ref(ref) if not ids.get("place_id") and not ids.get("data_id"): return {"status": "error", "errorMessage": "Paste a place ID (ChIJ...) or a feature ID (0x...:0x...)."} sort_by = sort_by if sort_by in SORT_OPTIONS else "newest" out: List[Dict[str, Any]] = [] topics: List[Dict[str, Any]] = [] info: Dict[str, Any] = {} token_next = "" calls = 0 for _ in range(max(1, min(5, int(pages or 1)))): res = self._call(token, "reviews", { "data_id": ids.get("data_id"), "place_id": None if ids.get("data_id") else ids.get("place_id"), "sort_by": None if sort_by == "most_relevant" else sort_by, "next_page_token": token_next or None, "hl": hl, }) if "_error" in res: if out: break return self._as_error(res) calls += 1 data = res["json"] info = info or data.get("place_info") or {} topics = topics or [{"keyword": t.get("keyword"), "mentions": t.get("mentions")} for t in data.get("topics") or []] out.extend(normalize_review(r) for r in data.get("reviews") or [] if isinstance(r, dict)) token_next = (data.get("pagination") or {}).get("next_page_token") or "" if not token_next: break return {"status": "success", "place": {k: info.get(k) for k in ("title", "address", "rating", "reviews")}, "topics": topics, "reviews": out, "credits_used": calls * CREDITS_PER_REQUEST} def forward(self, mode: str, query: str, location: Optional[str] = None, page: Optional[int] = None, sort_by: Optional[str] = None) -> str: key = (mode or "").strip().lower() if key == "search": result = self.search(query, location, page or 1) elif key == "place": result = self.place(query) elif key == "reviews": result = self.reviews(query, sort_by or "newest", page or 1) else: result = {"status": "error", "errorMessage": "mode must be 'search', 'place' or 'reviews'."} return json.dumps(result, ensure_ascii=False, indent=2)