Spaces:
Sleeping
Sleeping
Download tool.py from bugrahans/google-maps-leads-scraper: direct link, hf CLI and curl.
- Browser
- Download file 13.8 kB
-
https://huggingface.co/spaces/bugrahans/google-maps-leads-scraper/resolve/main/tool.py
- Command line
-
hf download hf://spaces/bugrahans/google-maps-leads-scraper/tool.py
-
curl -L -o tool.py https://huggingface.co/spaces/bugrahans/google-maps-leads-scraper/resolve/main/tool.py
13.8 kB
| from __future__ import annotations | |
| import json | |
| import os | |
| import re | |
| import time | |
| from typing import Any, Dict, List, Optional | |
| import requests | |
| from smolagents.tools import Tool | |
| SCRAPE_DO_SIGNUP_URL = "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=google-maps-leads-scraper" | |
| API_BASE = "https://api.scrape.do/plugin/google/maps" | |
| # The Scrape.do Google Maps API returns parsed JSON; every successful call is 10 credits. | |
| CREDITS_PER_REQUEST = 10 | |
| PAGE_SIZE = 20 | |
| MAX_PAGES = 6 | |
| SORT_OPTIONS = ("newest", "most_relevant", "highest_rating", "lowest_rating") | |
| DAYS = ("monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday") | |
| # ------------------------------------------------------------------ helpers | |
| # (same mapping as `pip install google-maps-leads-scraper`) | |
| _SPACES = re.compile(r"[ ]") | |
| _LL_RE = re.compile(r"^@?\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)(?:\s*,\s*(\d+(?:\.\d+)?)z?)?\s*$") | |
| def clean(value: Any) -> str: | |
| return "" if value is None else _SPACES.sub(" ", str(value)).strip() | |
| def format_ll(value: str, zoom: int = 14) -> Optional[str]: | |
| match = _LL_RE.match(value or "") | |
| if not match: | |
| return None | |
| lat, lng, z = match.groups() | |
| if abs(float(lat)) > 90 or abs(float(lng)) > 180: | |
| return None | |
| return f"@{lat},{lng},{z or zoom}z" | |
| def format_hours(hours: Any) -> str: | |
| merged: Dict[str, str] = {} | |
| if isinstance(hours, dict): | |
| merged = {str(k).lower(): clean(v) for k, v in hours.items()} | |
| elif isinstance(hours, list): | |
| for item in hours: | |
| if isinstance(item, dict): | |
| merged.update({str(k).lower(): clean(v) for k, v in item.items()}) | |
| return "; ".join(f"{d[:3].title()}: {merged[d]}" for d in DAYS if d in merged) | |
| def _num(value: Any) -> Optional[float]: | |
| try: | |
| return float(str(value).replace(",", "")) if value not in (None, "") else None | |
| except ValueError: | |
| return None | |
| def normalize_place(item: Dict[str, Any]) -> Dict[str, Any]: | |
| coords = item.get("gps_coordinates") or {} | |
| raw_type = item.get("type") | |
| types = [clean(t) for t in (item.get("types") or (raw_type if isinstance(raw_type, list) else [])) if t] | |
| category = (clean(raw_type) if isinstance(raw_type, str) else "") or (types[0] if types else "") | |
| if category and category not in types: | |
| types.insert(0, category) | |
| place_id = clean(item.get("place_id")) | |
| cid = clean(item.get("data_cid")) | |
| rating = _num(item.get("rating")) | |
| reviews = _num(item.get("reviews")) | |
| attributes = {} | |
| for ext in item.get("extensions") or []: | |
| if isinstance(ext, dict): | |
| for k, v in ext.items(): | |
| if isinstance(v, list): | |
| attributes[str(k)] = [clean(x) for x in v] | |
| return { | |
| "name": clean(item.get("title")), | |
| "category": category, | |
| "categories": types, | |
| "rating": rating, | |
| "reviews": int(reviews) if reviews is not None else None, | |
| "phone": clean(item.get("phone")), | |
| "website": clean(item.get("website")), | |
| "address": clean(item.get("address")), | |
| "latitude": _num(coords.get("latitude")), | |
| "longitude": _num(coords.get("longitude")), | |
| "hours": format_hours(item.get("operating_hours") or (item.get("hours") if not isinstance(item.get("hours"), str) else None)), | |
| "open_state": clean(item.get("open_state")), | |
| "plus_code": clean(item.get("plus_code")), | |
| "rating_breakdown": {str(r.get("stars")): int(r.get("amount") or 0) for r in item.get("rating_summary") or [] if isinstance(r, dict)}, | |
| "attributes": attributes, | |
| "place_id": place_id, | |
| "data_id": clean(item.get("data_id")), | |
| "cid": cid, | |
| "maps_url": f"https://maps.google.com/?cid={cid}" if cid else (f"https://www.google.com/maps/place/?q=place_id:{place_id}" if place_id else ""), | |
| "thumbnail": clean(item.get("thumbnail")), | |
| } | |
| def normalize_review(item: Dict[str, Any]) -> Dict[str, Any]: | |
| user = item.get("user") or {} | |
| resp = item.get("response") or {} | |
| return { | |
| "rating": item.get("rating"), | |
| "date": clean(item.get("date")), | |
| "iso_date": clean(item.get("iso_date")), | |
| "text": clean(item.get("snippet") or (item.get("extracted_snippet") or {}).get("original")), | |
| "likes": item.get("likes") or 0, | |
| "author": clean(user.get("name")), | |
| "local_guide": bool(user.get("local_guide")), | |
| "owner_reply": clean(resp.get("snippet") or (resp.get("extracted_snippet") or {}).get("original")), | |
| "review_id": clean(item.get("review_id")), | |
| } | |
| def parse_place_ref(value: str) -> Dict[str, str]: | |
| value = (value or "").strip() | |
| if re.fullmatch(r"ChI[A-Za-z0-9_-]{10,}", value): | |
| return {"place_id": value} | |
| if re.fullmatch(r"\d{6,25}", value): | |
| return {"data_cid": value} | |
| m = re.fullmatch(r"(0x[0-9a-f]+):0x([0-9a-f]+)", value, re.I) | |
| if m: | |
| return {"data_id": value.lower(), "data_cid": str(int(m.group(2), 16))} | |
| m = re.search(r"[?&]cid=(\d+)", value) | |
| if m: | |
| return {"data_cid": m.group(1)} | |
| m = re.search(r"place_id[:=](ChI[A-Za-z0-9_-]+)", value) | |
| if m: | |
| return {"place_id": m.group(1)} | |
| m = re.search(r"!1s(0x[0-9a-f]+):0x([0-9a-f]+)", value, re.I) | |
| if m: | |
| return {"data_id": f"{m.group(1)}:0x{m.group(2)}".lower(), "data_cid": str(int(m.group(2), 16))} | |
| return {} | |
| # ------------------------------------------------------------------ tool | |
| class ScrapeDoGoogleMapsTool(Tool): | |
| name = "google_maps_scraper" | |
| description = """ | |
| Get Google Maps business data through the Scrape.do Google Maps API. Three modes: | |
| - 'search': a query like 'dentist' or 'coffee shop in Austin, TX', optionally centred on `location` | |
| ('lat,lng'). Returns up to 20 places per page (page 1-6): name, category, rating, review count, | |
| phone, website, address, coordinates, hours, place_id and data_id. | |
| - 'place': a place_id (ChIJ...), CID, feature id (0x..:0x..) or Google Maps place URL. Returns full | |
| details: hours, plus code, rating breakdown, attributes. | |
| - 'reviews': a place_id or data_id. Returns about 10 reviews per page with owner replies. | |
| Each call is 10 Scrape.do credits. | |
| """ | |
| output_type = "string" | |
| def __init__(self, api_token: Optional[str] = None) -> None: | |
| self._api_token = api_token | |
| self.inputs = { | |
| "mode": {"type": "string", "description": "'search', 'place' or 'reviews'."}, | |
| "query": {"type": "string", "description": "Search terms for 'search'; a place_id, CID, data_id or Maps URL for 'place' and 'reviews'."}, | |
| "location": {"type": "string", "description": "Optional 'lat,lng' centre for 'search', e.g. '30.2672,-97.7431'.", "nullable": True}, | |
| "page": {"type": "integer", "description": "Result page (1-6) for 'search', review page for 'reviews'. Default 1.", "nullable": True}, | |
| "sort_by": {"type": "string", "description": "For 'reviews': newest, most_relevant, highest_rating or lowest_rating.", "nullable": True}, | |
| } | |
| super().__init__() | |
| def _token(self) -> Optional[str]: | |
| token = (self._api_token or os.environ.get("SCRAPE_DO_API_TOKEN") or "").strip() | |
| return token or None | |
| def _missing_token() -> Dict[str, Any]: | |
| return {"status": "error", "errorMessage": "A Scrape.do API token is required.", "hint": f"Get a free one at {SCRAPE_DO_SIGNUP_URL}"} | |
| def _call(self, token: str, endpoint: str, params: Dict[str, Any]) -> Dict[str, Any]: | |
| query = {k: v for k, v in params.items() if v not in (None, "")} | |
| query["token"] = token | |
| last: Dict[str, Any] = {"_error": "No response."} | |
| for attempt in range(3): | |
| started = time.time() | |
| try: | |
| r = requests.get(f"{API_BASE}/{endpoint}", params=query, timeout=90) | |
| except requests.RequestException as exc: | |
| last = {"_error": f"Request failed: {exc.__class__.__name__}"} | |
| continue | |
| print(f"[{endpoint}] HTTP {r.status_code} in {time.time() - started:.1f}s", flush=True) | |
| if r.status_code == 401: | |
| return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True} | |
| if r.status_code == 200: | |
| try: | |
| return {"json": r.json()} | |
| except ValueError: | |
| last = {"_error": "The API returned a body that is not JSON."} | |
| continue | |
| last = {"_error": f"The Google Maps API returned HTTP {r.status_code}."} | |
| if r.status_code not in (429, 500, 502, 503, 504): | |
| break | |
| time.sleep(1 + attempt) | |
| return last | |
| def _as_error(res: Dict[str, Any]) -> Dict[str, Any]: | |
| out = {"status": "error", "errorMessage": res.get("_error", "Unknown error")} | |
| if res.get("_auth"): | |
| out["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" | |
| return out | |
| def search(self, query: str, location: Optional[str] = None, page: int = 1, gl: str = "us", hl: str = "en") -> Dict[str, Any]: | |
| token = self._token() | |
| if not token: | |
| return self._missing_token() | |
| if not (query or "").strip(): | |
| return {"status": "error", "errorMessage": "Type what to search for, e.g. 'dentist'."} | |
| ll = None | |
| if location and location.strip(): | |
| ll = format_ll(location) | |
| if not ll: | |
| return {"status": "error", "errorMessage": "Location must be 'lat,lng', e.g. 30.2672,-97.7431."} | |
| page = max(1, min(MAX_PAGES, int(page or 1))) | |
| res = self._call(token, "search", {"q": query.strip(), "ll": ll, "gl": gl, "hl": hl, "start": (page - 1) * PAGE_SIZE or None}) | |
| if "_error" in res: | |
| return self._as_error(res) | |
| results = res["json"].get("local_results") or [] | |
| if isinstance(results, dict): | |
| results = [results] | |
| places = [normalize_place(x) for x in results if isinstance(x, dict)] | |
| return {"status": "success", "query": query.strip(), "location": ll, "page": page, "results": places, | |
| "maps_url": (res["json"].get("search_metadata") or {}).get("google_maps_url", ""), | |
| "credits_used": CREDITS_PER_REQUEST} | |
| def place(self, ref: str, hl: str = "en") -> Dict[str, Any]: | |
| token = self._token() | |
| if not token: | |
| return self._missing_token() | |
| ids = parse_place_ref(ref) | |
| if not ids.get("place_id") and not ids.get("data_cid"): | |
| return {"status": "error", "errorMessage": "Paste a place ID (ChIJ...), a CID, a feature ID (0x...:0x...) or a Google Maps place URL."} | |
| params = {"place_id": ids.get("place_id"), "data_cid": None if ids.get("place_id") else ids.get("data_cid"), "hl": hl} | |
| res = self._call(token, "place", params) | |
| if "_error" in res: | |
| return self._as_error(res) | |
| item = res["json"].get("place_results") | |
| if not isinstance(item, dict): | |
| return {"status": "error", "errorMessage": "Google Maps has no place for that ID."} | |
| return {"status": "success", "place": normalize_place(item), "credits_used": CREDITS_PER_REQUEST} | |
| def reviews(self, ref: str, sort_by: str = "newest", pages: int = 1, hl: str = "en") -> Dict[str, Any]: | |
| token = self._token() | |
| if not token: | |
| return self._missing_token() | |
| ids = parse_place_ref(ref) | |
| if not ids.get("place_id") and not ids.get("data_id"): | |
| return {"status": "error", "errorMessage": "Paste a place ID (ChIJ...) or a feature ID (0x...:0x...)."} | |
| sort_by = sort_by if sort_by in SORT_OPTIONS else "newest" | |
| out: List[Dict[str, Any]] = [] | |
| topics: List[Dict[str, Any]] = [] | |
| info: Dict[str, Any] = {} | |
| token_next = "" | |
| calls = 0 | |
| for _ in range(max(1, min(5, int(pages or 1)))): | |
| res = self._call(token, "reviews", { | |
| "data_id": ids.get("data_id"), "place_id": None if ids.get("data_id") else ids.get("place_id"), | |
| "sort_by": None if sort_by == "most_relevant" else sort_by, "next_page_token": token_next or None, "hl": hl, | |
| }) | |
| if "_error" in res: | |
| if out: | |
| break | |
| return self._as_error(res) | |
| calls += 1 | |
| data = res["json"] | |
| info = info or data.get("place_info") or {} | |
| topics = topics or [{"keyword": t.get("keyword"), "mentions": t.get("mentions")} for t in data.get("topics") or []] | |
| out.extend(normalize_review(r) for r in data.get("reviews") or [] if isinstance(r, dict)) | |
| token_next = (data.get("pagination") or {}).get("next_page_token") or "" | |
| if not token_next: | |
| break | |
| return {"status": "success", "place": {k: info.get(k) for k in ("title", "address", "rating", "reviews")}, | |
| "topics": topics, "reviews": out, "credits_used": calls * CREDITS_PER_REQUEST} | |
| def forward(self, mode: str, query: str, location: Optional[str] = None, page: Optional[int] = None, sort_by: Optional[str] = None) -> str: | |
| key = (mode or "").strip().lower() | |
| if key == "search": | |
| result = self.search(query, location, page or 1) | |
| elif key == "place": | |
| result = self.place(query) | |
| elif key == "reviews": | |
| result = self.reviews(query, sort_by or "newest", page or 1) | |
| else: | |
| result = {"status": "error", "errorMessage": "mode must be 'search', 'place' or 'reviews'."} | |
| return json.dumps(result, ensure_ascii=False, indent=2) | |