Spaces:
Sleeping
Sleeping
Download tool.py from bugrahans/google-maps-scraper: direct link, hf CLI and curl.
- Browser
- Download file 24.2 kB
-
https://huggingface.co/spaces/bugrahans/google-maps-scraper/resolve/main/tool.py
- Command line
-
hf download hf://spaces/bugrahans/google-maps-scraper/tool.py
-
curl -L -o tool.py https://huggingface.co/spaces/bugrahans/google-maps-scraper/resolve/main/tool.py
24.2 kB
| from __future__ import annotations | |
| import json | |
| import os | |
| import re | |
| import time | |
| import urllib.parse | |
| from typing import Any, Dict, List, Optional, Tuple | |
| import requests | |
| from smolagents.tools import Tool | |
| SCRAPE_DO_SIGNUP_URL = ( | |
| "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=google-maps-scraper" | |
| ) | |
| MAPS_BASE = "https://api.scrape.do/plugin/google/maps" | |
| SEARCH_ENDPOINT = MAPS_BASE + "/search" | |
| PLACE_ENDPOINT = MAPS_BASE + "/place" | |
| REVIEWS_ENDPOINT = MAPS_BASE + "/reviews" | |
| # From the Scrape.do docs: every Maps call costs 10 credits. | |
| CREDITS_PER_CALL = 10 | |
| PLACES_PER_PAGE = 20 | |
| MAX_SEARCH_PAGES = 6 # Google Maps stops at ~120 places. | |
| REVIEWS_PER_CALL = 20 | |
| MAX_REVIEWS = 100 | |
| # (label, gl, hl, google_domain) | |
| COUNTRIES: List[tuple] = [ | |
| ("United States", "us", "en", "google.com"), | |
| ("United Kingdom", "gb", "en", "google.co.uk"), | |
| ("Germany", "de", "de", "google.de"), | |
| ("France", "fr", "fr", "google.fr"), | |
| ("Spain", "es", "es", "google.es"), | |
| ("Italy", "it", "it", "google.it"), | |
| ("Netherlands", "nl", "nl", "google.nl"), | |
| ("Turkey", "tr", "tr", "google.com.tr"), | |
| ("Canada", "ca", "en", "google.ca"), | |
| ("Australia", "au", "en", "google.com.au"), | |
| ("India", "in", "en", "google.co.in"), | |
| ("Japan", "jp", "ja", "google.co.jp"), | |
| ("Brazil", "br", "pt", "google.com.br"), | |
| ("Mexico", "mx", "es", "google.com.mx"), | |
| ] | |
| COUNTRY_BY_GL = {gl: (label, gl, hl, domain) for label, gl, hl, domain in COUNTRIES} | |
| SORT_OPTIONS = { | |
| "relevance": None, | |
| "newest": "newestFirst", | |
| "highest": "ratingHigh", | |
| "lowest": "ratingLow", | |
| } | |
| SHORT_LINK_HOSTS = ("maps.app.goo.gl", "goo.gl", "g.co") | |
| # ------------------------------------------------------------------ helpers | |
| def _locale(country: Optional[str]) -> Tuple[str, str, str]: | |
| gl = (country or "us").strip().lower() | |
| _, gl, hl, domain = COUNTRY_BY_GL.get(gl, ("", gl, "en", "google.com")) | |
| return gl, hl, domain | |
| def normalize_map_center(value: Optional[str]) -> Tuple[Optional[str], Optional[str]]: | |
| """Turn '41.0082,28.9784' or '@41.0082,28.9784,14z' into Google's `ll` format.""" | |
| if not value or not str(value).strip(): | |
| return None, None | |
| parts = str(value).strip().lstrip("@").replace(" ", "").split(",") | |
| try: | |
| lat, lng = float(parts[0]), float(parts[1]) | |
| except (IndexError, ValueError): | |
| return None, "Map center ignored: use the format 41.0082,28.9784" | |
| if not (-90 <= lat <= 90 and -180 <= lng <= 180): | |
| return None, "Map center ignored: those coordinates are out of range." | |
| zoom = 14 | |
| if len(parts) > 2: | |
| match = re.match(r"(\d+(?:\.\d+)?)", parts[2]) | |
| if match: | |
| zoom = int(float(match.group(1))) | |
| zoom = max(1, min(zoom, 21)) | |
| return f"@{lat},{lng},{zoom}z", None | |
| def parse_place_ids(text: str) -> Dict[str, str]: | |
| """Pull whatever place identifiers a Google Maps link or pasted ID contains.""" | |
| decoded = urllib.parse.unquote(text or "").strip() | |
| ids: Dict[str, str] = {} | |
| # data_id, e.g. 0x89c259606c152b2d:0xe230af1b18e542a7 (inside `!1s...` in place links). | |
| # Its second half, read as a number, is the place's CID. | |
| match = re.search(r"(0x[0-9a-fA-F]{4,}):(0x[0-9a-fA-F]{4,})", decoded) | |
| if match: | |
| ids["data_id"] = f"{match.group(1)}:{match.group(2)}".lower() | |
| ids["data_cid"] = str(int(match.group(2), 16)) | |
| match = re.search(r"[?&]cid=(\d{5,})", decoded) | |
| if match and "data_cid" not in ids: | |
| ids["data_cid"] = match.group(1) | |
| match = re.search(r"(?:place_id[:=]|query_place_id=|placeid=)([A-Za-z0-9_-]{20,})", decoded) | |
| if match: | |
| ids["place_id"] = match.group(1) | |
| else: | |
| match = re.search(r"\b(ChIJ[A-Za-z0-9_-]{15,})", decoded) | |
| if match: | |
| ids["place_id"] = match.group(1) | |
| if not ids and re.fullmatch(r"\d{10,20}", decoded): | |
| ids["data_cid"] = decoded | |
| return ids | |
| def _expand_short_link(url: str) -> Optional[str]: | |
| """Follow a maps.app.goo.gl link's redirects without downloading the final page.""" | |
| current = url | |
| for _ in range(6): | |
| try: | |
| response = requests.get( | |
| current, | |
| allow_redirects=False, | |
| timeout=15, | |
| headers={"User-Agent": "Mozilla/5.0 (compatible; maps-link-resolver)"}, | |
| ) | |
| except requests.exceptions.RequestException: | |
| return None | |
| location = response.headers.get("Location") | |
| if not location: | |
| return current if current != url else None | |
| location = urllib.parse.urljoin(current, location) | |
| # EU visitors get bounced through a consent page that carries the real link. | |
| if "consent.google" in location: | |
| params = urllib.parse.parse_qs(urllib.parse.urlparse(location).query) | |
| if params.get("continue"): | |
| return params["continue"][0] | |
| current = location | |
| if parse_place_ids(current): | |
| return current | |
| return current | |
| def _is_url(text: str) -> bool: | |
| return text.lower().startswith(("http://", "https://")) | |
| # --------------------------------------------------------------------- tool | |
| class ScrapeDoGoogleMapsTool(Tool): | |
| name = "google_maps_scraper" | |
| description = """ | |
| Scrape Google Maps. Three modes: | |
| - 'search': find businesses for a query such as 'dentists in Kadikoy Istanbul' | |
| and get up to 120 places with name, category, rating, review count, phone, | |
| website, address, opening hours and GPS coordinates. Good for lead lists. | |
| - 'place': full details for one business: hours, rating breakdown, popular | |
| times, service options, photos, menu link and review highlights. | |
| - 'reviews': a business's user reviews with rating, date, text, reviewer | |
| and owner replies, sortable by newest or rating, filterable by keyword. | |
| For 'place' and 'reviews', pass a Google Maps link, a place ID (ChIJ...), | |
| a data ID (0x...:0x...), a CID, or just the business name. | |
| Powered by Scrape.do's Google Maps API, which handles proxies and CAPTCHAs. | |
| """ | |
| output_type = "string" | |
| def __init__(self, api_token: Optional[str] = None) -> None: | |
| self._api_token = api_token | |
| self.inputs = { | |
| "mode": { | |
| "type": "string", | |
| "description": "'search' to find businesses, 'place' for one business's details, 'reviews' for its user reviews.", | |
| }, | |
| "query": { | |
| "type": "string", | |
| "description": ( | |
| "For 'search': what to look for, ideally with a place, e.g. 'coffee shops in Austin TX'. " | |
| "For 'place' and 'reviews': a Google Maps link, place ID, data ID, CID or business name." | |
| ), | |
| }, | |
| "country": { | |
| "type": "string", | |
| "description": "Two-letter country code such as 'us', 'gb', 'de', 'tr'. Sets language and Google domain. Defaults to 'us'.", | |
| "nullable": True, | |
| }, | |
| "pages": { | |
| "type": "integer", | |
| "description": "'search' only: result pages to fetch (1-6, 20 places each, 10 credits each). Defaults to 1.", | |
| "nullable": True, | |
| }, | |
| "max_reviews": { | |
| "type": "integer", | |
| "description": "'reviews' only: how many reviews to collect (1-100, 20 per 10-credit call). Defaults to 20.", | |
| "nullable": True, | |
| }, | |
| "sort_by": { | |
| "type": "string", | |
| "description": "'reviews' only: 'relevance', 'newest', 'highest' or 'lowest'. Defaults to relevance.", | |
| "nullable": True, | |
| }, | |
| "keyword": { | |
| "type": "string", | |
| "description": "'reviews' only: only return reviews that mention this word, e.g. 'parking'.", | |
| "nullable": True, | |
| }, | |
| "location": { | |
| "type": "string", | |
| "description": "'search' only: map center as 'latitude,longitude' (optionally ',zoom'), e.g. '41.0082,28.9784'.", | |
| "nullable": True, | |
| }, | |
| } | |
| super().__init__() | |
| # ------------------------------------------------------------ plumbing | |
| def _resolve_token(self) -> Optional[str]: | |
| if self._api_token: | |
| return self._api_token.strip() | |
| env_token = os.getenv("SCRAPE_DO_API_TOKEN") | |
| return env_token.strip() if env_token else None | |
| def _missing_token() -> Dict[str, Any]: | |
| return { | |
| "status": "error", | |
| "errorMessage": "Missing Scrape.do API token.", | |
| "hint": f"Get a free token at {SCRAPE_DO_SIGNUP_URL}", | |
| } | |
| def _as_error(data: Dict[str, Any]) -> Dict[str, Any]: | |
| result = {"status": "error", "errorMessage": data.get("_error", "Unknown error")} | |
| if data.get("_auth"): | |
| result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" | |
| return result | |
| def _get(self, url: str, params: Dict[str, Any], retries: int = 1) -> Dict[str, Any]: | |
| """GET a Scrape.do endpoint. Retries once on the transient 5xx errors the docs mention.""" | |
| token = str(params.get("token") or "") | |
| def redact(text: str) -> str: | |
| return text.replace(token, "***") if token else text | |
| full_url = url + "?" + urllib.parse.urlencode(params, quote_via=urllib.parse.quote_plus) | |
| last: Dict[str, Any] = {"_error": "Request failed."} | |
| for attempt in range(retries + 1): | |
| try: | |
| response = requests.get(full_url, timeout=120) | |
| except requests.exceptions.RequestException as exc: | |
| last = {"_error": redact(f"Request failed: {exc}")} | |
| if attempt < retries: | |
| time.sleep(1.5) | |
| continue | |
| return last | |
| if response.status_code == 401: | |
| return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True} | |
| try: | |
| data = response.json() | |
| except ValueError: | |
| data = None | |
| if response.ok and isinstance(data, dict): | |
| return data | |
| message = "" | |
| if isinstance(data, dict): | |
| message = str(data.get("message") or data.get("error") or "") | |
| if not message: | |
| message = (response.text or "").strip()[:200] or "no details" | |
| last = { | |
| "_error": redact(f"Scrape.do returned HTTP {response.status_code}: {message}"), | |
| "_status": response.status_code, | |
| } | |
| if response.status_code in (500, 502, 503, 504) and attempt < retries: | |
| time.sleep(1.5) | |
| continue | |
| return last | |
| return last | |
| def _find_by_name( | |
| self, token: str, name: str, locale: Tuple[str, str, str], ll: Optional[str] | |
| ) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: | |
| gl, hl, domain = locale | |
| params: Dict[str, Any] = {"token": token, "q": name, "gl": gl, "hl": hl, "google_domain": domain} | |
| if ll: | |
| params["ll"] = ll | |
| data = self._get(SEARCH_ENDPOINT, params) | |
| if "_error" in data: | |
| return None, data["_error"] | |
| if isinstance(data.get("place_results"), dict): | |
| return data["place_results"], None | |
| results = data.get("local_results") or [] | |
| if results: | |
| return results[0], None | |
| return None, f"Google Maps found no place for “{name}”." | |
| def _resolve_place( | |
| self, token: str, ref: str, locale: Tuple[str, str, str] | |
| ) -> Dict[str, Any]: | |
| """Work out place IDs from a link, an ID, or a business name. | |
| Returns {"ids": {...}, "credits": int, "note": str|None} or {"error": str}. | |
| """ | |
| text = (ref or "").strip() | |
| if not text: | |
| return {"error": "Paste a Google Maps link, a place ID, or a business name."} | |
| host = urllib.parse.urlparse(text).netloc.lower() if _is_url(text) else "" | |
| if host in SHORT_LINK_HOSTS: | |
| expanded = _expand_short_link(text) | |
| if not expanded: | |
| return { | |
| "error": "Couldn't open that short link. Open it in your browser, then copy " | |
| "the full link from the address bar and paste that instead." | |
| } | |
| text = expanded | |
| ids = parse_place_ids(text) | |
| if ids: | |
| return {"ids": ids, "credits": 0, "note": None} | |
| ll = None | |
| if _is_url(text): | |
| decoded = urllib.parse.unquote(text) | |
| if "/maps/search/" in decoded or "/maps/dir/" in decoded: | |
| return { | |
| "error": "That's a search or directions link, not a single place. Click one " | |
| "business in Google Maps and copy that link, or use the Find businesses tab." | |
| } | |
| match = re.search(r"/maps/place/([^/@?]+)", decoded) | |
| if not match: | |
| return {"error": "Couldn't find a place in that link. Try the business name instead."} | |
| name = urllib.parse.unquote_plus(match.group(1)).strip() | |
| coords = re.search(r"@(-?\d+\.\d+),(-?\d+\.\d+),(\d+(?:\.\d+)?)z", decoded) | |
| if coords: | |
| ll = f"@{coords.group(1)},{coords.group(2)},{int(float(coords.group(3)))}z" | |
| else: | |
| name = text | |
| place, error = self._find_by_name(token, name, locale, ll) | |
| if error: | |
| return {"error": error} | |
| ids = {key: str(place[key]) for key in ("place_id", "data_id", "data_cid") if place.get(key)} | |
| if not ids: | |
| return {"error": f"Found “{place.get('title')}” but Google gave no ID for it."} | |
| note = ( | |
| f"Looked up “{name}” and used the first match: {place.get('title')}" | |
| + (f", {place.get('address')}" if place.get("address") else "") | |
| + f" ({CREDITS_PER_CALL} credits)." | |
| ) | |
| return {"ids": ids, "credits": CREDITS_PER_CALL, "note": note} | |
| # --------------------------------------------------------------- search | |
| def search_places( | |
| self, | |
| query: str, | |
| country: str = "us", | |
| pages: int = 1, | |
| location: Optional[str] = None, | |
| ) -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| if not query or not query.strip(): | |
| return {"status": "error", "errorMessage": "Type what you're looking for, e.g. 'dentists in Kadikoy Istanbul'."} | |
| gl, hl, domain = _locale(country) | |
| pages = max(1, min(int(pages or 1), MAX_SEARCH_PAGES)) | |
| base: Dict[str, Any] = {"token": token, "q": query.strip(), "gl": gl, "hl": hl, "google_domain": domain} | |
| warnings: List[str] = [] | |
| ll, ll_warning = normalize_map_center(location) | |
| if ll: | |
| base["ll"] = ll | |
| if ll_warning: | |
| warnings.append(ll_warning) | |
| places: List[Dict[str, Any]] = [] | |
| seen: set = set() | |
| credits = 0 | |
| pages_fetched = 0 | |
| maps_url = None | |
| for page_index in range(pages): | |
| data = self._get(SEARCH_ENDPOINT, dict(base, start=page_index * PLACES_PER_PAGE)) | |
| if "_error" in data: | |
| if page_index == 0: | |
| return self._as_error(data) | |
| # Deep pages sometimes answer 502 when Google has nothing more. | |
| warnings.append(f"Stopped after page {page_index}: {data['_error']}") | |
| break | |
| credits += CREDITS_PER_CALL | |
| pages_fetched += 1 | |
| if page_index == 0: | |
| maps_url = (data.get("search_metadata") or {}).get("google_maps_url") | |
| batch = data.get("local_results") | |
| if not batch and isinstance(data.get("place_results"), dict): | |
| batch = [data["place_results"]] # The query matched one exact place. | |
| batch = batch if isinstance(batch, list) else [] | |
| for item in batch: | |
| key = item.get("place_id") or item.get("data_id") or (item.get("title"), item.get("address")) | |
| if key in seen: | |
| continue | |
| seen.add(key) | |
| item = dict(item) | |
| item["page"] = page_index + 1 | |
| item["position"] = len(places) + 1 | |
| places.append(item) | |
| if len(batch) < PLACES_PER_PAGE: | |
| break | |
| result: Dict[str, Any] = { | |
| "status": "success", | |
| "query": query.strip(), | |
| "country": gl, | |
| "google_maps_url": maps_url, | |
| "places": places, | |
| "pages_fetched": pages_fetched, | |
| "credits_used": credits, | |
| } | |
| if warnings: | |
| result["warnings"] = warnings | |
| return result | |
| # ---------------------------------------------------------------- place | |
| def place_details(self, place: str, country: str = "us") -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| locale = _locale(country) | |
| resolved = self._resolve_place(token, place, locale) | |
| if "error" in resolved: | |
| return {"status": "error", "errorMessage": resolved["error"]} | |
| ids, credits = resolved["ids"], resolved["credits"] | |
| gl, hl, domain = locale | |
| base = {"token": token, "gl": gl, "hl": hl, "google_domain": domain} | |
| attempts = [] | |
| if ids.get("place_id"): | |
| attempts.append({"place_id": ids["place_id"]}) | |
| if ids.get("data_cid"): | |
| attempts.append({"data_cid": ids["data_cid"]}) | |
| if not attempts: | |
| return {"status": "error", "errorMessage": "No usable place ID found."} | |
| data: Dict[str, Any] = {} | |
| for identifier in attempts: | |
| data = self._get(PLACE_ENDPOINT, dict(base, **identifier)) | |
| if "_error" not in data: | |
| credits += CREDITS_PER_CALL | |
| break | |
| if "_error" in data: | |
| return self._as_error(data) | |
| details = data.get("place_results") | |
| if not isinstance(details, dict) or not details: | |
| return {"status": "error", "errorMessage": "Google Maps returned no details for this place."} | |
| result: Dict[str, Any] = { | |
| "status": "success", | |
| "place": details, | |
| "google_maps_url": (data.get("search_metadata") or {}).get("google_maps_url"), | |
| "credits_used": credits, | |
| } | |
| if resolved.get("note"): | |
| result["note"] = resolved["note"] | |
| return result | |
| # -------------------------------------------------------------- reviews | |
| def reviews( | |
| self, | |
| place: str, | |
| country: str = "us", | |
| max_reviews: int = 20, | |
| sort_by: Optional[str] = None, | |
| keyword: Optional[str] = None, | |
| ) -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| locale = _locale(country) | |
| resolved = self._resolve_place(token, place, locale) | |
| if "error" in resolved: | |
| return {"status": "error", "errorMessage": resolved["error"]} | |
| ids, credits = resolved["ids"], resolved["credits"] | |
| gl, hl, domain = locale | |
| warnings: List[str] = [] | |
| fallback_info: Optional[Dict[str, Any]] = None | |
| # The reviews endpoint needs a data ID or place ID. A bare CID has | |
| # neither, so look the place up first. | |
| if not ids.get("data_id") and not ids.get("place_id"): | |
| data = self._get( | |
| PLACE_ENDPOINT, | |
| {"token": token, "gl": gl, "hl": hl, "google_domain": domain, "data_cid": ids.get("data_cid", "")}, | |
| ) | |
| if "_error" in data: | |
| return self._as_error(data) | |
| credits += CREDITS_PER_CALL | |
| fallback_info = data.get("place_results") or {} | |
| for key in ("place_id", "data_id"): | |
| if fallback_info.get(key): | |
| ids[key] = str(fallback_info[key]) | |
| if not ids.get("data_id") and not ids.get("place_id"): | |
| return {"status": "error", "errorMessage": "Couldn't find this place's review ID."} | |
| max_reviews = max(1, min(int(max_reviews or 20), MAX_REVIEWS)) | |
| base: Dict[str, Any] = {"token": token, "gl": gl, "hl": hl, "google_domain": domain} | |
| if ids.get("data_id"): | |
| base["data_id"] = ids["data_id"] | |
| else: | |
| base["place_id"] = ids["place_id"] | |
| sort_key = (sort_by or "").strip() | |
| sort_value = SORT_OPTIONS.get(sort_key.lower(), sort_key if sort_key in SORT_OPTIONS.values() else None) | |
| if sort_value: | |
| base["sort_by"] = sort_value | |
| if keyword and keyword.strip(): | |
| base["query"] = keyword.strip() | |
| collected: List[Dict[str, Any]] = [] | |
| place_info: Optional[Dict[str, Any]] = None | |
| topics: List[Dict[str, Any]] = [] | |
| next_token: Optional[str] = None | |
| pages_fetched = 0 | |
| while len(collected) < max_reviews and pages_fetched < 12: | |
| params = dict(base, num=min(REVIEWS_PER_CALL, max_reviews - len(collected))) | |
| if next_token: | |
| params["next_page_token"] = next_token | |
| data = self._get(REVIEWS_ENDPOINT, params) | |
| if "_error" in data: | |
| if pages_fetched == 0: | |
| return self._as_error(data) | |
| warnings.append(f"Stopped after {len(collected)} reviews: {data['_error']}") | |
| break | |
| credits += CREDITS_PER_CALL | |
| pages_fetched += 1 | |
| if pages_fetched == 1: | |
| place_info = data.get("place_info") | |
| topics = data.get("topics") or [] | |
| batch = data.get("reviews") or [] | |
| for review in batch: | |
| if len(collected) >= max_reviews: | |
| break | |
| review = dict(review) | |
| review["position"] = len(collected) + 1 | |
| collected.append(review) | |
| next_token = (data.get("pagination") or {}).get("next_page_token") | |
| if not next_token or not batch: | |
| break | |
| if not place_info and fallback_info: | |
| place_info = { | |
| key: fallback_info.get(key) for key in ("title", "address", "rating", "reviews", "type") | |
| } | |
| result: Dict[str, Any] = { | |
| "status": "success", | |
| "place_info": place_info or {}, | |
| "topics": topics, | |
| "reviews": collected, | |
| "reviews_fetched": len(collected), | |
| "place_ids": ids, | |
| "pages_fetched": pages_fetched, | |
| "credits_used": credits, | |
| } | |
| if resolved.get("note"): | |
| result["note"] = resolved["note"] | |
| if warnings: | |
| result["warnings"] = warnings | |
| return result | |
| # -------------------------------------------------------------- forward | |
| def forward( | |
| self, | |
| mode: str, | |
| query: str, | |
| country: Optional[str] = None, | |
| pages: Optional[int] = None, | |
| max_reviews: Optional[int] = None, | |
| sort_by: Optional[str] = None, | |
| keyword: Optional[str] = None, | |
| location: Optional[str] = None, | |
| ) -> str: | |
| mode_key = (mode or "").strip().lower() | |
| if mode_key in ("search", "find", "businesses"): | |
| result = self.search_places(query, country or "us", pages or 1, location) | |
| elif mode_key in ("place", "details", "place_details"): | |
| result = self.place_details(query, country or "us") | |
| elif mode_key in ("reviews", "review"): | |
| result = self.reviews(query, country or "us", max_reviews or 20, sort_by, keyword) | |
| else: | |
| result = {"status": "error", "errorMessage": "mode must be 'search', 'place' or 'reviews'."} | |
| return json.dumps(result, ensure_ascii=False, indent=2) | |