bugrahans's picture
Google Maps Leads Scraper Space
b5bc75e verified
Raw History Blame Contribute Delete
13.8 kB
from __future__ import annotations
import json
import os
import re
import time
from typing import Any, Dict, List, Optional
import requests
from smolagents.tools import Tool
SCRAPE_DO_SIGNUP_URL = "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=google-maps-leads-scraper"
API_BASE = "https://api.scrape.do/plugin/google/maps"
# The Scrape.do Google Maps API returns parsed JSON; every successful call is 10 credits.
CREDITS_PER_REQUEST = 10
PAGE_SIZE = 20
MAX_PAGES = 6
SORT_OPTIONS = ("newest", "most_relevant", "highest_rating", "lowest_rating")
DAYS = ("monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday")
# ------------------------------------------------------------------ helpers
# (same mapping as `pip install google-maps-leads-scraper`)
_SPACES = re.compile(r"[   ]")
_LL_RE = re.compile(r"^@?\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)(?:\s*,\s*(\d+(?:\.\d+)?)z?)?\s*$")
def clean(value: Any) -> str:
return "" if value is None else _SPACES.sub(" ", str(value)).strip()
def format_ll(value: str, zoom: int = 14) -> Optional[str]:
match = _LL_RE.match(value or "")
if not match:
return None
lat, lng, z = match.groups()
if abs(float(lat)) > 90 or abs(float(lng)) > 180:
return None
return f"@{lat},{lng},{z or zoom}z"
def format_hours(hours: Any) -> str:
merged: Dict[str, str] = {}
if isinstance(hours, dict):
merged = {str(k).lower(): clean(v) for k, v in hours.items()}
elif isinstance(hours, list):
for item in hours:
if isinstance(item, dict):
merged.update({str(k).lower(): clean(v) for k, v in item.items()})
return "; ".join(f"{d[:3].title()}: {merged[d]}" for d in DAYS if d in merged)
def _num(value: Any) -> Optional[float]:
try:
return float(str(value).replace(",", "")) if value not in (None, "") else None
except ValueError:
return None
def normalize_place(item: Dict[str, Any]) -> Dict[str, Any]:
coords = item.get("gps_coordinates") or {}
raw_type = item.get("type")
types = [clean(t) for t in (item.get("types") or (raw_type if isinstance(raw_type, list) else [])) if t]
category = (clean(raw_type) if isinstance(raw_type, str) else "") or (types[0] if types else "")
if category and category not in types:
types.insert(0, category)
place_id = clean(item.get("place_id"))
cid = clean(item.get("data_cid"))
rating = _num(item.get("rating"))
reviews = _num(item.get("reviews"))
attributes = {}
for ext in item.get("extensions") or []:
if isinstance(ext, dict):
for k, v in ext.items():
if isinstance(v, list):
attributes[str(k)] = [clean(x) for x in v]
return {
"name": clean(item.get("title")),
"category": category,
"categories": types,
"rating": rating,
"reviews": int(reviews) if reviews is not None else None,
"phone": clean(item.get("phone")),
"website": clean(item.get("website")),
"address": clean(item.get("address")),
"latitude": _num(coords.get("latitude")),
"longitude": _num(coords.get("longitude")),
"hours": format_hours(item.get("operating_hours") or (item.get("hours") if not isinstance(item.get("hours"), str) else None)),
"open_state": clean(item.get("open_state")),
"plus_code": clean(item.get("plus_code")),
"rating_breakdown": {str(r.get("stars")): int(r.get("amount") or 0) for r in item.get("rating_summary") or [] if isinstance(r, dict)},
"attributes": attributes,
"place_id": place_id,
"data_id": clean(item.get("data_id")),
"cid": cid,
"maps_url": f"https://maps.google.com/?cid={cid}" if cid else (f"https://www.google.com/maps/place/?q=place_id:{place_id}" if place_id else ""),
"thumbnail": clean(item.get("thumbnail")),
}
def normalize_review(item: Dict[str, Any]) -> Dict[str, Any]:
user = item.get("user") or {}
resp = item.get("response") or {}
return {
"rating": item.get("rating"),
"date": clean(item.get("date")),
"iso_date": clean(item.get("iso_date")),
"text": clean(item.get("snippet") or (item.get("extracted_snippet") or {}).get("original")),
"likes": item.get("likes") or 0,
"author": clean(user.get("name")),
"local_guide": bool(user.get("local_guide")),
"owner_reply": clean(resp.get("snippet") or (resp.get("extracted_snippet") or {}).get("original")),
"review_id": clean(item.get("review_id")),
}
def parse_place_ref(value: str) -> Dict[str, str]:
value = (value or "").strip()
if re.fullmatch(r"ChI[A-Za-z0-9_-]{10,}", value):
return {"place_id": value}
if re.fullmatch(r"\d{6,25}", value):
return {"data_cid": value}
m = re.fullmatch(r"(0x[0-9a-f]+):0x([0-9a-f]+)", value, re.I)
if m:
return {"data_id": value.lower(), "data_cid": str(int(m.group(2), 16))}
m = re.search(r"[?&]cid=(\d+)", value)
if m:
return {"data_cid": m.group(1)}
m = re.search(r"place_id[:=](ChI[A-Za-z0-9_-]+)", value)
if m:
return {"place_id": m.group(1)}
m = re.search(r"!1s(0x[0-9a-f]+):0x([0-9a-f]+)", value, re.I)
if m:
return {"data_id": f"{m.group(1)}:0x{m.group(2)}".lower(), "data_cid": str(int(m.group(2), 16))}
return {}
# ------------------------------------------------------------------ tool
class ScrapeDoGoogleMapsTool(Tool):
name = "google_maps_scraper"
description = """
Get Google Maps business data through the Scrape.do Google Maps API. Three modes:
- 'search': a query like 'dentist' or 'coffee shop in Austin, TX', optionally centred on `location`
('lat,lng'). Returns up to 20 places per page (page 1-6): name, category, rating, review count,
phone, website, address, coordinates, hours, place_id and data_id.
- 'place': a place_id (ChIJ...), CID, feature id (0x..:0x..) or Google Maps place URL. Returns full
details: hours, plus code, rating breakdown, attributes.
- 'reviews': a place_id or data_id. Returns about 10 reviews per page with owner replies.
Each call is 10 Scrape.do credits.
"""
output_type = "string"
def __init__(self, api_token: Optional[str] = None) -> None:
self._api_token = api_token
self.inputs = {
"mode": {"type": "string", "description": "'search', 'place' or 'reviews'."},
"query": {"type": "string", "description": "Search terms for 'search'; a place_id, CID, data_id or Maps URL for 'place' and 'reviews'."},
"location": {"type": "string", "description": "Optional 'lat,lng' centre for 'search', e.g. '30.2672,-97.7431'.", "nullable": True},
"page": {"type": "integer", "description": "Result page (1-6) for 'search', review page for 'reviews'. Default 1.", "nullable": True},
"sort_by": {"type": "string", "description": "For 'reviews': newest, most_relevant, highest_rating or lowest_rating.", "nullable": True},
}
super().__init__()
def _token(self) -> Optional[str]:
token = (self._api_token or os.environ.get("SCRAPE_DO_API_TOKEN") or "").strip()
return token or None
@staticmethod
def _missing_token() -> Dict[str, Any]:
return {"status": "error", "errorMessage": "A Scrape.do API token is required.", "hint": f"Get a free one at {SCRAPE_DO_SIGNUP_URL}"}
def _call(self, token: str, endpoint: str, params: Dict[str, Any]) -> Dict[str, Any]:
query = {k: v for k, v in params.items() if v not in (None, "")}
query["token"] = token
last: Dict[str, Any] = {"_error": "No response."}
for attempt in range(3):
started = time.time()
try:
r = requests.get(f"{API_BASE}/{endpoint}", params=query, timeout=90)
except requests.RequestException as exc:
last = {"_error": f"Request failed: {exc.__class__.__name__}"}
continue
print(f"[{endpoint}] HTTP {r.status_code} in {time.time() - started:.1f}s", flush=True)
if r.status_code == 401:
return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True}
if r.status_code == 200:
try:
return {"json": r.json()}
except ValueError:
last = {"_error": "The API returned a body that is not JSON."}
continue
last = {"_error": f"The Google Maps API returned HTTP {r.status_code}."}
if r.status_code not in (429, 500, 502, 503, 504):
break
time.sleep(1 + attempt)
return last
@staticmethod
def _as_error(res: Dict[str, Any]) -> Dict[str, Any]:
out = {"status": "error", "errorMessage": res.get("_error", "Unknown error")}
if res.get("_auth"):
out["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}"
return out
def search(self, query: str, location: Optional[str] = None, page: int = 1, gl: str = "us", hl: str = "en") -> Dict[str, Any]:
token = self._token()
if not token:
return self._missing_token()
if not (query or "").strip():
return {"status": "error", "errorMessage": "Type what to search for, e.g. 'dentist'."}
ll = None
if location and location.strip():
ll = format_ll(location)
if not ll:
return {"status": "error", "errorMessage": "Location must be 'lat,lng', e.g. 30.2672,-97.7431."}
page = max(1, min(MAX_PAGES, int(page or 1)))
res = self._call(token, "search", {"q": query.strip(), "ll": ll, "gl": gl, "hl": hl, "start": (page - 1) * PAGE_SIZE or None})
if "_error" in res:
return self._as_error(res)
results = res["json"].get("local_results") or []
if isinstance(results, dict):
results = [results]
places = [normalize_place(x) for x in results if isinstance(x, dict)]
return {"status": "success", "query": query.strip(), "location": ll, "page": page, "results": places,
"maps_url": (res["json"].get("search_metadata") or {}).get("google_maps_url", ""),
"credits_used": CREDITS_PER_REQUEST}
def place(self, ref: str, hl: str = "en") -> Dict[str, Any]:
token = self._token()
if not token:
return self._missing_token()
ids = parse_place_ref(ref)
if not ids.get("place_id") and not ids.get("data_cid"):
return {"status": "error", "errorMessage": "Paste a place ID (ChIJ...), a CID, a feature ID (0x...:0x...) or a Google Maps place URL."}
params = {"place_id": ids.get("place_id"), "data_cid": None if ids.get("place_id") else ids.get("data_cid"), "hl": hl}
res = self._call(token, "place", params)
if "_error" in res:
return self._as_error(res)
item = res["json"].get("place_results")
if not isinstance(item, dict):
return {"status": "error", "errorMessage": "Google Maps has no place for that ID."}
return {"status": "success", "place": normalize_place(item), "credits_used": CREDITS_PER_REQUEST}
def reviews(self, ref: str, sort_by: str = "newest", pages: int = 1, hl: str = "en") -> Dict[str, Any]:
token = self._token()
if not token:
return self._missing_token()
ids = parse_place_ref(ref)
if not ids.get("place_id") and not ids.get("data_id"):
return {"status": "error", "errorMessage": "Paste a place ID (ChIJ...) or a feature ID (0x...:0x...)."}
sort_by = sort_by if sort_by in SORT_OPTIONS else "newest"
out: List[Dict[str, Any]] = []
topics: List[Dict[str, Any]] = []
info: Dict[str, Any] = {}
token_next = ""
calls = 0
for _ in range(max(1, min(5, int(pages or 1)))):
res = self._call(token, "reviews", {
"data_id": ids.get("data_id"), "place_id": None if ids.get("data_id") else ids.get("place_id"),
"sort_by": None if sort_by == "most_relevant" else sort_by, "next_page_token": token_next or None, "hl": hl,
})
if "_error" in res:
if out:
break
return self._as_error(res)
calls += 1
data = res["json"]
info = info or data.get("place_info") or {}
topics = topics or [{"keyword": t.get("keyword"), "mentions": t.get("mentions")} for t in data.get("topics") or []]
out.extend(normalize_review(r) for r in data.get("reviews") or [] if isinstance(r, dict))
token_next = (data.get("pagination") or {}).get("next_page_token") or ""
if not token_next:
break
return {"status": "success", "place": {k: info.get(k) for k in ("title", "address", "rating", "reviews")},
"topics": topics, "reviews": out, "credits_used": calls * CREDITS_PER_REQUEST}
def forward(self, mode: str, query: str, location: Optional[str] = None, page: Optional[int] = None, sort_by: Optional[str] = None) -> str:
key = (mode or "").strip().lower()
if key == "search":
result = self.search(query, location, page or 1)
elif key == "place":
result = self.place(query)
elif key == "reviews":
result = self.reviews(query, sort_by or "newest", page or 1)
else:
result = {"status": "error", "errorMessage": "mode must be 'search', 'place' or 'reviews'."}
return json.dumps(result, ensure_ascii=False, indent=2)