from __future__ import annotations import json import os import urllib.parse from typing import Any, Dict, List, Optional import requests from smolagents.tools import Tool SCRAPE_DO_SIGNUP_URL = ( "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=google-search-scraper" ) SEARCH_ENDPOINT = "https://api.scrape.do/plugin/google/search" AI_OVERVIEW_ENDPOINT = "https://api.scrape.do/plugin/google/search/ai-overview" # Credit costs from the Scrape.do docs. CREDITS_PER_SEARCH = 10 CREDITS_PER_AI_OVERVIEW = 5 # (label, gl, hl, google_domain) COUNTRIES: List[tuple] = [ ("United States", "us", "en", "google.com"), ("United Kingdom", "gb", "en", "google.co.uk"), ("Germany", "de", "de", "google.de"), ("France", "fr", "fr", "google.fr"), ("Spain", "es", "es", "google.es"), ("Italy", "it", "it", "google.it"), ("Netherlands", "nl", "nl", "google.nl"), ("Turkey", "tr", "tr", "google.com.tr"), ("Canada", "ca", "en", "google.ca"), ("Australia", "au", "en", "google.com.au"), ("India", "in", "en", "google.co.in"), ("Japan", "jp", "ja", "google.co.jp"), ("Brazil", "br", "pt", "google.com.br"), ("Mexico", "mx", "es", "google.com.mx"), ] COUNTRY_BY_GL = {gl: (label, gl, hl, domain) for label, gl, hl, domain in COUNTRIES} TIME_PERIODS = ["last_hour", "last_day", "last_week", "last_month", "last_year"] class ScrapeDoGoogleSearchTool(Tool): name = "google_search_scraper" description = """ Run a Google search and get the whole results page back as structured JSON: organic results with positions, top and bottom ads, People Also Ask questions, related searches, the AI Overview, knowledge panel, local map pack, top stories, videos and forum discussions. Supports any country and language, desktop or mobile layouts, time filters and pagination. Powered by Scrape.do's Google Search API, which handles proxies, CAPTCHAs and Google's changing HTML. Use this for rank tracking, SEO research, ad monitoring, or grounding an answer in live search results. """ output_type = "string" def __init__(self, api_token: Optional[str] = None) -> None: self._api_token = api_token self.inputs = { "query": { "type": "string", "description": "The search query, exactly as you would type it into Google.", }, "country": { "type": "string", "description": ( "Two-letter country code to search from, e.g. 'us', 'gb', 'de', 'tr'. " "Sets the ranking country, interface language and Google domain. Defaults to 'us'." ), "nullable": True, }, "pages": { "type": "integer", "description": "How many result pages to fetch (1-5). Each page costs 10 credits. Defaults to 1.", "nullable": True, }, "device": { "type": "string", "description": "'desktop' or 'mobile' results layout. Defaults to 'desktop'.", "nullable": True, }, "time_period": { "type": "string", "description": ( "Only return results from a recent window: last_hour, last_day, " "last_week, last_month or last_year." ), "nullable": True, }, "location": { "type": "string", "description": ( "City-level location in Google's format, e.g. 'Austin,Texas,United States' " "or 'Istanbul,Istanbul,Turkey'. Must match the country code." ), "nullable": True, }, } super().__init__() # ------------------------------------------------------------------ token def _resolve_token(self) -> Optional[str]: if self._api_token: return self._api_token.strip() env_token = os.getenv("SCRAPE_DO_API_TOKEN") return env_token.strip() if env_token else None # ------------------------------------------------------------------ calls def _get(self, url: str, params: Dict[str, Any]) -> Dict[str, Any]: full_url = url + "?" + urllib.parse.urlencode(params, quote_via=urllib.parse.quote_plus) try: response = requests.get(full_url, timeout=120) except requests.exceptions.RequestException as exc: return {"_error": f"Request failed: {exc}"} try: data = response.json() except ValueError: return {"_error": f"Unexpected non-JSON response (HTTP {response.status_code})."} if response.status_code == 401: return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True} if not response.ok: message = data.get("message") or data.get("error") or f"HTTP {response.status_code}" return {"_error": f"Scrape.do returned an error: {message}", "_status": response.status_code} return data def search( self, query: str, country: str = "us", pages: int = 1, device: str = "desktop", time_period: Optional[str] = None, location: Optional[str] = None, fetch_deferred_ai: bool = True, ) -> Dict[str, Any]: token = self._resolve_token() if not token: return { "status": "error", "errorMessage": "Missing Scrape.do API token.", "hint": f"Get a free token at {SCRAPE_DO_SIGNUP_URL}", } if not query or not query.strip(): return {"status": "error", "errorMessage": "Enter a search query."} gl = (country or "us").strip().lower() _, gl, hl, domain = COUNTRY_BY_GL.get(gl, ("", gl, "en", "google.com")) pages = max(1, min(int(pages or 1), 5)) base: Dict[str, Any] = { "token": token, "q": query.strip(), "gl": gl, "hl": hl, "google_domain": domain, "device": device if device in ("desktop", "mobile") else "desktop", "resolveGoto": "true", } if time_period in TIME_PERIODS: base["time_period"] = time_period if location and location.strip(): base["location"] = location.strip() combined: Dict[str, Any] = {} organic: List[Dict[str, Any]] = [] credits = 0 pages_fetched = 0 warnings: List[str] = [] for page_index in range(pages): # The docs say to send `start` from the very first request of a # multi-page sweep so all pages come from one coherent session. params = dict(base, start=page_index * 10) data = self._get(SEARCH_ENDPOINT, params) if "_error" in data: if page_index == 0: result = {"status": "error", "errorMessage": data["_error"]} if data.get("_auth"): result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" return result warnings.append(f"Page {page_index + 1}: {data['_error']}") break credits += CREDITS_PER_SEARCH pages_fetched += 1 for item in data.get("organic_results") or []: item = dict(item) item["page"] = page_index + 1 item["overall_position"] = len(organic) + 1 organic.append(item) if page_index == 0: combined = data # The AI Overview can come back "deferred". Its session key # expires after 90 seconds, so fetch it before any more pages. ai = data.get("ai_overview") if ( fetch_deferred_ai and isinstance(ai, dict) and ai.get("state") == "deferred" and ai.get("session_key") ): follow = self._get( AI_OVERVIEW_ENDPOINT, {"token": token, "session_key": ai["session_key"]} ) if "_error" in follow: warnings.append(f"AI Overview follow-up: {follow['_error']}") else: credits += CREDITS_PER_AI_OVERVIEW combined["ai_overview"] = follow if not (data.get("pagination") or {}).get("next"): break combined["organic_results"] = organic combined["status"] = "success" combined["pages_fetched"] = pages_fetched combined["credits_used"] = credits if warnings: combined["warnings"] = warnings return combined # ---------------------------------------------------------------- forward def forward( self, query: str, country: Optional[str] = None, pages: Optional[int] = None, device: Optional[str] = None, time_period: Optional[str] = None, location: Optional[str] = None, ) -> str: result = self.search( query=query, country=country or "us", pages=pages or 1, device=device or "desktop", time_period=time_period, location=location, ) return json.dumps(result, ensure_ascii=False, indent=2)