Spaces:
Running on Zero
Running on Zero
Download tool.py from bugrahans/google-search-scraper: direct link, hf CLI and curl.
- Browser
- Download file 9.61 kB
-
https://huggingface.co/spaces/bugrahans/google-search-scraper/resolve/main/tool.py
- Command line
-
hf download hf://spaces/bugrahans/google-search-scraper/tool.py
-
curl -L -o tool.py https://huggingface.co/spaces/bugrahans/google-search-scraper/resolve/main/tool.py
9.61 kB
| from __future__ import annotations | |
| import json | |
| import os | |
| import urllib.parse | |
| from typing import Any, Dict, List, Optional | |
| import requests | |
| from smolagents.tools import Tool | |
| SCRAPE_DO_SIGNUP_URL = ( | |
| "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=google-search-scraper" | |
| ) | |
| SEARCH_ENDPOINT = "https://api.scrape.do/plugin/google/search" | |
| AI_OVERVIEW_ENDPOINT = "https://api.scrape.do/plugin/google/search/ai-overview" | |
| # Credit costs from the Scrape.do docs. | |
| CREDITS_PER_SEARCH = 10 | |
| CREDITS_PER_AI_OVERVIEW = 5 | |
| # (label, gl, hl, google_domain) | |
| COUNTRIES: List[tuple] = [ | |
| ("United States", "us", "en", "google.com"), | |
| ("United Kingdom", "gb", "en", "google.co.uk"), | |
| ("Germany", "de", "de", "google.de"), | |
| ("France", "fr", "fr", "google.fr"), | |
| ("Spain", "es", "es", "google.es"), | |
| ("Italy", "it", "it", "google.it"), | |
| ("Netherlands", "nl", "nl", "google.nl"), | |
| ("Turkey", "tr", "tr", "google.com.tr"), | |
| ("Canada", "ca", "en", "google.ca"), | |
| ("Australia", "au", "en", "google.com.au"), | |
| ("India", "in", "en", "google.co.in"), | |
| ("Japan", "jp", "ja", "google.co.jp"), | |
| ("Brazil", "br", "pt", "google.com.br"), | |
| ("Mexico", "mx", "es", "google.com.mx"), | |
| ] | |
| COUNTRY_BY_GL = {gl: (label, gl, hl, domain) for label, gl, hl, domain in COUNTRIES} | |
| TIME_PERIODS = ["last_hour", "last_day", "last_week", "last_month", "last_year"] | |
| class ScrapeDoGoogleSearchTool(Tool): | |
| name = "google_search_scraper" | |
| description = """ | |
| Run a Google search and get the whole results page back as structured JSON: | |
| organic results with positions, top and bottom ads, People Also Ask | |
| questions, related searches, the AI Overview, knowledge panel, local map | |
| pack, top stories, videos and forum discussions. Supports any country and | |
| language, desktop or mobile layouts, time filters and pagination. | |
| Powered by Scrape.do's Google Search API, which handles proxies, CAPTCHAs | |
| and Google's changing HTML. Use this for rank tracking, SEO research, | |
| ad monitoring, or grounding an answer in live search results. | |
| """ | |
| output_type = "string" | |
| def __init__(self, api_token: Optional[str] = None) -> None: | |
| self._api_token = api_token | |
| self.inputs = { | |
| "query": { | |
| "type": "string", | |
| "description": "The search query, exactly as you would type it into Google.", | |
| }, | |
| "country": { | |
| "type": "string", | |
| "description": ( | |
| "Two-letter country code to search from, e.g. 'us', 'gb', 'de', 'tr'. " | |
| "Sets the ranking country, interface language and Google domain. Defaults to 'us'." | |
| ), | |
| "nullable": True, | |
| }, | |
| "pages": { | |
| "type": "integer", | |
| "description": "How many result pages to fetch (1-5). Each page costs 10 credits. Defaults to 1.", | |
| "nullable": True, | |
| }, | |
| "device": { | |
| "type": "string", | |
| "description": "'desktop' or 'mobile' results layout. Defaults to 'desktop'.", | |
| "nullable": True, | |
| }, | |
| "time_period": { | |
| "type": "string", | |
| "description": ( | |
| "Only return results from a recent window: last_hour, last_day, " | |
| "last_week, last_month or last_year." | |
| ), | |
| "nullable": True, | |
| }, | |
| "location": { | |
| "type": "string", | |
| "description": ( | |
| "City-level location in Google's format, e.g. 'Austin,Texas,United States' " | |
| "or 'Istanbul,Istanbul,Turkey'. Must match the country code." | |
| ), | |
| "nullable": True, | |
| }, | |
| } | |
| super().__init__() | |
| # ------------------------------------------------------------------ token | |
| def _resolve_token(self) -> Optional[str]: | |
| if self._api_token: | |
| return self._api_token.strip() | |
| env_token = os.getenv("SCRAPE_DO_API_TOKEN") | |
| return env_token.strip() if env_token else None | |
| # ------------------------------------------------------------------ calls | |
| def _get(self, url: str, params: Dict[str, Any]) -> Dict[str, Any]: | |
| full_url = url + "?" + urllib.parse.urlencode(params, quote_via=urllib.parse.quote_plus) | |
| try: | |
| response = requests.get(full_url, timeout=120) | |
| except requests.exceptions.RequestException as exc: | |
| return {"_error": f"Request failed: {exc}"} | |
| try: | |
| data = response.json() | |
| except ValueError: | |
| return {"_error": f"Unexpected non-JSON response (HTTP {response.status_code})."} | |
| if response.status_code == 401: | |
| return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True} | |
| if not response.ok: | |
| message = data.get("message") or data.get("error") or f"HTTP {response.status_code}" | |
| return {"_error": f"Scrape.do returned an error: {message}", "_status": response.status_code} | |
| return data | |
| def search( | |
| self, | |
| query: str, | |
| country: str = "us", | |
| pages: int = 1, | |
| device: str = "desktop", | |
| time_period: Optional[str] = None, | |
| location: Optional[str] = None, | |
| fetch_deferred_ai: bool = True, | |
| ) -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return { | |
| "status": "error", | |
| "errorMessage": "Missing Scrape.do API token.", | |
| "hint": f"Get a free token at {SCRAPE_DO_SIGNUP_URL}", | |
| } | |
| if not query or not query.strip(): | |
| return {"status": "error", "errorMessage": "Enter a search query."} | |
| gl = (country or "us").strip().lower() | |
| _, gl, hl, domain = COUNTRY_BY_GL.get(gl, ("", gl, "en", "google.com")) | |
| pages = max(1, min(int(pages or 1), 5)) | |
| base: Dict[str, Any] = { | |
| "token": token, | |
| "q": query.strip(), | |
| "gl": gl, | |
| "hl": hl, | |
| "google_domain": domain, | |
| "device": device if device in ("desktop", "mobile") else "desktop", | |
| "resolveGoto": "true", | |
| } | |
| if time_period in TIME_PERIODS: | |
| base["time_period"] = time_period | |
| if location and location.strip(): | |
| base["location"] = location.strip() | |
| combined: Dict[str, Any] = {} | |
| organic: List[Dict[str, Any]] = [] | |
| credits = 0 | |
| pages_fetched = 0 | |
| warnings: List[str] = [] | |
| for page_index in range(pages): | |
| # The docs say to send `start` from the very first request of a | |
| # multi-page sweep so all pages come from one coherent session. | |
| params = dict(base, start=page_index * 10) | |
| data = self._get(SEARCH_ENDPOINT, params) | |
| if "_error" in data: | |
| if page_index == 0: | |
| result = {"status": "error", "errorMessage": data["_error"]} | |
| if data.get("_auth"): | |
| result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" | |
| return result | |
| warnings.append(f"Page {page_index + 1}: {data['_error']}") | |
| break | |
| credits += CREDITS_PER_SEARCH | |
| pages_fetched += 1 | |
| for item in data.get("organic_results") or []: | |
| item = dict(item) | |
| item["page"] = page_index + 1 | |
| item["overall_position"] = len(organic) + 1 | |
| organic.append(item) | |
| if page_index == 0: | |
| combined = data | |
| # The AI Overview can come back "deferred". Its session key | |
| # expires after 90 seconds, so fetch it before any more pages. | |
| ai = data.get("ai_overview") | |
| if ( | |
| fetch_deferred_ai | |
| and isinstance(ai, dict) | |
| and ai.get("state") == "deferred" | |
| and ai.get("session_key") | |
| ): | |
| follow = self._get( | |
| AI_OVERVIEW_ENDPOINT, {"token": token, "session_key": ai["session_key"]} | |
| ) | |
| if "_error" in follow: | |
| warnings.append(f"AI Overview follow-up: {follow['_error']}") | |
| else: | |
| credits += CREDITS_PER_AI_OVERVIEW | |
| combined["ai_overview"] = follow | |
| if not (data.get("pagination") or {}).get("next"): | |
| break | |
| combined["organic_results"] = organic | |
| combined["status"] = "success" | |
| combined["pages_fetched"] = pages_fetched | |
| combined["credits_used"] = credits | |
| if warnings: | |
| combined["warnings"] = warnings | |
| return combined | |
| # ---------------------------------------------------------------- forward | |
| def forward( | |
| self, | |
| query: str, | |
| country: Optional[str] = None, | |
| pages: Optional[int] = None, | |
| device: Optional[str] = None, | |
| time_period: Optional[str] = None, | |
| location: Optional[str] = None, | |
| ) -> str: | |
| result = self.search( | |
| query=query, | |
| country=country or "us", | |
| pages=pages or 1, | |
| device=device or "desktop", | |
| time_period=time_period, | |
| location=location, | |
| ) | |
| return json.dumps(result, ensure_ascii=False, indent=2) | |