Spaces:
Sleeping
Sleeping
Download tool.py from bugrahans/google-trends-scraper: direct link, hf CLI and curl.
- Browser
- Download file 21.6 kB
-
https://huggingface.co/spaces/bugrahans/google-trends-scraper/resolve/main/tool.py
- Command line
-
hf download hf://spaces/bugrahans/google-trends-scraper/tool.py
-
curl -L -o tool.py https://huggingface.co/spaces/bugrahans/google-trends-scraper/resolve/main/tool.py
21.6 kB
| from __future__ import annotations | |
| import json | |
| import os | |
| import re | |
| import time | |
| import urllib.parse | |
| from typing import Any, Dict, List, Optional, Tuple | |
| import requests | |
| from smolagents.tools import Tool | |
| SCRAPE_DO_SIGNUP_URL = ( | |
| "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=google-trends-scraper" | |
| ) | |
| TRENDS_ENDPOINT = "https://api.scrape.do/plugin/google/trends" | |
| TRENDING_ENDPOINT = "https://api.scrape.do/plugin/google/trending" | |
| # From the Scrape.do docs: 10 credits per successful call. | |
| CREDITS_PER_CALL = 10 | |
| MAX_COMPARE = 5 | |
| # (label, geo code). An empty code means worldwide. | |
| COUNTRIES: List[Tuple[str, str]] = [ | |
| ("Worldwide", ""), | |
| ("United States", "US"), | |
| ("United Kingdom", "GB"), | |
| ("Germany", "DE"), | |
| ("France", "FR"), | |
| ("Spain", "ES"), | |
| ("Italy", "IT"), | |
| ("Netherlands", "NL"), | |
| ("Turkey", "TR"), | |
| ("Canada", "CA"), | |
| ("Australia", "AU"), | |
| ("India", "IN"), | |
| ("Japan", "JP"), | |
| ("Brazil", "BR"), | |
| ("Mexico", "MX"), | |
| ] | |
| # Friendly names accepted for the `date` parameter. | |
| DATE_ALIASES = { | |
| "7d": "now 7-d", | |
| "past 7 days": "now 7-d", | |
| "30d": "today 1-m", | |
| "1m": "today 1-m", | |
| "past 30 days": "today 1-m", | |
| "90d": "today 3-m", | |
| "3m": "today 3-m", | |
| "past 90 days": "today 3-m", | |
| "12m": "today 12-m", | |
| "1y": "today 12-m", | |
| "past 12 months": "today 12-m", | |
| "5y": "today 5-y", | |
| "past 5 years": "today 5-y", | |
| "all": "all", | |
| "2004": "all", | |
| } | |
| SEARCH_TYPES = { | |
| "web": "", | |
| "": "", | |
| "youtube": "youtube", | |
| "news": "news", | |
| "images": "images", | |
| "image": "images", | |
| "shopping": "froogle", | |
| "froogle": "froogle", | |
| } | |
| REGION_RESOLUTIONS = {"country": "COUNTRY", "region": "REGION", "metro": "DMA", "dma": "DMA", "city": "CITY"} | |
| TRENDING_CATEGORIES: Dict[int, str] = { | |
| 0: "All categories", | |
| 1: "Autos and Vehicles", | |
| 2: "Beauty and Fashion", | |
| 3: "Business and Finance", | |
| 4: "Entertainment", | |
| 5: "Food and Drink", | |
| 6: "Games", | |
| 7: "Health", | |
| 8: "Hobbies and Leisure", | |
| 9: "Jobs and Education", | |
| 10: "Law and Government", | |
| 11: "Other", | |
| 13: "Pets and Animals", | |
| 14: "Politics", | |
| 15: "Science", | |
| 16: "Shopping", | |
| 17: "Sports", | |
| 18: "Technology", | |
| 19: "Travel and Transportation", | |
| 20: "Climate", | |
| } | |
| TRENDING_HOURS = (4, 24, 48, 168) | |
| TRENDING_SORTS = ("relevance", "search_volume", "recency", "title") | |
| # ------------------------------------------------------------------ helpers | |
| def normalize_date(value: Optional[str]) -> str: | |
| text = (value or "").strip() | |
| if not text: | |
| return "today 12-m" | |
| return DATE_ALIASES.get(text.lower(), text) | |
| def normalize_geo(value: Optional[str]) -> str: | |
| text = (value or "").strip().upper() | |
| if text in ("", "WORLDWIDE", "WORLD", "GLOBAL", "ALL"): | |
| return "" | |
| return text | |
| def split_keywords(value: Optional[str]) -> List[str]: | |
| keywords: List[str] = [] | |
| for part in re.split(r"[,;\n]+", value or ""): | |
| part = part.strip() | |
| if part and part.lower() not in [k.lower() for k in keywords]: | |
| keywords.append(part) | |
| return keywords | |
| def _number(value: Any) -> Optional[float]: | |
| if isinstance(value, bool): | |
| return None | |
| if isinstance(value, (int, float)): | |
| return float(value) | |
| text = str(value or "").strip().replace(",", "").replace("%", "").replace("+", "") | |
| if text.startswith("<"): | |
| return 0.0 | |
| try: | |
| return float(text) | |
| except ValueError: | |
| return None | |
| def parse_timeline(data: Dict[str, Any]) -> List[Dict[str, Any]]: | |
| """Flatten interest_over_time into [{date, timestamp, values: {query: number}}].""" | |
| timeline = (data.get("interest_over_time") or {}).get("timeline_data") or [] | |
| rows: List[Dict[str, Any]] = [] | |
| for point in timeline: | |
| if not isinstance(point, dict): | |
| continue | |
| values: Dict[str, float] = {} | |
| for entry in point.get("values") or []: | |
| if not isinstance(entry, dict): | |
| continue | |
| number = _number(entry.get("extracted_value")) | |
| if number is None: | |
| number = _number(entry.get("value")) | |
| values[str(entry.get("query") or "")] = number if number is not None else 0.0 | |
| try: | |
| stamp = int(str(point.get("timestamp") or "0")) | |
| except ValueError: | |
| stamp = 0 | |
| rows.append({"date": point.get("date"), "timestamp": stamp, "values": values}) | |
| return rows | |
| def series_stats(timeline: List[Dict[str, Any]], query: str) -> Dict[str, Any]: | |
| points = [(row, row["values"].get(query)) for row in timeline if row["values"].get(query) is not None] | |
| if not points: | |
| return {} | |
| values = [v for _, v in points] | |
| peak_row, peak = max(points, key=lambda p: p[1]) | |
| quarter = max(1, len(values) // 4) | |
| first, last = sum(values[:quarter]) / quarter, sum(values[-quarter:]) / quarter | |
| change = ((last - first) / first * 100) if first > 0 else None | |
| return { | |
| "average": round(sum(values) / len(values), 1), | |
| "peak": peak, | |
| "peak_date": peak_row.get("date"), | |
| "latest": values[-1], | |
| "change_percent": round(change, 1) if change is not None else None, | |
| } | |
| # --------------------------------------------------------------------- tool | |
| class ScrapeDoGoogleTrendsTool(Tool): | |
| name = "google_trends_scraper" | |
| description = """ | |
| Get Google Trends data as JSON. Modes: | |
| - 'explore': interest over time (0-100) and interest by region for one | |
| keyword, plus its top and rising related searches. | |
| - 'compare': up to 5 comma-separated keywords on the same 0-100 scale. | |
| - 'related_topics': top and rising related topics for one keyword. | |
| - 'trending': what's trending right now in a country (no query needed), | |
| with search volume, growth and related searches. | |
| Supports any country or region code, time ranges from 7 days to 2004-now, | |
| and web, YouTube, News, Images or Shopping search. Powered by Scrape.do's | |
| Google Trends API, a working alternative to the unmaintained pytrends. | |
| """ | |
| output_type = "string" | |
| def __init__(self, api_token: Optional[str] = None) -> None: | |
| self._api_token = api_token | |
| self.inputs = { | |
| "mode": { | |
| "type": "string", | |
| "description": "'explore', 'compare', 'related_topics' or 'trending'.", | |
| }, | |
| "query": { | |
| "type": "string", | |
| "description": ( | |
| "The keyword, e.g. 'bitcoin'. For 'compare', up to 5 keywords separated by commas. " | |
| "Not used for 'trending'." | |
| ), | |
| "nullable": True, | |
| }, | |
| "geo": { | |
| "type": "string", | |
| "description": ( | |
| "Country or region code such as 'US', 'GB', 'TR' or 'US-CA'. Empty means worldwide. " | |
| "'trending' needs a country and defaults to 'US'." | |
| ), | |
| "nullable": True, | |
| }, | |
| "date": { | |
| "type": "string", | |
| "description": ( | |
| "Time range: '7d', '30d', '90d', '12m' (default), '5y', 'all', or Google's own format " | |
| "such as 'today 3-m' or '2024-01-01 2024-12-31'." | |
| ), | |
| "nullable": True, | |
| }, | |
| "search_type": { | |
| "type": "string", | |
| "description": "'web' (default), 'youtube', 'news', 'images' or 'shopping'.", | |
| "nullable": True, | |
| }, | |
| "category": { | |
| "type": "integer", | |
| "description": "'trending' only: category id, e.g. 17 Sports, 18 Technology, 3 Business. Defaults to all.", | |
| "nullable": True, | |
| }, | |
| "hours": { | |
| "type": "integer", | |
| "description": "'trending' only: time window in hours: 4, 24 (default), 48 or 168.", | |
| "nullable": True, | |
| }, | |
| } | |
| super().__init__() | |
| # ------------------------------------------------------------ plumbing | |
| def _resolve_token(self) -> Optional[str]: | |
| if self._api_token: | |
| return self._api_token.strip() | |
| env_token = os.getenv("SCRAPE_DO_API_TOKEN") | |
| return env_token.strip() if env_token else None | |
| def _missing_token() -> Dict[str, Any]: | |
| return { | |
| "status": "error", | |
| "errorMessage": "Missing Scrape.do API token.", | |
| "hint": f"Get a free token at {SCRAPE_DO_SIGNUP_URL}", | |
| } | |
| def _as_error(data: Dict[str, Any]) -> Dict[str, Any]: | |
| result = {"status": "error", "errorMessage": data.get("_error", "Unknown error")} | |
| if data.get("_auth"): | |
| result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" | |
| return result | |
| def _get(self, url: str, params: Dict[str, Any], retries: int = 1) -> Dict[str, Any]: | |
| """GET a Scrape.do endpoint. Retries once on 5xx, as the docs recommend.""" | |
| token = str(params.get("token") or "") | |
| def redact(text: str) -> str: | |
| return text.replace(token, "***") if token else text | |
| clean = {k: v for k, v in params.items() if v not in (None, "")} | |
| full_url = url + "?" + urllib.parse.urlencode(clean, quote_via=urllib.parse.quote) | |
| last: Dict[str, Any] = {"_error": "Request failed."} | |
| for attempt in range(retries + 1): | |
| try: | |
| response = requests.get(full_url, timeout=120) | |
| except requests.exceptions.RequestException as exc: | |
| last = {"_error": redact(f"Request failed: {exc}")} | |
| if attempt < retries: | |
| time.sleep(1.5) | |
| continue | |
| return last | |
| if response.status_code == 401: | |
| return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True} | |
| try: | |
| data = response.json() | |
| except ValueError: | |
| data = None | |
| if response.ok and isinstance(data, dict): | |
| return data | |
| message = "" | |
| if isinstance(data, dict): | |
| parts = [str(data.get(key)).strip().rstrip(".") for key in ("message", "error") if data.get(key)] | |
| message = parts[0] if parts else "" | |
| if not message: | |
| message = (response.text or "").strip()[:200] or "no details" | |
| if response.status_code >= 500: | |
| text = f"Google Trends didn't answer this time (HTTP {response.status_code}): {message}. Try again in a moment." | |
| else: | |
| text = f"Scrape.do returned HTTP {response.status_code}: {message}" | |
| last = {"_error": redact(text), "_status": response.status_code} | |
| if response.status_code in (500, 502, 503, 504) and attempt < retries: | |
| time.sleep(1.5) | |
| continue | |
| return last | |
| return last | |
| def _trends_params( | |
| self, token: str, q: str, geo: str, date: str, search_type: Optional[str], hl: str = "en" | |
| ) -> Dict[str, Any]: | |
| gprop = SEARCH_TYPES.get((search_type or "web").strip().lower(), "") | |
| return {"token": token, "q": q, "geo": geo, "hl": hl, "date": date, "gprop": gprop} | |
| # -------------------------------------------------------------- explore | |
| def explore( | |
| self, | |
| query: str, | |
| geo: Optional[str] = None, | |
| date: Optional[str] = None, | |
| search_type: Optional[str] = None, | |
| related_queries: bool = True, | |
| related_topics: bool = False, | |
| resolution: Optional[str] = None, | |
| ) -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| keyword = (query or "").strip() | |
| if not keyword: | |
| return {"status": "error", "errorMessage": "Type a keyword, e.g. 'bitcoin'."} | |
| if "," in keyword: | |
| keyword = split_keywords(keyword)[0] | |
| geo_code, date_value = normalize_geo(geo), normalize_date(date) | |
| base = self._trends_params(token, keyword, geo_code, date_value, search_type) | |
| region = REGION_RESOLUTIONS.get((resolution or "").strip().lower()) | |
| main_params = dict(base, region=region) if region else base | |
| data = self._get(TRENDS_ENDPOINT, main_params) | |
| if "_error" in data: | |
| return self._as_error(data) | |
| credits = CREDITS_PER_CALL | |
| warnings: List[str] = [] | |
| timeline = parse_timeline(data) | |
| result: Dict[str, Any] = { | |
| "status": "success", | |
| "query": keyword, | |
| "geo": geo_code or "worldwide", | |
| "date": date_value, | |
| "search_type": search_type or "web", | |
| "timeline": timeline, | |
| "stats": series_stats(timeline, keyword) or series_stats(timeline, next(iter(timeline[0]["values"]), "")) if timeline else {}, | |
| "interest_by_region": data.get("interest_by_region") or [], | |
| } | |
| for flag, data_type, key in ( | |
| (related_queries, "RELATED_QUERIES", "related_queries"), | |
| (related_topics, "RELATED_TOPICS", "related_topics"), | |
| ): | |
| if not flag: | |
| continue | |
| extra = self._get(TRENDS_ENDPOINT, dict(base, data_type=data_type)) | |
| if "_error" in extra: | |
| warnings.append(f"{key.replace('_', ' ').capitalize()}: {extra['_error']}") | |
| continue | |
| credits += CREDITS_PER_CALL | |
| result[key] = extra.get(key) or {"top": [], "rising": []} | |
| result["credits_used"] = credits | |
| if not timeline: | |
| warnings.append("Google Trends has too little data for this keyword, place and time range.") | |
| if warnings: | |
| result["warnings"] = warnings | |
| return result | |
| # -------------------------------------------------------------- compare | |
| def compare( | |
| self, | |
| keywords: str, | |
| geo: Optional[str] = None, | |
| date: Optional[str] = None, | |
| search_type: Optional[str] = None, | |
| ) -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| terms = split_keywords(keywords) | |
| warnings: List[str] = [] | |
| if len(terms) < 2: | |
| return {"status": "error", "errorMessage": "Enter at least two keywords separated by commas."} | |
| if len(terms) > MAX_COMPARE: | |
| warnings.append(f"Google Trends compares up to {MAX_COMPARE} keywords; the rest were dropped.") | |
| terms = terms[:MAX_COMPARE] | |
| geo_code, date_value = normalize_geo(geo), normalize_date(date) | |
| data = self._get( | |
| TRENDS_ENDPOINT, self._trends_params(token, ",".join(terms), geo_code, date_value, search_type) | |
| ) | |
| if "_error" in data: | |
| return self._as_error(data) | |
| credits = CREDITS_PER_CALL | |
| timeline = parse_timeline(data) | |
| regions = data.get("interest_by_region") or [] | |
| returned = {q.lower() for row in timeline for q in row["values"]} | |
| scaled_separately = False | |
| if not {t.lower() for t in terms} <= returned: | |
| # The combined request didn't come back as separate lines, so fetch | |
| # each keyword on its own. Each line is then scaled to its own peak. | |
| scaled_separately = True | |
| regions = [] | |
| merged: Dict[int, Dict[str, Any]] = {} | |
| for term in terms: | |
| one = self._get( | |
| TRENDS_ENDPOINT, | |
| dict(self._trends_params(token, term, geo_code, date_value, search_type), data_type="TIMESERIES"), | |
| ) | |
| if "_error" in one: | |
| warnings.append(f"“{term}”: {one['_error']}") | |
| continue | |
| credits += CREDITS_PER_CALL | |
| for row in parse_timeline(one): | |
| slot = merged.setdefault(row["timestamp"], {"date": row["date"], "timestamp": row["timestamp"], "values": {}}) | |
| value = next(iter(row["values"].values()), None) | |
| if value is not None: | |
| slot["values"][term] = value | |
| timeline = [merged[k] for k in sorted(merged)] | |
| warnings.append( | |
| "These keywords came back as separate requests, so each line is scaled to its own peak " | |
| "(100 = that keyword's busiest point). Compare shapes and timing, not heights." | |
| ) | |
| else: | |
| # Use the exact spelling Google echoed back. | |
| echoed = {q.lower(): q for row in timeline for q in row["values"]} | |
| terms = [echoed.get(t.lower(), t) for t in terms] | |
| stats = {term: series_stats(timeline, term) for term in terms} | |
| averages = {term: s.get("average") for term, s in stats.items() if s} | |
| result: Dict[str, Any] = { | |
| "status": "success", | |
| "keywords": terms, | |
| "geo": geo_code or "worldwide", | |
| "date": date_value, | |
| "search_type": search_type or "web", | |
| "timeline": timeline, | |
| "stats": stats, | |
| "averages": averages, | |
| "interest_by_region": regions, | |
| "scaled_separately": scaled_separately, | |
| "credits_used": credits, | |
| } | |
| if averages and not scaled_separately: | |
| result["leader"] = max(averages, key=lambda k: averages[k] or 0) | |
| if warnings: | |
| result["warnings"] = warnings | |
| return result | |
| # ------------------------------------------------------- related topics | |
| def related_topics( | |
| self, query: str, geo: Optional[str] = None, date: Optional[str] = None, search_type: Optional[str] = None | |
| ) -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| keyword = (query or "").strip() | |
| if not keyword: | |
| return {"status": "error", "errorMessage": "Type a keyword, e.g. 'bitcoin'."} | |
| base = self._trends_params(token, keyword, normalize_geo(geo), normalize_date(date), search_type) | |
| data = self._get(TRENDS_ENDPOINT, dict(base, data_type="RELATED_TOPICS")) | |
| if "_error" in data: | |
| return self._as_error(data) | |
| return { | |
| "status": "success", | |
| "query": keyword, | |
| "related_topics": data.get("related_topics") or {"top": [], "rising": []}, | |
| "credits_used": CREDITS_PER_CALL, | |
| } | |
| # ------------------------------------------------------------- trending | |
| def trending( | |
| self, | |
| geo: Optional[str] = "US", | |
| hours: Optional[int] = 24, | |
| category: Optional[int] = 0, | |
| sort: Optional[str] = "relevance", | |
| active_only: bool = False, | |
| ) -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| geo_code = normalize_geo(geo) or "US" | |
| geo_code = geo_code.split("-")[0] | |
| hours_value = int(hours or 24) | |
| if hours_value not in TRENDING_HOURS: | |
| hours_value = min(TRENDING_HOURS, key=lambda h: abs(h - hours_value)) | |
| try: | |
| cat = int(category or 0) | |
| except (TypeError, ValueError): | |
| cat = 0 | |
| if cat not in TRENDING_CATEGORIES: | |
| cat = 0 | |
| sort_value = sort if sort in TRENDING_SORTS else "relevance" | |
| params = { | |
| "token": token, "geo": geo_code, "hl": "en", "hours": hours_value, "cat": cat, | |
| "sort": sort_value, "status": "active" if active_only else "all", | |
| } | |
| data = self._get(TRENDING_ENDPOINT, params) | |
| if "_error" in data: | |
| return self._as_error(data) | |
| trends = [t for t in data.get("trends") or [] if isinstance(t, dict)] | |
| for trend in trends: | |
| trend["categories"] = [TRENDING_CATEGORIES.get(c, str(c)) for c in trend.get("category_ids") or []] | |
| return { | |
| "status": "success", | |
| "geo": geo_code, | |
| "hours": hours_value, | |
| "category": TRENDING_CATEGORIES[cat], | |
| "trends": trends, | |
| "credits_used": CREDITS_PER_CALL, | |
| } | |
| # -------------------------------------------------------------- forward | |
| def forward( | |
| self, | |
| mode: str, | |
| query: Optional[str] = None, | |
| geo: Optional[str] = None, | |
| date: Optional[str] = None, | |
| search_type: Optional[str] = None, | |
| category: Optional[int] = None, | |
| hours: Optional[int] = None, | |
| ) -> str: | |
| mode_key = (mode or "").strip().lower() | |
| if mode_key in ("explore", "interest", "keyword"): | |
| result = self.explore(query or "", geo, date, search_type, related_queries=True) | |
| elif mode_key in ("compare", "comparison"): | |
| result = self.compare(query or "", geo, date, search_type) | |
| elif mode_key in ("related_topics", "topics"): | |
| result = self.related_topics(query or "", geo, date, search_type) | |
| elif mode_key in ("trending", "trending_now", "trends"): | |
| result = self.trending(geo or "US", hours or 24, category or 0) | |
| else: | |
| result = { | |
| "status": "error", | |
| "errorMessage": "mode must be 'explore', 'compare', 'related_topics' or 'trending'.", | |
| } | |
| return json.dumps(result, ensure_ascii=False, indent=2) | |