from __future__ import annotations import json import os import re import time import urllib.parse from typing import Any, Dict, List, Optional, Tuple import requests from smolagents.tools import Tool SCRAPE_DO_SIGNUP_URL = ( "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=google-trends-scraper" ) TRENDS_ENDPOINT = "https://api.scrape.do/plugin/google/trends" TRENDING_ENDPOINT = "https://api.scrape.do/plugin/google/trending" # From the Scrape.do docs: 10 credits per successful call. CREDITS_PER_CALL = 10 MAX_COMPARE = 5 # (label, geo code). An empty code means worldwide. COUNTRIES: List[Tuple[str, str]] = [ ("Worldwide", ""), ("United States", "US"), ("United Kingdom", "GB"), ("Germany", "DE"), ("France", "FR"), ("Spain", "ES"), ("Italy", "IT"), ("Netherlands", "NL"), ("Turkey", "TR"), ("Canada", "CA"), ("Australia", "AU"), ("India", "IN"), ("Japan", "JP"), ("Brazil", "BR"), ("Mexico", "MX"), ] # Friendly names accepted for the `date` parameter. DATE_ALIASES = { "7d": "now 7-d", "past 7 days": "now 7-d", "30d": "today 1-m", "1m": "today 1-m", "past 30 days": "today 1-m", "90d": "today 3-m", "3m": "today 3-m", "past 90 days": "today 3-m", "12m": "today 12-m", "1y": "today 12-m", "past 12 months": "today 12-m", "5y": "today 5-y", "past 5 years": "today 5-y", "all": "all", "2004": "all", } SEARCH_TYPES = { "web": "", "": "", "youtube": "youtube", "news": "news", "images": "images", "image": "images", "shopping": "froogle", "froogle": "froogle", } REGION_RESOLUTIONS = {"country": "COUNTRY", "region": "REGION", "metro": "DMA", "dma": "DMA", "city": "CITY"} TRENDING_CATEGORIES: Dict[int, str] = { 0: "All categories", 1: "Autos and Vehicles", 2: "Beauty and Fashion", 3: "Business and Finance", 4: "Entertainment", 5: "Food and Drink", 6: "Games", 7: "Health", 8: "Hobbies and Leisure", 9: "Jobs and Education", 10: "Law and Government", 11: "Other", 13: "Pets and Animals", 14: "Politics", 15: "Science", 16: "Shopping", 17: "Sports", 18: "Technology", 19: "Travel and Transportation", 20: "Climate", } TRENDING_HOURS = (4, 24, 48, 168) TRENDING_SORTS = ("relevance", "search_volume", "recency", "title") # ------------------------------------------------------------------ helpers def normalize_date(value: Optional[str]) -> str: text = (value or "").strip() if not text: return "today 12-m" return DATE_ALIASES.get(text.lower(), text) def normalize_geo(value: Optional[str]) -> str: text = (value or "").strip().upper() if text in ("", "WORLDWIDE", "WORLD", "GLOBAL", "ALL"): return "" return text def split_keywords(value: Optional[str]) -> List[str]: keywords: List[str] = [] for part in re.split(r"[,;\n]+", value or ""): part = part.strip() if part and part.lower() not in [k.lower() for k in keywords]: keywords.append(part) return keywords def _number(value: Any) -> Optional[float]: if isinstance(value, bool): return None if isinstance(value, (int, float)): return float(value) text = str(value or "").strip().replace(",", "").replace("%", "").replace("+", "") if text.startswith("<"): return 0.0 try: return float(text) except ValueError: return None def parse_timeline(data: Dict[str, Any]) -> List[Dict[str, Any]]: """Flatten interest_over_time into [{date, timestamp, values: {query: number}}].""" timeline = (data.get("interest_over_time") or {}).get("timeline_data") or [] rows: List[Dict[str, Any]] = [] for point in timeline: if not isinstance(point, dict): continue values: Dict[str, float] = {} for entry in point.get("values") or []: if not isinstance(entry, dict): continue number = _number(entry.get("extracted_value")) if number is None: number = _number(entry.get("value")) values[str(entry.get("query") or "")] = number if number is not None else 0.0 try: stamp = int(str(point.get("timestamp") or "0")) except ValueError: stamp = 0 rows.append({"date": point.get("date"), "timestamp": stamp, "values": values}) return rows def series_stats(timeline: List[Dict[str, Any]], query: str) -> Dict[str, Any]: points = [(row, row["values"].get(query)) for row in timeline if row["values"].get(query) is not None] if not points: return {} values = [v for _, v in points] peak_row, peak = max(points, key=lambda p: p[1]) quarter = max(1, len(values) // 4) first, last = sum(values[:quarter]) / quarter, sum(values[-quarter:]) / quarter change = ((last - first) / first * 100) if first > 0 else None return { "average": round(sum(values) / len(values), 1), "peak": peak, "peak_date": peak_row.get("date"), "latest": values[-1], "change_percent": round(change, 1) if change is not None else None, } # --------------------------------------------------------------------- tool class ScrapeDoGoogleTrendsTool(Tool): name = "google_trends_scraper" description = """ Get Google Trends data as JSON. Modes: - 'explore': interest over time (0-100) and interest by region for one keyword, plus its top and rising related searches. - 'compare': up to 5 comma-separated keywords on the same 0-100 scale. - 'related_topics': top and rising related topics for one keyword. - 'trending': what's trending right now in a country (no query needed), with search volume, growth and related searches. Supports any country or region code, time ranges from 7 days to 2004-now, and web, YouTube, News, Images or Shopping search. Powered by Scrape.do's Google Trends API, a working alternative to the unmaintained pytrends. """ output_type = "string" def __init__(self, api_token: Optional[str] = None) -> None: self._api_token = api_token self.inputs = { "mode": { "type": "string", "description": "'explore', 'compare', 'related_topics' or 'trending'.", }, "query": { "type": "string", "description": ( "The keyword, e.g. 'bitcoin'. For 'compare', up to 5 keywords separated by commas. " "Not used for 'trending'." ), "nullable": True, }, "geo": { "type": "string", "description": ( "Country or region code such as 'US', 'GB', 'TR' or 'US-CA'. Empty means worldwide. " "'trending' needs a country and defaults to 'US'." ), "nullable": True, }, "date": { "type": "string", "description": ( "Time range: '7d', '30d', '90d', '12m' (default), '5y', 'all', or Google's own format " "such as 'today 3-m' or '2024-01-01 2024-12-31'." ), "nullable": True, }, "search_type": { "type": "string", "description": "'web' (default), 'youtube', 'news', 'images' or 'shopping'.", "nullable": True, }, "category": { "type": "integer", "description": "'trending' only: category id, e.g. 17 Sports, 18 Technology, 3 Business. Defaults to all.", "nullable": True, }, "hours": { "type": "integer", "description": "'trending' only: time window in hours: 4, 24 (default), 48 or 168.", "nullable": True, }, } super().__init__() # ------------------------------------------------------------ plumbing def _resolve_token(self) -> Optional[str]: if self._api_token: return self._api_token.strip() env_token = os.getenv("SCRAPE_DO_API_TOKEN") return env_token.strip() if env_token else None @staticmethod def _missing_token() -> Dict[str, Any]: return { "status": "error", "errorMessage": "Missing Scrape.do API token.", "hint": f"Get a free token at {SCRAPE_DO_SIGNUP_URL}", } @staticmethod def _as_error(data: Dict[str, Any]) -> Dict[str, Any]: result = {"status": "error", "errorMessage": data.get("_error", "Unknown error")} if data.get("_auth"): result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" return result def _get(self, url: str, params: Dict[str, Any], retries: int = 1) -> Dict[str, Any]: """GET a Scrape.do endpoint. Retries once on 5xx, as the docs recommend.""" token = str(params.get("token") or "") def redact(text: str) -> str: return text.replace(token, "***") if token else text clean = {k: v for k, v in params.items() if v not in (None, "")} full_url = url + "?" + urllib.parse.urlencode(clean, quote_via=urllib.parse.quote) last: Dict[str, Any] = {"_error": "Request failed."} for attempt in range(retries + 1): try: response = requests.get(full_url, timeout=120) except requests.exceptions.RequestException as exc: last = {"_error": redact(f"Request failed: {exc}")} if attempt < retries: time.sleep(1.5) continue return last if response.status_code == 401: return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True} try: data = response.json() except ValueError: data = None if response.ok and isinstance(data, dict): return data message = "" if isinstance(data, dict): parts = [str(data.get(key)).strip().rstrip(".") for key in ("message", "error") if data.get(key)] message = parts[0] if parts else "" if not message: message = (response.text or "").strip()[:200] or "no details" if response.status_code >= 500: text = f"Google Trends didn't answer this time (HTTP {response.status_code}): {message}. Try again in a moment." else: text = f"Scrape.do returned HTTP {response.status_code}: {message}" last = {"_error": redact(text), "_status": response.status_code} if response.status_code in (500, 502, 503, 504) and attempt < retries: time.sleep(1.5) continue return last return last def _trends_params( self, token: str, q: str, geo: str, date: str, search_type: Optional[str], hl: str = "en" ) -> Dict[str, Any]: gprop = SEARCH_TYPES.get((search_type or "web").strip().lower(), "") return {"token": token, "q": q, "geo": geo, "hl": hl, "date": date, "gprop": gprop} # -------------------------------------------------------------- explore def explore( self, query: str, geo: Optional[str] = None, date: Optional[str] = None, search_type: Optional[str] = None, related_queries: bool = True, related_topics: bool = False, resolution: Optional[str] = None, ) -> Dict[str, Any]: token = self._resolve_token() if not token: return self._missing_token() keyword = (query or "").strip() if not keyword: return {"status": "error", "errorMessage": "Type a keyword, e.g. 'bitcoin'."} if "," in keyword: keyword = split_keywords(keyword)[0] geo_code, date_value = normalize_geo(geo), normalize_date(date) base = self._trends_params(token, keyword, geo_code, date_value, search_type) region = REGION_RESOLUTIONS.get((resolution or "").strip().lower()) main_params = dict(base, region=region) if region else base data = self._get(TRENDS_ENDPOINT, main_params) if "_error" in data: return self._as_error(data) credits = CREDITS_PER_CALL warnings: List[str] = [] timeline = parse_timeline(data) result: Dict[str, Any] = { "status": "success", "query": keyword, "geo": geo_code or "worldwide", "date": date_value, "search_type": search_type or "web", "timeline": timeline, "stats": series_stats(timeline, keyword) or series_stats(timeline, next(iter(timeline[0]["values"]), "")) if timeline else {}, "interest_by_region": data.get("interest_by_region") or [], } for flag, data_type, key in ( (related_queries, "RELATED_QUERIES", "related_queries"), (related_topics, "RELATED_TOPICS", "related_topics"), ): if not flag: continue extra = self._get(TRENDS_ENDPOINT, dict(base, data_type=data_type)) if "_error" in extra: warnings.append(f"{key.replace('_', ' ').capitalize()}: {extra['_error']}") continue credits += CREDITS_PER_CALL result[key] = extra.get(key) or {"top": [], "rising": []} result["credits_used"] = credits if not timeline: warnings.append("Google Trends has too little data for this keyword, place and time range.") if warnings: result["warnings"] = warnings return result # -------------------------------------------------------------- compare def compare( self, keywords: str, geo: Optional[str] = None, date: Optional[str] = None, search_type: Optional[str] = None, ) -> Dict[str, Any]: token = self._resolve_token() if not token: return self._missing_token() terms = split_keywords(keywords) warnings: List[str] = [] if len(terms) < 2: return {"status": "error", "errorMessage": "Enter at least two keywords separated by commas."} if len(terms) > MAX_COMPARE: warnings.append(f"Google Trends compares up to {MAX_COMPARE} keywords; the rest were dropped.") terms = terms[:MAX_COMPARE] geo_code, date_value = normalize_geo(geo), normalize_date(date) data = self._get( TRENDS_ENDPOINT, self._trends_params(token, ",".join(terms), geo_code, date_value, search_type) ) if "_error" in data: return self._as_error(data) credits = CREDITS_PER_CALL timeline = parse_timeline(data) regions = data.get("interest_by_region") or [] returned = {q.lower() for row in timeline for q in row["values"]} scaled_separately = False if not {t.lower() for t in terms} <= returned: # The combined request didn't come back as separate lines, so fetch # each keyword on its own. Each line is then scaled to its own peak. scaled_separately = True regions = [] merged: Dict[int, Dict[str, Any]] = {} for term in terms: one = self._get( TRENDS_ENDPOINT, dict(self._trends_params(token, term, geo_code, date_value, search_type), data_type="TIMESERIES"), ) if "_error" in one: warnings.append(f"“{term}”: {one['_error']}") continue credits += CREDITS_PER_CALL for row in parse_timeline(one): slot = merged.setdefault(row["timestamp"], {"date": row["date"], "timestamp": row["timestamp"], "values": {}}) value = next(iter(row["values"].values()), None) if value is not None: slot["values"][term] = value timeline = [merged[k] for k in sorted(merged)] warnings.append( "These keywords came back as separate requests, so each line is scaled to its own peak " "(100 = that keyword's busiest point). Compare shapes and timing, not heights." ) else: # Use the exact spelling Google echoed back. echoed = {q.lower(): q for row in timeline for q in row["values"]} terms = [echoed.get(t.lower(), t) for t in terms] stats = {term: series_stats(timeline, term) for term in terms} averages = {term: s.get("average") for term, s in stats.items() if s} result: Dict[str, Any] = { "status": "success", "keywords": terms, "geo": geo_code or "worldwide", "date": date_value, "search_type": search_type or "web", "timeline": timeline, "stats": stats, "averages": averages, "interest_by_region": regions, "scaled_separately": scaled_separately, "credits_used": credits, } if averages and not scaled_separately: result["leader"] = max(averages, key=lambda k: averages[k] or 0) if warnings: result["warnings"] = warnings return result # ------------------------------------------------------- related topics def related_topics( self, query: str, geo: Optional[str] = None, date: Optional[str] = None, search_type: Optional[str] = None ) -> Dict[str, Any]: token = self._resolve_token() if not token: return self._missing_token() keyword = (query or "").strip() if not keyword: return {"status": "error", "errorMessage": "Type a keyword, e.g. 'bitcoin'."} base = self._trends_params(token, keyword, normalize_geo(geo), normalize_date(date), search_type) data = self._get(TRENDS_ENDPOINT, dict(base, data_type="RELATED_TOPICS")) if "_error" in data: return self._as_error(data) return { "status": "success", "query": keyword, "related_topics": data.get("related_topics") or {"top": [], "rising": []}, "credits_used": CREDITS_PER_CALL, } # ------------------------------------------------------------- trending def trending( self, geo: Optional[str] = "US", hours: Optional[int] = 24, category: Optional[int] = 0, sort: Optional[str] = "relevance", active_only: bool = False, ) -> Dict[str, Any]: token = self._resolve_token() if not token: return self._missing_token() geo_code = normalize_geo(geo) or "US" geo_code = geo_code.split("-")[0] hours_value = int(hours or 24) if hours_value not in TRENDING_HOURS: hours_value = min(TRENDING_HOURS, key=lambda h: abs(h - hours_value)) try: cat = int(category or 0) except (TypeError, ValueError): cat = 0 if cat not in TRENDING_CATEGORIES: cat = 0 sort_value = sort if sort in TRENDING_SORTS else "relevance" params = { "token": token, "geo": geo_code, "hl": "en", "hours": hours_value, "cat": cat, "sort": sort_value, "status": "active" if active_only else "all", } data = self._get(TRENDING_ENDPOINT, params) if "_error" in data: return self._as_error(data) trends = [t for t in data.get("trends") or [] if isinstance(t, dict)] for trend in trends: trend["categories"] = [TRENDING_CATEGORIES.get(c, str(c)) for c in trend.get("category_ids") or []] return { "status": "success", "geo": geo_code, "hours": hours_value, "category": TRENDING_CATEGORIES[cat], "trends": trends, "credits_used": CREDITS_PER_CALL, } # -------------------------------------------------------------- forward def forward( self, mode: str, query: Optional[str] = None, geo: Optional[str] = None, date: Optional[str] = None, search_type: Optional[str] = None, category: Optional[int] = None, hours: Optional[int] = None, ) -> str: mode_key = (mode or "").strip().lower() if mode_key in ("explore", "interest", "keyword"): result = self.explore(query or "", geo, date, search_type, related_queries=True) elif mode_key in ("compare", "comparison"): result = self.compare(query or "", geo, date, search_type) elif mode_key in ("related_topics", "topics"): result = self.related_topics(query or "", geo, date, search_type) elif mode_key in ("trending", "trending_now", "trends"): result = self.trending(geo or "US", hours or 24, category or 0) else: result = { "status": "error", "errorMessage": "mode must be 'explore', 'compare', 'related_topics' or 'trending'.", } return json.dumps(result, ensure_ascii=False, indent=2)