bugrahans's picture
Upload 4 files
a86ae8d verified
Raw History Blame Contribute Delete
21.6 kB
from __future__ import annotations
import json
import os
import re
import time
import urllib.parse
from typing import Any, Dict, List, Optional, Tuple
import requests
from smolagents.tools import Tool
SCRAPE_DO_SIGNUP_URL = (
"https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=google-trends-scraper"
)
TRENDS_ENDPOINT = "https://api.scrape.do/plugin/google/trends"
TRENDING_ENDPOINT = "https://api.scrape.do/plugin/google/trending"
# From the Scrape.do docs: 10 credits per successful call.
CREDITS_PER_CALL = 10
MAX_COMPARE = 5
# (label, geo code). An empty code means worldwide.
COUNTRIES: List[Tuple[str, str]] = [
("Worldwide", ""),
("United States", "US"),
("United Kingdom", "GB"),
("Germany", "DE"),
("France", "FR"),
("Spain", "ES"),
("Italy", "IT"),
("Netherlands", "NL"),
("Turkey", "TR"),
("Canada", "CA"),
("Australia", "AU"),
("India", "IN"),
("Japan", "JP"),
("Brazil", "BR"),
("Mexico", "MX"),
]
# Friendly names accepted for the `date` parameter.
DATE_ALIASES = {
"7d": "now 7-d",
"past 7 days": "now 7-d",
"30d": "today 1-m",
"1m": "today 1-m",
"past 30 days": "today 1-m",
"90d": "today 3-m",
"3m": "today 3-m",
"past 90 days": "today 3-m",
"12m": "today 12-m",
"1y": "today 12-m",
"past 12 months": "today 12-m",
"5y": "today 5-y",
"past 5 years": "today 5-y",
"all": "all",
"2004": "all",
}
SEARCH_TYPES = {
"web": "",
"": "",
"youtube": "youtube",
"news": "news",
"images": "images",
"image": "images",
"shopping": "froogle",
"froogle": "froogle",
}
REGION_RESOLUTIONS = {"country": "COUNTRY", "region": "REGION", "metro": "DMA", "dma": "DMA", "city": "CITY"}
TRENDING_CATEGORIES: Dict[int, str] = {
0: "All categories",
1: "Autos and Vehicles",
2: "Beauty and Fashion",
3: "Business and Finance",
4: "Entertainment",
5: "Food and Drink",
6: "Games",
7: "Health",
8: "Hobbies and Leisure",
9: "Jobs and Education",
10: "Law and Government",
11: "Other",
13: "Pets and Animals",
14: "Politics",
15: "Science",
16: "Shopping",
17: "Sports",
18: "Technology",
19: "Travel and Transportation",
20: "Climate",
}
TRENDING_HOURS = (4, 24, 48, 168)
TRENDING_SORTS = ("relevance", "search_volume", "recency", "title")
# ------------------------------------------------------------------ helpers
def normalize_date(value: Optional[str]) -> str:
text = (value or "").strip()
if not text:
return "today 12-m"
return DATE_ALIASES.get(text.lower(), text)
def normalize_geo(value: Optional[str]) -> str:
text = (value or "").strip().upper()
if text in ("", "WORLDWIDE", "WORLD", "GLOBAL", "ALL"):
return ""
return text
def split_keywords(value: Optional[str]) -> List[str]:
keywords: List[str] = []
for part in re.split(r"[,;\n]+", value or ""):
part = part.strip()
if part and part.lower() not in [k.lower() for k in keywords]:
keywords.append(part)
return keywords
def _number(value: Any) -> Optional[float]:
if isinstance(value, bool):
return None
if isinstance(value, (int, float)):
return float(value)
text = str(value or "").strip().replace(",", "").replace("%", "").replace("+", "")
if text.startswith("<"):
return 0.0
try:
return float(text)
except ValueError:
return None
def parse_timeline(data: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Flatten interest_over_time into [{date, timestamp, values: {query: number}}]."""
timeline = (data.get("interest_over_time") or {}).get("timeline_data") or []
rows: List[Dict[str, Any]] = []
for point in timeline:
if not isinstance(point, dict):
continue
values: Dict[str, float] = {}
for entry in point.get("values") or []:
if not isinstance(entry, dict):
continue
number = _number(entry.get("extracted_value"))
if number is None:
number = _number(entry.get("value"))
values[str(entry.get("query") or "")] = number if number is not None else 0.0
try:
stamp = int(str(point.get("timestamp") or "0"))
except ValueError:
stamp = 0
rows.append({"date": point.get("date"), "timestamp": stamp, "values": values})
return rows
def series_stats(timeline: List[Dict[str, Any]], query: str) -> Dict[str, Any]:
points = [(row, row["values"].get(query)) for row in timeline if row["values"].get(query) is not None]
if not points:
return {}
values = [v for _, v in points]
peak_row, peak = max(points, key=lambda p: p[1])
quarter = max(1, len(values) // 4)
first, last = sum(values[:quarter]) / quarter, sum(values[-quarter:]) / quarter
change = ((last - first) / first * 100) if first > 0 else None
return {
"average": round(sum(values) / len(values), 1),
"peak": peak,
"peak_date": peak_row.get("date"),
"latest": values[-1],
"change_percent": round(change, 1) if change is not None else None,
}
# --------------------------------------------------------------------- tool
class ScrapeDoGoogleTrendsTool(Tool):
name = "google_trends_scraper"
description = """
Get Google Trends data as JSON. Modes:
- 'explore': interest over time (0-100) and interest by region for one
keyword, plus its top and rising related searches.
- 'compare': up to 5 comma-separated keywords on the same 0-100 scale.
- 'related_topics': top and rising related topics for one keyword.
- 'trending': what's trending right now in a country (no query needed),
with search volume, growth and related searches.
Supports any country or region code, time ranges from 7 days to 2004-now,
and web, YouTube, News, Images or Shopping search. Powered by Scrape.do's
Google Trends API, a working alternative to the unmaintained pytrends.
"""
output_type = "string"
def __init__(self, api_token: Optional[str] = None) -> None:
self._api_token = api_token
self.inputs = {
"mode": {
"type": "string",
"description": "'explore', 'compare', 'related_topics' or 'trending'.",
},
"query": {
"type": "string",
"description": (
"The keyword, e.g. 'bitcoin'. For 'compare', up to 5 keywords separated by commas. "
"Not used for 'trending'."
),
"nullable": True,
},
"geo": {
"type": "string",
"description": (
"Country or region code such as 'US', 'GB', 'TR' or 'US-CA'. Empty means worldwide. "
"'trending' needs a country and defaults to 'US'."
),
"nullable": True,
},
"date": {
"type": "string",
"description": (
"Time range: '7d', '30d', '90d', '12m' (default), '5y', 'all', or Google's own format "
"such as 'today 3-m' or '2024-01-01 2024-12-31'."
),
"nullable": True,
},
"search_type": {
"type": "string",
"description": "'web' (default), 'youtube', 'news', 'images' or 'shopping'.",
"nullable": True,
},
"category": {
"type": "integer",
"description": "'trending' only: category id, e.g. 17 Sports, 18 Technology, 3 Business. Defaults to all.",
"nullable": True,
},
"hours": {
"type": "integer",
"description": "'trending' only: time window in hours: 4, 24 (default), 48 or 168.",
"nullable": True,
},
}
super().__init__()
# ------------------------------------------------------------ plumbing
def _resolve_token(self) -> Optional[str]:
if self._api_token:
return self._api_token.strip()
env_token = os.getenv("SCRAPE_DO_API_TOKEN")
return env_token.strip() if env_token else None
@staticmethod
def _missing_token() -> Dict[str, Any]:
return {
"status": "error",
"errorMessage": "Missing Scrape.do API token.",
"hint": f"Get a free token at {SCRAPE_DO_SIGNUP_URL}",
}
@staticmethod
def _as_error(data: Dict[str, Any]) -> Dict[str, Any]:
result = {"status": "error", "errorMessage": data.get("_error", "Unknown error")}
if data.get("_auth"):
result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}"
return result
def _get(self, url: str, params: Dict[str, Any], retries: int = 1) -> Dict[str, Any]:
"""GET a Scrape.do endpoint. Retries once on 5xx, as the docs recommend."""
token = str(params.get("token") or "")
def redact(text: str) -> str:
return text.replace(token, "***") if token else text
clean = {k: v for k, v in params.items() if v not in (None, "")}
full_url = url + "?" + urllib.parse.urlencode(clean, quote_via=urllib.parse.quote)
last: Dict[str, Any] = {"_error": "Request failed."}
for attempt in range(retries + 1):
try:
response = requests.get(full_url, timeout=120)
except requests.exceptions.RequestException as exc:
last = {"_error": redact(f"Request failed: {exc}")}
if attempt < retries:
time.sleep(1.5)
continue
return last
if response.status_code == 401:
return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True}
try:
data = response.json()
except ValueError:
data = None
if response.ok and isinstance(data, dict):
return data
message = ""
if isinstance(data, dict):
parts = [str(data.get(key)).strip().rstrip(".") for key in ("message", "error") if data.get(key)]
message = parts[0] if parts else ""
if not message:
message = (response.text or "").strip()[:200] or "no details"
if response.status_code >= 500:
text = f"Google Trends didn't answer this time (HTTP {response.status_code}): {message}. Try again in a moment."
else:
text = f"Scrape.do returned HTTP {response.status_code}: {message}"
last = {"_error": redact(text), "_status": response.status_code}
if response.status_code in (500, 502, 503, 504) and attempt < retries:
time.sleep(1.5)
continue
return last
return last
def _trends_params(
self, token: str, q: str, geo: str, date: str, search_type: Optional[str], hl: str = "en"
) -> Dict[str, Any]:
gprop = SEARCH_TYPES.get((search_type or "web").strip().lower(), "")
return {"token": token, "q": q, "geo": geo, "hl": hl, "date": date, "gprop": gprop}
# -------------------------------------------------------------- explore
def explore(
self,
query: str,
geo: Optional[str] = None,
date: Optional[str] = None,
search_type: Optional[str] = None,
related_queries: bool = True,
related_topics: bool = False,
resolution: Optional[str] = None,
) -> Dict[str, Any]:
token = self._resolve_token()
if not token:
return self._missing_token()
keyword = (query or "").strip()
if not keyword:
return {"status": "error", "errorMessage": "Type a keyword, e.g. 'bitcoin'."}
if "," in keyword:
keyword = split_keywords(keyword)[0]
geo_code, date_value = normalize_geo(geo), normalize_date(date)
base = self._trends_params(token, keyword, geo_code, date_value, search_type)
region = REGION_RESOLUTIONS.get((resolution or "").strip().lower())
main_params = dict(base, region=region) if region else base
data = self._get(TRENDS_ENDPOINT, main_params)
if "_error" in data:
return self._as_error(data)
credits = CREDITS_PER_CALL
warnings: List[str] = []
timeline = parse_timeline(data)
result: Dict[str, Any] = {
"status": "success",
"query": keyword,
"geo": geo_code or "worldwide",
"date": date_value,
"search_type": search_type or "web",
"timeline": timeline,
"stats": series_stats(timeline, keyword) or series_stats(timeline, next(iter(timeline[0]["values"]), "")) if timeline else {},
"interest_by_region": data.get("interest_by_region") or [],
}
for flag, data_type, key in (
(related_queries, "RELATED_QUERIES", "related_queries"),
(related_topics, "RELATED_TOPICS", "related_topics"),
):
if not flag:
continue
extra = self._get(TRENDS_ENDPOINT, dict(base, data_type=data_type))
if "_error" in extra:
warnings.append(f"{key.replace('_', ' ').capitalize()}: {extra['_error']}")
continue
credits += CREDITS_PER_CALL
result[key] = extra.get(key) or {"top": [], "rising": []}
result["credits_used"] = credits
if not timeline:
warnings.append("Google Trends has too little data for this keyword, place and time range.")
if warnings:
result["warnings"] = warnings
return result
# -------------------------------------------------------------- compare
def compare(
self,
keywords: str,
geo: Optional[str] = None,
date: Optional[str] = None,
search_type: Optional[str] = None,
) -> Dict[str, Any]:
token = self._resolve_token()
if not token:
return self._missing_token()
terms = split_keywords(keywords)
warnings: List[str] = []
if len(terms) < 2:
return {"status": "error", "errorMessage": "Enter at least two keywords separated by commas."}
if len(terms) > MAX_COMPARE:
warnings.append(f"Google Trends compares up to {MAX_COMPARE} keywords; the rest were dropped.")
terms = terms[:MAX_COMPARE]
geo_code, date_value = normalize_geo(geo), normalize_date(date)
data = self._get(
TRENDS_ENDPOINT, self._trends_params(token, ",".join(terms), geo_code, date_value, search_type)
)
if "_error" in data:
return self._as_error(data)
credits = CREDITS_PER_CALL
timeline = parse_timeline(data)
regions = data.get("interest_by_region") or []
returned = {q.lower() for row in timeline for q in row["values"]}
scaled_separately = False
if not {t.lower() for t in terms} <= returned:
# The combined request didn't come back as separate lines, so fetch
# each keyword on its own. Each line is then scaled to its own peak.
scaled_separately = True
regions = []
merged: Dict[int, Dict[str, Any]] = {}
for term in terms:
one = self._get(
TRENDS_ENDPOINT,
dict(self._trends_params(token, term, geo_code, date_value, search_type), data_type="TIMESERIES"),
)
if "_error" in one:
warnings.append(f"“{term}”: {one['_error']}")
continue
credits += CREDITS_PER_CALL
for row in parse_timeline(one):
slot = merged.setdefault(row["timestamp"], {"date": row["date"], "timestamp": row["timestamp"], "values": {}})
value = next(iter(row["values"].values()), None)
if value is not None:
slot["values"][term] = value
timeline = [merged[k] for k in sorted(merged)]
warnings.append(
"These keywords came back as separate requests, so each line is scaled to its own peak "
"(100 = that keyword's busiest point). Compare shapes and timing, not heights."
)
else:
# Use the exact spelling Google echoed back.
echoed = {q.lower(): q for row in timeline for q in row["values"]}
terms = [echoed.get(t.lower(), t) for t in terms]
stats = {term: series_stats(timeline, term) for term in terms}
averages = {term: s.get("average") for term, s in stats.items() if s}
result: Dict[str, Any] = {
"status": "success",
"keywords": terms,
"geo": geo_code or "worldwide",
"date": date_value,
"search_type": search_type or "web",
"timeline": timeline,
"stats": stats,
"averages": averages,
"interest_by_region": regions,
"scaled_separately": scaled_separately,
"credits_used": credits,
}
if averages and not scaled_separately:
result["leader"] = max(averages, key=lambda k: averages[k] or 0)
if warnings:
result["warnings"] = warnings
return result
# ------------------------------------------------------- related topics
def related_topics(
self, query: str, geo: Optional[str] = None, date: Optional[str] = None, search_type: Optional[str] = None
) -> Dict[str, Any]:
token = self._resolve_token()
if not token:
return self._missing_token()
keyword = (query or "").strip()
if not keyword:
return {"status": "error", "errorMessage": "Type a keyword, e.g. 'bitcoin'."}
base = self._trends_params(token, keyword, normalize_geo(geo), normalize_date(date), search_type)
data = self._get(TRENDS_ENDPOINT, dict(base, data_type="RELATED_TOPICS"))
if "_error" in data:
return self._as_error(data)
return {
"status": "success",
"query": keyword,
"related_topics": data.get("related_topics") or {"top": [], "rising": []},
"credits_used": CREDITS_PER_CALL,
}
# ------------------------------------------------------------- trending
def trending(
self,
geo: Optional[str] = "US",
hours: Optional[int] = 24,
category: Optional[int] = 0,
sort: Optional[str] = "relevance",
active_only: bool = False,
) -> Dict[str, Any]:
token = self._resolve_token()
if not token:
return self._missing_token()
geo_code = normalize_geo(geo) or "US"
geo_code = geo_code.split("-")[0]
hours_value = int(hours or 24)
if hours_value not in TRENDING_HOURS:
hours_value = min(TRENDING_HOURS, key=lambda h: abs(h - hours_value))
try:
cat = int(category or 0)
except (TypeError, ValueError):
cat = 0
if cat not in TRENDING_CATEGORIES:
cat = 0
sort_value = sort if sort in TRENDING_SORTS else "relevance"
params = {
"token": token, "geo": geo_code, "hl": "en", "hours": hours_value, "cat": cat,
"sort": sort_value, "status": "active" if active_only else "all",
}
data = self._get(TRENDING_ENDPOINT, params)
if "_error" in data:
return self._as_error(data)
trends = [t for t in data.get("trends") or [] if isinstance(t, dict)]
for trend in trends:
trend["categories"] = [TRENDING_CATEGORIES.get(c, str(c)) for c in trend.get("category_ids") or []]
return {
"status": "success",
"geo": geo_code,
"hours": hours_value,
"category": TRENDING_CATEGORIES[cat],
"trends": trends,
"credits_used": CREDITS_PER_CALL,
}
# -------------------------------------------------------------- forward
def forward(
self,
mode: str,
query: Optional[str] = None,
geo: Optional[str] = None,
date: Optional[str] = None,
search_type: Optional[str] = None,
category: Optional[int] = None,
hours: Optional[int] = None,
) -> str:
mode_key = (mode or "").strip().lower()
if mode_key in ("explore", "interest", "keyword"):
result = self.explore(query or "", geo, date, search_type, related_queries=True)
elif mode_key in ("compare", "comparison"):
result = self.compare(query or "", geo, date, search_type)
elif mode_key in ("related_topics", "topics"):
result = self.related_topics(query or "", geo, date, search_type)
elif mode_key in ("trending", "trending_now", "trends"):
result = self.trending(geo or "US", hours or 24, category or 0)
else:
result = {
"status": "error",
"errorMessage": "mode must be 'explore', 'compare', 'related_topics' or 'trending'.",
}
return json.dumps(result, ensure_ascii=False, indent=2)