from __future__ import annotations
import csv
import html
import tempfile
from typing import Any, Dict, List, Optional, Tuple
from urllib.parse import urlparse
import gradio as gr
from tool import COUNTRIES, SCRAPE_DO_SIGNUP_URL, TIME_PERIODS, ScrapeDoGoogleSearchTool
# ZeroGPU hardware refuses to keep a Space alive unless it finds at least one
# @spaces.GPU function at startup. This app is pure HTTP and never touches a GPU,
# so the probe below exists only to satisfy that check. It is never called.
try:
import spaces
@spaces.GPU(duration=1)
def _zerogpu_probe() -> str:
return "ok"
except ImportError:
pass
COUNTRY_CHOICES = [(label, gl) for label, gl, _, _ in COUNTRIES]
TIME_CHOICES = [("Any time", "")] + [
(value.replace("_", " ").capitalize(), value) for value in TIME_PERIODS
]
NO_TOKEN_MD = (
"### Paste your Scrape.do API token above\n\n"
f"No token yet? [Create a free account]({SCRAPE_DO_SIGNUP_URL}) — "
"1,000 free credits a month, no credit card. One search costs 10 credits."
)
# --------------------------------------------------------------------- helpers
def _esc(value: Any) -> str:
return html.escape(str(value or ""), quote=True)
def _safe_link(url: Any) -> str:
url = str(url or "")
return url if url.startswith(("http://", "https://")) else "#"
def _domain(url: Any) -> str:
try:
host = urlparse(str(url or "")).netloc.lower()
except ValueError:
return ""
return host[4:] if host.startswith("www.") else host
def _tool_or_none(token: str) -> Optional[ScrapeDoGoogleSearchTool]:
if not token or not token.strip():
return None
return ScrapeDoGoogleSearchTool(api_token=token.strip())
def _write_csv(rows: List[Dict[str, Any]], prefix: str) -> Optional[str]:
if not rows:
return None
fieldnames: List[str] = []
for row in rows:
for key in row:
if key not in fieldnames:
fieldnames.append(key)
handle = tempfile.NamedTemporaryFile(
mode="w", suffix=".csv", prefix=f"{prefix}-", delete=False,
newline="", encoding="utf-8-sig",
)
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
for row in rows:
writer.writerow({key: row.get(key, "") for key in fieldnames})
handle.close()
return handle.name
def _error_md(data: Dict[str, Any]) -> str:
out = f"**That search didn't go through.**\n\n{data.get('errorMessage') or 'Unknown error'}"
if data.get("hint"):
out += f"\n\n{data['hint']}"
return out
# ------------------------------------------------------------ SERP rendering
def _render_ai_blocks(blocks: List[Dict[str, Any]]) -> str:
parts: List[str] = []
def render_list(items: List[Dict[str, Any]]) -> str:
inner = ""
for item in items or []:
inner += f"
{_esc(item.get('snippet'))}"
if item.get("list"):
inner += render_list(item["list"])
inner += ""
return f""
for block in blocks or []:
kind = block.get("type")
if kind == "heading":
parts.append(f"{_esc(block.get('snippet'))}
")
elif kind == "list":
if block.get("snippet"):
parts.append(f"{_esc(block['snippet'])}
")
parts.append(render_list(block.get("list") or []))
elif kind == "table":
head = "".join(f"{_esc(h)} | " for h in block.get("table_headers") or [])
rows = "".join(
"" + "".join(f"| {_esc(c)} | " for c in row) + "
"
for row in block.get("table_rows") or []
)
parts.append(f"")
else:
parts.append(f"{_esc(block.get('snippet'))}
")
return "".join(parts)
def _render_ad(ad: Dict[str, Any]) -> str:
link = _safe_link(ad.get("url") or ad.get("link"))
return (
''
f'
{_esc(ad.get("displayed_url") or _domain(link))}
'
f'
{_esc(ad.get("title"))}'
f'
{_esc(ad.get("description"))}
'
"
"
)
def render_serp(data: Dict[str, Any], highlight_domain: str = "") -> str:
info = data.get("search_information") or {}
out: List[str] = ['']
stats = []
if info.get("total_results"):
stats.append(f"About {int(info['total_results']):,} results")
if info.get("time_taken_displayed"):
stats.append(f"({info['time_taken_displayed']} seconds)")
if stats:
out.append(f'
{_esc(" ".join(stats))}
')
if info.get("results_for"):
out.append(f'
Showing results for {_esc(info["results_for"])}
')
ai = data.get("ai_overview")
if isinstance(ai, dict) and ai.get("state") == "complete" and ai.get("text_blocks"):
refs = "".join(
f'
'
f'{_esc(ref.get("title") or ref.get("source"))} {_esc(ref.get("source"))}'
for ref in (ai.get("references") or [])[:8]
)
out.append(
'
✦ AI Overview
'
f"{_render_ai_blocks(ai['text_blocks'])}"
+ (f'
' if refs else "")
+ "
"
)
for ad in data.get("top_ads") or []:
out.append(_render_ad(ad))
kg = data.get("knowledge_graph")
if isinstance(kg, dict) and kg.get("title"):
facts = "".join(
f"
{_esc(f.get('label'))}: {_esc(f.get('value'))}
"
for f in (kg.get("facts") or [])[:6]
)
out.append(
'
'
f"
{_esc(kg.get('title'))}
"
f"
{_esc(kg.get('type'))}
"
f"
{_esc(kg.get('description'))}
{facts}
"
)
local = data.get("local_results")
if isinstance(local, dict) and local.get("places"):
places = "".join(
f"
{_esc(p.get('title'))} · {_esc(p.get('rating') or '—')}★ {_esc(p.get('price'))}"
for p in local["places"][:5]
)
out.append(f'
')
paa = data.get("related_questions") or []
organic = data.get("organic_results") or []
highlight = highlight_domain.lower().removeprefix("www.")
for index, result in enumerate(organic):
link = _safe_link(result.get("link"))
is_hit = bool(highlight) and (
_domain(link) == highlight or _domain(link).endswith("." + highlight)
)
cls = "gs-result gs-hit" if is_hit else "gs-result"
page_badge = (
f'
#{result.get("overall_position", index + 1)}'
)
sitelinks = ""
sl = result.get("sitelinks") or {}
sl_items = (sl.get("inline") or []) + (sl.get("expanded") or [])
if sl_items:
sitelinks = '
"
date = f'
{_esc(result.get("date"))} — ' if result.get("date") else ""
out.append(
f'
'
f'
{page_badge}{_esc(result.get("source") or _domain(link))}'
f'{_esc(result.get("displayed_link") or link)}
'
f'
{_esc(result.get("title"))}'
f'
{date}{_esc(result.get("snippet"))}
{sitelinks}
'
)
# Google puts People Also Ask after the first few results.
if index == 2 and paa:
qs = "".join(f"
{_esc(q.get('question'))}" for q in paa[:6])
out.append(f'
')
paa = []
if paa:
qs = "".join(f"
{_esc(q.get('question'))}" for q in paa[:6])
out.append(f'
')
for ad in data.get("bottom_ads") or []:
out.append(_render_ad(ad))
related = data.get("related_searches") or []
if related:
chips = "".join(f'
🔍 {_esc(r.get("query"))}' for r in related[:10])
out.append(f'
People also search for
{chips}
')
if not organic:
out.append('
No organic results came back for this query.
')
out.append("
")
return "".join(out)
def _organic_rows(data: Dict[str, Any]) -> List[Dict[str, Any]]:
return [
{
"position": r.get("overall_position"),
"page": r.get("page"),
"title": r.get("title"),
"link": r.get("link"),
"domain": _domain(r.get("link")),
"snippet": r.get("snippet"),
"date": r.get("date"),
"source": r.get("source"),
}
for r in data.get("organic_results") or []
]
# ---------------------------------------------------------------- handlers
TABLE_HEADERS = ["#", "Domain", "Title", "Link"]
def run_search(
token: str, query: str, country: str, pages: int, device: str, time_period: str, location: str
) -> Tuple[str, str, List[List[Any]], Dict[str, Any], Optional[str]]:
tool = _tool_or_none(token)
if tool is None:
return NO_TOKEN_MD, "", [], {}, None
if not query or not query.strip():
return "Type a search query to begin.", "", [], {}, None
data = tool.search(query, country, int(pages or 1), device, time_period or None, location)
if data.get("status") != "success":
return _error_md(data), "", [], data, None
rows = _organic_rows(data)
counts = [
f"**{len(rows)}** organic results",
f"**{len(data.get('top_ads') or []) + len(data.get('bottom_ads') or [])}** ads",
f"**{len(data.get('related_questions') or [])}** People Also Ask",
f"**{len(data.get('related_searches') or [])}** related searches",
]
ai = data.get("ai_overview")
counts.append("AI Overview **yes**" if isinstance(ai, dict) and ai.get("state") == "complete" else "AI Overview **no**")
summary = (
" · ".join(counts)
+ f"\n\n{data.get('pages_fetched')} page(s) fetched · {data.get('credits_used')} credits used"
)
if data.get("warnings"):
summary += "\n\n" + "\n".join(f"- {w}" for w in data["warnings"])
table = [[r["position"], r["domain"], (r["title"] or "")[:90], r["link"]] for r in rows]
csv_path = _write_csv(rows, "google-serp")
return summary, render_serp(data), table, data, csv_path
def run_rank_check(
token: str, query: str, target_domain: str, country: str, pages: int,
device: str, time_period: str, location: str,
) -> Tuple[str, str, Dict[str, Any]]:
tool = _tool_or_none(token)
if tool is None:
return NO_TOKEN_MD, "", {}
if not query or not query.strip():
return "Type the keyword you want to check.", "", {}
domain = _domain(target_domain if "://" in (target_domain or "") else f"https://{(target_domain or '').strip()}")
if not domain:
return "Enter the domain to look for, e.g. `scrape.do`.", "", {}
data = tool.search(query, country, int(pages or 1), device, time_period or None, location)
if data.get("status") != "success":
return _error_md(data), "", data
organic = data.get("organic_results") or []
hits = [
r for r in organic
if _domain(r.get("link")) == domain or _domain(r.get("link")).endswith("." + domain)
]
looked = len(organic)
country_label = dict((gl, label) for label, gl in COUNTRY_CHOICES).get(country, country)
if hits:
best = hits[0]
lines = [
f"# #{best.get('overall_position')}",
f"**{domain}** ranks **#{best.get('overall_position')}** for “{query.strip()}” "
f"in {country_label} ({device}), page {best.get('page')}.",
]
if len(hits) > 1:
others = ", ".join(f"#{h.get('overall_position')}" for h in hits[1:])
lines.append(f"It also appears at {others}.")
above = [r for r in organic if r.get("overall_position", 0) < best.get("overall_position", 0)]
if above:
lines += ["", "**Ranking above you:**"]
lines += [f"{r.get('overall_position')}. {_domain(r.get('link'))}" for r in above[:10]]
else:
lines = [
"# Not found",
f"**{domain}** is not in the top {looked} organic results for “{query.strip()}” "
f"in {country_label}. Fetch more pages to look deeper.",
]
lines.append(f"\n{data.get('credits_used')} credits used.")
return "\n".join(lines), render_serp(data, highlight_domain=domain), data
# ----------------------------------------------------------------------- UI
STYLE = """
"""
HERO = f"""
Google Search Scraper
Any Google results page as structured data — organic rankings, ads, People Also Ask,
related searches, the AI Overview, knowledge panel and local pack, from any country and in any language.
No selectors to maintain: Google changes its page code constantly, so this uses the
Scrape.do Google Search API,
which returns the same clean fields whatever Google ships. Check where a site ranks, watch competitors' ads,
or feed live results to an AI agent.
Get a free API token →
"""
CREDIT_NOTE = """
Each results page costs 10 credits, so the free 1,000 credits cover about 100 searches.
If Google is still writing its AI Overview when the page comes back, it's fetched with one
follow-up call for 5 more credits.
"""
FOOTER_MD = f"""
---
### Use it as a smolagents Tool
```python
from smolagents import CodeAgent, InferenceClientModel
from tool import ScrapeDoGoogleSearchTool
agent = CodeAgent(
tools=[ScrapeDoGoogleSearchTool(api_token="YOUR_SCRAPEDO_TOKEN")],
model=InferenceClientModel(),
)
agent.run("Who ranks in the top 5 for 'best web scraping api' in Germany, and do any run ads?")
```
Or set `SCRAPE_DO_API_TOKEN` in the environment and instantiate with no arguments.
Want to parse the raw HTML yourself instead? The
[Google search scraping guide](https://scrape.do/blog/scraping-google-search-results/) covers both approaches.
[Scrape.do](https://scrape.do/) · [Google Search API docs](https://scrape.do/documentation/google-scraper-api/search/) · [Sign up free]({SCRAPE_DO_SIGNUP_URL})
"""
def build_demo() -> gr.Blocks:
with gr.Blocks(title="Google Search Scraper — Scrape.do") as demo:
gr.HTML(STYLE)
gr.HTML(HERO)
api_token = gr.Textbox(
label="Scrape.do API token",
placeholder="Paste your token here",
type="password",
info="Free tier: 1,000 credits a month, about 100 searches.",
)
with gr.Row():
country = gr.Dropdown(label="Search from", choices=COUNTRY_CHOICES, value="us")
device = gr.Radio(label="Device", choices=["desktop", "mobile"], value="desktop")
pages = gr.Slider(label="Result pages", minimum=1, maximum=5, step=1, value=1,
info="10 credits per page.")
with gr.Accordion("More filters", open=False):
with gr.Row():
time_period = gr.Dropdown(label="Time", choices=TIME_CHOICES, value="")
location = gr.Textbox(
label="City (optional)",
placeholder="Austin,Texas,United States",
info="Google's own place name. Must be in the country above.",
)
gr.HTML(CREDIT_NOTE)
with gr.Tabs():
with gr.Tab("Search results"):
query = gr.Textbox(label="Search Google", placeholder="best web scraping api")
search_btn = gr.Button("Google Search", variant="primary")
search_summary = gr.Markdown()
serp_view = gr.HTML()
serp_table = gr.Dataframe(headers=TABLE_HEADERS, wrap=True, label="Organic results")
serp_csv = gr.File(label="Download CSV")
with gr.Accordion("Raw JSON response", open=False):
serp_json = gr.JSON()
search_btn.click(
fn=run_search,
inputs=[api_token, query, country, pages, device, time_period, location],
outputs=[search_summary, serp_view, serp_table, serp_json, serp_csv],
)
query.submit(
fn=run_search,
inputs=[api_token, query, country, pages, device, time_period, location],
outputs=[search_summary, serp_view, serp_table, serp_json, serp_csv],
)
with gr.Tab("Rank checker"):
gr.Markdown(
"Where does a website rank for a keyword? Enter both, and the matching "
"result is highlighted on the page with everyone ranking above it."
)
with gr.Row():
rank_query = gr.Textbox(label="Keyword", placeholder="web scraping api", scale=3)
rank_domain = gr.Textbox(label="Your domain", placeholder="scrape.do", scale=2)
rank_btn = gr.Button("Check ranking", variant="primary")
rank_summary = gr.Markdown()
rank_view = gr.HTML()
with gr.Accordion("Raw JSON response", open=False):
rank_json = gr.JSON()
rank_btn.click(
fn=run_rank_check,
inputs=[api_token, rank_query, rank_domain, country, pages, device, time_period, location],
outputs=[rank_summary, rank_view, rank_json],
)
gr.Markdown(FOOTER_MD)
return demo
# Hugging Face Spaces launches the module-level `demo` object rather than
# calling launch() itself, so build it at import time.
demo = build_demo()
if __name__ == "__main__":
demo.launch()