bugrahans's picture
Upload 4 files
4aeb13f verified
Raw History Blame Contribute Delete
23.9 kB
from __future__ import annotations
import csv
import html
import tempfile
from typing import Any, Dict, List, Optional, Tuple
from urllib.parse import urlparse
import gradio as gr
from tool import COUNTRIES, SCRAPE_DO_SIGNUP_URL, TIME_PERIODS, ScrapeDoGoogleSearchTool
# ZeroGPU hardware refuses to keep a Space alive unless it finds at least one
# @spaces.GPU function at startup. This app is pure HTTP and never touches a GPU,
# so the probe below exists only to satisfy that check. It is never called.
try:
import spaces
@spaces.GPU(duration=1)
def _zerogpu_probe() -> str:
return "ok"
except ImportError:
pass
COUNTRY_CHOICES = [(label, gl) for label, gl, _, _ in COUNTRIES]
TIME_CHOICES = [("Any time", "")] + [
(value.replace("_", " ").capitalize(), value) for value in TIME_PERIODS
]
NO_TOKEN_MD = (
"### Paste your Scrape.do API token above\n\n"
f"No token yet? [Create a free account]({SCRAPE_DO_SIGNUP_URL}) — "
"1,000 free credits a month, no credit card. One search costs 10 credits."
)
# --------------------------------------------------------------------- helpers
def _esc(value: Any) -> str:
return html.escape(str(value or ""), quote=True)
def _safe_link(url: Any) -> str:
url = str(url or "")
return url if url.startswith(("http://", "https://")) else "#"
def _domain(url: Any) -> str:
try:
host = urlparse(str(url or "")).netloc.lower()
except ValueError:
return ""
return host[4:] if host.startswith("www.") else host
def _tool_or_none(token: str) -> Optional[ScrapeDoGoogleSearchTool]:
if not token or not token.strip():
return None
return ScrapeDoGoogleSearchTool(api_token=token.strip())
def _write_csv(rows: List[Dict[str, Any]], prefix: str) -> Optional[str]:
if not rows:
return None
fieldnames: List[str] = []
for row in rows:
for key in row:
if key not in fieldnames:
fieldnames.append(key)
handle = tempfile.NamedTemporaryFile(
mode="w", suffix=".csv", prefix=f"{prefix}-", delete=False,
newline="", encoding="utf-8-sig",
)
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
for row in rows:
writer.writerow({key: row.get(key, "") for key in fieldnames})
handle.close()
return handle.name
def _error_md(data: Dict[str, Any]) -> str:
out = f"**That search didn't go through.**\n\n{data.get('errorMessage') or 'Unknown error'}"
if data.get("hint"):
out += f"\n\n{data['hint']}"
return out
# ------------------------------------------------------------ SERP rendering
def _render_ai_blocks(blocks: List[Dict[str, Any]]) -> str:
parts: List[str] = []
def render_list(items: List[Dict[str, Any]]) -> str:
inner = ""
for item in items or []:
inner += f"<li>{_esc(item.get('snippet'))}"
if item.get("list"):
inner += render_list(item["list"])
inner += "</li>"
return f"<ul>{inner}</ul>"
for block in blocks or []:
kind = block.get("type")
if kind == "heading":
parts.append(f"<h4>{_esc(block.get('snippet'))}</h4>")
elif kind == "list":
if block.get("snippet"):
parts.append(f"<p>{_esc(block['snippet'])}</p>")
parts.append(render_list(block.get("list") or []))
elif kind == "table":
head = "".join(f"<th>{_esc(h)}</th>" for h in block.get("table_headers") or [])
rows = "".join(
"<tr>" + "".join(f"<td>{_esc(c)}</td>" for c in row) + "</tr>"
for row in block.get("table_rows") or []
)
parts.append(f"<table><thead><tr>{head}</tr></thead><tbody>{rows}</tbody></table>")
else:
parts.append(f"<p>{_esc(block.get('snippet'))}</p>")
return "".join(parts)
def _render_ad(ad: Dict[str, Any]) -> str:
link = _safe_link(ad.get("url") or ad.get("link"))
return (
'<div class="gs-result">'
f'<div class="gs-cite"><span class="gs-sponsored">Sponsored</span> {_esc(ad.get("displayed_url") or _domain(link))}</div>'
f'<a class="gs-title" href="{_esc(link)}" target="_blank" rel="noopener noreferrer">{_esc(ad.get("title"))}</a>'
f'<div class="gs-snippet">{_esc(ad.get("description"))}</div>'
"</div>"
)
def render_serp(data: Dict[str, Any], highlight_domain: str = "") -> str:
info = data.get("search_information") or {}
out: List[str] = ['<div class="gs-serp">']
stats = []
if info.get("total_results"):
stats.append(f"About {int(info['total_results']):,} results")
if info.get("time_taken_displayed"):
stats.append(f"({info['time_taken_displayed']} seconds)")
if stats:
out.append(f'<div class="gs-stats">{_esc(" ".join(stats))}</div>')
if info.get("results_for"):
out.append(f'<div class="gs-stats">Showing results for <b>{_esc(info["results_for"])}</b></div>')
ai = data.get("ai_overview")
if isinstance(ai, dict) and ai.get("state") == "complete" and ai.get("text_blocks"):
refs = "".join(
f'<li><a href="{_esc(_safe_link(ref.get("link")))}" target="_blank" rel="noopener noreferrer">'
f'{_esc(ref.get("title") or ref.get("source"))}</a> <span>{_esc(ref.get("source"))}</span></li>'
for ref in (ai.get("references") or [])[:8]
)
out.append(
'<div class="gs-ai"><div class="gs-ai-label">✦ AI Overview</div>'
f"{_render_ai_blocks(ai['text_blocks'])}"
+ (f'<div class="gs-ai-refs"><b>Sources</b><ol>{refs}</ol></div>' if refs else "")
+ "</div>"
)
for ad in data.get("top_ads") or []:
out.append(_render_ad(ad))
kg = data.get("knowledge_graph")
if isinstance(kg, dict) and kg.get("title"):
facts = "".join(
f"<div><b>{_esc(f.get('label'))}:</b> {_esc(f.get('value'))}</div>"
for f in (kg.get("facts") or [])[:6]
)
out.append(
'<div class="gs-kg">'
f"<div class=\"gs-kg-title\">{_esc(kg.get('title'))}</div>"
f"<div class=\"gs-kg-type\">{_esc(kg.get('type'))}</div>"
f"<p>{_esc(kg.get('description'))}</p>{facts}</div>"
)
local = data.get("local_results")
if isinstance(local, dict) and local.get("places"):
places = "".join(
f"<li><b>{_esc(p.get('title'))}</b> · {_esc(p.get('rating') or '—')}★ {_esc(p.get('price'))}</li>"
for p in local["places"][:5]
)
out.append(f'<div class="gs-box"><div class="gs-box-title">Places</div><ul>{places}</ul></div>')
paa = data.get("related_questions") or []
organic = data.get("organic_results") or []
highlight = highlight_domain.lower().removeprefix("www.")
for index, result in enumerate(organic):
link = _safe_link(result.get("link"))
is_hit = bool(highlight) and (
_domain(link) == highlight or _domain(link).endswith("." + highlight)
)
cls = "gs-result gs-hit" if is_hit else "gs-result"
page_badge = (
f'<span class="gs-pos">#{result.get("overall_position", index + 1)}</span>'
)
sitelinks = ""
sl = result.get("sitelinks") or {}
sl_items = (sl.get("inline") or []) + (sl.get("expanded") or [])
if sl_items:
sitelinks = '<div class="gs-sitelinks">' + " · ".join(
f'<a href="{_esc(_safe_link(s.get("link")))}" target="_blank" rel="noopener noreferrer">{_esc(s.get("title"))}</a>'
for s in sl_items[:4]
) + "</div>"
date = f'<span class="gs-date">{_esc(result.get("date"))} — </span>' if result.get("date") else ""
out.append(
f'<div class="{cls}">'
f'<div class="gs-cite">{page_badge}<span class="gs-source">{_esc(result.get("source") or _domain(link))}</span>'
f'<span class="gs-url">{_esc(result.get("displayed_link") or link)}</span></div>'
f'<a class="gs-title" href="{_esc(link)}" target="_blank" rel="noopener noreferrer">{_esc(result.get("title"))}</a>'
f'<div class="gs-snippet">{date}{_esc(result.get("snippet"))}</div>{sitelinks}</div>'
)
# Google puts People Also Ask after the first few results.
if index == 2 and paa:
qs = "".join(f"<li>{_esc(q.get('question'))}</li>" for q in paa[:6])
out.append(f'<div class="gs-box"><div class="gs-box-title">People also ask</div><ul class="gs-paa">{qs}</ul></div>')
paa = []
if paa:
qs = "".join(f"<li>{_esc(q.get('question'))}</li>" for q in paa[:6])
out.append(f'<div class="gs-box"><div class="gs-box-title">People also ask</div><ul class="gs-paa">{qs}</ul></div>')
for ad in data.get("bottom_ads") or []:
out.append(_render_ad(ad))
related = data.get("related_searches") or []
if related:
chips = "".join(f'<span class="gs-chip">🔍 {_esc(r.get("query"))}</span>' for r in related[:10])
out.append(f'<div class="gs-box"><div class="gs-box-title">People also search for</div><div class="gs-chips">{chips}</div></div>')
if not organic:
out.append('<div class="gs-stats">No organic results came back for this query.</div>')
out.append("</div>")
return "".join(out)
def _organic_rows(data: Dict[str, Any]) -> List[Dict[str, Any]]:
return [
{
"position": r.get("overall_position"),
"page": r.get("page"),
"title": r.get("title"),
"link": r.get("link"),
"domain": _domain(r.get("link")),
"snippet": r.get("snippet"),
"date": r.get("date"),
"source": r.get("source"),
}
for r in data.get("organic_results") or []
]
# ---------------------------------------------------------------- handlers
TABLE_HEADERS = ["#", "Domain", "Title", "Link"]
def run_search(
token: str, query: str, country: str, pages: int, device: str, time_period: str, location: str
) -> Tuple[str, str, List[List[Any]], Dict[str, Any], Optional[str]]:
tool = _tool_or_none(token)
if tool is None:
return NO_TOKEN_MD, "", [], {}, None
if not query or not query.strip():
return "Type a search query to begin.", "", [], {}, None
data = tool.search(query, country, int(pages or 1), device, time_period or None, location)
if data.get("status") != "success":
return _error_md(data), "", [], data, None
rows = _organic_rows(data)
counts = [
f"**{len(rows)}** organic results",
f"**{len(data.get('top_ads') or []) + len(data.get('bottom_ads') or [])}** ads",
f"**{len(data.get('related_questions') or [])}** People Also Ask",
f"**{len(data.get('related_searches') or [])}** related searches",
]
ai = data.get("ai_overview")
counts.append("AI Overview **yes**" if isinstance(ai, dict) and ai.get("state") == "complete" else "AI Overview **no**")
summary = (
" · ".join(counts)
+ f"\n\n{data.get('pages_fetched')} page(s) fetched · {data.get('credits_used')} credits used"
)
if data.get("warnings"):
summary += "\n\n" + "\n".join(f"- {w}" for w in data["warnings"])
table = [[r["position"], r["domain"], (r["title"] or "")[:90], r["link"]] for r in rows]
csv_path = _write_csv(rows, "google-serp")
return summary, render_serp(data), table, data, csv_path
def run_rank_check(
token: str, query: str, target_domain: str, country: str, pages: int,
device: str, time_period: str, location: str,
) -> Tuple[str, str, Dict[str, Any]]:
tool = _tool_or_none(token)
if tool is None:
return NO_TOKEN_MD, "", {}
if not query or not query.strip():
return "Type the keyword you want to check.", "", {}
domain = _domain(target_domain if "://" in (target_domain or "") else f"https://{(target_domain or '').strip()}")
if not domain:
return "Enter the domain to look for, e.g. `scrape.do`.", "", {}
data = tool.search(query, country, int(pages or 1), device, time_period or None, location)
if data.get("status") != "success":
return _error_md(data), "", data
organic = data.get("organic_results") or []
hits = [
r for r in organic
if _domain(r.get("link")) == domain or _domain(r.get("link")).endswith("." + domain)
]
looked = len(organic)
country_label = dict((gl, label) for label, gl in COUNTRY_CHOICES).get(country, country)
if hits:
best = hits[0]
lines = [
f"# #{best.get('overall_position')}",
f"**{domain}** ranks **#{best.get('overall_position')}** for “{query.strip()}” "
f"in {country_label} ({device}), page {best.get('page')}.",
]
if len(hits) > 1:
others = ", ".join(f"#{h.get('overall_position')}" for h in hits[1:])
lines.append(f"It also appears at {others}.")
above = [r for r in organic if r.get("overall_position", 0) < best.get("overall_position", 0)]
if above:
lines += ["", "**Ranking above you:**"]
lines += [f"{r.get('overall_position')}. {_domain(r.get('link'))}" for r in above[:10]]
else:
lines = [
"# Not found",
f"**{domain}** is not in the top {looked} organic results for “{query.strip()}” "
f"in {country_label}. Fetch more pages to look deeper.",
]
lines.append(f"\n{data.get('credits_used')} credits used.")
return "\n".join(lines), render_serp(data, highlight_domain=domain), data
# ----------------------------------------------------------------------- UI
STYLE = """
<style>
@import url('https://fonts.googleapis.com/css2?family=Roboto:wght@400;500;700&display=swap');
.gradio-container, .gradio-container button, .gradio-container input,
.gradio-container textarea, .gradio-container select {
font-family: 'Roboto', Arial, sans-serif;
}
.gradio-container { color: #202124; }
.gs-hero {
background: #ffffff; border: 1px solid #dadce0; border-radius: 16px;
padding: 28px 32px 24px 32px; margin-bottom: 16px;
}
.gs-hero h1 { color: #202124 !important; font-size: 2rem !important; font-weight: 500 !important;
margin: 0 0 10px 0 !important; letter-spacing: -0.01em; }
.gs-dots { display: flex; gap: 6px; margin-bottom: 14px; }
.gs-dots span { width: 12px; height: 12px; border-radius: 50%; display: inline-block; }
.gs-hero p { color: #4d5156 !important; margin: 0 0 8px 0 !important; font-size: 1rem !important; line-height: 1.55 !important; }
.gs-hero a { color: #1a0dab !important; }
.gs-cta {
display: inline-block; margin-top: 10px; padding: 11px 24px;
background: #1a73e8 !important; color: #ffffff !important;
border-radius: 999px; font-weight: 500 !important; text-decoration: none !important;
}
.gs-note { border-left: 3px solid #fbbc05; background: #fef7e0; padding: 12px 16px;
border-radius: 0 8px 8px 0; font-size: 0.9rem; margin-bottom: 12px; color: #202124; }
.gradio-container .primary, .gradio-container button.primary,
.gradio-container button[variant="primary"] {
background: #1a73e8 !important; background-image: none !important;
border-color: #1a73e8 !important; color: #ffffff !important;
border-radius: 999px !important; font-weight: 500 !important;
}
.gradio-container .primary:hover, .gradio-container button.primary:hover,
.gradio-container button[variant="primary"]:hover { background: #1765cc !important; }
.gradio-container .tab-nav button.selected { color: #1a73e8 !important; border-bottom-color: #1a73e8 !important; }
/* results page preview */
.gs-serp { background: #ffffff; border: 1px solid #dadce0; border-radius: 12px;
padding: 20px 24px; max-width: 760px; color: #202124; }
.gs-stats { color: #70757a; font-size: 0.85rem; margin-bottom: 14px; }
.gs-result { margin: 0 0 24px 0; padding: 2px 0; }
.gs-hit { background: #e8f0fe; border-radius: 8px; padding: 10px 12px; margin-left: -12px; }
.gs-cite { font-size: 0.85rem; color: #202124; display: flex; gap: 8px; align-items: baseline; flex-wrap: wrap; }
.gs-source { font-weight: 500; }
.gs-url { color: #4d5156; overflow-wrap: anywhere; }
.gs-pos { background: #f1f3f4; color: #5f6368; border-radius: 999px; padding: 1px 8px; font-size: 0.75rem; }
.gs-sponsored { font-weight: 700; }
.gs-title { display: block; color: #1a0dab !important; font-size: 1.25rem; line-height: 1.3;
margin: 4px 0 3px 0; text-decoration: none !important; }
.gs-title:hover { text-decoration: underline !important; }
.gs-snippet { color: #4d5156; font-size: 0.9rem; line-height: 1.58; }
.gs-date { color: #70757a; }
.gs-sitelinks { margin-top: 6px; font-size: 0.85rem; }
.gs-sitelinks a { color: #1a0dab !important; }
.gs-ai { border: 1px solid #dadce0; border-radius: 12px; padding: 16px 18px; margin-bottom: 24px;
background: linear-gradient(180deg, #f8faff 0%, #ffffff 100%); }
.gs-ai-label { font-weight: 500; color: #1a73e8; margin-bottom: 8px; }
.gs-ai p, .gs-ai li { color: #202124; font-size: 0.92rem; line-height: 1.6; }
.gs-ai table { border-collapse: collapse; margin: 8px 0; font-size: 0.85rem; }
.gs-ai th, .gs-ai td { border: 1px solid #dadce0; padding: 4px 8px; }
.gs-ai-refs { margin-top: 10px; font-size: 0.82rem; }
.gs-ai-refs a { color: #1a0dab !important; }
.gs-ai-refs span { color: #70757a; }
.gs-box { border-top: 1px solid #ebebeb; border-bottom: 1px solid #ebebeb; padding: 12px 0; margin-bottom: 24px; }
.gs-box-title { font-size: 1.1rem; margin-bottom: 8px; }
.gs-box ul { margin: 0; padding-left: 18px; }
.gs-paa { list-style: none; padding-left: 0 !important; }
.gs-paa li { border-bottom: 1px solid #ebebeb; padding: 10px 0; }
.gs-chips { display: flex; flex-wrap: wrap; gap: 8px; }
.gs-chip { background: #f1f3f4; border-radius: 999px; padding: 6px 12px; font-size: 0.88rem; }
.gs-kg { border: 1px solid #dadce0; border-radius: 12px; padding: 14px 16px; margin-bottom: 24px; }
.gs-kg-title { font-size: 1.4rem; }
.gs-kg-type { color: #70757a; margin-bottom: 6px; }
footer { display: none !important; }
</style>
"""
HERO = f"""
<div class="gs-hero">
<div class="gs-dots"><span style="background:#4285F4"></span><span style="background:#EA4335"></span><span style="background:#FBBC05"></span><span style="background:#34A853"></span></div>
<h1>Google Search Scraper</h1>
<p><strong>Any Google results page as structured data</strong> — organic rankings, ads, People Also Ask,
related searches, the AI Overview, knowledge panel and local pack, from any country and in any language.</p>
<p>No selectors to maintain: Google changes its page code constantly, so this uses the
<a href="https://scrape.do/documentation/google-scraper-api/search/" target="_blank" rel="noopener noreferrer">Scrape.do Google Search API</a>,
which returns the same clean fields whatever Google ships. Check where a site ranks, watch competitors' ads,
or feed live results to an AI agent.</p>
<a class="gs-cta" href="{SCRAPE_DO_SIGNUP_URL}" target="_blank" rel="noopener noreferrer">Get a free API token →</a>
</div>
"""
CREDIT_NOTE = """
<div class="gs-note">
Each results page costs 10 credits, so the free 1,000 credits cover about 100 searches.
If Google is still writing its AI Overview when the page comes back, it's fetched with one
follow-up call for 5 more credits.
</div>
"""
FOOTER_MD = f"""
---
### Use it as a smolagents Tool
```python
from smolagents import CodeAgent, InferenceClientModel
from tool import ScrapeDoGoogleSearchTool
agent = CodeAgent(
tools=[ScrapeDoGoogleSearchTool(api_token="YOUR_SCRAPEDO_TOKEN")],
model=InferenceClientModel(),
)
agent.run("Who ranks in the top 5 for 'best web scraping api' in Germany, and do any run ads?")
```
Or set `SCRAPE_DO_API_TOKEN` in the environment and instantiate with no arguments.
Want to parse the raw HTML yourself instead? The
[Google search scraping guide](https://scrape.do/blog/scraping-google-search-results/) covers both approaches.
[Scrape.do](https://scrape.do/) · [Google Search API docs](https://scrape.do/documentation/google-scraper-api/search/) · [Sign up free]({SCRAPE_DO_SIGNUP_URL})
"""
def build_demo() -> gr.Blocks:
with gr.Blocks(title="Google Search Scraper — Scrape.do") as demo:
gr.HTML(STYLE)
gr.HTML(HERO)
api_token = gr.Textbox(
label="Scrape.do API token",
placeholder="Paste your token here",
type="password",
info="Free tier: 1,000 credits a month, about 100 searches.",
)
with gr.Row():
country = gr.Dropdown(label="Search from", choices=COUNTRY_CHOICES, value="us")
device = gr.Radio(label="Device", choices=["desktop", "mobile"], value="desktop")
pages = gr.Slider(label="Result pages", minimum=1, maximum=5, step=1, value=1,
info="10 credits per page.")
with gr.Accordion("More filters", open=False):
with gr.Row():
time_period = gr.Dropdown(label="Time", choices=TIME_CHOICES, value="")
location = gr.Textbox(
label="City (optional)",
placeholder="Austin,Texas,United States",
info="Google's own place name. Must be in the country above.",
)
gr.HTML(CREDIT_NOTE)
with gr.Tabs():
with gr.Tab("Search results"):
query = gr.Textbox(label="Search Google", placeholder="best web scraping api")
search_btn = gr.Button("Google Search", variant="primary")
search_summary = gr.Markdown()
serp_view = gr.HTML()
serp_table = gr.Dataframe(headers=TABLE_HEADERS, wrap=True, label="Organic results")
serp_csv = gr.File(label="Download CSV")
with gr.Accordion("Raw JSON response", open=False):
serp_json = gr.JSON()
search_btn.click(
fn=run_search,
inputs=[api_token, query, country, pages, device, time_period, location],
outputs=[search_summary, serp_view, serp_table, serp_json, serp_csv],
)
query.submit(
fn=run_search,
inputs=[api_token, query, country, pages, device, time_period, location],
outputs=[search_summary, serp_view, serp_table, serp_json, serp_csv],
)
with gr.Tab("Rank checker"):
gr.Markdown(
"Where does a website rank for a keyword? Enter both, and the matching "
"result is highlighted on the page with everyone ranking above it."
)
with gr.Row():
rank_query = gr.Textbox(label="Keyword", placeholder="web scraping api", scale=3)
rank_domain = gr.Textbox(label="Your domain", placeholder="scrape.do", scale=2)
rank_btn = gr.Button("Check ranking", variant="primary")
rank_summary = gr.Markdown()
rank_view = gr.HTML()
with gr.Accordion("Raw JSON response", open=False):
rank_json = gr.JSON()
rank_btn.click(
fn=run_rank_check,
inputs=[api_token, rank_query, rank_domain, country, pages, device, time_period, location],
outputs=[rank_summary, rank_view, rank_json],
)
gr.Markdown(FOOTER_MD)
return demo
# Hugging Face Spaces launches the module-level `demo` object rather than
# calling launch() itself, so build it at import time.
demo = build_demo()
if __name__ == "__main__":
demo.launch()