from __future__ import annotations import csv import html import tempfile from typing import Any, Dict, List, Optional, Tuple from urllib.parse import urlparse import gradio as gr from tool import COUNTRIES, SCRAPE_DO_SIGNUP_URL, TIME_PERIODS, ScrapeDoGoogleSearchTool # ZeroGPU hardware refuses to keep a Space alive unless it finds at least one # @spaces.GPU function at startup. This app is pure HTTP and never touches a GPU, # so the probe below exists only to satisfy that check. It is never called. try: import spaces @spaces.GPU(duration=1) def _zerogpu_probe() -> str: return "ok" except ImportError: pass COUNTRY_CHOICES = [(label, gl) for label, gl, _, _ in COUNTRIES] TIME_CHOICES = [("Any time", "")] + [ (value.replace("_", " ").capitalize(), value) for value in TIME_PERIODS ] NO_TOKEN_MD = ( "### Paste your Scrape.do API token above\n\n" f"No token yet? [Create a free account]({SCRAPE_DO_SIGNUP_URL}) — " "1,000 free credits a month, no credit card. One search costs 10 credits." ) # --------------------------------------------------------------------- helpers def _esc(value: Any) -> str: return html.escape(str(value or ""), quote=True) def _safe_link(url: Any) -> str: url = str(url or "") return url if url.startswith(("http://", "https://")) else "#" def _domain(url: Any) -> str: try: host = urlparse(str(url or "")).netloc.lower() except ValueError: return "" return host[4:] if host.startswith("www.") else host def _tool_or_none(token: str) -> Optional[ScrapeDoGoogleSearchTool]: if not token or not token.strip(): return None return ScrapeDoGoogleSearchTool(api_token=token.strip()) def _write_csv(rows: List[Dict[str, Any]], prefix: str) -> Optional[str]: if not rows: return None fieldnames: List[str] = [] for row in rows: for key in row: if key not in fieldnames: fieldnames.append(key) handle = tempfile.NamedTemporaryFile( mode="w", suffix=".csv", prefix=f"{prefix}-", delete=False, newline="", encoding="utf-8-sig", ) writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() for row in rows: writer.writerow({key: row.get(key, "") for key in fieldnames}) handle.close() return handle.name def _error_md(data: Dict[str, Any]) -> str: out = f"**That search didn't go through.**\n\n{data.get('errorMessage') or 'Unknown error'}" if data.get("hint"): out += f"\n\n{data['hint']}" return out # ------------------------------------------------------------ SERP rendering def _render_ai_blocks(blocks: List[Dict[str, Any]]) -> str: parts: List[str] = [] def render_list(items: List[Dict[str, Any]]) -> str: inner = "" for item in items or []: inner += f"
  • {_esc(item.get('snippet'))}" if item.get("list"): inner += render_list(item["list"]) inner += "
  • " return f"" for block in blocks or []: kind = block.get("type") if kind == "heading": parts.append(f"

    {_esc(block.get('snippet'))}

    ") elif kind == "list": if block.get("snippet"): parts.append(f"

    {_esc(block['snippet'])}

    ") parts.append(render_list(block.get("list") or [])) elif kind == "table": head = "".join(f"{_esc(h)}" for h in block.get("table_headers") or []) rows = "".join( "" + "".join(f"{_esc(c)}" for c in row) + "" for row in block.get("table_rows") or [] ) parts.append(f"{head}{rows}
    ") else: parts.append(f"

    {_esc(block.get('snippet'))}

    ") return "".join(parts) def _render_ad(ad: Dict[str, Any]) -> str: link = _safe_link(ad.get("url") or ad.get("link")) return ( '
    ' f'
    Sponsored {_esc(ad.get("displayed_url") or _domain(link))}
    ' f'{_esc(ad.get("title"))}' f'
    {_esc(ad.get("description"))}
    ' "
    " ) def render_serp(data: Dict[str, Any], highlight_domain: str = "") -> str: info = data.get("search_information") or {} out: List[str] = ['
    '] stats = [] if info.get("total_results"): stats.append(f"About {int(info['total_results']):,} results") if info.get("time_taken_displayed"): stats.append(f"({info['time_taken_displayed']} seconds)") if stats: out.append(f'
    {_esc(" ".join(stats))}
    ') if info.get("results_for"): out.append(f'
    Showing results for {_esc(info["results_for"])}
    ') ai = data.get("ai_overview") if isinstance(ai, dict) and ai.get("state") == "complete" and ai.get("text_blocks"): refs = "".join( f'
  • ' f'{_esc(ref.get("title") or ref.get("source"))} {_esc(ref.get("source"))}
  • ' for ref in (ai.get("references") or [])[:8] ) out.append( '
    ✦ AI Overview
    ' f"{_render_ai_blocks(ai['text_blocks'])}" + (f'
    Sources
      {refs}
    ' if refs else "") + "
    " ) for ad in data.get("top_ads") or []: out.append(_render_ad(ad)) kg = data.get("knowledge_graph") if isinstance(kg, dict) and kg.get("title"): facts = "".join( f"
    {_esc(f.get('label'))}: {_esc(f.get('value'))}
    " for f in (kg.get("facts") or [])[:6] ) out.append( '
    ' f"
    {_esc(kg.get('title'))}
    " f"
    {_esc(kg.get('type'))}
    " f"

    {_esc(kg.get('description'))}

    {facts}
    " ) local = data.get("local_results") if isinstance(local, dict) and local.get("places"): places = "".join( f"
  • {_esc(p.get('title'))} · {_esc(p.get('rating') or '—')}★ {_esc(p.get('price'))}
  • " for p in local["places"][:5] ) out.append(f'
    Places
    ') paa = data.get("related_questions") or [] organic = data.get("organic_results") or [] highlight = highlight_domain.lower().removeprefix("www.") for index, result in enumerate(organic): link = _safe_link(result.get("link")) is_hit = bool(highlight) and ( _domain(link) == highlight or _domain(link).endswith("." + highlight) ) cls = "gs-result gs-hit" if is_hit else "gs-result" page_badge = ( f'#{result.get("overall_position", index + 1)}' ) sitelinks = "" sl = result.get("sitelinks") or {} sl_items = (sl.get("inline") or []) + (sl.get("expanded") or []) if sl_items: sitelinks = '" date = f'{_esc(result.get("date"))} — ' if result.get("date") else "" out.append( f'
    ' f'
    {page_badge}{_esc(result.get("source") or _domain(link))}' f'{_esc(result.get("displayed_link") or link)}
    ' f'{_esc(result.get("title"))}' f'
    {date}{_esc(result.get("snippet"))}
    {sitelinks}
    ' ) # Google puts People Also Ask after the first few results. if index == 2 and paa: qs = "".join(f"
  • {_esc(q.get('question'))}
  • " for q in paa[:6]) out.append(f'
    People also ask
    ') paa = [] if paa: qs = "".join(f"
  • {_esc(q.get('question'))}
  • " for q in paa[:6]) out.append(f'
    People also ask
    ') for ad in data.get("bottom_ads") or []: out.append(_render_ad(ad)) related = data.get("related_searches") or [] if related: chips = "".join(f'🔍 {_esc(r.get("query"))}' for r in related[:10]) out.append(f'
    People also search for
    {chips}
    ') if not organic: out.append('
    No organic results came back for this query.
    ') out.append("
    ") return "".join(out) def _organic_rows(data: Dict[str, Any]) -> List[Dict[str, Any]]: return [ { "position": r.get("overall_position"), "page": r.get("page"), "title": r.get("title"), "link": r.get("link"), "domain": _domain(r.get("link")), "snippet": r.get("snippet"), "date": r.get("date"), "source": r.get("source"), } for r in data.get("organic_results") or [] ] # ---------------------------------------------------------------- handlers TABLE_HEADERS = ["#", "Domain", "Title", "Link"] def run_search( token: str, query: str, country: str, pages: int, device: str, time_period: str, location: str ) -> Tuple[str, str, List[List[Any]], Dict[str, Any], Optional[str]]: tool = _tool_or_none(token) if tool is None: return NO_TOKEN_MD, "", [], {}, None if not query or not query.strip(): return "Type a search query to begin.", "", [], {}, None data = tool.search(query, country, int(pages or 1), device, time_period or None, location) if data.get("status") != "success": return _error_md(data), "", [], data, None rows = _organic_rows(data) counts = [ f"**{len(rows)}** organic results", f"**{len(data.get('top_ads') or []) + len(data.get('bottom_ads') or [])}** ads", f"**{len(data.get('related_questions') or [])}** People Also Ask", f"**{len(data.get('related_searches') or [])}** related searches", ] ai = data.get("ai_overview") counts.append("AI Overview **yes**" if isinstance(ai, dict) and ai.get("state") == "complete" else "AI Overview **no**") summary = ( " · ".join(counts) + f"\n\n{data.get('pages_fetched')} page(s) fetched · {data.get('credits_used')} credits used" ) if data.get("warnings"): summary += "\n\n" + "\n".join(f"- {w}" for w in data["warnings"]) table = [[r["position"], r["domain"], (r["title"] or "")[:90], r["link"]] for r in rows] csv_path = _write_csv(rows, "google-serp") return summary, render_serp(data), table, data, csv_path def run_rank_check( token: str, query: str, target_domain: str, country: str, pages: int, device: str, time_period: str, location: str, ) -> Tuple[str, str, Dict[str, Any]]: tool = _tool_or_none(token) if tool is None: return NO_TOKEN_MD, "", {} if not query or not query.strip(): return "Type the keyword you want to check.", "", {} domain = _domain(target_domain if "://" in (target_domain or "") else f"https://{(target_domain or '').strip()}") if not domain: return "Enter the domain to look for, e.g. `scrape.do`.", "", {} data = tool.search(query, country, int(pages or 1), device, time_period or None, location) if data.get("status") != "success": return _error_md(data), "", data organic = data.get("organic_results") or [] hits = [ r for r in organic if _domain(r.get("link")) == domain or _domain(r.get("link")).endswith("." + domain) ] looked = len(organic) country_label = dict((gl, label) for label, gl in COUNTRY_CHOICES).get(country, country) if hits: best = hits[0] lines = [ f"# #{best.get('overall_position')}", f"**{domain}** ranks **#{best.get('overall_position')}** for “{query.strip()}” " f"in {country_label} ({device}), page {best.get('page')}.", ] if len(hits) > 1: others = ", ".join(f"#{h.get('overall_position')}" for h in hits[1:]) lines.append(f"It also appears at {others}.") above = [r for r in organic if r.get("overall_position", 0) < best.get("overall_position", 0)] if above: lines += ["", "**Ranking above you:**"] lines += [f"{r.get('overall_position')}. {_domain(r.get('link'))}" for r in above[:10]] else: lines = [ "# Not found", f"**{domain}** is not in the top {looked} organic results for “{query.strip()}” " f"in {country_label}. Fetch more pages to look deeper.", ] lines.append(f"\n{data.get('credits_used')} credits used.") return "\n".join(lines), render_serp(data, highlight_domain=domain), data # ----------------------------------------------------------------------- UI STYLE = """ """ HERO = f"""

    Google Search Scraper

    Any Google results page as structured data — organic rankings, ads, People Also Ask, related searches, the AI Overview, knowledge panel and local pack, from any country and in any language.

    No selectors to maintain: Google changes its page code constantly, so this uses the Scrape.do Google Search API, which returns the same clean fields whatever Google ships. Check where a site ranks, watch competitors' ads, or feed live results to an AI agent.

    Get a free API token →
    """ CREDIT_NOTE = """
    Each results page costs 10 credits, so the free 1,000 credits cover about 100 searches. If Google is still writing its AI Overview when the page comes back, it's fetched with one follow-up call for 5 more credits.
    """ FOOTER_MD = f""" --- ### Use it as a smolagents Tool ```python from smolagents import CodeAgent, InferenceClientModel from tool import ScrapeDoGoogleSearchTool agent = CodeAgent( tools=[ScrapeDoGoogleSearchTool(api_token="YOUR_SCRAPEDO_TOKEN")], model=InferenceClientModel(), ) agent.run("Who ranks in the top 5 for 'best web scraping api' in Germany, and do any run ads?") ``` Or set `SCRAPE_DO_API_TOKEN` in the environment and instantiate with no arguments. Want to parse the raw HTML yourself instead? The [Google search scraping guide](https://scrape.do/blog/scraping-google-search-results/) covers both approaches. [Scrape.do](https://scrape.do/) · [Google Search API docs](https://scrape.do/documentation/google-scraper-api/search/) · [Sign up free]({SCRAPE_DO_SIGNUP_URL}) """ def build_demo() -> gr.Blocks: with gr.Blocks(title="Google Search Scraper — Scrape.do") as demo: gr.HTML(STYLE) gr.HTML(HERO) api_token = gr.Textbox( label="Scrape.do API token", placeholder="Paste your token here", type="password", info="Free tier: 1,000 credits a month, about 100 searches.", ) with gr.Row(): country = gr.Dropdown(label="Search from", choices=COUNTRY_CHOICES, value="us") device = gr.Radio(label="Device", choices=["desktop", "mobile"], value="desktop") pages = gr.Slider(label="Result pages", minimum=1, maximum=5, step=1, value=1, info="10 credits per page.") with gr.Accordion("More filters", open=False): with gr.Row(): time_period = gr.Dropdown(label="Time", choices=TIME_CHOICES, value="") location = gr.Textbox( label="City (optional)", placeholder="Austin,Texas,United States", info="Google's own place name. Must be in the country above.", ) gr.HTML(CREDIT_NOTE) with gr.Tabs(): with gr.Tab("Search results"): query = gr.Textbox(label="Search Google", placeholder="best web scraping api") search_btn = gr.Button("Google Search", variant="primary") search_summary = gr.Markdown() serp_view = gr.HTML() serp_table = gr.Dataframe(headers=TABLE_HEADERS, wrap=True, label="Organic results") serp_csv = gr.File(label="Download CSV") with gr.Accordion("Raw JSON response", open=False): serp_json = gr.JSON() search_btn.click( fn=run_search, inputs=[api_token, query, country, pages, device, time_period, location], outputs=[search_summary, serp_view, serp_table, serp_json, serp_csv], ) query.submit( fn=run_search, inputs=[api_token, query, country, pages, device, time_period, location], outputs=[search_summary, serp_view, serp_table, serp_json, serp_csv], ) with gr.Tab("Rank checker"): gr.Markdown( "Where does a website rank for a keyword? Enter both, and the matching " "result is highlighted on the page with everyone ranking above it." ) with gr.Row(): rank_query = gr.Textbox(label="Keyword", placeholder="web scraping api", scale=3) rank_domain = gr.Textbox(label="Your domain", placeholder="scrape.do", scale=2) rank_btn = gr.Button("Check ranking", variant="primary") rank_summary = gr.Markdown() rank_view = gr.HTML() with gr.Accordion("Raw JSON response", open=False): rank_json = gr.JSON() rank_btn.click( fn=run_rank_check, inputs=[api_token, rank_query, rank_domain, country, pages, device, time_period, location], outputs=[rank_summary, rank_view, rank_json], ) gr.Markdown(FOOTER_MD) return demo # Hugging Face Spaces launches the module-level `demo` object rather than # calling launch() itself, so build it at import time. demo = build_demo() if __name__ == "__main__": demo.launch()