bugrahans's picture
Upload 4 files
dca809d verified
Raw History Blame Contribute Delete
17.4 kB
from __future__ import annotations
import csv
import os
import tempfile
from typing import Any, Dict, List, Optional, Tuple
import gradio as gr
from tool import (
MARKETPLACES,
SCRAPE_DO_SIGNUP_URL,
ScrapeDoAmazonTool,
ZIP_LEVEL_MARKETS,
extract_asin,
)
# ZeroGPU hardware refuses to keep a Space alive unless it finds at least one
# @spaces.GPU function at startup. This app is pure HTTP and never touches a GPU,
# so the probe below exists only to satisfy that check. It is never called by the UI.
try:
import spaces
@spaces.GPU(duration=1)
def _zerogpu_probe() -> str:
return "ok"
except ImportError:
pass
MARKET_CHOICES = [(label, code) for code, label in MARKETPLACES.items()]
NO_TOKEN_MD = (
"⚠️ **Please paste your Scrape.do API token above.**\n\n"
f"Don't have one yet? [Get a free token in 30 seconds → ]({SCRAPE_DO_SIGNUP_URL})\n\n"
"Free tier includes 1,000 successful requests / month, no credit card required."
)
# --------------------------------------------------------------------- helpers
def _write_csv(rows: List[Dict[str, Any]], prefix: str) -> Optional[str]:
"""Write rows to a temp CSV and return its path."""
if not rows:
return None
fieldnames: List[str] = []
for row in rows:
for key in row:
if key not in fieldnames:
fieldnames.append(key)
handle = tempfile.NamedTemporaryFile(
mode="w", suffix=".csv", prefix=f"{prefix}-", delete=False,
newline="", encoding="utf-8-sig",
)
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
for row in rows:
writer.writerow({key: row.get(key, "") for key in fieldnames})
handle.close()
return handle.name
def _error_md(data: Dict[str, Any]) -> str:
message = data.get("errorMessage") or data.get("message") or data.get("error") or "Unknown error"
hint = data.get("hint")
text = f"❌ **{message}**"
if hint:
text += f"\n\n{hint}"
return text
def _money(amount: Any, symbol: str = "", code: str = "") -> str:
if amount in (None, ""):
return "—"
if symbol:
return f"{symbol}{amount}"
if code:
return f"{amount} {code}"
return str(amount)
def _tool_or_none(token: str) -> Optional[ScrapeDoAmazonTool]:
if not token or not token.strip():
return None
return ScrapeDoAmazonTool(api_token=token.strip())
# --------------------------------------------------------------- product (PDP)
def scrape_product(
token: str, query: str, geocode: str, zipcode: str, language: str
) -> Tuple[str, List[str], Dict[str, Any], Optional[str]]:
tool = _tool_or_none(token)
if tool is None:
return NO_TOKEN_MD, [], {}, None
if not query or not query.strip():
return "⚠️ **Paste an Amazon product URL or ASIN.**", [], {}, None
data = tool.request("product", query, geocode, zipcode, language=language)
if data.get("status") != "success":
return _error_md(data), [], data, None
symbol = data.get("currency_symbol") or ""
code = data.get("currency") or ""
price = _money(data.get("price"), symbol, code)
list_price = data.get("list_price")
lines = [f"## {data.get('name') or 'Untitled product'}", ""]
facts = [
f"**ASIN** `{data.get('asin', '—')}`",
f"**Brand** {data.get('brand') or '—'}",
f"**Price** {price}",
]
if list_price:
facts.append(f"**List price** ~~{_money(list_price, symbol, code)}~~")
if data.get("rating") is not None:
facts.append(
f"**Rating** {data.get('rating')} ★ ({data.get('total_ratings') or 0:,} ratings)"
)
facts.append("**Prime** " + ("yes" if data.get("is_prime") else "no"))
lines.append(" · ".join(facts))
shipping = data.get("shipping_info") or []
if shipping:
lines += ["", "**Delivery**"] + [f"- {item}" for item in shipping[:4]]
rankings = data.get("best_seller_rankings") or []
if rankings:
lines += ["", "**Best seller rankings**"]
for rank in rankings[:6]:
lines.append(f"- #{rank.get('rank')} in {rank.get('category')}")
details = data.get("technical_details") or {}
if isinstance(details, dict) and details:
lines += ["", "**Technical details**", "", "| Field | Value |", "| --- | --- |"]
for key, value in list(details.items())[:15]:
lines.append(f"| {key} | {value} |")
description = data.get("description")
if description:
lines += ["", "**Description**", "", str(description)[:1200]]
if data.get("url"):
lines += ["", f"[Open on Amazon →]({data['url']})"]
images = [
image.get("url")
for image in (data.get("images") or [])
if isinstance(image, dict) and image.get("url")
]
if not images and data.get("thumbnail"):
images = [data["thumbnail"]]
flat = {
"asin": data.get("asin"),
"name": data.get("name"),
"brand": data.get("brand"),
"price": data.get("price"),
"list_price": data.get("list_price"),
"currency": data.get("currency"),
"rating": data.get("rating"),
"total_ratings": data.get("total_ratings"),
"is_prime": data.get("is_prime"),
"url": data.get("url"),
"thumbnail": data.get("thumbnail"),
}
csv_path = _write_csv([flat], f"amazon-product-{data.get('asin', 'result')}")
return "\n".join(lines), images[:12], data, csv_path
# ------------------------------------------------------------------- offers
OFFER_HEADERS = [
"Condition", "Seller", "Price", "Currency",
"Ships from", "FBA", "Prime", "Buy Box",
]
def scrape_offers(
token: str, query: str, geocode: str, zipcode: str, language: str
) -> Tuple[str, List[List[Any]], Dict[str, Any], Optional[str]]:
tool = _tool_or_none(token)
if tool is None:
return NO_TOKEN_MD, [], {}, None
if not query or not query.strip():
return "⚠️ **Paste an Amazon product URL or ASIN.**", [], {}, None
data = tool.request("offers", query, geocode, zipcode, language=language)
if data.get("status") != "success":
return _error_md(data), [], data, None
offers = data.get("offers") or []
if not offers:
return (
f"No competing offers returned for `{data.get('asin')}` in this "
"marketplace and delivery location. That is a valid result, not an "
"error — try another ZIP code or marketplace.",
[], data, None,
)
rows: List[List[Any]] = []
flat_rows: List[Dict[str, Any]] = []
for offer in offers:
listing = offer.get("listingPrice") or {}
prime = (offer.get("primeInformation") or {}).get("isPrime")
row = {
"asin": data.get("asin"),
"condition": offer.get("condition") or "",
"seller": offer.get("merchantName") or "",
"price": listing.get("amount", ""),
"currency": listing.get("currency", ""),
"ships_from": offer.get("shipsFrom") or "",
"fulfilled_by_amazon": bool(offer.get("isFulfilledByAmazon")),
"is_prime": bool(prime),
"is_buy_box_winner": bool(offer.get("isBuyBoxWinner")),
}
flat_rows.append(row)
rows.append([
row["condition"], row["seller"], row["price"], row["currency"],
row["ships_from"],
"✓" if row["fulfilled_by_amazon"] else "",
"✓" if row["is_prime"] else "",
"✓" if row["is_buy_box_winner"] else "",
])
prices = [
offer["price"] for offer in flat_rows
if isinstance(offer["price"], (int, float))
]
summary = f"**{len(rows)} offer(s)** for ASIN `{data.get('asin')}`"
if prices:
summary += f" · lowest {min(prices)} · highest {max(prices)}"
buy_box = next((o["seller"] for o in flat_rows if o["is_buy_box_winner"]), None)
if buy_box:
summary += f" · Buy Box: **{buy_box}**"
csv_path = _write_csv(flat_rows, f"amazon-offers-{data.get('asin', 'result')}")
return summary, rows, data, csv_path
# ------------------------------------------------------------------- search
SEARCH_HEADERS = [
"#", "ASIN", "Title", "Price", "Rating", "Reviews", "Prime", "Sponsored", "Badge",
]
def scrape_search(
token: str, keyword: str, geocode: str, zipcode: str, page: int, language: str
) -> Tuple[str, List[List[Any]], Dict[str, Any], Optional[str]]:
tool = _tool_or_none(token)
if tool is None:
return NO_TOKEN_MD, [], {}, None
if not keyword or not keyword.strip():
return "⚠️ **Enter a search keyword.**", [], {}, None
data = tool.request("search", keyword, geocode, zipcode, page=page, language=language)
if data.get("status") != "success":
return _error_md(data), [], data, None
products = data.get("products") or []
if not products:
return "No products returned for that keyword.", [], data, None
rows: List[List[Any]] = []
flat_rows: List[Dict[str, Any]] = []
for product in products:
price = product.get("price") or {}
rating = product.get("rating") or {}
row = {
"position": product.get("position"),
"asin": product.get("asin"),
"title": product.get("title"),
"price": price.get("amount", ""),
"currency": price.get("currencyCode", ""),
"rating": rating.get("value", ""),
"rating_count": rating.get("count", ""),
"review_count": product.get("reviewCount", ""),
"is_prime": bool(product.get("isPrime")),
"is_sponsored": bool(product.get("isSponsored")),
"badge": product.get("badge") or "",
"url": product.get("url"),
"image_url": product.get("imageUrl"),
}
flat_rows.append(row)
title = row["title"] or ""
rows.append([
row["position"], row["asin"],
title[:90] + ("…" if len(title) > 90 else ""),
f"{row['price']} {row['currency']}".strip(),
row["rating"], row["review_count"],
"✓" if row["is_prime"] else "",
"✓" if row["is_sponsored"] else "",
row["badge"],
])
sponsored = sum(1 for product in flat_rows if product["is_sponsored"])
summary = (
f"**{len(rows)} results** for “{data.get('keyword')}” · page {data.get('page')} · "
f"{sponsored} sponsored, {len(rows) - sponsored} organic"
)
if data.get("totalResults"):
summary += f" · {data['totalResults']}"
csv_path = _write_csv(flat_rows, "amazon-search")
return summary, rows, data, csv_path
# ----------------------------------------------------------------------- UI
HEADER_MD = f"""
# 📦 Amazon Product Scraper
**Pull structured JSON from any Amazon marketplace — product pages, every seller
offer, and keyword search results.** No HTML parsing, no ASIN spreadsheets, no blocks.
AWS WAF bypass, proxy rotation, session handling and CAPTCHAs are handled for you
across 21 marketplaces, with ZIP-code level delivery targeting.
Powered by [Scrape.do](https://scrape.do/).
### 🔑 Need an API token?
👉 **[Sign up free on Scrape.do]({SCRAPE_DO_SIGNUP_URL})** — 1,000 free requests / month, no credit card.
"""
FOOTER_MD = f"""
---
### Using this as a smolagents Tool
```python
from smolagents import CodeAgent, InferenceClientModel
from tool import ScrapeDoAmazonTool
agent = CodeAgent(
tools=[ScrapeDoAmazonTool(api_token="YOUR_SCRAPEDO_TOKEN")],
model=InferenceClientModel(),
)
agent.run("Compare the price and Buy Box seller for ASIN B0BLRJ4R8F in the US and the UK")
```
Or set `SCRAPE_DO_API_TOKEN` in your environment and instantiate without args.
Only publicly available product data is in scope. Anything behind an Amazon login —
account pages, full customer review archives — is out of bounds; see the
[Amazon scraping guide](https://scrape.do/blog/amazon-scraping/) for the legal and
ethical boundaries.
[Scrape.do](https://scrape.do/) · [Amazon Scraper API docs](https://scrape.do/documentation/amazon-scraper-api/) · [Sign up free]({SCRAPE_DO_SIGNUP_URL})
"""
ZIP_INFO = (
"ZIP code for us / gb / de / fr / it / es / ca / jp / in. "
"For other marketplaces, enter a delivery country name instead (or leave blank)."
)
def build_demo() -> gr.Blocks:
with gr.Blocks(title="Amazon Product Scraper — Scrape.do") as demo:
gr.Markdown(HEADER_MD)
api_token = gr.Textbox(
label="Scrape.do API Token",
placeholder="Paste your token here…",
type="password",
info="Don't have one? Get a free token from the link above.",
)
with gr.Row():
geocode = gr.Dropdown(
label="Marketplace",
choices=MARKET_CHOICES,
value="us",
)
zipcode = gr.Textbox(
label="Delivery location",
placeholder="10001",
info=ZIP_INFO,
)
language = gr.Textbox(
label="Language",
placeholder="EN",
info="Optional ISO 639-1 code, uppercase. Must be supported by the marketplace.",
)
with gr.Tabs():
# ------------------------------------------------ product tab
with gr.Tab("Product page"):
gr.Markdown(
"Paste a product URL or a bare ASIN. Returns title, brand, price, "
"list price, rating, images, best seller rankings and specs."
)
product_query = gr.Textbox(
label="Amazon product URL or ASIN",
placeholder="https://www.amazon.com/dp/B0BLRJ4R8F · or just B0BLRJ4R8F",
)
product_btn = gr.Button("Scrape product", variant="primary")
product_summary = gr.Markdown()
product_gallery = gr.Gallery(
label="Product images", columns=6, height=180, preview=False
)
product_csv = gr.File(label="Download CSV")
with gr.Accordion("Raw JSON response", open=False):
product_json = gr.JSON()
product_btn.click(
fn=scrape_product,
inputs=[api_token, product_query, geocode, zipcode, language],
outputs=[product_summary, product_gallery, product_json, product_csv],
)
# ------------------------------------------------- offers tab
with gr.Tab("Seller offers"):
gr.Markdown(
"Every active offer on an ASIN: price, condition, seller, "
"fulfilment and who currently holds the Buy Box."
)
offers_query = gr.Textbox(
label="Amazon product URL or ASIN",
placeholder="B0DZDBWM5B",
)
offers_btn = gr.Button("Scrape offers", variant="primary")
offers_summary = gr.Markdown()
offers_table = gr.Dataframe(
headers=OFFER_HEADERS, wrap=True, label="Offers"
)
offers_csv = gr.File(label="Download CSV")
with gr.Accordion("Raw JSON response", open=False):
offers_json = gr.JSON()
offers_btn.click(
fn=scrape_offers,
inputs=[api_token, offers_query, geocode, zipcode, language],
outputs=[offers_summary, offers_table, offers_json, offers_csv],
)
# ------------------------------------------------- search tab
with gr.Tab("Search results"):
gr.Markdown(
"Keyword and category search with organic vs. sponsored "
"placements and exact on-page positions."
)
with gr.Row():
search_query = gr.Textbox(
label="Search keyword",
placeholder="laptop stands",
scale=3,
)
search_page = gr.Number(
label="Page", value=1, precision=0, minimum=1, scale=1
)
search_btn = gr.Button("Scrape search results", variant="primary")
search_summary = gr.Markdown()
search_table = gr.Dataframe(
headers=SEARCH_HEADERS, wrap=True, label="Results"
)
search_csv = gr.File(label="Download CSV")
with gr.Accordion("Raw JSON response", open=False):
search_json = gr.JSON()
search_btn.click(
fn=scrape_search,
inputs=[api_token, search_query, geocode, zipcode, search_page, language],
outputs=[search_summary, search_table, search_json, search_csv],
)
gr.Markdown(FOOTER_MD)
return demo
if __name__ == "__main__":
build_demo().launch()