from __future__ import annotations
import csv
import html
import re
import tempfile
from typing import Any, Dict, List, Optional, Tuple
import gradio as gr
from tool import (
DEFAULT_COMPARE_LOCATIONS,
DEFAULT_LOCATION,
MAX_COMPARE_LOCATIONS,
SCRAPE_DO_SIGNUP_URL,
ScrapeDoWalmartTool,
describe_store,
)
# ZeroGPU hardware refuses to keep a Space alive unless it finds at least one
# @spaces.GPU function at startup. This app is pure HTTP and never touches a GPU,
# so the probe below exists only to satisfy that check. It is never called.
try:
import spaces
@spaces.GPU(duration=1)
def _zerogpu_probe() -> str:
return "ok"
except ImportError:
pass
NO_TOKEN_MD = (
"### Paste your Scrape.do API token above\n\n"
f"No token yet? [Create a free account]({SCRAPE_DO_SIGNUP_URL}) — "
"1,000 free credits a month, no credit card. Each Walmart call costs 10 credits."
)
# --------------------------------------------------------------------- helpers
def _esc(value: Any) -> str:
return html.escape("" if value is None else str(value), quote=True)
def _safe_link(url: Any) -> str:
url = str(url or "")
return url if url.startswith(("http://", "https://")) else ""
def _num(value: Any) -> Optional[float]:
if isinstance(value, bool):
return None
try:
return float(str(value).replace(",", "").replace("$", ""))
except (TypeError, ValueError):
return None
def _fmt_int(value: Any) -> str:
number = _num(value)
return f"{int(number):,}" if number is not None else _esc(value)
def _money(value: Any, currency: Any = "USD") -> str:
number = _num(value)
if number is None:
return ""
code = str(currency or "USD").upper()
if code in ("USD", "CAD"):
return f"${number:,.2f}"
return f"{number:,.2f} {code}"
def _tool_or_none(token: str) -> Optional[ScrapeDoWalmartTool]:
if not token or not token.strip():
return None
return ScrapeDoWalmartTool(api_token=token.strip())
def _write_csv(rows: List[Dict[str, Any]], prefix: str) -> Optional[str]:
if not rows:
return None
fieldnames: List[str] = []
for row in rows:
for key in row:
if key not in fieldnames:
fieldnames.append(key)
handle = tempfile.NamedTemporaryFile(
mode="w", suffix=".csv", prefix=f"{prefix}-", delete=False, newline="", encoding="utf-8-sig",
)
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
for row in rows:
writer.writerow({key: row.get(key, "") for key in fieldnames})
handle.close()
return handle.name
def _error_md(data: Dict[str, Any]) -> str:
out = f"**That didn't go through.**\n\n{data.get('errorMessage') or 'Unknown error'}"
if data.get("hint"):
out += f"\n\n{data['hint']}"
return out
def _img(src: Any, cls: str) -> str:
link = _safe_link(src)
if not link:
return ""
return (
f'"
)
def _stars(rating: Any) -> str:
value = _num(rating)
if value is None:
return ""
full = max(0, min(5, int(round(value))))
return f'{"★" * full}{"★" * (5 - full)}'
def _rating_html(rating: Any, count: Any) -> str:
if _num(rating) is None:
return ""
count_html = f'({_fmt_int(count)})' if count not in (None, "") else ""
return f'
{_stars(rating)}{_num(rating):.1f}{count_html}
'
def _price_html(price: Any, list_price: Any, currency: Any = "USD", big: bool = False) -> str:
value = _num(price)
if value is None:
return '
Price not shown at this store
'
was = _num(list_price)
dollars, cents = f"{value:,.2f}".split(".")
amount = (
f'${dollars}{cents}'
if str(currency or "USD").upper() in ("USD", "CAD")
else _esc(_money(value, currency))
)
cls = "wm-price wm-price-big" if big else "wm-price"
if was and was > value:
return (
f'
Now{amount}'
f' {_esc(_money(was, currency))}
'
)
return f'
{amount}
'
def _stock_html(availability: Any) -> str:
code = str(availability or "").upper()
if not code:
return ""
if code == "IN_STOCK":
return 'In stock'
if "OUT" in code:
return 'Out of stock'
label = code.replace("_", " ").capitalize()
return f'{_esc(label)}'
def _fulfillment_html(methods: Any) -> str:
if not isinstance(methods, list) or not methods:
return ""
chips = "".join(f'{_esc(str(m).replace("_", " ").capitalize())}' for m in methods)
return f'
{chips}
'
def _html_to_text(raw: Any, limit: int = 3000) -> str:
text = re.sub(r"(?is)<\s*(script|style)[^>]*>.*?<\s*/\s*\1\s*>", "", str(raw or ""))
text = re.sub(r"(?i)<\s*li[^>]*>", "\n• ", text)
text = re.sub(r"(?i)<\s*(br|/p|/div|/ul|/ol|/h\d)[^>]*>", "\n", text)
text = re.sub(r"<[^>]+>", "", text)
text = html.unescape(text)
lines = [line.strip() for line in text.splitlines()]
text = "\n".join(line for line in lines if line)
return text[:limit] + ("…" if len(text) > limit else "")
def _store_banner(store: Dict[str, Any], prefix: str = "Prices and stock at") -> str:
label = describe_store(store)
if not label:
return ""
return f'
{PIN_SVG} {prefix} {_esc(label)}
'
CART_SVG = (
''
)
PIN_SVG = (
''
)
# ------------------------------------------------------------ search results
def render_results(data: Dict[str, Any], hide_sponsored: bool) -> str:
results = data.get("results") or []
shown = [r for r in results if not (hide_sponsored and r.get("sponsored"))]
cards: List[str] = []
for item in shown:
link = _safe_link(item.get("url"))
name = _esc(item.get("name"))
title = (
f'{name}'
if link else f'{name}'
)
seller = item.get("seller")
seller_html = (
f'
Sold by {_esc(seller)}
'
if seller and "walmart" not in str(seller).lower() else ""
)
cards.append(
'
'
f'
{_img(item.get("image"), "wm-img")}'
+ ('Sponsored' if item.get("sponsored") else "")
+ "
"
f"{_price_html(item.get('price'), item.get('list_price'), item.get('currency'))}"
f"{title}"
f"{_rating_html(item.get('rating'), item.get('reviews_count'))}"
f'
{_stock_html(item.get("availability"))}
'
f"{_fulfillment_html(item.get('fulfillment'))}"
f"{seller_html}"
f'
No products came back for this search at this store.
')
return (
f'
{_store_banner(data.get("store") or {})}'
f'
{"".join(cards)}
'
)
def _result_rows(data: Dict[str, Any]) -> List[Dict[str, Any]]:
store = data.get("store") or {}
rows = []
for item in data.get("results") or []:
rows.append(
{
"position": item.get("position"),
"item_id": item.get("item_id"),
"name": item.get("name"),
"brand": item.get("brand"),
"price": item.get("price"),
"list_price": item.get("list_price"),
"currency": item.get("currency"),
"rating": item.get("rating"),
"reviews_count": item.get("reviews_count"),
"availability": item.get("availability"),
"fulfillment": ", ".join(item.get("fulfillment") or []),
"seller": item.get("seller"),
"sponsored": item.get("sponsored"),
"category": item.get("category"),
"url": item.get("url"),
"image": item.get("image"),
"store_id": store.get("store_id"),
"store_city": store.get("city"),
"store_state": store.get("state"),
"store_zip": store.get("postal_code"),
}
)
return rows
# ------------------------------------------------------------- product page
def render_product(data: Dict[str, Any]) -> str:
product = data.get("product") or {}
images = [src for src in (product.get("images") or []) if _safe_link(src)]
main_img = _img(images[0], "wm-pdp-img") if images else ""
thumbs = "".join(_img(src, "wm-pdp-thumb") for src in images[1:6])
price = _num(product.get("price"))
was = _num(product.get("list_price"))
savings = ""
if price is not None and was and was > price:
saved = was - price
savings = (
f'You save {_esc(_money(saved, product.get("currency")))} '
f"({saved / was * 100:.0f}%)"
)
seller = product.get("seller")
seller_html = f'
Sold and shipped by {_esc(seller)}
' if seller else ""
limit = (
f'
Limit {_esc(product.get("order_limit"))} per order
'
if product.get("order_limit") else ""
)
link = _safe_link(product.get("url"))
button = (
f'View on Walmart'
if link else ""
)
def name_values(items: Any, cls: str) -> str:
rows = "".join(
f'
{_esc(i.get("name"))}
{_esc(i.get("value"))}
'
for i in (items or []) if isinstance(i, dict) and i.get("name")
)
return f'
{rows}
' if rows else ""
highlights = name_values(product.get("highlights"), "wm-specs")
specs = name_values(product.get("specifications"), "wm-specs")
about = product.get("short_description") or ""
long_text = _html_to_text(product.get("long_description"))
about_html = ""
if about or long_text:
about_html = '
About this item
'
if about:
about_html += f'
{_esc(_html_to_text(about, 1200))}
'
if long_text:
about_html += f'
{_esc(long_text).replace(chr(10), " ")}
'
about_html += "
"
facts = []
for label, key in [
("Condition", "condition"), ("Model number", "model_number"), ("UPC", "upc"),
("GTIN", "gtin"), ("Returns", "return_policy"), ("Item ID", "item_id"),
]:
if product.get(key):
facts.append({"name": label, "value": product[key]})
facts_html = name_values(facts, "wm-specs")
category = product.get("category") or ""
crumbs = " › ".join(_esc(part) for part in str(category).split("/") if part and part != "Home Page")
return (
f'
{_store_banner(data.get("store") or {})}'
'
'
f'
{main_img}
{thumbs}
'
'
'
+ (f'
{crumbs}
' if crumbs else "")
+ (f'
{_esc(product.get("brand"))}
' if product.get("brand") else "")
+ f'
{_esc(product.get("name"))}
'
f"{_rating_html(product.get('rating'), product.get('reviews_count'))}"
f"{_price_html(product.get('price'), product.get('list_price'), product.get('currency'), big=True)}"
f"{savings}"
f'
"
if (highlights or about_html or specs or facts_html) else ""
)
+ "
"
)
# ------------------------------------------------------------------- compare
def _place(row: Dict[str, Any]) -> str:
place = ", ".join(p for p in [row.get("city"), row.get("state")] if p)
return place or f"ZIP/store {row.get('location')}"
def render_compare(data: Dict[str, Any]) -> str:
product = data.get("product") or {}
rows = data.get("stores") or []
priced = sorted(
(r for r in rows if _num(r.get("price")) is not None), key=lambda r: _num(r.get("price")) or 0
)
unpriced = [r for r in rows if _num(r.get("price")) is None]
top = max((_num(r.get("price")) or 0 for r in priced), default=0)
best = _num(priced[0].get("price")) if priced else None
link = _safe_link(product.get("url"))
name = _esc(product.get("name") or f"Item {data.get('item_id')}")
name_html = (
f'{name}' if link
else f'{name}'
)
head = (
'
'
f'
{_img(product.get("image"), "wm-img")}
'
'
'
+ (f'
{_esc(product.get("brand"))}
' if product.get("brand") else "")
+ f"{name_html}{_rating_html(product.get('rating'), product.get('reviews_count'))}"
f'
Item {_esc(data.get("item_id"))}
'
"
"
)
lines: List[str] = []
for row in priced:
price = _num(row.get("price")) or 0
width = (price / top * 100) if top else 0
is_best = best is not None and abs(price - best) < 0.005
store_bits = " · ".join(
p for p in [f"Walmart #{row['store_id']}" if row.get("store_id") else "", str(row.get("postal_code") or "")] if p
)
lines.append(
f'
'
f'
{_esc(_place(row))}{_esc(store_bits)}
'
f'
'
f'
{_esc(_money(price, row.get("currency")))}'
+ (
f' {_esc(_money(row.get("list_price"), row.get("currency")))}'
if (_num(row.get("list_price")) or 0) > price else ""
)
+ "
'
+ ('Lowest price' if is_best and len(priced) > 1 else "")
+ "
"
)
for row in unpriced:
reason = row.get("error") or "No price shown at this store."
lines.append(
'
'
f'
{_esc(_place(row))}{_esc(row.get("store_id") and "Walmart #" + str(row["store_id"]) or "")}
'
f'
{_esc(reason)}
'
f'
{_stock_html(row.get("availability"))}
'
"
"
)
return f'
{head}
{"".join(lines)}
'
# ---------------------------------------------------------------- handlers
RESULT_HEADERS = ["#", "Product", "Price", "List price", "Rating", "Reviews", "Stock", "Item ID"]
COMPARE_HEADERS = ["Store", "Location", "ZIP", "Price", "List price", "Stock", "Fulfillment"]
def _picker(choices: List[Tuple[str, str]]) -> Any:
return gr.Dropdown(choices=choices, value=None, visible=bool(choices))
def run_search(
token: str, query: str, location: str, hide_sponsored: bool
) -> Tuple[str, str, List[List[Any]], Dict[str, Any], Optional[str], Any]:
tool = _tool_or_none(token)
if tool is None:
return NO_TOKEN_MD, "", [], {}, None, _picker([])
if not query or not query.strip():
return "Type what you're looking for, e.g. `whole milk`.", "", [], {}, None, _picker([])
data = tool.search(query, location)
if data.get("status") != "success":
return _error_md(data), "", [], data, None, _picker([])
results = data.get("results") or []
prices = [_num(r.get("price")) for r in results if _num(r.get("price")) is not None]
in_stock = sum(1 for r in results if str(r.get("availability") or "").upper() == "IN_STOCK")
on_sale = sum(
1 for r in results
if _num(r.get("list_price")) and _num(r.get("price")) is not None and _num(r["list_price"]) > _num(r["price"])
)
sponsored = sum(1 for r in results if r.get("sponsored"))
parts = [f"**{len(results)}** products", f"**{in_stock}** in stock", f"**{on_sale}** on sale"]
if prices:
parts.append(f"prices **{_money(min(prices))}–{_money(max(prices))}**")
if sponsored:
parts.append(f"{sponsored} sponsored" + (" (hidden)" if hide_sponsored else ""))
summary = " · ".join(parts)
store_label = describe_store(data.get("store"))
footer = []
if store_label:
footer.append(store_label)
if data.get("total_results"):
footer.append(f"{_fmt_int(data['total_results'])} matches on Walmart")
footer.append(f"{data.get('credits_used')} credits used")
summary += "\n\n" + " · ".join(footer)
rows = _result_rows(data)
table = [
[
r["position"], (r["name"] or "")[:90], _money(r["price"], r["currency"]),
_money(r["list_price"], r["currency"]), r["rating"], r["reviews_count"],
str(r["availability"] or "").replace("_", " ").lower(), r["item_id"],
]
for r in rows
]
choices = [
(f"{r.get('position')}. {str(r.get('name') or '')[:70]} — {_money(r.get('price'), r.get('currency')) or 'no price'}",
str(r["item_id"]))
for r in results if r.get("item_id")
]
csv_path = _write_csv(rows, "walmart-search")
return summary, render_results(data, bool(hide_sponsored)), table, data, csv_path, _picker(choices)
def send_to_tabs(picked: Optional[str], product_ref: str, compare_ref: str) -> Tuple[str, str, str]:
if not picked:
return product_ref, compare_ref, ""
return picked, picked, "Sent. Open the **Product details** or **Compare prices** tab and press the button."
def run_product(token: str, ref: str, location: str) -> Tuple[str, str, Dict[str, Any]]:
tool = _tool_or_none(token)
if tool is None:
return NO_TOKEN_MD, "", {}
if not ref or not ref.strip():
return "Paste a Walmart product link or item id, or type a product name.", "", {}
data = tool.product(ref, location)
if data.get("status") != "success":
return _error_md(data), "", data
lines = []
if data.get("note"):
lines.append(data["note"])
lines.append(f"{data.get('credits_used')} credits used.")
return "\n\n".join(lines), render_product(data), data
def run_compare(
token: str, ref: str, locations: str
) -> Tuple[str, str, List[List[Any]], Dict[str, Any], Optional[str]]:
tool = _tool_or_none(token)
if tool is None:
return NO_TOKEN_MD, "", [], {}, None
if not ref or not ref.strip():
return "Paste a Walmart product link or item id, or type a product name.", "", [], {}, None
data = tool.compare(ref, locations)
if data.get("status") != "success":
return _error_md(data), "", [], data, None
rows = data.get("stores") or []
priced = [r for r in rows if _num(r.get("price")) is not None]
lines = []
if data.get("note"):
lines.append(data["note"])
cheapest, dearest = data.get("cheapest"), data.get("most_expensive")
if cheapest and dearest:
low, high = _num(cheapest["price"]) or 0, _num(dearest["price"]) or 0
line = f"Cheapest at **{_place(cheapest)}**: **{_money(low)}**."
if high - low >= 0.01:
line += (
f" That's **{_money(high - low)} ({(high - low) / high * 100:.0f}%) less** than "
f"**{_place(dearest)}** at {_money(high)}."
)
elif len(priced) > 1:
line = f"Same price, **{_money(low)}**, at every store that showed one."
lines.append(line)
else:
lines.append("None of these stores showed a price for this product.")
lines.append(f"Prices found at {len(priced)} of {len(rows)} stores · {data.get('credits_used')} credits used")
if data.get("warnings"):
lines.append("\n".join(f"- {w}" for w in data["warnings"]))
table = [
[
f"#{r.get('store_id')}" if r.get("store_id") else "",
_place(r),
r.get("postal_code") or r.get("location"),
_money(r.get("price"), r.get("currency")) or "—",
_money(r.get("list_price"), r.get("currency")),
str(r.get("availability") or r.get("error") or "").replace("_", " ").lower(),
", ".join(r.get("fulfillment") or []),
]
for r in rows
]
product = data.get("product") or {}
csv_rows = [
{
"item_id": data.get("item_id"),
"product": product.get("name"),
"location_input": r.get("location"),
"store_id": r.get("store_id"),
"city": r.get("city"),
"state": r.get("state"),
"zip": r.get("postal_code"),
"price": r.get("price"),
"list_price": r.get("list_price"),
"currency": r.get("currency"),
"availability": r.get("availability"),
"fulfillment": ", ".join(r.get("fulfillment") or []),
"seller": r.get("seller"),
"error": r.get("error"),
}
for r in rows
]
csv_path = _write_csv(csv_rows, "walmart-price-compare")
return "\n\n".join(lines), render_compare(data), table, data, csv_path
# ----------------------------------------------------------------------- UI
STYLE = """
"""
HERO = f"""
{CART_SVG.format(size=34)} Walmart Scraper
Walmart prices change from store to store. Search products at any US store by ZIP code,
pull a product's full details, or check one product's price and stock at up to five stores side by side.
Built on the Scrape.do Walmart API,
which picks the store for you and gets past Walmart's bot protection. Use it for price monitoring,
assortment research, MAP checks, or to give an AI agent live retail prices.
Every call costs 10 credits: one search, one product, or one store in a price comparison. Failed calls
are free. The free 1,000 credits cover about 100 calls. Typing a product name instead of pasting a link
adds one search (10 credits) to find it.
"""
FOOTER_MD = f"""
---
### Use it as a smolagents Tool
```python
from smolagents import CodeAgent, InferenceClientModel
from tool import ScrapeDoWalmartTool
agent = CodeAgent(
tools=[ScrapeDoWalmartTool(api_token="YOUR_SCRAPEDO_TOKEN")],
model=InferenceClientModel(),
)
agent.run("Find the cheapest gallon of whole milk at the Walmart nearest ZIP 60074, then compare its price in Dallas and Los Angeles.")
```
Or set `SCRAPE_DO_API_TOKEN` in the environment and instantiate with no arguments.
Want to scrape category pages and track price changes over time? The
[Walmart scraping guide](https://scrape.do/blog/walmart-scraping/) walks through it in Python.
[Scrape.do](https://scrape.do/) · [Walmart API docs](https://scrape.do/documentation/walmart-scraper-api/) · [Sign up free]({SCRAPE_DO_SIGNUP_URL})
"""
def build_demo() -> gr.Blocks:
with gr.Blocks(title="Walmart Scraper — Scrape.do") as demo:
gr.HTML(STYLE)
gr.HTML(HERO)
with gr.Row():
api_token = gr.Textbox(
label="Scrape.do API token",
placeholder="Paste your token here",
type="password",
info="Free tier: 1,000 credits a month, about 100 calls.",
scale=3,
)
location = gr.Textbox(
label="Store: ZIP code or Walmart store number",
value=DEFAULT_LOCATION,
placeholder="07094",
info="Prices and stock are for the nearest store.",
scale=2,
)
gr.HTML(CREDIT_NOTE)
with gr.Tabs():
with gr.Tab("Search products"):
with gr.Row():
query = gr.Textbox(label="Search Walmart", placeholder="whole milk", scale=4)
hide_sponsored = gr.Checkbox(label="Hide sponsored", value=False, scale=1)
search_btn = gr.Button("Search Walmart", variant="primary")
search_summary = gr.Markdown()
results_view = gr.HTML()
results_table = gr.Dataframe(headers=RESULT_HEADERS, wrap=True, label="Products")
results_csv = gr.File(label="Download results as CSV")
pick = gr.Dropdown(
label="Open one of these products in the Details and Compare tabs",
choices=[],
value=None,
visible=False,
)
pick_note = gr.Markdown()
with gr.Accordion("Raw JSON response", open=False):
results_json = gr.JSON()
with gr.Tab("Product details"):
product_ref = gr.Textbox(
label="Walmart product link, item id, or product name",
placeholder="https://www.walmart.com/ip/…/17221971947 or 17221971947",
)
product_btn = gr.Button("Get product details", variant="primary")
product_summary = gr.Markdown()
product_view = gr.HTML()
with gr.Accordion("Raw JSON response", open=False):
product_json = gr.JSON()
with gr.Tab("Compare prices by ZIP"):
gr.Markdown(
"Same product, different stores. Enter up to "
f"{MAX_COMPARE_LOCATIONS} ZIP codes or store numbers; each one costs 10 credits."
)
compare_ref = gr.Textbox(
label="Walmart product link, item id, or product name",
placeholder="https://www.walmart.com/ip/…/17221971947 or 17221971947",
)
compare_locations = gr.Textbox(
label="ZIP codes or store numbers, separated by commas",
value=", ".join(DEFAULT_COMPARE_LOCATIONS),
info="Default: Secaucus NJ, Deer Park IL, Dallas TX, Los Angeles CA.",
)
compare_btn = gr.Button("Compare prices", variant="primary")
compare_summary = gr.Markdown()
compare_view = gr.HTML()
compare_table = gr.Dataframe(headers=COMPARE_HEADERS, wrap=True, label="Price by store")
compare_csv = gr.File(label="Download comparison as CSV")
with gr.Accordion("Raw JSON response", open=False):
compare_json = gr.JSON()
search_inputs = [api_token, query, location, hide_sponsored]
search_outputs = [search_summary, results_view, results_table, results_json, results_csv, pick]
search_btn.click(fn=run_search, inputs=search_inputs, outputs=search_outputs)
query.submit(fn=run_search, inputs=search_inputs, outputs=search_outputs)
pick.change(fn=send_to_tabs, inputs=[pick, product_ref, compare_ref], outputs=[product_ref, compare_ref, pick_note])
product_inputs = [api_token, product_ref, location]
product_outputs = [product_summary, product_view, product_json]
product_btn.click(fn=run_product, inputs=product_inputs, outputs=product_outputs)
product_ref.submit(fn=run_product, inputs=product_inputs, outputs=product_outputs)
compare_inputs = [api_token, compare_ref, compare_locations]
compare_outputs = [compare_summary, compare_view, compare_table, compare_json, compare_csv]
compare_btn.click(fn=run_compare, inputs=compare_inputs, outputs=compare_outputs)
compare_ref.submit(fn=run_compare, inputs=compare_inputs, outputs=compare_outputs)
gr.Markdown(FOOTER_MD)
return demo
# Hugging Face Spaces launches the module-level `demo` object rather than
# calling launch() itself, so build it at import time.
demo = build_demo()
if __name__ == "__main__":
demo.launch()