from __future__ import annotations import csv import html import re import tempfile from typing import Any, Dict, List, Optional, Tuple import gradio as gr from tool import ( DEFAULT_COMPARE_LOCATIONS, DEFAULT_LOCATION, MAX_COMPARE_LOCATIONS, SCRAPE_DO_SIGNUP_URL, ScrapeDoWalmartTool, describe_store, ) # ZeroGPU hardware refuses to keep a Space alive unless it finds at least one # @spaces.GPU function at startup. This app is pure HTTP and never touches a GPU, # so the probe below exists only to satisfy that check. It is never called. try: import spaces @spaces.GPU(duration=1) def _zerogpu_probe() -> str: return "ok" except ImportError: pass NO_TOKEN_MD = ( "### Paste your Scrape.do API token above\n\n" f"No token yet? [Create a free account]({SCRAPE_DO_SIGNUP_URL}) — " "1,000 free credits a month, no credit card. Each Walmart call costs 10 credits." ) # --------------------------------------------------------------------- helpers def _esc(value: Any) -> str: return html.escape("" if value is None else str(value), quote=True) def _safe_link(url: Any) -> str: url = str(url or "") return url if url.startswith(("http://", "https://")) else "" def _num(value: Any) -> Optional[float]: if isinstance(value, bool): return None try: return float(str(value).replace(",", "").replace("$", "")) except (TypeError, ValueError): return None def _fmt_int(value: Any) -> str: number = _num(value) return f"{int(number):,}" if number is not None else _esc(value) def _money(value: Any, currency: Any = "USD") -> str: number = _num(value) if number is None: return "" code = str(currency or "USD").upper() if code in ("USD", "CAD"): return f"${number:,.2f}" return f"{number:,.2f} {code}" def _tool_or_none(token: str) -> Optional[ScrapeDoWalmartTool]: if not token or not token.strip(): return None return ScrapeDoWalmartTool(api_token=token.strip()) def _write_csv(rows: List[Dict[str, Any]], prefix: str) -> Optional[str]: if not rows: return None fieldnames: List[str] = [] for row in rows: for key in row: if key not in fieldnames: fieldnames.append(key) handle = tempfile.NamedTemporaryFile( mode="w", suffix=".csv", prefix=f"{prefix}-", delete=False, newline="", encoding="utf-8-sig", ) writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() for row in rows: writer.writerow({key: row.get(key, "") for key in fieldnames}) handle.close() return handle.name def _error_md(data: Dict[str, Any]) -> str: out = f"**That didn't go through.**\n\n{data.get('errorMessage') or 'Unknown error'}" if data.get("hint"): out += f"\n\n{data['hint']}" return out def _img(src: Any, cls: str) -> str: link = _safe_link(src) if not link: return "" return ( f'" ) def _stars(rating: Any) -> str: value = _num(rating) if value is None: return "" full = max(0, min(5, int(round(value)))) return f'{"★" * full}{"★" * (5 - full)}' def _rating_html(rating: Any, count: Any) -> str: if _num(rating) is None: return "" count_html = f'({_fmt_int(count)})' if count not in (None, "") else "" return f'
{_stars(rating)}{_num(rating):.1f}{count_html}
' def _price_html(price: Any, list_price: Any, currency: Any = "USD", big: bool = False) -> str: value = _num(price) if value is None: return '
Price not shown at this store
' was = _num(list_price) dollars, cents = f"{value:,.2f}".split(".") amount = ( f'${dollars}{cents}' if str(currency or "USD").upper() in ("USD", "CAD") else _esc(_money(value, currency)) ) cls = "wm-price wm-price-big" if big else "wm-price" if was and was > value: return ( f'
Now {amount}' f' {_esc(_money(was, currency))}
' ) return f'
{amount}
' def _stock_html(availability: Any) -> str: code = str(availability or "").upper() if not code: return "" if code == "IN_STOCK": return 'In stock' if "OUT" in code: return 'Out of stock' label = code.replace("_", " ").capitalize() return f'{_esc(label)}' def _fulfillment_html(methods: Any) -> str: if not isinstance(methods, list) or not methods: return "" chips = "".join(f'{_esc(str(m).replace("_", " ").capitalize())}' for m in methods) return f'
{chips}
' def _html_to_text(raw: Any, limit: int = 3000) -> str: text = re.sub(r"(?is)<\s*(script|style)[^>]*>.*?<\s*/\s*\1\s*>", "", str(raw or "")) text = re.sub(r"(?i)<\s*li[^>]*>", "\n• ", text) text = re.sub(r"(?i)<\s*(br|/p|/div|/ul|/ol|/h\d)[^>]*>", "\n", text) text = re.sub(r"<[^>]+>", "", text) text = html.unescape(text) lines = [line.strip() for line in text.splitlines()] text = "\n".join(line for line in lines if line) return text[:limit] + ("…" if len(text) > limit else "") def _store_banner(store: Dict[str, Any], prefix: str = "Prices and stock at") -> str: label = describe_store(store) if not label: return "" return f'
{PIN_SVG} {prefix} {_esc(label)}
' CART_SVG = ( '' ) PIN_SVG = ( '' ) # ------------------------------------------------------------ search results def render_results(data: Dict[str, Any], hide_sponsored: bool) -> str: results = data.get("results") or [] shown = [r for r in results if not (hide_sponsored and r.get("sponsored"))] cards: List[str] = [] for item in shown: link = _safe_link(item.get("url")) name = _esc(item.get("name")) title = ( f'{name}' if link else f'{name}' ) seller = item.get("seller") seller_html = ( f'
Sold by {_esc(seller)}
' if seller and "walmart" not in str(seller).lower() else "" ) cards.append( '
' f'
{_img(item.get("image"), "wm-img")}' + ('Sponsored' if item.get("sponsored") else "") + "
" f"{_price_html(item.get('price'), item.get('list_price'), item.get('currency'))}" f"{title}" f"{_rating_html(item.get('rating'), item.get('reviews_count'))}" f'
{_stock_html(item.get("availability"))}
' f"{_fulfillment_html(item.get('fulfillment'))}" f"{seller_html}" f'
#{_esc(item.get("position"))} · Item {_esc(item.get("item_id"))}
' "
" ) if not cards: cards.append('
No products came back for this search at this store.
') return ( f'
{_store_banner(data.get("store") or {})}' f'
{"".join(cards)}
' ) def _result_rows(data: Dict[str, Any]) -> List[Dict[str, Any]]: store = data.get("store") or {} rows = [] for item in data.get("results") or []: rows.append( { "position": item.get("position"), "item_id": item.get("item_id"), "name": item.get("name"), "brand": item.get("brand"), "price": item.get("price"), "list_price": item.get("list_price"), "currency": item.get("currency"), "rating": item.get("rating"), "reviews_count": item.get("reviews_count"), "availability": item.get("availability"), "fulfillment": ", ".join(item.get("fulfillment") or []), "seller": item.get("seller"), "sponsored": item.get("sponsored"), "category": item.get("category"), "url": item.get("url"), "image": item.get("image"), "store_id": store.get("store_id"), "store_city": store.get("city"), "store_state": store.get("state"), "store_zip": store.get("postal_code"), } ) return rows # ------------------------------------------------------------- product page def render_product(data: Dict[str, Any]) -> str: product = data.get("product") or {} images = [src for src in (product.get("images") or []) if _safe_link(src)] main_img = _img(images[0], "wm-pdp-img") if images else "" thumbs = "".join(_img(src, "wm-pdp-thumb") for src in images[1:6]) price = _num(product.get("price")) was = _num(product.get("list_price")) savings = "" if price is not None and was and was > price: saved = was - price savings = ( f'You save {_esc(_money(saved, product.get("currency")))} ' f"({saved / was * 100:.0f}%)" ) seller = product.get("seller") seller_html = f'
Sold and shipped by {_esc(seller)}
' if seller else "" limit = ( f'
Limit {_esc(product.get("order_limit"))} per order
' if product.get("order_limit") else "" ) link = _safe_link(product.get("url")) button = ( f'View on Walmart' if link else "" ) def name_values(items: Any, cls: str) -> str: rows = "".join( f'
{_esc(i.get("name"))}
{_esc(i.get("value"))}
' for i in (items or []) if isinstance(i, dict) and i.get("name") ) return f'
{rows}
' if rows else "" highlights = name_values(product.get("highlights"), "wm-specs") specs = name_values(product.get("specifications"), "wm-specs") about = product.get("short_description") or "" long_text = _html_to_text(product.get("long_description")) about_html = "" if about or long_text: about_html = '
About this item
' if about: about_html += f'
{_esc(_html_to_text(about, 1200))}
' if long_text: about_html += f'
{_esc(long_text).replace(chr(10), "
")}
' about_html += "
" facts = [] for label, key in [ ("Condition", "condition"), ("Model number", "model_number"), ("UPC", "upc"), ("GTIN", "gtin"), ("Returns", "return_policy"), ("Item ID", "item_id"), ]: if product.get(key): facts.append({"name": label, "value": product[key]}) facts_html = name_values(facts, "wm-specs") category = product.get("category") or "" crumbs = " › ".join(_esc(part) for part in str(category).split("/") if part and part != "Home Page") return ( f'
{_store_banner(data.get("store") or {})}' '
' f'
{main_img}
{thumbs}
' '
' + (f'
{crumbs}
' if crumbs else "") + (f'
{_esc(product.get("brand"))}
' if product.get("brand") else "") + f'
{_esc(product.get("name"))}
' f"{_rating_html(product.get('rating'), product.get('reviews_count'))}" f"{_price_html(product.get('price'), product.get('list_price'), product.get('currency'), big=True)}" f"{savings}" f'
{_stock_html(product.get("availability"))}
' f"{_fulfillment_html(product.get('fulfillment'))}" f"{seller_html}{limit}{button}" "
" + ( '
' + (f'
Highlights
{highlights}
' if highlights else "") + about_html + (f'
Specifications
{specs}
' if specs else "") + (f'
Product details
{facts_html}
' if facts_html else "") + "
" if (highlights or about_html or specs or facts_html) else "" ) + "
" ) # ------------------------------------------------------------------- compare def _place(row: Dict[str, Any]) -> str: place = ", ".join(p for p in [row.get("city"), row.get("state")] if p) return place or f"ZIP/store {row.get('location')}" def render_compare(data: Dict[str, Any]) -> str: product = data.get("product") or {} rows = data.get("stores") or [] priced = sorted( (r for r in rows if _num(r.get("price")) is not None), key=lambda r: _num(r.get("price")) or 0 ) unpriced = [r for r in rows if _num(r.get("price")) is None] top = max((_num(r.get("price")) or 0 for r in priced), default=0) best = _num(priced[0].get("price")) if priced else None link = _safe_link(product.get("url")) name = _esc(product.get("name") or f"Item {data.get('item_id')}") name_html = ( f'{name}' if link else f'{name}' ) head = ( '
' f'
{_img(product.get("image"), "wm-img")}
' '
' + (f'
{_esc(product.get("brand"))}
' if product.get("brand") else "") + f"{name_html}{_rating_html(product.get('rating'), product.get('reviews_count'))}" f'
Item {_esc(data.get("item_id"))}
' "
" ) lines: List[str] = [] for row in priced: price = _num(row.get("price")) or 0 width = (price / top * 100) if top else 0 is_best = best is not None and abs(price - best) < 0.005 store_bits = " · ".join( p for p in [f"Walmart #{row['store_id']}" if row.get("store_id") else "", str(row.get("postal_code") or "")] if p ) lines.append( f'
' f'
{_esc(_place(row))}{_esc(store_bits)}
' f'
' f'
{_esc(_money(price, row.get("currency")))}' + ( f' {_esc(_money(row.get("list_price"), row.get("currency")))}' if (_num(row.get("list_price")) or 0) > price else "" ) + "
" f'
{_stock_html(row.get("availability"))}{_fulfillment_html(row.get("fulfillment"))}
' + ('Lowest price' if is_best and len(priced) > 1 else "") + "
" ) for row in unpriced: reason = row.get("error") or "No price shown at this store." lines.append( '
' f'
{_esc(_place(row))}{_esc(row.get("store_id") and "Walmart #" + str(row["store_id"]) or "")}
' f'
{_esc(reason)}
' f'
{_stock_html(row.get("availability"))}
' "
" ) return f'
{head}
{"".join(lines)}
' # ---------------------------------------------------------------- handlers RESULT_HEADERS = ["#", "Product", "Price", "List price", "Rating", "Reviews", "Stock", "Item ID"] COMPARE_HEADERS = ["Store", "Location", "ZIP", "Price", "List price", "Stock", "Fulfillment"] def _picker(choices: List[Tuple[str, str]]) -> Any: return gr.Dropdown(choices=choices, value=None, visible=bool(choices)) def run_search( token: str, query: str, location: str, hide_sponsored: bool ) -> Tuple[str, str, List[List[Any]], Dict[str, Any], Optional[str], Any]: tool = _tool_or_none(token) if tool is None: return NO_TOKEN_MD, "", [], {}, None, _picker([]) if not query or not query.strip(): return "Type what you're looking for, e.g. `whole milk`.", "", [], {}, None, _picker([]) data = tool.search(query, location) if data.get("status") != "success": return _error_md(data), "", [], data, None, _picker([]) results = data.get("results") or [] prices = [_num(r.get("price")) for r in results if _num(r.get("price")) is not None] in_stock = sum(1 for r in results if str(r.get("availability") or "").upper() == "IN_STOCK") on_sale = sum( 1 for r in results if _num(r.get("list_price")) and _num(r.get("price")) is not None and _num(r["list_price"]) > _num(r["price"]) ) sponsored = sum(1 for r in results if r.get("sponsored")) parts = [f"**{len(results)}** products", f"**{in_stock}** in stock", f"**{on_sale}** on sale"] if prices: parts.append(f"prices **{_money(min(prices))}–{_money(max(prices))}**") if sponsored: parts.append(f"{sponsored} sponsored" + (" (hidden)" if hide_sponsored else "")) summary = " · ".join(parts) store_label = describe_store(data.get("store")) footer = [] if store_label: footer.append(store_label) if data.get("total_results"): footer.append(f"{_fmt_int(data['total_results'])} matches on Walmart") footer.append(f"{data.get('credits_used')} credits used") summary += "\n\n" + " · ".join(footer) rows = _result_rows(data) table = [ [ r["position"], (r["name"] or "")[:90], _money(r["price"], r["currency"]), _money(r["list_price"], r["currency"]), r["rating"], r["reviews_count"], str(r["availability"] or "").replace("_", " ").lower(), r["item_id"], ] for r in rows ] choices = [ (f"{r.get('position')}. {str(r.get('name') or '')[:70]} — {_money(r.get('price'), r.get('currency')) or 'no price'}", str(r["item_id"])) for r in results if r.get("item_id") ] csv_path = _write_csv(rows, "walmart-search") return summary, render_results(data, bool(hide_sponsored)), table, data, csv_path, _picker(choices) def send_to_tabs(picked: Optional[str], product_ref: str, compare_ref: str) -> Tuple[str, str, str]: if not picked: return product_ref, compare_ref, "" return picked, picked, "Sent. Open the **Product details** or **Compare prices** tab and press the button." def run_product(token: str, ref: str, location: str) -> Tuple[str, str, Dict[str, Any]]: tool = _tool_or_none(token) if tool is None: return NO_TOKEN_MD, "", {} if not ref or not ref.strip(): return "Paste a Walmart product link or item id, or type a product name.", "", {} data = tool.product(ref, location) if data.get("status") != "success": return _error_md(data), "", data lines = [] if data.get("note"): lines.append(data["note"]) lines.append(f"{data.get('credits_used')} credits used.") return "\n\n".join(lines), render_product(data), data def run_compare( token: str, ref: str, locations: str ) -> Tuple[str, str, List[List[Any]], Dict[str, Any], Optional[str]]: tool = _tool_or_none(token) if tool is None: return NO_TOKEN_MD, "", [], {}, None if not ref or not ref.strip(): return "Paste a Walmart product link or item id, or type a product name.", "", [], {}, None data = tool.compare(ref, locations) if data.get("status") != "success": return _error_md(data), "", [], data, None rows = data.get("stores") or [] priced = [r for r in rows if _num(r.get("price")) is not None] lines = [] if data.get("note"): lines.append(data["note"]) cheapest, dearest = data.get("cheapest"), data.get("most_expensive") if cheapest and dearest: low, high = _num(cheapest["price"]) or 0, _num(dearest["price"]) or 0 line = f"Cheapest at **{_place(cheapest)}**: **{_money(low)}**." if high - low >= 0.01: line += ( f" That's **{_money(high - low)} ({(high - low) / high * 100:.0f}%) less** than " f"**{_place(dearest)}** at {_money(high)}." ) elif len(priced) > 1: line = f"Same price, **{_money(low)}**, at every store that showed one." lines.append(line) else: lines.append("None of these stores showed a price for this product.") lines.append(f"Prices found at {len(priced)} of {len(rows)} stores · {data.get('credits_used')} credits used") if data.get("warnings"): lines.append("\n".join(f"- {w}" for w in data["warnings"])) table = [ [ f"#{r.get('store_id')}" if r.get("store_id") else "", _place(r), r.get("postal_code") or r.get("location"), _money(r.get("price"), r.get("currency")) or "—", _money(r.get("list_price"), r.get("currency")), str(r.get("availability") or r.get("error") or "").replace("_", " ").lower(), ", ".join(r.get("fulfillment") or []), ] for r in rows ] product = data.get("product") or {} csv_rows = [ { "item_id": data.get("item_id"), "product": product.get("name"), "location_input": r.get("location"), "store_id": r.get("store_id"), "city": r.get("city"), "state": r.get("state"), "zip": r.get("postal_code"), "price": r.get("price"), "list_price": r.get("list_price"), "currency": r.get("currency"), "availability": r.get("availability"), "fulfillment": ", ".join(r.get("fulfillment") or []), "seller": r.get("seller"), "error": r.get("error"), } for r in rows ] csv_path = _write_csv(csv_rows, "walmart-price-compare") return "\n\n".join(lines), render_compare(data), table, data, csv_path # ----------------------------------------------------------------------- UI STYLE = """ """ HERO = f"""

{CART_SVG.format(size=34)} Walmart Scraper

Walmart prices change from store to store. Search products at any US store by ZIP code, pull a product's full details, or check one product's price and stock at up to five stores side by side.

Built on the Scrape.do Walmart API, which picks the store for you and gets past Walmart's bot protection. Use it for price monitoring, assortment research, MAP checks, or to give an AI agent live retail prices.

Get a free API token →
""" CREDIT_NOTE = """
Every call costs 10 credits: one search, one product, or one store in a price comparison. Failed calls are free. The free 1,000 credits cover about 100 calls. Typing a product name instead of pasting a link adds one search (10 credits) to find it.
""" FOOTER_MD = f""" --- ### Use it as a smolagents Tool ```python from smolagents import CodeAgent, InferenceClientModel from tool import ScrapeDoWalmartTool agent = CodeAgent( tools=[ScrapeDoWalmartTool(api_token="YOUR_SCRAPEDO_TOKEN")], model=InferenceClientModel(), ) agent.run("Find the cheapest gallon of whole milk at the Walmart nearest ZIP 60074, then compare its price in Dallas and Los Angeles.") ``` Or set `SCRAPE_DO_API_TOKEN` in the environment and instantiate with no arguments. Want to scrape category pages and track price changes over time? The [Walmart scraping guide](https://scrape.do/blog/walmart-scraping/) walks through it in Python. [Scrape.do](https://scrape.do/) · [Walmart API docs](https://scrape.do/documentation/walmart-scraper-api/) · [Sign up free]({SCRAPE_DO_SIGNUP_URL}) """ def build_demo() -> gr.Blocks: with gr.Blocks(title="Walmart Scraper — Scrape.do") as demo: gr.HTML(STYLE) gr.HTML(HERO) with gr.Row(): api_token = gr.Textbox( label="Scrape.do API token", placeholder="Paste your token here", type="password", info="Free tier: 1,000 credits a month, about 100 calls.", scale=3, ) location = gr.Textbox( label="Store: ZIP code or Walmart store number", value=DEFAULT_LOCATION, placeholder="07094", info="Prices and stock are for the nearest store.", scale=2, ) gr.HTML(CREDIT_NOTE) with gr.Tabs(): with gr.Tab("Search products"): with gr.Row(): query = gr.Textbox(label="Search Walmart", placeholder="whole milk", scale=4) hide_sponsored = gr.Checkbox(label="Hide sponsored", value=False, scale=1) search_btn = gr.Button("Search Walmart", variant="primary") search_summary = gr.Markdown() results_view = gr.HTML() results_table = gr.Dataframe(headers=RESULT_HEADERS, wrap=True, label="Products") results_csv = gr.File(label="Download results as CSV") pick = gr.Dropdown( label="Open one of these products in the Details and Compare tabs", choices=[], value=None, visible=False, ) pick_note = gr.Markdown() with gr.Accordion("Raw JSON response", open=False): results_json = gr.JSON() with gr.Tab("Product details"): product_ref = gr.Textbox( label="Walmart product link, item id, or product name", placeholder="https://www.walmart.com/ip/…/17221971947 or 17221971947", ) product_btn = gr.Button("Get product details", variant="primary") product_summary = gr.Markdown() product_view = gr.HTML() with gr.Accordion("Raw JSON response", open=False): product_json = gr.JSON() with gr.Tab("Compare prices by ZIP"): gr.Markdown( "Same product, different stores. Enter up to " f"{MAX_COMPARE_LOCATIONS} ZIP codes or store numbers; each one costs 10 credits." ) compare_ref = gr.Textbox( label="Walmart product link, item id, or product name", placeholder="https://www.walmart.com/ip/…/17221971947 or 17221971947", ) compare_locations = gr.Textbox( label="ZIP codes or store numbers, separated by commas", value=", ".join(DEFAULT_COMPARE_LOCATIONS), info="Default: Secaucus NJ, Deer Park IL, Dallas TX, Los Angeles CA.", ) compare_btn = gr.Button("Compare prices", variant="primary") compare_summary = gr.Markdown() compare_view = gr.HTML() compare_table = gr.Dataframe(headers=COMPARE_HEADERS, wrap=True, label="Price by store") compare_csv = gr.File(label="Download comparison as CSV") with gr.Accordion("Raw JSON response", open=False): compare_json = gr.JSON() search_inputs = [api_token, query, location, hide_sponsored] search_outputs = [search_summary, results_view, results_table, results_json, results_csv, pick] search_btn.click(fn=run_search, inputs=search_inputs, outputs=search_outputs) query.submit(fn=run_search, inputs=search_inputs, outputs=search_outputs) pick.change(fn=send_to_tabs, inputs=[pick, product_ref, compare_ref], outputs=[product_ref, compare_ref, pick_note]) product_inputs = [api_token, product_ref, location] product_outputs = [product_summary, product_view, product_json] product_btn.click(fn=run_product, inputs=product_inputs, outputs=product_outputs) product_ref.submit(fn=run_product, inputs=product_inputs, outputs=product_outputs) compare_inputs = [api_token, compare_ref, compare_locations] compare_outputs = [compare_summary, compare_view, compare_table, compare_json, compare_csv] compare_btn.click(fn=run_compare, inputs=compare_inputs, outputs=compare_outputs) compare_ref.submit(fn=run_compare, inputs=compare_inputs, outputs=compare_outputs) gr.Markdown(FOOTER_MD) return demo # Hugging Face Spaces launches the module-level `demo` object rather than # calling launch() itself, so build it at import time. demo = build_demo() if __name__ == "__main__": demo.launch()