Spaces:
Sleeping
Sleeping
Download app.py from bugrahans/walmart-scraper: direct link, hf CLI and curl.
- Browser
- Download file 42 kB
-
https://huggingface.co/spaces/bugrahans/walmart-scraper/resolve/main/app.py
- Command line
-
hf download hf://spaces/bugrahans/walmart-scraper/app.py
-
curl -L -o app.py https://huggingface.co/spaces/bugrahans/walmart-scraper/resolve/main/app.py
42 kB
| from __future__ import annotations | |
| import csv | |
| import html | |
| import re | |
| import tempfile | |
| from typing import Any, Dict, List, Optional, Tuple | |
| import gradio as gr | |
| from tool import ( | |
| DEFAULT_COMPARE_LOCATIONS, | |
| DEFAULT_LOCATION, | |
| MAX_COMPARE_LOCATIONS, | |
| SCRAPE_DO_SIGNUP_URL, | |
| ScrapeDoWalmartTool, | |
| describe_store, | |
| ) | |
| # ZeroGPU hardware refuses to keep a Space alive unless it finds at least one | |
| # @spaces.GPU function at startup. This app is pure HTTP and never touches a GPU, | |
| # so the probe below exists only to satisfy that check. It is never called. | |
| try: | |
| import spaces | |
| def _zerogpu_probe() -> str: | |
| return "ok" | |
| except ImportError: | |
| pass | |
| NO_TOKEN_MD = ( | |
| "### Paste your Scrape.do API token above\n\n" | |
| f"No token yet? [Create a free account]({SCRAPE_DO_SIGNUP_URL}) — " | |
| "1,000 free credits a month, no credit card. Each Walmart call costs 10 credits." | |
| ) | |
| # --------------------------------------------------------------------- helpers | |
| def _esc(value: Any) -> str: | |
| return html.escape("" if value is None else str(value), quote=True) | |
| def _safe_link(url: Any) -> str: | |
| url = str(url or "") | |
| return url if url.startswith(("http://", "https://")) else "" | |
| def _num(value: Any) -> Optional[float]: | |
| if isinstance(value, bool): | |
| return None | |
| try: | |
| return float(str(value).replace(",", "").replace("$", "")) | |
| except (TypeError, ValueError): | |
| return None | |
| def _fmt_int(value: Any) -> str: | |
| number = _num(value) | |
| return f"{int(number):,}" if number is not None else _esc(value) | |
| def _money(value: Any, currency: Any = "USD") -> str: | |
| number = _num(value) | |
| if number is None: | |
| return "" | |
| code = str(currency or "USD").upper() | |
| if code in ("USD", "CAD"): | |
| return f"${number:,.2f}" | |
| return f"{number:,.2f} {code}" | |
| def _tool_or_none(token: str) -> Optional[ScrapeDoWalmartTool]: | |
| if not token or not token.strip(): | |
| return None | |
| return ScrapeDoWalmartTool(api_token=token.strip()) | |
| def _write_csv(rows: List[Dict[str, Any]], prefix: str) -> Optional[str]: | |
| if not rows: | |
| return None | |
| fieldnames: List[str] = [] | |
| for row in rows: | |
| for key in row: | |
| if key not in fieldnames: | |
| fieldnames.append(key) | |
| handle = tempfile.NamedTemporaryFile( | |
| mode="w", suffix=".csv", prefix=f"{prefix}-", delete=False, newline="", encoding="utf-8-sig", | |
| ) | |
| writer = csv.DictWriter(handle, fieldnames=fieldnames) | |
| writer.writeheader() | |
| for row in rows: | |
| writer.writerow({key: row.get(key, "") for key in fieldnames}) | |
| handle.close() | |
| return handle.name | |
| def _error_md(data: Dict[str, Any]) -> str: | |
| out = f"**That didn't go through.**\n\n{data.get('errorMessage') or 'Unknown error'}" | |
| if data.get("hint"): | |
| out += f"\n\n{data['hint']}" | |
| return out | |
| def _img(src: Any, cls: str) -> str: | |
| link = _safe_link(src) | |
| if not link: | |
| return "" | |
| return ( | |
| f'<img class="{cls}" src="{_esc(link)}" alt="" loading="lazy" referrerpolicy="no-referrer" ' | |
| "onerror=\"this.style.visibility='hidden'\">" | |
| ) | |
| def _stars(rating: Any) -> str: | |
| value = _num(rating) | |
| if value is None: | |
| return "" | |
| full = max(0, min(5, int(round(value)))) | |
| return f'<span class="wm-stars">{"★" * full}<span class="wm-stars-off">{"★" * (5 - full)}</span></span>' | |
| def _rating_html(rating: Any, count: Any) -> str: | |
| if _num(rating) is None: | |
| return "" | |
| count_html = f'<span class="wm-count">({_fmt_int(count)})</span>' if count not in (None, "") else "" | |
| return f'<div class="wm-rating">{_stars(rating)}<span class="wm-rating-num">{_num(rating):.1f}</span>{count_html}</div>' | |
| def _price_html(price: Any, list_price: Any, currency: Any = "USD", big: bool = False) -> str: | |
| value = _num(price) | |
| if value is None: | |
| return '<div class="wm-price-na">Price not shown at this store</div>' | |
| was = _num(list_price) | |
| dollars, cents = f"{value:,.2f}".split(".") | |
| amount = ( | |
| f'<span class="wm-cur">$</span><span class="wm-dollars">{dollars}</span><span class="wm-cents">{cents}</span>' | |
| if str(currency or "USD").upper() in ("USD", "CAD") | |
| else _esc(_money(value, currency)) | |
| ) | |
| cls = "wm-price wm-price-big" if big else "wm-price" | |
| if was and was > value: | |
| return ( | |
| f'<div class="{cls}"><span class="wm-now">Now</span> <span class="wm-amount wm-sale">{amount}</span>' | |
| f' <span class="wm-was">{_esc(_money(was, currency))}</span></div>' | |
| ) | |
| return f'<div class="{cls}"><span class="wm-amount">{amount}</span></div>' | |
| def _stock_html(availability: Any) -> str: | |
| code = str(availability or "").upper() | |
| if not code: | |
| return "" | |
| if code == "IN_STOCK": | |
| return '<span class="wm-stock wm-in">In stock</span>' | |
| if "OUT" in code: | |
| return '<span class="wm-stock wm-out">Out of stock</span>' | |
| label = code.replace("_", " ").capitalize() | |
| return f'<span class="wm-stock wm-limited">{_esc(label)}</span>' | |
| def _fulfillment_html(methods: Any) -> str: | |
| if not isinstance(methods, list) or not methods: | |
| return "" | |
| chips = "".join(f'<span class="wm-chip">{_esc(str(m).replace("_", " ").capitalize())}</span>' for m in methods) | |
| return f'<div class="wm-chips">{chips}</div>' | |
| def _html_to_text(raw: Any, limit: int = 3000) -> str: | |
| text = re.sub(r"(?is)<\s*(script|style)[^>]*>.*?<\s*/\s*\1\s*>", "", str(raw or "")) | |
| text = re.sub(r"(?i)<\s*li[^>]*>", "\n• ", text) | |
| text = re.sub(r"(?i)<\s*(br|/p|/div|/ul|/ol|/h\d)[^>]*>", "\n", text) | |
| text = re.sub(r"<[^>]+>", "", text) | |
| text = html.unescape(text) | |
| lines = [line.strip() for line in text.splitlines()] | |
| text = "\n".join(line for line in lines if line) | |
| return text[:limit] + ("…" if len(text) > limit else "") | |
| def _store_banner(store: Dict[str, Any], prefix: str = "Prices and stock at") -> str: | |
| label = describe_store(store) | |
| if not label: | |
| return "" | |
| return f'<div class="wm-store">{PIN_SVG} {prefix} <b>{_esc(label)}</b></div>' | |
| CART_SVG = ( | |
| '<svg viewBox="0 0 24 24" width="{size}" height="{size}" aria-hidden="true"><path fill="#ffc220" ' | |
| 'd="M7 18c-1.1 0-1.99.9-1.99 2S5.9 22 7 22s2-.9 2-2-.9-2-2-2zM1 2v2h2l3.6 7.59-1.35 2.45c-.16.28-.25.61-.25.96 ' | |
| "0 1.1.9 2 2 2h12v-2H7.42c-.14 0-.25-.11-.25-.25l.03-.12.9-1.63h7.45c.75 0 1.41-.41 1.75-1.03l3.58-6.49A1.003 " | |
| '1.003 0 0 0 20 4H5.21l-.94-2H1zm16 16c-1.1 0-1.99.9-1.99 2s.89 2 1.99 2 2-.9 2-2-.9-2-2-2z"/></svg>' | |
| ) | |
| PIN_SVG = ( | |
| '<svg viewBox="0 0 24 24" width="16" height="16" aria-hidden="true" style="vertical-align:-3px">' | |
| '<path fill="#0071dc" d="M12 2C8.13 2 5 5.13 5 9c0 5.25 7 13 7 13s7-7.75 7-13c0-3.87-3.13-7-7-7zm0 9.5' | |
| 'a2.5 2.5 0 110-5 2.5 2.5 0 010 5z"/></svg>' | |
| ) | |
| # ------------------------------------------------------------ search results | |
| def render_results(data: Dict[str, Any], hide_sponsored: bool) -> str: | |
| results = data.get("results") or [] | |
| shown = [r for r in results if not (hide_sponsored and r.get("sponsored"))] | |
| cards: List[str] = [] | |
| for item in shown: | |
| link = _safe_link(item.get("url")) | |
| name = _esc(item.get("name")) | |
| title = ( | |
| f'<a class="wm-name" href="{_esc(link)}" target="_blank" rel="noopener noreferrer">{name}</a>' | |
| if link else f'<span class="wm-name">{name}</span>' | |
| ) | |
| seller = item.get("seller") | |
| seller_html = ( | |
| f'<div class="wm-muted">Sold by {_esc(seller)}</div>' | |
| if seller and "walmart" not in str(seller).lower() else "" | |
| ) | |
| cards.append( | |
| '<div class="wm-card">' | |
| f'<div class="wm-img-box">{_img(item.get("image"), "wm-img")}' | |
| + ('<span class="wm-sponsored">Sponsored</span>' if item.get("sponsored") else "") | |
| + "</div>" | |
| f"{_price_html(item.get('price'), item.get('list_price'), item.get('currency'))}" | |
| f"{title}" | |
| f"{_rating_html(item.get('rating'), item.get('reviews_count'))}" | |
| f'<div class="wm-line">{_stock_html(item.get("availability"))}</div>' | |
| f"{_fulfillment_html(item.get('fulfillment'))}" | |
| f"{seller_html}" | |
| f'<div class="wm-id">#{_esc(item.get("position"))} · Item {_esc(item.get("item_id"))}</div>' | |
| "</div>" | |
| ) | |
| if not cards: | |
| cards.append('<div class="wm-empty">No products came back for this search at this store.</div>') | |
| return ( | |
| f'<div class="wm-wrap">{_store_banner(data.get("store") or {})}' | |
| f'<div class="wm-grid">{"".join(cards)}</div></div>' | |
| ) | |
| def _result_rows(data: Dict[str, Any]) -> List[Dict[str, Any]]: | |
| store = data.get("store") or {} | |
| rows = [] | |
| for item in data.get("results") or []: | |
| rows.append( | |
| { | |
| "position": item.get("position"), | |
| "item_id": item.get("item_id"), | |
| "name": item.get("name"), | |
| "brand": item.get("brand"), | |
| "price": item.get("price"), | |
| "list_price": item.get("list_price"), | |
| "currency": item.get("currency"), | |
| "rating": item.get("rating"), | |
| "reviews_count": item.get("reviews_count"), | |
| "availability": item.get("availability"), | |
| "fulfillment": ", ".join(item.get("fulfillment") or []), | |
| "seller": item.get("seller"), | |
| "sponsored": item.get("sponsored"), | |
| "category": item.get("category"), | |
| "url": item.get("url"), | |
| "image": item.get("image"), | |
| "store_id": store.get("store_id"), | |
| "store_city": store.get("city"), | |
| "store_state": store.get("state"), | |
| "store_zip": store.get("postal_code"), | |
| } | |
| ) | |
| return rows | |
| # ------------------------------------------------------------- product page | |
| def render_product(data: Dict[str, Any]) -> str: | |
| product = data.get("product") or {} | |
| images = [src for src in (product.get("images") or []) if _safe_link(src)] | |
| main_img = _img(images[0], "wm-pdp-img") if images else "" | |
| thumbs = "".join(_img(src, "wm-pdp-thumb") for src in images[1:6]) | |
| price = _num(product.get("price")) | |
| was = _num(product.get("list_price")) | |
| savings = "" | |
| if price is not None and was and was > price: | |
| saved = was - price | |
| savings = ( | |
| f'<span class="wm-save">You save {_esc(_money(saved, product.get("currency")))} ' | |
| f"({saved / was * 100:.0f}%)</span>" | |
| ) | |
| seller = product.get("seller") | |
| seller_html = f'<div class="wm-line">Sold and shipped by <b>{_esc(seller)}</b></div>' if seller else "" | |
| limit = ( | |
| f'<div class="wm-muted">Limit {_esc(product.get("order_limit"))} per order</div>' | |
| if product.get("order_limit") else "" | |
| ) | |
| link = _safe_link(product.get("url")) | |
| button = ( | |
| f'<a class="wm-btn" href="{_esc(link)}" target="_blank" rel="noopener noreferrer">View on Walmart</a>' | |
| if link else "" | |
| ) | |
| def name_values(items: Any, cls: str) -> str: | |
| rows = "".join( | |
| f'<div class="wm-spec-name">{_esc(i.get("name"))}</div><div class="wm-spec-value">{_esc(i.get("value"))}</div>' | |
| for i in (items or []) if isinstance(i, dict) and i.get("name") | |
| ) | |
| return f'<div class="{cls}">{rows}</div>' if rows else "" | |
| highlights = name_values(product.get("highlights"), "wm-specs") | |
| specs = name_values(product.get("specifications"), "wm-specs") | |
| about = product.get("short_description") or "" | |
| long_text = _html_to_text(product.get("long_description")) | |
| about_html = "" | |
| if about or long_text: | |
| about_html = '<div class="wm-section"><div class="wm-h">About this item</div>' | |
| if about: | |
| about_html += f'<div class="wm-text">{_esc(_html_to_text(about, 1200))}</div>' | |
| if long_text: | |
| about_html += f'<div class="wm-text">{_esc(long_text).replace(chr(10), "<br>")}</div>' | |
| about_html += "</div>" | |
| facts = [] | |
| for label, key in [ | |
| ("Condition", "condition"), ("Model number", "model_number"), ("UPC", "upc"), | |
| ("GTIN", "gtin"), ("Returns", "return_policy"), ("Item ID", "item_id"), | |
| ]: | |
| if product.get(key): | |
| facts.append({"name": label, "value": product[key]}) | |
| facts_html = name_values(facts, "wm-specs") | |
| category = product.get("category") or "" | |
| crumbs = " › ".join(_esc(part) for part in str(category).split("/") if part and part != "Home Page") | |
| return ( | |
| f'<div class="wm-wrap">{_store_banner(data.get("store") or {})}' | |
| '<div class="wm-pdp">' | |
| f'<div class="wm-pdp-media">{main_img}<div class="wm-pdp-thumbs">{thumbs}</div></div>' | |
| '<div class="wm-pdp-info">' | |
| + (f'<div class="wm-crumbs">{crumbs}</div>' if crumbs else "") | |
| + (f'<div class="wm-brand">{_esc(product.get("brand"))}</div>' if product.get("brand") else "") | |
| + f'<div class="wm-pdp-name">{_esc(product.get("name"))}</div>' | |
| f"{_rating_html(product.get('rating'), product.get('reviews_count'))}" | |
| f"{_price_html(product.get('price'), product.get('list_price'), product.get('currency'), big=True)}" | |
| f"{savings}" | |
| f'<div class="wm-line">{_stock_html(product.get("availability"))}</div>' | |
| f"{_fulfillment_html(product.get('fulfillment'))}" | |
| f"{seller_html}{limit}{button}" | |
| "</div></div>" | |
| + ( | |
| '<div class="wm-panel">' | |
| + (f'<div class="wm-section"><div class="wm-h">Highlights</div>{highlights}</div>' if highlights else "") | |
| + about_html | |
| + (f'<div class="wm-section"><div class="wm-h">Specifications</div>{specs}</div>' if specs else "") | |
| + (f'<div class="wm-section"><div class="wm-h">Product details</div>{facts_html}</div>' if facts_html else "") | |
| + "</div>" | |
| if (highlights or about_html or specs or facts_html) else "" | |
| ) | |
| + "</div>" | |
| ) | |
| # ------------------------------------------------------------------- compare | |
| def _place(row: Dict[str, Any]) -> str: | |
| place = ", ".join(p for p in [row.get("city"), row.get("state")] if p) | |
| return place or f"ZIP/store {row.get('location')}" | |
| def render_compare(data: Dict[str, Any]) -> str: | |
| product = data.get("product") or {} | |
| rows = data.get("stores") or [] | |
| priced = sorted( | |
| (r for r in rows if _num(r.get("price")) is not None), key=lambda r: _num(r.get("price")) or 0 | |
| ) | |
| unpriced = [r for r in rows if _num(r.get("price")) is None] | |
| top = max((_num(r.get("price")) or 0 for r in priced), default=0) | |
| best = _num(priced[0].get("price")) if priced else None | |
| link = _safe_link(product.get("url")) | |
| name = _esc(product.get("name") or f"Item {data.get('item_id')}") | |
| name_html = ( | |
| f'<a class="wm-name" href="{_esc(link)}" target="_blank" rel="noopener noreferrer">{name}</a>' if link | |
| else f'<span class="wm-name">{name}</span>' | |
| ) | |
| head = ( | |
| '<div class="wm-cmp-head">' | |
| f'<div class="wm-cmp-img">{_img(product.get("image"), "wm-img")}</div>' | |
| '<div>' | |
| + (f'<div class="wm-brand">{_esc(product.get("brand"))}</div>' if product.get("brand") else "") | |
| + f"{name_html}{_rating_html(product.get('rating'), product.get('reviews_count'))}" | |
| f'<div class="wm-muted">Item {_esc(data.get("item_id"))}</div>' | |
| "</div></div>" | |
| ) | |
| lines: List[str] = [] | |
| for row in priced: | |
| price = _num(row.get("price")) or 0 | |
| width = (price / top * 100) if top else 0 | |
| is_best = best is not None and abs(price - best) < 0.005 | |
| store_bits = " · ".join( | |
| p for p in [f"Walmart #{row['store_id']}" if row.get("store_id") else "", str(row.get("postal_code") or "")] if p | |
| ) | |
| lines.append( | |
| f'<div class="wm-cmp-row{" wm-best" if is_best else ""}">' | |
| f'<div class="wm-cmp-store"><b>{_esc(_place(row))}</b><span>{_esc(store_bits)}</span></div>' | |
| f'<div class="wm-cmp-bar"><span style="width:{width:.0f}%"></span></div>' | |
| f'<div class="wm-cmp-price">{_esc(_money(price, row.get("currency")))}' | |
| + ( | |
| f' <span class="wm-was">{_esc(_money(row.get("list_price"), row.get("currency")))}</span>' | |
| if (_num(row.get("list_price")) or 0) > price else "" | |
| ) | |
| + "</div>" | |
| f'<div class="wm-cmp-stock">{_stock_html(row.get("availability"))}{_fulfillment_html(row.get("fulfillment"))}</div>' | |
| + ('<span class="wm-badge">Lowest price</span>' if is_best and len(priced) > 1 else "") | |
| + "</div>" | |
| ) | |
| for row in unpriced: | |
| reason = row.get("error") or "No price shown at this store." | |
| lines.append( | |
| '<div class="wm-cmp-row wm-cmp-missing">' | |
| f'<div class="wm-cmp-store"><b>{_esc(_place(row))}</b><span>{_esc(row.get("store_id") and "Walmart #" + str(row["store_id"]) or "")}</span></div>' | |
| f'<div class="wm-cmp-note">{_esc(reason)}</div>' | |
| f'<div class="wm-cmp-stock">{_stock_html(row.get("availability"))}</div>' | |
| "</div>" | |
| ) | |
| return f'<div class="wm-wrap">{head}<div class="wm-cmp-list">{"".join(lines)}</div></div>' | |
| # ---------------------------------------------------------------- handlers | |
| RESULT_HEADERS = ["#", "Product", "Price", "List price", "Rating", "Reviews", "Stock", "Item ID"] | |
| COMPARE_HEADERS = ["Store", "Location", "ZIP", "Price", "List price", "Stock", "Fulfillment"] | |
| def _picker(choices: List[Tuple[str, str]]) -> Any: | |
| return gr.Dropdown(choices=choices, value=None, visible=bool(choices)) | |
| def run_search( | |
| token: str, query: str, location: str, hide_sponsored: bool | |
| ) -> Tuple[str, str, List[List[Any]], Dict[str, Any], Optional[str], Any]: | |
| tool = _tool_or_none(token) | |
| if tool is None: | |
| return NO_TOKEN_MD, "", [], {}, None, _picker([]) | |
| if not query or not query.strip(): | |
| return "Type what you're looking for, e.g. `whole milk`.", "", [], {}, None, _picker([]) | |
| data = tool.search(query, location) | |
| if data.get("status") != "success": | |
| return _error_md(data), "", [], data, None, _picker([]) | |
| results = data.get("results") or [] | |
| prices = [_num(r.get("price")) for r in results if _num(r.get("price")) is not None] | |
| in_stock = sum(1 for r in results if str(r.get("availability") or "").upper() == "IN_STOCK") | |
| on_sale = sum( | |
| 1 for r in results | |
| if _num(r.get("list_price")) and _num(r.get("price")) is not None and _num(r["list_price"]) > _num(r["price"]) | |
| ) | |
| sponsored = sum(1 for r in results if r.get("sponsored")) | |
| parts = [f"**{len(results)}** products", f"**{in_stock}** in stock", f"**{on_sale}** on sale"] | |
| if prices: | |
| parts.append(f"prices **{_money(min(prices))}–{_money(max(prices))}**") | |
| if sponsored: | |
| parts.append(f"{sponsored} sponsored" + (" (hidden)" if hide_sponsored else "")) | |
| summary = " · ".join(parts) | |
| store_label = describe_store(data.get("store")) | |
| footer = [] | |
| if store_label: | |
| footer.append(store_label) | |
| if data.get("total_results"): | |
| footer.append(f"{_fmt_int(data['total_results'])} matches on Walmart") | |
| footer.append(f"{data.get('credits_used')} credits used") | |
| summary += "\n\n" + " · ".join(footer) | |
| rows = _result_rows(data) | |
| table = [ | |
| [ | |
| r["position"], (r["name"] or "")[:90], _money(r["price"], r["currency"]), | |
| _money(r["list_price"], r["currency"]), r["rating"], r["reviews_count"], | |
| str(r["availability"] or "").replace("_", " ").lower(), r["item_id"], | |
| ] | |
| for r in rows | |
| ] | |
| choices = [ | |
| (f"{r.get('position')}. {str(r.get('name') or '')[:70]} — {_money(r.get('price'), r.get('currency')) or 'no price'}", | |
| str(r["item_id"])) | |
| for r in results if r.get("item_id") | |
| ] | |
| csv_path = _write_csv(rows, "walmart-search") | |
| return summary, render_results(data, bool(hide_sponsored)), table, data, csv_path, _picker(choices) | |
| def send_to_tabs(picked: Optional[str], product_ref: str, compare_ref: str) -> Tuple[str, str, str]: | |
| if not picked: | |
| return product_ref, compare_ref, "" | |
| return picked, picked, "Sent. Open the **Product details** or **Compare prices** tab and press the button." | |
| def run_product(token: str, ref: str, location: str) -> Tuple[str, str, Dict[str, Any]]: | |
| tool = _tool_or_none(token) | |
| if tool is None: | |
| return NO_TOKEN_MD, "", {} | |
| if not ref or not ref.strip(): | |
| return "Paste a Walmart product link or item id, or type a product name.", "", {} | |
| data = tool.product(ref, location) | |
| if data.get("status") != "success": | |
| return _error_md(data), "", data | |
| lines = [] | |
| if data.get("note"): | |
| lines.append(data["note"]) | |
| lines.append(f"{data.get('credits_used')} credits used.") | |
| return "\n\n".join(lines), render_product(data), data | |
| def run_compare( | |
| token: str, ref: str, locations: str | |
| ) -> Tuple[str, str, List[List[Any]], Dict[str, Any], Optional[str]]: | |
| tool = _tool_or_none(token) | |
| if tool is None: | |
| return NO_TOKEN_MD, "", [], {}, None | |
| if not ref or not ref.strip(): | |
| return "Paste a Walmart product link or item id, or type a product name.", "", [], {}, None | |
| data = tool.compare(ref, locations) | |
| if data.get("status") != "success": | |
| return _error_md(data), "", [], data, None | |
| rows = data.get("stores") or [] | |
| priced = [r for r in rows if _num(r.get("price")) is not None] | |
| lines = [] | |
| if data.get("note"): | |
| lines.append(data["note"]) | |
| cheapest, dearest = data.get("cheapest"), data.get("most_expensive") | |
| if cheapest and dearest: | |
| low, high = _num(cheapest["price"]) or 0, _num(dearest["price"]) or 0 | |
| line = f"Cheapest at **{_place(cheapest)}**: **{_money(low)}**." | |
| if high - low >= 0.01: | |
| line += ( | |
| f" That's **{_money(high - low)} ({(high - low) / high * 100:.0f}%) less** than " | |
| f"**{_place(dearest)}** at {_money(high)}." | |
| ) | |
| elif len(priced) > 1: | |
| line = f"Same price, **{_money(low)}**, at every store that showed one." | |
| lines.append(line) | |
| else: | |
| lines.append("None of these stores showed a price for this product.") | |
| lines.append(f"Prices found at {len(priced)} of {len(rows)} stores · {data.get('credits_used')} credits used") | |
| if data.get("warnings"): | |
| lines.append("\n".join(f"- {w}" for w in data["warnings"])) | |
| table = [ | |
| [ | |
| f"#{r.get('store_id')}" if r.get("store_id") else "", | |
| _place(r), | |
| r.get("postal_code") or r.get("location"), | |
| _money(r.get("price"), r.get("currency")) or "—", | |
| _money(r.get("list_price"), r.get("currency")), | |
| str(r.get("availability") or r.get("error") or "").replace("_", " ").lower(), | |
| ", ".join(r.get("fulfillment") or []), | |
| ] | |
| for r in rows | |
| ] | |
| product = data.get("product") or {} | |
| csv_rows = [ | |
| { | |
| "item_id": data.get("item_id"), | |
| "product": product.get("name"), | |
| "location_input": r.get("location"), | |
| "store_id": r.get("store_id"), | |
| "city": r.get("city"), | |
| "state": r.get("state"), | |
| "zip": r.get("postal_code"), | |
| "price": r.get("price"), | |
| "list_price": r.get("list_price"), | |
| "currency": r.get("currency"), | |
| "availability": r.get("availability"), | |
| "fulfillment": ", ".join(r.get("fulfillment") or []), | |
| "seller": r.get("seller"), | |
| "error": r.get("error"), | |
| } | |
| for r in rows | |
| ] | |
| csv_path = _write_csv(csv_rows, "walmart-price-compare") | |
| return "\n\n".join(lines), render_compare(data), table, data, csv_path | |
| # ----------------------------------------------------------------------- UI | |
| STYLE = """ | |
| <style> | |
| .gradio-container, .gradio-container button, .gradio-container input, | |
| .gradio-container textarea, .gradio-container select { | |
| font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Helvetica Neue", Arial, sans-serif; | |
| } | |
| /* hero */ | |
| .wm-hero { position: relative; overflow: hidden; border-radius: 16px; padding: 30px 32px 28px 32px; margin-bottom: 16px; | |
| background: radial-gradient(circle at 85% 20%, #1a8cff 0%, rgba(26,140,255,0) 45%), #0071dc; | |
| font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Helvetica Neue", Arial, sans-serif; } | |
| .wm-hero, .wm-hero * { color: #ffffff !important; } | |
| .wm-hero-inner { position: relative; z-index: 1; max-width: 620px; } | |
| .wm-hero h1 { font-size: 2.05rem !important; font-weight: 700 !important; margin: 0 0 10px 0 !important; | |
| display: flex; align-items: center; gap: 10px; letter-spacing: -0.01em; } | |
| .wm-hero p { margin: 0 0 8px 0 !important; font-size: 1rem !important; line-height: 1.55 !important; opacity: 0.96; } | |
| .wm-hero strong { font-weight: 700 !important; } | |
| .wm-hero a { text-decoration: underline !important; } | |
| .wm-hero a.wm-cta { display: inline-block; margin-top: 12px; padding: 11px 24px; border-radius: 999px; | |
| background: #ffc220 !important; color: #1a1a1a !important; font-weight: 700 !important; text-decoration: none !important; } | |
| .wm-hero a.wm-cta:hover { background: #ffd451 !important; } | |
| .wm-hero-card { position: absolute; right: 34px; top: 50%; transform: translateY(-50%); width: 270px; z-index: 1; | |
| background: #ffffff; border-radius: 14px; padding: 14px 16px; box-shadow: 0 8px 24px rgba(0,0,0,.18); } | |
| .wm-hero-card, .wm-hero-card * { color: #2e2f32 !important; } | |
| .wm-hc-title { font-size: 0.8rem; font-weight: 700; margin-bottom: 10px; color: #74767c !important; text-transform: uppercase; letter-spacing: .04em; } | |
| .wm-hc-row { display: flex; align-items: center; gap: 8px; margin: 7px 0; font-size: 0.82rem; } | |
| .wm-hc-row span:first-child { width: 92px; } | |
| .wm-hc-bar { flex: 1; height: 8px; border-radius: 999px; background: #e3e4e5; overflow: hidden; } | |
| .wm-hc-bar i { display: block; height: 100%; background: #0071dc; border-radius: 999px; } | |
| .wm-hc-row.best .wm-hc-bar i { background: #2a8703; } | |
| .wm-hc-tag { font-size: 0.7rem; font-weight: 700; color: #2a8703 !important; } | |
| @media (max-width: 900px) { .wm-hero-card { display: none; } } | |
| .wm-panel { background: #ffffff; border: 1px solid #e3e4e5; border-radius: 16px; padding: 4px 22px 22px 22px; margin-top: 14px; } | |
| .wm-note { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Helvetica Neue", Arial, sans-serif; border-left: 3px solid #ffc220; background: #fff8e1; padding: 12px 16px; border-radius: 0 8px 8px 0; | |
| font-size: 0.9rem; margin-bottom: 12px; color: #2e2f32 !important; } | |
| .gradio-container .primary, .gradio-container button.primary, | |
| .gradio-container button[variant="primary"] { | |
| background: #0071dc !important; background-image: none !important; border-color: #0071dc !important; | |
| color: #ffffff !important; border-radius: 999px !important; font-weight: 700 !important; | |
| } | |
| .gradio-container .primary:hover, .gradio-container button.primary:hover, | |
| .gradio-container button[variant="primary"]:hover { background: #004f9a !important; } | |
| .gradio-container .tab-nav button.selected { color: #0071dc !important; border-bottom-color: #0071dc !important; } | |
| /* shared */ | |
| .wm-wrap { color: #2e2f32 !important; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Helvetica Neue", Arial, sans-serif; } | |
| .wm-wrap img { margin: 0 !important; max-width: none; } | |
| .wm-muted { color: #74767c !important; font-size: 0.8rem; margin-top: 3px; } | |
| .wm-empty { background: #ffffff; border: 1px solid #e3e4e5; border-radius: 12px; padding: 20px; color: #74767c !important; } | |
| .wm-store { background: #f2f8fd; border: 1px solid #cfe3f6; border-radius: 10px; padding: 9px 14px; margin-bottom: 12px; | |
| font-size: 0.9rem; color: #2e2f32 !important; } | |
| .wm-store b { color: #2e2f32 !important; } | |
| .wm-line { margin-top: 4px; font-size: 0.85rem; color: #2e2f32 !important; } | |
| .wm-line b { color: #2e2f32 !important; } | |
| .wm-price { display: flex; align-items: baseline; flex-wrap: wrap; gap: 4px; margin: 8px 0 4px 0; color: #2e2f32 !important; } | |
| .wm-amount { color: #2e2f32 !important; font-weight: 700; display: inline-flex; align-items: flex-start; } | |
| .wm-cur, .wm-cents { font-size: 0.8rem; line-height: 1.5; color: inherit !important; } | |
| .wm-dollars { font-size: 1.45rem; line-height: 1; color: inherit !important; } | |
| .wm-price-big .wm-dollars { font-size: 2.2rem; } | |
| .wm-price-big .wm-cur, .wm-price-big .wm-cents { font-size: 1rem; } | |
| .wm-now { color: #2a8703 !important; font-weight: 700; font-size: 0.95rem; } | |
| .wm-amount.wm-sale { color: #2a8703 !important; } | |
| .wm-was { color: #74767c !important; text-decoration: line-through; font-size: 0.85rem; } | |
| .wm-price-na { color: #74767c !important; font-size: 0.85rem; margin: 8px 0 4px 0; } | |
| .wm-save { display: inline-block; background: #eaf3e6; color: #2a8703 !important; font-weight: 700; font-size: 0.8rem; | |
| border-radius: 4px; padding: 3px 8px; margin-bottom: 4px; } | |
| .wm-rating { display: flex; align-items: center; gap: 5px; margin-top: 4px; font-size: 0.8rem; } | |
| .wm-stars { color: #2e2f32 !important; letter-spacing: 0.5px; font-size: 0.85rem; } | |
| .wm-stars-off { color: #d0d1d3 !important; } | |
| .wm-rating-num { color: #2e2f32 !important; font-weight: 600; } | |
| .wm-count { color: #74767c !important; } | |
| .wm-stock { font-size: 0.8rem; font-weight: 600; } | |
| .wm-in { color: #2a8703 !important; } | |
| .wm-out { color: #de1c24 !important; } | |
| .wm-limited { color: #b35c00 !important; } | |
| .wm-chips { display: flex; flex-wrap: wrap; gap: 5px; margin-top: 6px; } | |
| .wm-chip { background: #f2f8fd; color: #004f9a !important; border: 1px solid #cfe3f6; border-radius: 999px; padding: 2px 9px; font-size: 0.75rem; font-weight: 600; } | |
| .wm-name { color: #2e2f32 !important; font-size: 0.9rem; line-height: 1.35; text-decoration: none !important; | |
| display: -webkit-box; -webkit-line-clamp: 3; -webkit-box-orient: vertical; overflow: hidden; } | |
| a.wm-name:hover { text-decoration: underline !important; color: #0071dc !important; } | |
| .wm-id { color: #a4a5a8 !important; font-size: 0.72rem; margin-top: 8px; } | |
| .wm-brand { color: #0071dc !important; font-size: 0.85rem; font-weight: 600; text-decoration: underline; } | |
| .wm-section { margin-top: 22px; } | |
| .wm-h { font-size: 1.1rem; font-weight: 700; color: #2e2f32 !important; margin-bottom: 10px; } | |
| .wm-text { color: #2e2f32 !important; font-size: 0.9rem; line-height: 1.6; margin-bottom: 10px; } | |
| .wm-btn { display: inline-block; margin-top: 14px; padding: 10px 22px; border-radius: 999px; background: #0071dc !important; | |
| color: #ffffff !important; font-weight: 700; text-decoration: none !important; font-size: 0.9rem; } | |
| .wm-btn:hover { background: #004f9a !important; } | |
| /* search grid */ | |
| .wm-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(min(210px, 100%), 1fr)); gap: 12px; } | |
| .wm-card { background: #ffffff; border: 1px solid #e3e4e5; border-radius: 12px; padding: 12px 14px 12px 14px; } | |
| .wm-card:hover { box-shadow: 0 2px 10px rgba(0,0,0,.10); } | |
| .wm-img-box { position: relative; height: 170px; display: flex; align-items: center; justify-content: center; | |
| background: #ffffff; border-radius: 8px; } | |
| .wm-img { max-width: 100%; max-height: 100%; object-fit: contain; } | |
| .wm-sponsored { position: absolute; top: 0; left: 0; background: #f2f2f2; color: #74767c !important; | |
| font-size: 0.68rem; border-radius: 4px; padding: 1px 6px; } | |
| /* product page */ | |
| .wm-pdp { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1.1fr); gap: 28px; background: #ffffff; | |
| border: 1px solid #e3e4e5; border-radius: 16px; padding: 22px; } | |
| @media (max-width: 760px) { .wm-pdp { grid-template-columns: 1fr; } } | |
| .wm-pdp-media { display: flex; flex-direction: column; gap: 10px; } | |
| .wm-pdp-img { width: 100%; max-height: 380px; object-fit: contain; } | |
| .wm-pdp-thumbs { display: flex; gap: 8px; flex-wrap: wrap; } | |
| .wm-pdp-thumb { width: 62px; height: 62px; object-fit: contain; border: 1px solid #e3e4e5; border-radius: 8px; background: #ffffff; } | |
| .wm-crumbs { color: #74767c !important; font-size: 0.75rem; margin-bottom: 8px; } | |
| .wm-pdp-name { font-size: 1.35rem; font-weight: 600; line-height: 1.3; color: #2e2f32 !important; margin-top: 4px; } | |
| .wm-specs { display: grid; grid-template-columns: minmax(120px, 0.6fr) 1fr; border-top: 1px solid #e3e4e5; background: #ffffff; } | |
| .wm-spec-name, .wm-spec-value { padding: 9px 12px; border-bottom: 1px solid #e3e4e5; font-size: 0.88rem; } | |
| .wm-spec-name { color: #74767c !important; } | |
| .wm-spec-value { color: #2e2f32 !important; } | |
| /* compare */ | |
| .wm-cmp-head { display: flex; gap: 16px; align-items: center; background: #ffffff; border: 1px solid #e3e4e5; | |
| border-radius: 12px; padding: 14px 16px; } | |
| .wm-cmp-img { width: 90px; height: 90px; display: flex; align-items: center; justify-content: center; flex-shrink: 0; } | |
| .wm-cmp-list { display: flex; flex-direction: column; gap: 8px; margin-top: 12px; } | |
| .wm-cmp-row { position: relative; display: grid; grid-template-columns: 200px 1fr 130px 210px; gap: 14px; align-items: center; | |
| background: #ffffff; border: 1px solid #e3e4e5; border-radius: 12px; padding: 12px 16px; } | |
| @media (max-width: 860px) { .wm-cmp-row { grid-template-columns: 1fr; gap: 6px; } } | |
| .wm-cmp-row.wm-best { border-color: #2a8703; box-shadow: 0 0 0 1px #2a8703 inset; } | |
| .wm-cmp-store b { display: block; color: #2e2f32 !important; font-size: 0.95rem; } | |
| .wm-cmp-store span { color: #74767c !important; font-size: 0.78rem; } | |
| .wm-cmp-bar { height: 10px; background: #e3e4e5; border-radius: 999px; overflow: hidden; } | |
| .wm-cmp-bar span { display: block; height: 100%; background: #0071dc; border-radius: 999px; } | |
| .wm-best .wm-cmp-bar span { background: #2a8703; } | |
| .wm-cmp-price { font-size: 1.2rem; font-weight: 700; color: #2e2f32 !important; } | |
| .wm-best .wm-cmp-price { color: #2a8703 !important; } | |
| .wm-cmp-stock .wm-chips { margin-top: 3px; } | |
| .wm-badge { position: absolute; top: -9px; right: 14px; background: #2a8703; color: #ffffff !important; font-size: 0.7rem; | |
| font-weight: 700; border-radius: 4px; padding: 2px 8px; } | |
| .wm-cmp-missing { opacity: 0.85; } | |
| .wm-cmp-note { color: #74767c !important; font-size: 0.85rem; grid-column: span 2; } | |
| footer { display: none !important; } | |
| </style> | |
| """ | |
| HERO = f""" | |
| <div class="wm-hero"> | |
| <div class="wm-hero-inner"> | |
| <h1>{CART_SVG.format(size=34)} Walmart Scraper</h1> | |
| <p><strong>Walmart prices change from store to store.</strong> Search products at any US store by ZIP code, | |
| pull a product's full details, or check one product's price and stock at up to five stores side by side.</p> | |
| <p>Built on the <a href="https://scrape.do/documentation/walmart-scraper-api/" target="_blank" rel="noopener noreferrer">Scrape.do Walmart API</a>, | |
| which picks the store for you and gets past Walmart's bot protection. Use it for price monitoring, | |
| assortment research, MAP checks, or to give an AI agent live retail prices.</p> | |
| <a class="wm-cta" href="{SCRAPE_DO_SIGNUP_URL}" target="_blank" rel="noopener noreferrer">Get a free API token →</a> | |
| </div> | |
| <div class="wm-hero-card" aria-hidden="true"> | |
| <div class="wm-hc-title">One product · four stores</div> | |
| <div class="wm-hc-row"><span>Store A</span><span class="wm-hc-bar"><i style="width:92%"></i></span></div> | |
| <div class="wm-hc-row best"><span>Store B</span><span class="wm-hc-bar"><i style="width:71%"></i></span><span class="wm-hc-tag">Lowest</span></div> | |
| <div class="wm-hc-row"><span>Store C</span><span class="wm-hc-bar"><i style="width:100%"></i></span></div> | |
| <div class="wm-hc-row"><span>Store D</span><span class="wm-hc-bar"><i style="width:84%"></i></span></div> | |
| </div> | |
| </div> | |
| """ | |
| CREDIT_NOTE = """ | |
| <div class="wm-note"> | |
| Every call costs 10 credits: one search, one product, or one store in a price comparison. Failed calls | |
| are free. The free 1,000 credits cover about 100 calls. Typing a product name instead of pasting a link | |
| adds one search (10 credits) to find it. | |
| </div> | |
| """ | |
| FOOTER_MD = f""" | |
| --- | |
| ### Use it as a smolagents Tool | |
| ```python | |
| from smolagents import CodeAgent, InferenceClientModel | |
| from tool import ScrapeDoWalmartTool | |
| agent = CodeAgent( | |
| tools=[ScrapeDoWalmartTool(api_token="YOUR_SCRAPEDO_TOKEN")], | |
| model=InferenceClientModel(), | |
| ) | |
| agent.run("Find the cheapest gallon of whole milk at the Walmart nearest ZIP 60074, then compare its price in Dallas and Los Angeles.") | |
| ``` | |
| Or set `SCRAPE_DO_API_TOKEN` in the environment and instantiate with no arguments. | |
| Want to scrape category pages and track price changes over time? The | |
| [Walmart scraping guide](https://scrape.do/blog/walmart-scraping/) walks through it in Python. | |
| [Scrape.do](https://scrape.do/) · [Walmart API docs](https://scrape.do/documentation/walmart-scraper-api/) · [Sign up free]({SCRAPE_DO_SIGNUP_URL}) | |
| """ | |
| def build_demo() -> gr.Blocks: | |
| with gr.Blocks(title="Walmart Scraper — Scrape.do") as demo: | |
| gr.HTML(STYLE) | |
| gr.HTML(HERO) | |
| with gr.Row(): | |
| api_token = gr.Textbox( | |
| label="Scrape.do API token", | |
| placeholder="Paste your token here", | |
| type="password", | |
| info="Free tier: 1,000 credits a month, about 100 calls.", | |
| scale=3, | |
| ) | |
| location = gr.Textbox( | |
| label="Store: ZIP code or Walmart store number", | |
| value=DEFAULT_LOCATION, | |
| placeholder="07094", | |
| info="Prices and stock are for the nearest store.", | |
| scale=2, | |
| ) | |
| gr.HTML(CREDIT_NOTE) | |
| with gr.Tabs(): | |
| with gr.Tab("Search products"): | |
| with gr.Row(): | |
| query = gr.Textbox(label="Search Walmart", placeholder="whole milk", scale=4) | |
| hide_sponsored = gr.Checkbox(label="Hide sponsored", value=False, scale=1) | |
| search_btn = gr.Button("Search Walmart", variant="primary") | |
| search_summary = gr.Markdown() | |
| results_view = gr.HTML() | |
| results_table = gr.Dataframe(headers=RESULT_HEADERS, wrap=True, label="Products") | |
| results_csv = gr.File(label="Download results as CSV") | |
| pick = gr.Dropdown( | |
| label="Open one of these products in the Details and Compare tabs", | |
| choices=[], | |
| value=None, | |
| visible=False, | |
| ) | |
| pick_note = gr.Markdown() | |
| with gr.Accordion("Raw JSON response", open=False): | |
| results_json = gr.JSON() | |
| with gr.Tab("Product details"): | |
| product_ref = gr.Textbox( | |
| label="Walmart product link, item id, or product name", | |
| placeholder="https://www.walmart.com/ip/…/17221971947 or 17221971947", | |
| ) | |
| product_btn = gr.Button("Get product details", variant="primary") | |
| product_summary = gr.Markdown() | |
| product_view = gr.HTML() | |
| with gr.Accordion("Raw JSON response", open=False): | |
| product_json = gr.JSON() | |
| with gr.Tab("Compare prices by ZIP"): | |
| gr.Markdown( | |
| "Same product, different stores. Enter up to " | |
| f"{MAX_COMPARE_LOCATIONS} ZIP codes or store numbers; each one costs 10 credits." | |
| ) | |
| compare_ref = gr.Textbox( | |
| label="Walmart product link, item id, or product name", | |
| placeholder="https://www.walmart.com/ip/…/17221971947 or 17221971947", | |
| ) | |
| compare_locations = gr.Textbox( | |
| label="ZIP codes or store numbers, separated by commas", | |
| value=", ".join(DEFAULT_COMPARE_LOCATIONS), | |
| info="Default: Secaucus NJ, Deer Park IL, Dallas TX, Los Angeles CA.", | |
| ) | |
| compare_btn = gr.Button("Compare prices", variant="primary") | |
| compare_summary = gr.Markdown() | |
| compare_view = gr.HTML() | |
| compare_table = gr.Dataframe(headers=COMPARE_HEADERS, wrap=True, label="Price by store") | |
| compare_csv = gr.File(label="Download comparison as CSV") | |
| with gr.Accordion("Raw JSON response", open=False): | |
| compare_json = gr.JSON() | |
| search_inputs = [api_token, query, location, hide_sponsored] | |
| search_outputs = [search_summary, results_view, results_table, results_json, results_csv, pick] | |
| search_btn.click(fn=run_search, inputs=search_inputs, outputs=search_outputs) | |
| query.submit(fn=run_search, inputs=search_inputs, outputs=search_outputs) | |
| pick.change(fn=send_to_tabs, inputs=[pick, product_ref, compare_ref], outputs=[product_ref, compare_ref, pick_note]) | |
| product_inputs = [api_token, product_ref, location] | |
| product_outputs = [product_summary, product_view, product_json] | |
| product_btn.click(fn=run_product, inputs=product_inputs, outputs=product_outputs) | |
| product_ref.submit(fn=run_product, inputs=product_inputs, outputs=product_outputs) | |
| compare_inputs = [api_token, compare_ref, compare_locations] | |
| compare_outputs = [compare_summary, compare_view, compare_table, compare_json, compare_csv] | |
| compare_btn.click(fn=run_compare, inputs=compare_inputs, outputs=compare_outputs) | |
| compare_ref.submit(fn=run_compare, inputs=compare_inputs, outputs=compare_outputs) | |
| gr.Markdown(FOOTER_MD) | |
| return demo | |
| # Hugging Face Spaces launches the module-level `demo` object rather than | |
| # calling launch() itself, so build it at import time. | |
| demo = build_demo() | |
| if __name__ == "__main__": | |
| demo.launch() | |