from __future__ import annotations import json import os import re import time import urllib.parse from concurrent.futures import ThreadPoolExecutor from typing import Any, Dict, List, Optional, Tuple import requests from bs4 import BeautifulSoup from smolagents.tools import Tool SCRAPE_DO_SIGNUP_URL = ( "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=etsy-scraper" ) API_ENDPOINT = "https://api.scrape.do/" ETSY_BASE = "https://www.etsy.com" # Etsy is not in Scrape.do's per-domain price list, so the standard residential # price (super=true) applies. The real cost is read from the # `Scrape.do-Request-Cost` response header whenever it is present. CREDITS_PER_REQUEST = 10 MAX_SEARCH_PAGES = 10 MAX_BULK = 20 BULK_WORKERS = 3 # (label, geoCode, sd-x-detected-locale as CURRENCY|language|COUNTRY) COUNTRIES: List[tuple] = [ ("United States", "us", "USD|en-US|US"), ("United Kingdom", "gb", "GBP|en-GB|GB"), ("Canada", "ca", "CAD|en-CA|CA"), ("Australia", "au", "AUD|en-AU|AU"), ("Germany", "de", "EUR|de|DE"), ("France", "fr", "EUR|fr|FR"), ("Spain", "es", "EUR|es|ES"), ("Italy", "it", "EUR|it|IT"), ("Netherlands", "nl", "EUR|nl|NL"), ("Turkey", "tr", "TRY|tr|TR"), ] COUNTRY_BY_CODE = {code: (label, code, locale) for label, code, locale in COUNTRIES} SORT_OPTIONS = { "relevance": None, "newest": "date_desc", "price_low": "price_asc", "price_high": "price_desc", "top_reviewed": "highest_reviews", } # ------------------------------------------------------------------ helpers def _market(country: Optional[str]) -> Tuple[str, str]: code = (country or "us").strip().lower() _, code, locale = COUNTRY_BY_CODE.get(code, ("", code, "USD|en-US|US")) return code, locale def _is_url(text: str) -> bool: return text.lower().startswith(("http://", "https://")) def _host_is_etsy(host: str) -> bool: host = host.lower().split(":")[0] return host == "etsy.com" or host.endswith(".etsy.com") def parse_price(text: Any) -> Optional[float]: """Parse '1,234.56', '1.234,56', '12,50' or '19' into a float.""" if text is None: return None s = re.sub(r"[^\d.,]", "", str(text)) if not s or not re.search(r"\d", s): return None if "," in s and "." in s: if s.rfind(",") > s.rfind("."): s = s.replace(".", "").replace(",", ".") else: s = s.replace(",", "") elif "," in s: head, _, tail = s.rpartition(",") s = f"{head.replace(',', '')}.{tail}" if len(tail) in (1, 2) else s.replace(",", "") elif "." in s: head, _, tail = s.rpartition(".") if s.count(".") > 1 or len(tail) == 3: s = s.replace(".", "") try: return float(s) except ValueError: return None def _count(text: str, suffix: Optional[str]) -> Optional[int]: try: value = float(text.replace(",", "")) except ValueError: return None if suffix and suffix.lower() == "k": value *= 1000 return int(round(value)) def normalize_listing_ref(text: str) -> Tuple[Optional[str], Optional[str]]: """A listing link or numeric ID -> a clean listing URL (or an error message).""" raw = (text or "").strip() if not raw: return None, "Paste an Etsy listing link or its numeric listing ID." if re.fullmatch(r"\d{6,14}", raw): return f"{ETSY_BASE}/listing/{raw}", None if _is_url(raw): parts = urllib.parse.urlsplit(raw) host = parts.netloc.lower() if host == "etsy.me": # short share link; Scrape.do follows the redirect return f"https://etsy.me{parts.path}", None if not _host_is_etsy(host): return None, "Only etsy.com links are supported." if not re.search(r"/listing/\d+", parts.path): return None, "That link isn't a single listing. Use the Search tab for category, search and shop pages." return f"https://{parts.netloc}{parts.path}", None return None, "Paste an Etsy listing link or its numeric listing ID." def build_search_url( target: str, page: int = 1, sort_by: Optional[str] = None, min_price: Optional[float] = None, max_price: Optional[float] = None, free_shipping: bool = False, ) -> str: """Turn a search phrase, or an Etsy category/search/shop URL, into a paged URL.""" text = (target or "").strip() if _is_url(text): parts = urllib.parse.urlsplit(text) if not _host_is_etsy(parts.netloc): raise ValueError("Only etsy.com links are supported.") params = dict(urllib.parse.parse_qsl(parts.query)) base = f"https://{parts.netloc}{parts.path or '/'}" else: params = {"q": text} base = ETSY_BASE + "/search" params.pop("page", None) if page > 1: params["page"] = str(page) order = SORT_OPTIONS.get((sort_by or "").strip().lower(), None) if order: params["order"] = order if min_price is not None: params["min"] = f"{float(min_price):g}" if max_price is not None: params["max"] = f"{float(max_price):g}" if free_shipping: params["free_shipping"] = "true" return base + ("?" + urllib.parse.urlencode(params) if params else "") def _clean_listing_url(href: str) -> str: parts = urllib.parse.urlsplit(href) path = parts.path netloc = parts.netloc or "www.etsy.com" return f"https://{netloc}{path}" def parse_search_page(html: str) -> List[Dict[str, Any]]: """Read the listing cards out of an Etsy search, category or shop page.""" soup = BeautifulSoup(html, "html.parser") cards: Dict[str, Any] = {} first_href: Dict[str, str] = {} def is_card(tag: Any) -> bool: return tag.name in ("div", "li") and ( tag.has_attr("data-listing-id") or "v2-listing-card" in (tag.get("class") or []) ) for anchor in soup.select('a[href*="/listing/"]'): href = anchor.get("href") or "" match = re.search(r"/listing/(\d+)", href) if not match: continue lid = match.group(1) if lid in cards: continue card = anchor.find_parent(is_card) if card is None: card = anchor for _ in range(3): if card.parent is not None: card = card.parent cards[lid] = card first_href[lid] = href items: List[Dict[str, Any]] = [] for lid, card in cards.items(): text = card.get_text(" ", strip=True) heading = card.select_one("h3, h2") name = None if heading is not None: name = heading.get("title") or heading.get_text(" ", strip=True) if not name: for anchor in card.select('a[href*="/listing/"]'): name = anchor.get("title") or anchor.get_text(" ", strip=True) if name: break if not name: img_alt = card.select_one("img[alt]") name = img_alt.get("alt") if img_alt else None image = None img = card.select_one("img") if img is not None: image = img.get("src") or img.get("data-src") or img.get("data-srcset") if image and " " in image: found = [p for p in image.split() if p.startswith("http")] image = found[0] if found else None symbol_el = card.select_one(".currency-symbol") currency = symbol_el.get_text(strip=True) if symbol_el else None price_el = card.select_one(".lc-price .currency-value") or card.select_one(".currency-value") price = parse_price(price_el.get_text()) if price_el else None orig_el = card.select_one(".search-collage-promotion-price .currency-value") original = parse_price(orig_el.get_text()) if orig_el else None discount = None if price and original and original > price: discount = round(100 * (original - price) / original, 1) else: original = None off = re.search(r"(\d{1,2})%\s*off", text, re.I) if off: discount = float(off.group(1)) rating = None review_count = None rating_el = card.select_one('[aria-label*="out of 5"]') if rating_el is not None: found = re.search(r"([\d.]+)\s*out of 5", rating_el.get("aria-label", "")) if found: rating = float(found.group(1)) scope = rating_el.parent.get_text(" ", strip=True) if rating_el.parent else text count = re.search(r"\(([\d.,]+)\s*([kK])?\)", scope) or re.search(r"\(([\d.,]+)\s*([kK])?\)", text) if count: review_count = _count(count.group(1), count.group(2)) else: rating_input = card.select_one('input[name="rating"]') if rating_input is not None and rating_input.get("value"): try: rating = float(rating_input["value"]) except ValueError: pass shop_el = card.find(attrs={"data-shop-id": True}) shop_id = shop_el.get("data-shop-id") if shop_el is not None else None left = re.search(r"Only\s+(\d+)\s+left", text, re.I) items.append( { "listing_id": lid, "name": name or "", "price": price, "original_price": original, "discount_percent": discount, "currency": currency, "rating": rating, "review_count": review_count, "shop_id": shop_id, "star_seller": bool(re.search(r"\bStar Seller\b", text, re.I)), "free_shipping": bool(re.search(r"\bFree shipping\b", text, re.I)), "bestseller": bool(re.search(r"\bBestseller\b", text, re.I)), "is_ad": bool(re.search(r"\bAd (?:by|from)\b", text, re.I)), "only_left": int(left.group(1)) if left else None, "image": image, "url": _clean_listing_url(first_href[lid]), } ) return items def _walk_ld(data: Any) -> List[Dict[str, Any]]: objs: List[Dict[str, Any]] = [] if isinstance(data, list): for item in data: objs.extend(_walk_ld(item)) elif isinstance(data, dict): objs.append(data) if isinstance(data.get("@graph"), list): objs.extend(_walk_ld(data["@graph"])) return objs def _is_product(obj: Dict[str, Any]) -> bool: kind = obj.get("@type") return kind == "Product" or (isinstance(kind, list) and "Product" in kind) def parse_reviews(soup: BeautifulSoup, listing_id: Optional[str]) -> List[Dict[str, Any]]: """Reviews Etsy prints straight into the listing page (the first page of them).""" reviews: List[Dict[str, Any]] = [] for card in soup.find_all("div", class_="review-card"): rating = None rating_input = card.find("input", attrs={"name": "rating"}) if rating_input is not None and rating_input.get("value"): try: rating = float(rating_input["value"]) except ValueError: rating = None body = card.find("div", class_="wt-text-body") text = body.get_text(" ", strip=True) if body else None author = None created = None meta = card.find("p", class_="wt-text-body-small") if meta is not None: raw = meta.get_text(" ", strip=True) found = re.search(r"([A-Z][a-z]{2}\s+\d{1,2},\s+\d{4})$", raw) if found: created = found.group(1) author = raw[: found.start()].strip() or None else: author = raw or None reviews.append( { "position": len(reviews) + 1, "listing_id": listing_id, "review_id": card.get("data-review-region"), "rating": rating, "text": text, "author": author, "date": created, } ) return reviews def parse_listing_page(html: str, url: str) -> Dict[str, Any]: """Return {'listing': {...}, 'reviews': [...]} or {'_error': '...'}.""" soup = BeautifulSoup(html, "html.parser") product: Optional[Dict[str, Any]] = None for script in soup.find_all("script", attrs={"type": "application/ld+json"}): try: data = json.loads(script.string or script.get_text() or "") except ValueError: continue for obj in _walk_ld(data): if _is_product(obj): product = obj break if product: break if product is None: lowered = soup.get_text(" ", strip=True).lower() if "unavailable" in lowered or "no longer available" in lowered or "sold" in lowered[:600]: return {"_error": "This listing is unavailable (removed, sold out or expired)."} return {"_error": "Couldn't find product data on that page. Check the link and try again."} listing_id = None match = re.search(r"/listing/(\d+)", url) if match: listing_id = match.group(1) canonical = soup.find("link", attrs={"rel": "canonical"}) canonical_url = canonical.get("href") if canonical is not None else None if canonical_url: found = re.search(r"/listing/(\d+)", canonical_url) if found: listing_id = found.group(1) if not listing_id and product.get("sku"): listing_id = str(product["sku"]) listing: Dict[str, Any] = { "listing_id": listing_id, "url": canonical_url or url, "name": product.get("name"), "description": product.get("description"), "category": product.get("category"), } brand = product.get("brand") shop = brand.get("name") if isinstance(brand, dict) else brand if isinstance(brand, str) else None shop_url = None for anchor in soup.select('a[href*="/shop/"]'): found = re.search(r"/shop/([A-Za-z0-9_-]+)", anchor.get("href") or "") if found: shop = shop or found.group(1) shop_url = f"{ETSY_BASE}/shop/{found.group(1)}" break listing["shop"] = shop listing["shop_url"] = shop_url image = product.get("image") images = image if isinstance(image, list) else [image] if image else [] listing["images"] = [i.get("url") if isinstance(i, dict) else i for i in images if i] offers = product.get("offers") if isinstance(offers, list): offers = offers[0] if offers else {} offers = offers if isinstance(offers, dict) else {} price = offers.get("price") or offers.get("lowPrice") listing["price"] = parse_price(price) listing["price_high"] = parse_price(offers.get("highPrice")) if offers.get("highPrice") else None listing["currency"] = offers.get("priceCurrency") availability = str(offers.get("availability") or "") listing["availability"] = availability.rsplit("/", 1)[-1] if availability else None aggregate = product.get("aggregateRating") or {} listing["rating"] = parse_price(aggregate.get("ratingValue")) if aggregate.get("ratingValue") else None listing["review_count"] = ( int(parse_price(aggregate.get("reviewCount")) or 0) if aggregate.get("reviewCount") else None ) for key in ("sku", "material", "color"): if product.get(key): listing[key] = product[key] return {"listing": listing, "reviews": parse_reviews(soup, listing_id)} # --------------------------------------------------------------------- tool class ScrapeDoEtsyTool(Tool): name = "etsy_scraper" description = """ Scrape Etsy. Two modes: - 'search': find listings for a search phrase such as 'personalized leather wallet', or pass an Etsy category, search or shop URL. Returns up to ~48 listings per page with name, price, original price and discount, rating, review count, shop ID, Star Seller / Free shipping / Bestseller / Ad flags and URL. Supports sorting and price / free-shipping filters. Good for price tracking and niche research. - 'listing': full details for one listing (pass its link or numeric ID): name, description, category, shop, all images, price, availability, rating and the reviews Etsy prints on the page. Powered by Scrape.do, which gets past Etsy's DataDome protection with residential proxies and handles localized prices. """ output_type = "string" def __init__(self, api_token: Optional[str] = None) -> None: self._api_token = api_token self.inputs = { "mode": { "type": "string", "description": "'search' to find listings, 'listing' for one listing's details and reviews.", }, "query": { "type": "string", "description": ( "For 'search': a search phrase, or an Etsy category / search / shop URL. " "For 'listing': an Etsy listing link or numeric listing ID." ), }, "country": { "type": "string", "description": "Market for prices and shipping: 'us', 'gb', 'ca', 'au', 'de', 'fr', 'es', 'it', 'nl', 'tr'. Defaults to 'us'.", "nullable": True, }, "pages": { "type": "integer", "description": "'search' only: result pages to fetch (1-10, about 48 listings each, 10 credits each). Defaults to 1.", "nullable": True, }, "sort_by": { "type": "string", "description": "'search' only: 'relevance', 'newest', 'price_low', 'price_high' or 'top_reviewed'.", "nullable": True, }, "min_price": { "type": "number", "description": "'search' only: minimum price in the market's currency.", "nullable": True, }, "max_price": { "type": "number", "description": "'search' only: maximum price in the market's currency.", "nullable": True, }, "free_shipping": { "type": "boolean", "description": "'search' only: only listings with free shipping.", "nullable": True, }, } super().__init__() # ------------------------------------------------------------ plumbing def _resolve_token(self) -> Optional[str]: if self._api_token: return self._api_token.strip() env_token = os.getenv("SCRAPE_DO_API_TOKEN") return env_token.strip() if env_token else None @staticmethod def _missing_token() -> Dict[str, Any]: return { "status": "error", "errorMessage": "Missing Scrape.do API token.", "hint": f"Get a free token at {SCRAPE_DO_SIGNUP_URL}", } @staticmethod def _as_error(data: Dict[str, Any]) -> Dict[str, Any]: result = {"status": "error", "errorMessage": data.get("_error", "Unknown error")} if data.get("_auth"): result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" return result def _fetch(self, token: str, target_url: str, geo: str, locale: str, retries: int = 2) -> Dict[str, Any]: """Fetch one Etsy page through Scrape.do. Returns {'html', 'credits'} or {'_error'}.""" params = { "token": token, "url": target_url, "geoCode": geo, "super": "true", "extraHeaders": "true", } headers = {"sd-x-detected-locale": locale} def redact(text: str) -> str: return text.replace(token, "***") if token else text last: Dict[str, Any] = {"_error": "Request failed."} for attempt in range(retries + 1): try: response = requests.get(API_ENDPOINT, params=params, headers=headers, timeout=120) except requests.exceptions.RequestException as exc: last = {"_error": redact(f"Request failed: {exc}")} if attempt < retries: time.sleep(1.5) continue return last status = response.status_code if status == 401: return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True} if status in (404, 410): return {"_error": f"Etsy says this page doesn't exist (HTTP {status}).", "_status": status} if status == 429 or status >= 500: last = {"_error": f"Scrape.do returned HTTP {status}. Too many parallel requests or a temporary error.", "_status": status} if attempt < retries: time.sleep(2.0 * (attempt + 1)) continue return last if not response.ok: snippet = (response.text or "").strip()[:200] or "no details" return {"_error": redact(f"Scrape.do returned HTTP {status}: {snippet}"), "_status": status} text = response.text or "" if "captcha-delivery.com" in text[:6000] and len(text) < 20000: last = {"_error": "Etsy served a bot-check page instead of the content."} if attempt < retries: time.sleep(2.0) continue return last try: credits = int(response.headers.get("Scrape.do-Request-Cost", CREDITS_PER_REQUEST)) except (TypeError, ValueError): credits = CREDITS_PER_REQUEST return {"html": text, "credits": credits} return last # --------------------------------------------------------------- search def search_listings( self, query: str, country: str = "us", pages: int = 1, sort_by: Optional[str] = None, min_price: Optional[float] = None, max_price: Optional[float] = None, free_shipping: bool = False, ) -> Dict[str, Any]: token = self._resolve_token() if not token: return self._missing_token() if not query or not query.strip(): return {"status": "error", "errorMessage": "Type what you're looking for, e.g. 'personalized leather wallet'."} geo, locale = _market(country) pages = max(1, min(int(pages or 1), MAX_SEARCH_PAGES)) warnings: List[str] = [] listings: List[Dict[str, Any]] = [] seen: set = set() credits = 0 pages_fetched = 0 first_url = None for page_index in range(pages): try: url = build_search_url(query, page_index + 1, sort_by, min_price, max_price, bool(free_shipping)) except ValueError as exc: return {"status": "error", "errorMessage": str(exc)} if page_index == 0: first_url = url data = self._fetch(token, url, geo, locale) if "_error" in data: if page_index == 0: return self._as_error(data) warnings.append(f"Stopped after page {page_index}: {data['_error']}") break credits += data["credits"] pages_fetched += 1 fresh = 0 for item in parse_search_page(data["html"]): if item["listing_id"] in seen: continue seen.add(item["listing_id"]) item["page"] = page_index + 1 item["position"] = len(listings) + 1 listings.append(item) fresh += 1 if fresh == 0: if page_index == 0: warnings.append("No listings found on this page. The search may have no results.") break if page_index + 1 < pages: time.sleep(0.5) result: Dict[str, Any] = { "status": "success", "query": query.strip(), "country": geo, "etsy_url": first_url, "listings": listings, "pages_fetched": pages_fetched, "credits_used": credits, } if warnings: result["warnings"] = warnings return result # -------------------------------------------------------------- listing def listing_details(self, listing: str, country: str = "us") -> Dict[str, Any]: token = self._resolve_token() if not token: return self._missing_token() url, error = normalize_listing_ref(listing) if error: return {"status": "error", "errorMessage": error} geo, locale = _market(country) data = self._fetch(token, url, geo, locale) if "_error" in data: return self._as_error(data) parsed = parse_listing_page(data["html"], url) if "_error" in parsed: return {"status": "error", "errorMessage": parsed["_error"]} return { "status": "success", "listing": parsed["listing"], "reviews": parsed["reviews"], "reviews_note": "Etsy prints only the first page of reviews inside the listing page; those are the ones returned.", "credits_used": data["credits"], } def listings_bulk(self, refs: List[str], country: str = "us") -> Dict[str, Any]: """Details for several listings at once (10 credits each, a few in parallel).""" token = self._resolve_token() if not token: return self._missing_token() cleaned = [r.strip() for r in refs if r and r.strip()] seen: set = set() unique = [r for r in cleaned if not (r in seen or seen.add(r))] if not unique: return {"status": "error", "errorMessage": "Paste at least one Etsy listing link or ID."} warnings: List[str] = [] if len(unique) > MAX_BULK: warnings.append(f"Only the first {MAX_BULK} of {len(unique)} were fetched.") unique = unique[:MAX_BULK] def one(ref: str) -> Dict[str, Any]: result = self.listing_details(ref, country) result["ref"] = ref return result with ThreadPoolExecutor(max_workers=BULK_WORKERS) as pool: results = list(pool.map(one, unique)) listings = [r["listing"] for r in results if r.get("status") == "success"] errors = [{"ref": r["ref"], "error": r.get("errorMessage")} for r in results if r.get("status") != "success"] credits = sum(r.get("credits_used", 0) for r in results if r.get("status") == "success") out: Dict[str, Any] = { "status": "success", "listings": listings, "errors": errors, "credits_used": credits, } if warnings: out["warnings"] = warnings if not listings and errors: out = {"status": "error", "errorMessage": errors[0]["error"] or "No listing could be fetched."} return out # -------------------------------------------------------------- forward def forward( self, mode: str, query: str, country: Optional[str] = None, pages: Optional[int] = None, sort_by: Optional[str] = None, min_price: Optional[float] = None, max_price: Optional[float] = None, free_shipping: Optional[bool] = None, ) -> str: mode_key = (mode or "").strip().lower() if mode_key in ("search", "find", "listings"): result = self.search_listings( query, country or "us", pages or 1, sort_by, min_price, max_price, bool(free_shipping) ) elif mode_key in ("listing", "details", "product"): result = self.listing_details(query, country or "us") else: result = {"status": "error", "errorMessage": "mode must be 'search' or 'listing'."} return json.dumps(result, ensure_ascii=False, indent=2)