Spaces:
Sleeping
Sleeping
Download tool.py from bugrahans/etsy-scraper: direct link, hf CLI and curl.
- Browser
- Download file 28.3 kB
-
https://huggingface.co/spaces/bugrahans/etsy-scraper/resolve/main/tool.py
- Command line
-
hf download hf://spaces/bugrahans/etsy-scraper/tool.py
-
curl -L -o tool.py https://huggingface.co/spaces/bugrahans/etsy-scraper/resolve/main/tool.py
28.3 kB
| from __future__ import annotations | |
| import json | |
| import os | |
| import re | |
| import time | |
| import urllib.parse | |
| from concurrent.futures import ThreadPoolExecutor | |
| from typing import Any, Dict, List, Optional, Tuple | |
| import requests | |
| from bs4 import BeautifulSoup | |
| from smolagents.tools import Tool | |
| SCRAPE_DO_SIGNUP_URL = ( | |
| "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=etsy-scraper" | |
| ) | |
| API_ENDPOINT = "https://api.scrape.do/" | |
| ETSY_BASE = "https://www.etsy.com" | |
| # Etsy is not in Scrape.do's per-domain price list, so the standard residential | |
| # price (super=true) applies. The real cost is read from the | |
| # `Scrape.do-Request-Cost` response header whenever it is present. | |
| CREDITS_PER_REQUEST = 10 | |
| MAX_SEARCH_PAGES = 10 | |
| MAX_BULK = 20 | |
| BULK_WORKERS = 3 | |
| # (label, geoCode, sd-x-detected-locale as CURRENCY|language|COUNTRY) | |
| COUNTRIES: List[tuple] = [ | |
| ("United States", "us", "USD|en-US|US"), | |
| ("United Kingdom", "gb", "GBP|en-GB|GB"), | |
| ("Canada", "ca", "CAD|en-CA|CA"), | |
| ("Australia", "au", "AUD|en-AU|AU"), | |
| ("Germany", "de", "EUR|de|DE"), | |
| ("France", "fr", "EUR|fr|FR"), | |
| ("Spain", "es", "EUR|es|ES"), | |
| ("Italy", "it", "EUR|it|IT"), | |
| ("Netherlands", "nl", "EUR|nl|NL"), | |
| ("Turkey", "tr", "TRY|tr|TR"), | |
| ] | |
| COUNTRY_BY_CODE = {code: (label, code, locale) for label, code, locale in COUNTRIES} | |
| SORT_OPTIONS = { | |
| "relevance": None, | |
| "newest": "date_desc", | |
| "price_low": "price_asc", | |
| "price_high": "price_desc", | |
| "top_reviewed": "highest_reviews", | |
| } | |
| # ------------------------------------------------------------------ helpers | |
| def _market(country: Optional[str]) -> Tuple[str, str]: | |
| code = (country or "us").strip().lower() | |
| _, code, locale = COUNTRY_BY_CODE.get(code, ("", code, "USD|en-US|US")) | |
| return code, locale | |
| def _is_url(text: str) -> bool: | |
| return text.lower().startswith(("http://", "https://")) | |
| def _host_is_etsy(host: str) -> bool: | |
| host = host.lower().split(":")[0] | |
| return host == "etsy.com" or host.endswith(".etsy.com") | |
| def parse_price(text: Any) -> Optional[float]: | |
| """Parse '1,234.56', '1.234,56', '12,50' or '19' into a float.""" | |
| if text is None: | |
| return None | |
| s = re.sub(r"[^\d.,]", "", str(text)) | |
| if not s or not re.search(r"\d", s): | |
| return None | |
| if "," in s and "." in s: | |
| if s.rfind(",") > s.rfind("."): | |
| s = s.replace(".", "").replace(",", ".") | |
| else: | |
| s = s.replace(",", "") | |
| elif "," in s: | |
| head, _, tail = s.rpartition(",") | |
| s = f"{head.replace(',', '')}.{tail}" if len(tail) in (1, 2) else s.replace(",", "") | |
| elif "." in s: | |
| head, _, tail = s.rpartition(".") | |
| if s.count(".") > 1 or len(tail) == 3: | |
| s = s.replace(".", "") | |
| try: | |
| return float(s) | |
| except ValueError: | |
| return None | |
| def _count(text: str, suffix: Optional[str]) -> Optional[int]: | |
| try: | |
| value = float(text.replace(",", "")) | |
| except ValueError: | |
| return None | |
| if suffix and suffix.lower() == "k": | |
| value *= 1000 | |
| return int(round(value)) | |
| def normalize_listing_ref(text: str) -> Tuple[Optional[str], Optional[str]]: | |
| """A listing link or numeric ID -> a clean listing URL (or an error message).""" | |
| raw = (text or "").strip() | |
| if not raw: | |
| return None, "Paste an Etsy listing link or its numeric listing ID." | |
| if re.fullmatch(r"\d{6,14}", raw): | |
| return f"{ETSY_BASE}/listing/{raw}", None | |
| if _is_url(raw): | |
| parts = urllib.parse.urlsplit(raw) | |
| host = parts.netloc.lower() | |
| if host == "etsy.me": # short share link; Scrape.do follows the redirect | |
| return f"https://etsy.me{parts.path}", None | |
| if not _host_is_etsy(host): | |
| return None, "Only etsy.com links are supported." | |
| if not re.search(r"/listing/\d+", parts.path): | |
| return None, "That link isn't a single listing. Use the Search tab for category, search and shop pages." | |
| return f"https://{parts.netloc}{parts.path}", None | |
| return None, "Paste an Etsy listing link or its numeric listing ID." | |
| def build_search_url( | |
| target: str, | |
| page: int = 1, | |
| sort_by: Optional[str] = None, | |
| min_price: Optional[float] = None, | |
| max_price: Optional[float] = None, | |
| free_shipping: bool = False, | |
| ) -> str: | |
| """Turn a search phrase, or an Etsy category/search/shop URL, into a paged URL.""" | |
| text = (target or "").strip() | |
| if _is_url(text): | |
| parts = urllib.parse.urlsplit(text) | |
| if not _host_is_etsy(parts.netloc): | |
| raise ValueError("Only etsy.com links are supported.") | |
| params = dict(urllib.parse.parse_qsl(parts.query)) | |
| base = f"https://{parts.netloc}{parts.path or '/'}" | |
| else: | |
| params = {"q": text} | |
| base = ETSY_BASE + "/search" | |
| params.pop("page", None) | |
| if page > 1: | |
| params["page"] = str(page) | |
| order = SORT_OPTIONS.get((sort_by or "").strip().lower(), None) | |
| if order: | |
| params["order"] = order | |
| if min_price is not None: | |
| params["min"] = f"{float(min_price):g}" | |
| if max_price is not None: | |
| params["max"] = f"{float(max_price):g}" | |
| if free_shipping: | |
| params["free_shipping"] = "true" | |
| return base + ("?" + urllib.parse.urlencode(params) if params else "") | |
| def _clean_listing_url(href: str) -> str: | |
| parts = urllib.parse.urlsplit(href) | |
| path = parts.path | |
| netloc = parts.netloc or "www.etsy.com" | |
| return f"https://{netloc}{path}" | |
| def parse_search_page(html: str) -> List[Dict[str, Any]]: | |
| """Read the listing cards out of an Etsy search, category or shop page.""" | |
| soup = BeautifulSoup(html, "html.parser") | |
| cards: Dict[str, Any] = {} | |
| first_href: Dict[str, str] = {} | |
| def is_card(tag: Any) -> bool: | |
| return tag.name in ("div", "li") and ( | |
| tag.has_attr("data-listing-id") or "v2-listing-card" in (tag.get("class") or []) | |
| ) | |
| for anchor in soup.select('a[href*="/listing/"]'): | |
| href = anchor.get("href") or "" | |
| match = re.search(r"/listing/(\d+)", href) | |
| if not match: | |
| continue | |
| lid = match.group(1) | |
| if lid in cards: | |
| continue | |
| card = anchor.find_parent(is_card) | |
| if card is None: | |
| card = anchor | |
| for _ in range(3): | |
| if card.parent is not None: | |
| card = card.parent | |
| cards[lid] = card | |
| first_href[lid] = href | |
| items: List[Dict[str, Any]] = [] | |
| for lid, card in cards.items(): | |
| text = card.get_text(" ", strip=True) | |
| heading = card.select_one("h3, h2") | |
| name = None | |
| if heading is not None: | |
| name = heading.get("title") or heading.get_text(" ", strip=True) | |
| if not name: | |
| for anchor in card.select('a[href*="/listing/"]'): | |
| name = anchor.get("title") or anchor.get_text(" ", strip=True) | |
| if name: | |
| break | |
| if not name: | |
| img_alt = card.select_one("img[alt]") | |
| name = img_alt.get("alt") if img_alt else None | |
| image = None | |
| img = card.select_one("img") | |
| if img is not None: | |
| image = img.get("src") or img.get("data-src") or img.get("data-srcset") | |
| if image and " " in image: | |
| found = [p for p in image.split() if p.startswith("http")] | |
| image = found[0] if found else None | |
| symbol_el = card.select_one(".currency-symbol") | |
| currency = symbol_el.get_text(strip=True) if symbol_el else None | |
| price_el = card.select_one(".lc-price .currency-value") or card.select_one(".currency-value") | |
| price = parse_price(price_el.get_text()) if price_el else None | |
| orig_el = card.select_one(".search-collage-promotion-price .currency-value") | |
| original = parse_price(orig_el.get_text()) if orig_el else None | |
| discount = None | |
| if price and original and original > price: | |
| discount = round(100 * (original - price) / original, 1) | |
| else: | |
| original = None | |
| off = re.search(r"(\d{1,2})%\s*off", text, re.I) | |
| if off: | |
| discount = float(off.group(1)) | |
| rating = None | |
| review_count = None | |
| rating_el = card.select_one('[aria-label*="out of 5"]') | |
| if rating_el is not None: | |
| found = re.search(r"([\d.]+)\s*out of 5", rating_el.get("aria-label", "")) | |
| if found: | |
| rating = float(found.group(1)) | |
| scope = rating_el.parent.get_text(" ", strip=True) if rating_el.parent else text | |
| count = re.search(r"\(([\d.,]+)\s*([kK])?\)", scope) or re.search(r"\(([\d.,]+)\s*([kK])?\)", text) | |
| if count: | |
| review_count = _count(count.group(1), count.group(2)) | |
| else: | |
| rating_input = card.select_one('input[name="rating"]') | |
| if rating_input is not None and rating_input.get("value"): | |
| try: | |
| rating = float(rating_input["value"]) | |
| except ValueError: | |
| pass | |
| shop_el = card.find(attrs={"data-shop-id": True}) | |
| shop_id = shop_el.get("data-shop-id") if shop_el is not None else None | |
| left = re.search(r"Only\s+(\d+)\s+left", text, re.I) | |
| items.append( | |
| { | |
| "listing_id": lid, | |
| "name": name or "", | |
| "price": price, | |
| "original_price": original, | |
| "discount_percent": discount, | |
| "currency": currency, | |
| "rating": rating, | |
| "review_count": review_count, | |
| "shop_id": shop_id, | |
| "star_seller": bool(re.search(r"\bStar Seller\b", text, re.I)), | |
| "free_shipping": bool(re.search(r"\bFree shipping\b", text, re.I)), | |
| "bestseller": bool(re.search(r"\bBestseller\b", text, re.I)), | |
| "is_ad": bool(re.search(r"\bAd (?:by|from)\b", text, re.I)), | |
| "only_left": int(left.group(1)) if left else None, | |
| "image": image, | |
| "url": _clean_listing_url(first_href[lid]), | |
| } | |
| ) | |
| return items | |
| def _walk_ld(data: Any) -> List[Dict[str, Any]]: | |
| objs: List[Dict[str, Any]] = [] | |
| if isinstance(data, list): | |
| for item in data: | |
| objs.extend(_walk_ld(item)) | |
| elif isinstance(data, dict): | |
| objs.append(data) | |
| if isinstance(data.get("@graph"), list): | |
| objs.extend(_walk_ld(data["@graph"])) | |
| return objs | |
| def _is_product(obj: Dict[str, Any]) -> bool: | |
| kind = obj.get("@type") | |
| return kind == "Product" or (isinstance(kind, list) and "Product" in kind) | |
| def parse_reviews(soup: BeautifulSoup, listing_id: Optional[str]) -> List[Dict[str, Any]]: | |
| """Reviews Etsy prints straight into the listing page (the first page of them).""" | |
| reviews: List[Dict[str, Any]] = [] | |
| for card in soup.find_all("div", class_="review-card"): | |
| rating = None | |
| rating_input = card.find("input", attrs={"name": "rating"}) | |
| if rating_input is not None and rating_input.get("value"): | |
| try: | |
| rating = float(rating_input["value"]) | |
| except ValueError: | |
| rating = None | |
| body = card.find("div", class_="wt-text-body") | |
| text = body.get_text(" ", strip=True) if body else None | |
| author = None | |
| created = None | |
| meta = card.find("p", class_="wt-text-body-small") | |
| if meta is not None: | |
| raw = meta.get_text(" ", strip=True) | |
| found = re.search(r"([A-Z][a-z]{2}\s+\d{1,2},\s+\d{4})$", raw) | |
| if found: | |
| created = found.group(1) | |
| author = raw[: found.start()].strip() or None | |
| else: | |
| author = raw or None | |
| reviews.append( | |
| { | |
| "position": len(reviews) + 1, | |
| "listing_id": listing_id, | |
| "review_id": card.get("data-review-region"), | |
| "rating": rating, | |
| "text": text, | |
| "author": author, | |
| "date": created, | |
| } | |
| ) | |
| return reviews | |
| def parse_listing_page(html: str, url: str) -> Dict[str, Any]: | |
| """Return {'listing': {...}, 'reviews': [...]} or {'_error': '...'}.""" | |
| soup = BeautifulSoup(html, "html.parser") | |
| product: Optional[Dict[str, Any]] = None | |
| for script in soup.find_all("script", attrs={"type": "application/ld+json"}): | |
| try: | |
| data = json.loads(script.string or script.get_text() or "") | |
| except ValueError: | |
| continue | |
| for obj in _walk_ld(data): | |
| if _is_product(obj): | |
| product = obj | |
| break | |
| if product: | |
| break | |
| if product is None: | |
| lowered = soup.get_text(" ", strip=True).lower() | |
| if "unavailable" in lowered or "no longer available" in lowered or "sold" in lowered[:600]: | |
| return {"_error": "This listing is unavailable (removed, sold out or expired)."} | |
| return {"_error": "Couldn't find product data on that page. Check the link and try again."} | |
| listing_id = None | |
| match = re.search(r"/listing/(\d+)", url) | |
| if match: | |
| listing_id = match.group(1) | |
| canonical = soup.find("link", attrs={"rel": "canonical"}) | |
| canonical_url = canonical.get("href") if canonical is not None else None | |
| if canonical_url: | |
| found = re.search(r"/listing/(\d+)", canonical_url) | |
| if found: | |
| listing_id = found.group(1) | |
| if not listing_id and product.get("sku"): | |
| listing_id = str(product["sku"]) | |
| listing: Dict[str, Any] = { | |
| "listing_id": listing_id, | |
| "url": canonical_url or url, | |
| "name": product.get("name"), | |
| "description": product.get("description"), | |
| "category": product.get("category"), | |
| } | |
| brand = product.get("brand") | |
| shop = brand.get("name") if isinstance(brand, dict) else brand if isinstance(brand, str) else None | |
| shop_url = None | |
| for anchor in soup.select('a[href*="/shop/"]'): | |
| found = re.search(r"/shop/([A-Za-z0-9_-]+)", anchor.get("href") or "") | |
| if found: | |
| shop = shop or found.group(1) | |
| shop_url = f"{ETSY_BASE}/shop/{found.group(1)}" | |
| break | |
| listing["shop"] = shop | |
| listing["shop_url"] = shop_url | |
| image = product.get("image") | |
| images = image if isinstance(image, list) else [image] if image else [] | |
| listing["images"] = [i.get("url") if isinstance(i, dict) else i for i in images if i] | |
| offers = product.get("offers") | |
| if isinstance(offers, list): | |
| offers = offers[0] if offers else {} | |
| offers = offers if isinstance(offers, dict) else {} | |
| price = offers.get("price") or offers.get("lowPrice") | |
| listing["price"] = parse_price(price) | |
| listing["price_high"] = parse_price(offers.get("highPrice")) if offers.get("highPrice") else None | |
| listing["currency"] = offers.get("priceCurrency") | |
| availability = str(offers.get("availability") or "") | |
| listing["availability"] = availability.rsplit("/", 1)[-1] if availability else None | |
| aggregate = product.get("aggregateRating") or {} | |
| listing["rating"] = parse_price(aggregate.get("ratingValue")) if aggregate.get("ratingValue") else None | |
| listing["review_count"] = ( | |
| int(parse_price(aggregate.get("reviewCount")) or 0) if aggregate.get("reviewCount") else None | |
| ) | |
| for key in ("sku", "material", "color"): | |
| if product.get(key): | |
| listing[key] = product[key] | |
| return {"listing": listing, "reviews": parse_reviews(soup, listing_id)} | |
| # --------------------------------------------------------------------- tool | |
| class ScrapeDoEtsyTool(Tool): | |
| name = "etsy_scraper" | |
| description = """ | |
| Scrape Etsy. Two modes: | |
| - 'search': find listings for a search phrase such as 'personalized leather wallet', | |
| or pass an Etsy category, search or shop URL. Returns up to ~48 listings per page | |
| with name, price, original price and discount, rating, review count, shop ID, | |
| Star Seller / Free shipping / Bestseller / Ad flags and URL. Supports sorting | |
| and price / free-shipping filters. Good for price tracking and niche research. | |
| - 'listing': full details for one listing (pass its link or numeric ID): name, | |
| description, category, shop, all images, price, availability, rating and the | |
| reviews Etsy prints on the page. | |
| Powered by Scrape.do, which gets past Etsy's DataDome protection with residential | |
| proxies and handles localized prices. | |
| """ | |
| output_type = "string" | |
| def __init__(self, api_token: Optional[str] = None) -> None: | |
| self._api_token = api_token | |
| self.inputs = { | |
| "mode": { | |
| "type": "string", | |
| "description": "'search' to find listings, 'listing' for one listing's details and reviews.", | |
| }, | |
| "query": { | |
| "type": "string", | |
| "description": ( | |
| "For 'search': a search phrase, or an Etsy category / search / shop URL. " | |
| "For 'listing': an Etsy listing link or numeric listing ID." | |
| ), | |
| }, | |
| "country": { | |
| "type": "string", | |
| "description": "Market for prices and shipping: 'us', 'gb', 'ca', 'au', 'de', 'fr', 'es', 'it', 'nl', 'tr'. Defaults to 'us'.", | |
| "nullable": True, | |
| }, | |
| "pages": { | |
| "type": "integer", | |
| "description": "'search' only: result pages to fetch (1-10, about 48 listings each, 10 credits each). Defaults to 1.", | |
| "nullable": True, | |
| }, | |
| "sort_by": { | |
| "type": "string", | |
| "description": "'search' only: 'relevance', 'newest', 'price_low', 'price_high' or 'top_reviewed'.", | |
| "nullable": True, | |
| }, | |
| "min_price": { | |
| "type": "number", | |
| "description": "'search' only: minimum price in the market's currency.", | |
| "nullable": True, | |
| }, | |
| "max_price": { | |
| "type": "number", | |
| "description": "'search' only: maximum price in the market's currency.", | |
| "nullable": True, | |
| }, | |
| "free_shipping": { | |
| "type": "boolean", | |
| "description": "'search' only: only listings with free shipping.", | |
| "nullable": True, | |
| }, | |
| } | |
| super().__init__() | |
| # ------------------------------------------------------------ plumbing | |
| def _resolve_token(self) -> Optional[str]: | |
| if self._api_token: | |
| return self._api_token.strip() | |
| env_token = os.getenv("SCRAPE_DO_API_TOKEN") | |
| return env_token.strip() if env_token else None | |
| def _missing_token() -> Dict[str, Any]: | |
| return { | |
| "status": "error", | |
| "errorMessage": "Missing Scrape.do API token.", | |
| "hint": f"Get a free token at {SCRAPE_DO_SIGNUP_URL}", | |
| } | |
| def _as_error(data: Dict[str, Any]) -> Dict[str, Any]: | |
| result = {"status": "error", "errorMessage": data.get("_error", "Unknown error")} | |
| if data.get("_auth"): | |
| result["hint"] = f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}" | |
| return result | |
| def _fetch(self, token: str, target_url: str, geo: str, locale: str, retries: int = 2) -> Dict[str, Any]: | |
| """Fetch one Etsy page through Scrape.do. Returns {'html', 'credits'} or {'_error'}.""" | |
| params = { | |
| "token": token, | |
| "url": target_url, | |
| "geoCode": geo, | |
| "super": "true", | |
| "extraHeaders": "true", | |
| } | |
| headers = {"sd-x-detected-locale": locale} | |
| def redact(text: str) -> str: | |
| return text.replace(token, "***") if token else text | |
| last: Dict[str, Any] = {"_error": "Request failed."} | |
| for attempt in range(retries + 1): | |
| try: | |
| response = requests.get(API_ENDPOINT, params=params, headers=headers, timeout=120) | |
| except requests.exceptions.RequestException as exc: | |
| last = {"_error": redact(f"Request failed: {exc}")} | |
| if attempt < retries: | |
| time.sleep(1.5) | |
| continue | |
| return last | |
| status = response.status_code | |
| if status == 401: | |
| return {"_error": "Scrape.do rejected the token (HTTP 401).", "_auth": True} | |
| if status in (404, 410): | |
| return {"_error": f"Etsy says this page doesn't exist (HTTP {status}).", "_status": status} | |
| if status == 429 or status >= 500: | |
| last = {"_error": f"Scrape.do returned HTTP {status}. Too many parallel requests or a temporary error.", "_status": status} | |
| if attempt < retries: | |
| time.sleep(2.0 * (attempt + 1)) | |
| continue | |
| return last | |
| if not response.ok: | |
| snippet = (response.text or "").strip()[:200] or "no details" | |
| return {"_error": redact(f"Scrape.do returned HTTP {status}: {snippet}"), "_status": status} | |
| text = response.text or "" | |
| if "captcha-delivery.com" in text[:6000] and len(text) < 20000: | |
| last = {"_error": "Etsy served a bot-check page instead of the content."} | |
| if attempt < retries: | |
| time.sleep(2.0) | |
| continue | |
| return last | |
| try: | |
| credits = int(response.headers.get("Scrape.do-Request-Cost", CREDITS_PER_REQUEST)) | |
| except (TypeError, ValueError): | |
| credits = CREDITS_PER_REQUEST | |
| return {"html": text, "credits": credits} | |
| return last | |
| # --------------------------------------------------------------- search | |
| def search_listings( | |
| self, | |
| query: str, | |
| country: str = "us", | |
| pages: int = 1, | |
| sort_by: Optional[str] = None, | |
| min_price: Optional[float] = None, | |
| max_price: Optional[float] = None, | |
| free_shipping: bool = False, | |
| ) -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| if not query or not query.strip(): | |
| return {"status": "error", "errorMessage": "Type what you're looking for, e.g. 'personalized leather wallet'."} | |
| geo, locale = _market(country) | |
| pages = max(1, min(int(pages or 1), MAX_SEARCH_PAGES)) | |
| warnings: List[str] = [] | |
| listings: List[Dict[str, Any]] = [] | |
| seen: set = set() | |
| credits = 0 | |
| pages_fetched = 0 | |
| first_url = None | |
| for page_index in range(pages): | |
| try: | |
| url = build_search_url(query, page_index + 1, sort_by, min_price, max_price, bool(free_shipping)) | |
| except ValueError as exc: | |
| return {"status": "error", "errorMessage": str(exc)} | |
| if page_index == 0: | |
| first_url = url | |
| data = self._fetch(token, url, geo, locale) | |
| if "_error" in data: | |
| if page_index == 0: | |
| return self._as_error(data) | |
| warnings.append(f"Stopped after page {page_index}: {data['_error']}") | |
| break | |
| credits += data["credits"] | |
| pages_fetched += 1 | |
| fresh = 0 | |
| for item in parse_search_page(data["html"]): | |
| if item["listing_id"] in seen: | |
| continue | |
| seen.add(item["listing_id"]) | |
| item["page"] = page_index + 1 | |
| item["position"] = len(listings) + 1 | |
| listings.append(item) | |
| fresh += 1 | |
| if fresh == 0: | |
| if page_index == 0: | |
| warnings.append("No listings found on this page. The search may have no results.") | |
| break | |
| if page_index + 1 < pages: | |
| time.sleep(0.5) | |
| result: Dict[str, Any] = { | |
| "status": "success", | |
| "query": query.strip(), | |
| "country": geo, | |
| "etsy_url": first_url, | |
| "listings": listings, | |
| "pages_fetched": pages_fetched, | |
| "credits_used": credits, | |
| } | |
| if warnings: | |
| result["warnings"] = warnings | |
| return result | |
| # -------------------------------------------------------------- listing | |
| def listing_details(self, listing: str, country: str = "us") -> Dict[str, Any]: | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| url, error = normalize_listing_ref(listing) | |
| if error: | |
| return {"status": "error", "errorMessage": error} | |
| geo, locale = _market(country) | |
| data = self._fetch(token, url, geo, locale) | |
| if "_error" in data: | |
| return self._as_error(data) | |
| parsed = parse_listing_page(data["html"], url) | |
| if "_error" in parsed: | |
| return {"status": "error", "errorMessage": parsed["_error"]} | |
| return { | |
| "status": "success", | |
| "listing": parsed["listing"], | |
| "reviews": parsed["reviews"], | |
| "reviews_note": "Etsy prints only the first page of reviews inside the listing page; those are the ones returned.", | |
| "credits_used": data["credits"], | |
| } | |
| def listings_bulk(self, refs: List[str], country: str = "us") -> Dict[str, Any]: | |
| """Details for several listings at once (10 credits each, a few in parallel).""" | |
| token = self._resolve_token() | |
| if not token: | |
| return self._missing_token() | |
| cleaned = [r.strip() for r in refs if r and r.strip()] | |
| seen: set = set() | |
| unique = [r for r in cleaned if not (r in seen or seen.add(r))] | |
| if not unique: | |
| return {"status": "error", "errorMessage": "Paste at least one Etsy listing link or ID."} | |
| warnings: List[str] = [] | |
| if len(unique) > MAX_BULK: | |
| warnings.append(f"Only the first {MAX_BULK} of {len(unique)} were fetched.") | |
| unique = unique[:MAX_BULK] | |
| def one(ref: str) -> Dict[str, Any]: | |
| result = self.listing_details(ref, country) | |
| result["ref"] = ref | |
| return result | |
| with ThreadPoolExecutor(max_workers=BULK_WORKERS) as pool: | |
| results = list(pool.map(one, unique)) | |
| listings = [r["listing"] for r in results if r.get("status") == "success"] | |
| errors = [{"ref": r["ref"], "error": r.get("errorMessage")} for r in results if r.get("status") != "success"] | |
| credits = sum(r.get("credits_used", 0) for r in results if r.get("status") == "success") | |
| out: Dict[str, Any] = { | |
| "status": "success", | |
| "listings": listings, | |
| "errors": errors, | |
| "credits_used": credits, | |
| } | |
| if warnings: | |
| out["warnings"] = warnings | |
| if not listings and errors: | |
| out = {"status": "error", "errorMessage": errors[0]["error"] or "No listing could be fetched."} | |
| return out | |
| # -------------------------------------------------------------- forward | |
| def forward( | |
| self, | |
| mode: str, | |
| query: str, | |
| country: Optional[str] = None, | |
| pages: Optional[int] = None, | |
| sort_by: Optional[str] = None, | |
| min_price: Optional[float] = None, | |
| max_price: Optional[float] = None, | |
| free_shipping: Optional[bool] = None, | |
| ) -> str: | |
| mode_key = (mode or "").strip().lower() | |
| if mode_key in ("search", "find", "listings"): | |
| result = self.search_listings( | |
| query, country or "us", pages or 1, sort_by, min_price, max_price, bool(free_shipping) | |
| ) | |
| elif mode_key in ("listing", "details", "product"): | |
| result = self.listing_details(query, country or "us") | |
| else: | |
| result = {"status": "error", "errorMessage": "mode must be 'search' or 'listing'."} | |
| return json.dumps(result, ensure_ascii=False, indent=2) | |