bugrahans commited on
Commit
ff7ea74
·
verified ·
1 Parent(s): dca809d

Delete tool.py

Browse files
Files changed (1) hide show
  1. tool.py +0 -262
tool.py DELETED
@@ -1,262 +0,0 @@
1
- from __future__ import annotations
2
-
3
- import json
4
- import os
5
- import re
6
- import urllib.parse
7
- from typing import Any, Dict, Optional
8
-
9
- import requests
10
- from smolagents.tools import Tool
11
-
12
-
13
- SCRAPE_DO_SIGNUP_URL = (
14
- "https://dashboard.scrape.do/sign-up?utm_source=huggingface&utm_medium=amazon-scraper"
15
- )
16
- API_BASE = "https://api.scrape.do/plugin/amazon"
17
-
18
- # geocode -> human label, shown in the UI dropdown.
19
- MARKETPLACES: Dict[str, str] = {
20
- "us": "United States (amazon.com)",
21
- "gb": "United Kingdom (amazon.co.uk)",
22
- "de": "Germany (amazon.de)",
23
- "fr": "France (amazon.fr)",
24
- "it": "Italy (amazon.it)",
25
- "es": "Spain (amazon.es)",
26
- "ca": "Canada (amazon.ca)",
27
- "jp": "Japan (amazon.co.jp)",
28
- "in": "India (amazon.in)",
29
- "au": "Australia (amazon.com.au)",
30
- "br": "Brazil (amazon.com.br)",
31
- "mx": "Mexico (amazon.com.mx)",
32
- "nl": "Netherlands (amazon.nl)",
33
- "pl": "Poland (amazon.pl)",
34
- "se": "Sweden (amazon.se)",
35
- "tr": "Turkey (amazon.com.tr)",
36
- "ae": "UAE (amazon.ae)",
37
- "sa": "Saudi Arabia (amazon.sa)",
38
- "sg": "Singapore (amazon.sg)",
39
- "eg": "Egypt (amazon.eg)",
40
- "be": "Belgium (amazon.com.be)",
41
- }
42
-
43
- # Marketplaces where Amazon localises delivery by postal code. Everything else
44
- # takes a country-level location instead, via countryName.
45
- ZIP_LEVEL_MARKETS = {"us", "gb", "uk", "de", "fr", "it", "es", "ca", "jp", "in"}
46
-
47
- COUNTRY_NAMES: Dict[str, str] = {
48
- "tr": "Turkey",
49
- "br": "Brazil",
50
- "au": "Australia",
51
- "mx": "Mexico",
52
- "nl": "Netherlands",
53
- "pl": "Poland",
54
- "se": "Sweden",
55
- "ae": "United Arab Emirates",
56
- "sa": "Saudi Arabia",
57
- "sg": "Singapore",
58
- "eg": "Egypt",
59
- "be": "Belgium",
60
- }
61
-
62
- _ASIN_PATTERN = re.compile(r"(?:/dp/|/gp/product/|/gp/aw/d/|/product/|[?&]asin=)([A-Z0-9]{10})", re.I)
63
- _BARE_ASIN = re.compile(r"^[A-Z0-9]{10}$", re.I)
64
-
65
-
66
- def extract_asin(value: str) -> Optional[str]:
67
- """Accept a full Amazon URL or a bare ASIN and return the ASIN."""
68
- if not value:
69
- return None
70
- value = value.strip()
71
-
72
- if _BARE_ASIN.match(value):
73
- return value.upper()
74
-
75
- match = _ASIN_PATTERN.search(value)
76
- if match:
77
- return match.group(1).upper()
78
-
79
- return None
80
-
81
-
82
- class ScrapeDoAmazonTool(Tool):
83
- name = "amazon_product_scraper"
84
- description = """
85
- Scrape structured product data from any Amazon marketplace: product detail
86
- pages (title, brand, price, rating, images, rankings, technical specs),
87
- all seller offers for an ASIN, or keyword search results with organic and
88
- sponsored positions. Returns clean JSON, no HTML parsing required.
89
- Powered by Scrape.do's Amazon Scraper API, which handles AWS WAF, proxy
90
- rotation, sessions and CAPTCHAs automatically across 21 marketplaces.
91
- Use this when you need Amazon pricing, catalogue or competitive data.
92
- """
93
- output_type = "string"
94
-
95
- def __init__(self, api_token: Optional[str] = None) -> None:
96
- # Token can be passed explicitly (e.g. by a Gradio UI) or read from the
97
- # SCRAPE_DO_API_TOKEN environment variable for programmatic use.
98
- self._api_token = api_token
99
- self.inputs = {
100
- "mode": {
101
- "type": "string",
102
- "description": (
103
- "What to scrape: 'product' for a product detail page, "
104
- "'offers' for every seller offer on an ASIN, or 'search' "
105
- "for keyword/category search results."
106
- ),
107
- },
108
- "query": {
109
- "type": "string",
110
- "description": (
111
- "An Amazon product URL or ASIN for 'product' and 'offers'. "
112
- "A search keyword for 'search'."
113
- ),
114
- },
115
- "geocode": {
116
- "type": "string",
117
- "description": (
118
- "Marketplace country code: 'us', 'gb', 'de', 'jp', 'tr'… "
119
- "Defaults to 'us'."
120
- ),
121
- "nullable": True,
122
- },
123
- "zipcode": {
124
- "type": "string",
125
- "description": (
126
- "Postal code for ZIP-level marketplaces (us, gb, de, fr, it, "
127
- "es, ca, jp, in) to localise price, stock and delivery. "
128
- "Ignored elsewhere, where a country-level location is used."
129
- ),
130
- "nullable": True,
131
- },
132
- "page": {
133
- "type": "integer",
134
- "description": "Search results page number. Only used when mode='search'.",
135
- "nullable": True,
136
- },
137
- "language": {
138
- "type": "string",
139
- "description": (
140
- "ISO 639-1 language code in uppercase, e.g. 'EN', 'DE', 'FR'. "
141
- "Must be supported by the target marketplace."
142
- ),
143
- "nullable": True,
144
- },
145
- }
146
- super().__init__()
147
-
148
- # ------------------------------------------------------------------ token
149
-
150
- def _resolve_token(self) -> Optional[str]:
151
- if self._api_token:
152
- return self._api_token.strip()
153
- env_token = os.getenv("SCRAPE_DO_API_TOKEN")
154
- return env_token.strip() if env_token else None
155
-
156
- # ---------------------------------------------------------------- request
157
-
158
- def request(
159
- self,
160
- mode: str,
161
- query: str,
162
- geocode: str = "us",
163
- zipcode: Optional[str] = None,
164
- page: Optional[int] = None,
165
- language: Optional[str] = None,
166
- ) -> Dict[str, Any]:
167
- """Call the Amazon Scraper API and return a parsed dict."""
168
- token = self._resolve_token()
169
- if not token:
170
- return {
171
- "status": "error",
172
- "errorMessage": "Missing Scrape.do API token.",
173
- "hint": f"Get a free token at {SCRAPE_DO_SIGNUP_URL}",
174
- }
175
-
176
- if not query or not query.strip():
177
- return {"status": "error", "errorMessage": "No ASIN, URL or keyword provided."}
178
-
179
- geocode = (geocode or "us").strip().lower()
180
- params: Dict[str, Any] = {"token": token, "geocode": geocode}
181
-
182
- if mode in ("product", "offers"):
183
- asin = extract_asin(query)
184
- if not asin:
185
- return {
186
- "status": "error",
187
- "errorMessage": (
188
- "Could not read an ASIN from that input. Paste a full "
189
- "Amazon product URL (…/dp/B0BLRJ4R8F) or the 10-character "
190
- "ASIN on its own."
191
- ),
192
- }
193
- params["asin"] = asin
194
- endpoint = f"{API_BASE}/pdp" if mode == "product" else f"{API_BASE}/offer-listing"
195
- elif mode == "search":
196
- params["keyword"] = query.strip()
197
- params["page"] = int(page or 1)
198
- endpoint = f"{API_BASE}/search"
199
- else:
200
- return {"status": "error", "errorMessage": f"Unknown mode: {mode}"}
201
-
202
- # Location: ZIP for ZIP-level marketplaces, country name for the rest.
203
- if geocode in ZIP_LEVEL_MARKETS:
204
- if zipcode and zipcode.strip():
205
- params["zipcode"] = zipcode.strip()
206
- elif geocode in COUNTRY_NAMES:
207
- # Passing the marketplace's own country is a no-op, so only send it
208
- # when the caller explicitly asked for a different delivery country.
209
- if zipcode and zipcode.strip():
210
- params["countryName"] = zipcode.strip()
211
-
212
- if language and language.strip():
213
- params["language"] = language.strip().upper()
214
-
215
- url = endpoint + "?" + urllib.parse.urlencode(params, quote_via=urllib.parse.quote)
216
-
217
- try:
218
- response = requests.get(url, timeout=90)
219
- except requests.exceptions.RequestException as exc:
220
- return {"status": "error", "errorMessage": f"Request failed: {exc}"}
221
-
222
- try:
223
- data = response.json()
224
- except ValueError:
225
- return {
226
- "status": "error",
227
- "errorMessage": f"Unexpected non-JSON response (HTTP {response.status_code}).",
228
- "details": response.text[:500],
229
- }
230
-
231
- if response.status_code == 401 or "token" in str(data.get("error", "")).lower():
232
- data.setdefault(
233
- "hint", f"Check your token or create one at {SCRAPE_DO_SIGNUP_URL}"
234
- )
235
-
236
- if isinstance(data, dict) and "status" not in data:
237
- data["status"] = "success" if response.ok else "error"
238
- if isinstance(data, dict) and not response.ok and not data.get("errorMessage"):
239
- data["errorMessage"] = data.get("message") or f"HTTP {response.status_code}"
240
-
241
- return data
242
-
243
- # ---------------------------------------------------------------- forward
244
-
245
- def forward(
246
- self,
247
- mode: str = "product",
248
- query: str = "",
249
- geocode: Optional[str] = None,
250
- zipcode: Optional[str] = None,
251
- page: Optional[int] = None,
252
- language: Optional[str] = None,
253
- ) -> str:
254
- result = self.request(
255
- mode=mode,
256
- query=query,
257
- geocode=geocode or "us",
258
- zipcode=zipcode,
259
- page=page,
260
- language=language,
261
- )
262
- return json.dumps(result, ensure_ascii=False, indent=2)