Hammedalmodel commited on
Commit
2ffe051
·
verified ·
1 Parent(s): f4dfd2f

Update app/listing_scraper.py

Browse files
Files changed (1) hide show
  1. app/listing_scraper.py +65 -77
app/listing_scraper.py CHANGED
@@ -3,12 +3,10 @@ import re
3
  import time
4
  from typing import Callable, Any, Dict, List, Optional
5
  from urllib.parse import urljoin, urlparse, parse_qs, urlencode, urlunparse
6
-
7
  from browser_session import BrowserSession
8
 
9
  logger = logging.getLogger("bot")
10
 
11
-
12
  def _parse_price(text: str) -> Optional[float]:
13
  """Extract price from 'Lowest Price: $X.XX' format."""
14
  if not text:
@@ -27,7 +25,6 @@ def _parse_price(text: str) -> Optional[float]:
27
  pass
28
  return None
29
 
30
-
31
  class ListingScraper:
32
  def __init__(
33
  self,
@@ -55,6 +52,7 @@ class ListingScraper:
55
  self.total_timeout = total_timeout
56
  self.scan_start_time = None
57
  self.last_product_found_time = None
 
58
 
59
  @property
60
  def page(self):
@@ -63,32 +61,48 @@ class ListingScraper:
63
  def _get_products_on_page(self) -> List[Dict[str, Any]]:
64
  page = self.page
65
  products: List[Dict[str, Any]] = []
66
-
67
- logger.debug("Extracting products from div.row structure")
68
-
69
  try:
70
- rows = page.locator("div.row").all()
71
- logger.debug(f"Found {len(rows)} potential product rows")
72
-
 
73
  for idx, row in enumerate(rows):
74
  try:
75
- try:
76
- title_elem = row.locator("div.caption h4 a").first
77
- title = (title_elem.inner_text(timeout=500) or "").strip()
78
- link = title_elem.get_attribute("href") or ""
79
- except Exception:
80
- logger.debug(f"Row {idx}: Could not extract title")
 
 
 
 
 
 
81
  continue
82
-
83
- if not title or not link:
84
- logger.debug(f"Row {idx}: Missing title or link")
 
 
85
  continue
86
 
 
87
  price = None
88
  try:
89
- price_elem = row.locator("div.price strong").first
90
- price_text = (price_elem.inner_text(timeout=500) or "").strip()
91
- price = _parse_price(price_text)
 
 
 
 
 
 
 
92
  except Exception as e:
93
  logger.debug(f"Row {idx}: Could not extract price: {e}")
94
 
@@ -96,90 +110,65 @@ class ListingScraper:
96
  self.skipped_unknown_price += 1
97
  continue
98
 
 
99
  if price < self.min_price:
100
  self.skipped_below_threshold += 1
101
  continue
102
 
103
  products.append({"url": link, "title": title, "price": price})
104
-
105
  except Exception as e:
106
  logger.debug(f"Row {idx}: Error processing row: {e}")
107
  continue
108
 
109
  logger.info(f"Extracted {len(products)} qualifying product(s) from page")
110
  return products
111
-
112
  except Exception as e:
113
  logger.error(f"Failed to extract products: {e}")
114
  return []
115
 
116
- def _next_page_url(self, current_url: str) -> Optional[str]:
117
- """Find next page link in pagination."""
118
  page = self.page
119
-
120
- def _valid_href(href: str) -> bool:
121
- if not href or href.strip() in ("#", "javascript:void(0);", "javascript:;"):
122
- return False
123
- href_l = href.lower()
124
- if "dvd_view_" in href_l or "/dvd_view_" in href_l:
125
- return False
126
- return True
127
-
128
- # Check explicit 'Next' selectors
129
- next_selectors = [
130
- "nav ul.pager a:has-text('Next')",
131
- "ul.pagination a:has-text('Next')",
132
- "a[rel='next']",
133
- "a:has-text('Next')",
134
- "a:has-text('»')",
135
- "a:has-text('>')",
136
- ]
137
-
138
- for sel in next_selectors:
139
- try:
140
  el = page.locator(sel).first
141
  if el and el.is_visible(timeout=500):
142
  href = el.get_attribute("href") or ""
143
- if _valid_href(href):
144
  return urljoin(current_url, href)
145
- except Exception:
146
- continue
147
-
148
- return None
149
-
150
- def _resolve_next_page_url(self, current_url: str) -> Optional[str]:
151
- page = self.page
152
 
153
- # Method 1: Check UI text indicator e.g., "Page 5 of 12"
154
  try:
155
- results_el = page.locator("div.col-sm-3.results").first
156
- txt = (results_el.inner_text(timeout=800) or "").strip()
157
- m = re.search(r"Page\s*(\d+)\s*of\s*(\d+)", txt, re.IGNORECASE)
158
- if m:
159
- cur = int(m.group(1))
160
- total = int(m.group(2))
161
- if cur < total:
162
- parsed = urlparse(current_url)
163
- qs = parse_qs(parsed.query)
164
- qs["page"] = [str(cur + 1)]
165
- new_query = urlencode(qs, doseq=True)
166
- return urlunparse(
167
- (parsed.scheme, parsed.netloc, parsed.path, parsed.params, new_query, parsed.fragment)
168
- )
169
- return None
170
  except Exception:
171
  pass
172
 
173
- # Method 2: Link selector fallback
174
- return self._next_page_url(current_url)
175
 
176
  def iter_qualifying_products(self, studio_url: str):
177
  """Stream qualifying products page by page."""
178
  current_url = studio_url
179
  page_num = 1
180
  visited_urls = set()
181
- pages_without_products = 0
182
-
183
  self.scan_start_time = time.time()
184
 
185
  while current_url and page_num <= self.max_pages:
@@ -191,6 +180,7 @@ class ListingScraper:
191
  break
192
 
193
  visited_urls.add(current_url)
 
194
  logger.info(f"Scanning listing page {page_num}: {current_url}")
195
 
196
  if not self.session.goto(current_url):
@@ -198,8 +188,6 @@ class ListingScraper:
198
  break
199
 
200
  products = self._get_products_on_page()
201
-
202
- # Resolve next URL BEFORE yielding items so we maintain pagination state
203
  next_url = self._resolve_next_page_url(current_url)
204
 
205
  for p in products:
@@ -211,7 +199,7 @@ class ListingScraper:
211
  yield p
212
 
213
  if not next_url:
214
- logger.info("No next page found. Pagination complete.")
215
  break
216
 
217
  current_url = next_url
 
3
  import time
4
  from typing import Callable, Any, Dict, List, Optional
5
  from urllib.parse import urljoin, urlparse, parse_qs, urlencode, urlunparse
 
6
  from browser_session import BrowserSession
7
 
8
  logger = logging.getLogger("bot")
9
 
 
10
  def _parse_price(text: str) -> Optional[float]:
11
  """Extract price from 'Lowest Price: $X.XX' format."""
12
  if not text:
 
25
  pass
26
  return None
27
 
 
28
  class ListingScraper:
29
  def __init__(
30
  self,
 
52
  self.total_timeout = total_timeout
53
  self.scan_start_time = None
54
  self.last_product_found_time = None
55
+ self.sort_verified = True
56
 
57
  @property
58
  def page(self):
 
61
  def _get_products_on_page(self) -> List[Dict[str, Any]]:
62
  page = self.page
63
  products: List[Dict[str, Any]] = []
64
+ logger.debug("Extracting products from listing container structure")
65
+
 
66
  try:
67
+ # Updated selector matching the layout seen in your video
68
+ rows = page.locator("div.row, div.item, div.media, tr").all()
69
+ logger.debug(f"Found {len(rows)} potential product containers")
70
+
71
  for idx, row in enumerate(rows):
72
  try:
73
+ # Find title and link inside the product block
74
+ title_elem = None
75
+ for t_sel in ["h4 a", "h3 a", ".title a", "a[href*='dvd_view']"]:
76
+ try:
77
+ candidate = row.locator(t_sel).first
78
+ if candidate and candidate.count() > 0:
79
+ title_elem = candidate
80
+ break
81
+ except Exception:
82
+ continue
83
+
84
+ if not title_elem:
85
  continue
86
+
87
+ title = (title_elem.inner_text(timeout=300) or "").strip()
88
+ link = title_elem.get_attribute("href") or ""
89
+
90
+ if not title or not link or "dvd_view" not in link.lower():
91
  continue
92
 
93
+ # Extract price text container
94
  price = None
95
  try:
96
+ row_text = row.inner_text(timeout=300)
97
+ p_match = re.search(r"Lowest\s+Price\s*:\s*\$?\s*([\d,.]+)", row_text, re.IGNORECASE)
98
+ if p_match:
99
+ price = float(p_match.group(1).replace(",", ""))
100
+ else:
101
+ # Fallback price element search
102
+ price_elem = row.locator("div.price, span.price, strong").first
103
+ if price_elem.count() > 0:
104
+ price_text = price_elem.inner_text(timeout=300).strip()
105
+ price = _parse_price(price_text)
106
  except Exception as e:
107
  logger.debug(f"Row {idx}: Could not extract price: {e}")
108
 
 
110
  self.skipped_unknown_price += 1
111
  continue
112
 
113
+ # Filter by minimum price threshold
114
  if price < self.min_price:
115
  self.skipped_below_threshold += 1
116
  continue
117
 
118
  products.append({"url": link, "title": title, "price": price})
 
119
  except Exception as e:
120
  logger.debug(f"Row {idx}: Error processing row: {e}")
121
  continue
122
 
123
  logger.info(f"Extracted {len(products)} qualifying product(s) from page")
124
  return products
 
125
  except Exception as e:
126
  logger.error(f"Failed to extract products: {e}")
127
  return []
128
 
129
+ def _resolve_next_page_url(self, current_url: str) -> Optional[str]:
 
130
  page = self.page
131
+
132
+ # Method 1: Check pagination list/buttons (e.g., 1, 2, 3, 4, 5, Next)
133
+ try:
134
+ next_selectors = [
135
+ "ul.pagination li.active + li a",
136
+ "nav ul.pager a:has-text('Next')",
137
+ "ul.pagination a:has-text('Next')",
138
+ "a[rel='next']",
139
+ "a:has-text('Next')",
140
+ "a:has-text('»')",
141
+ "a:has-text('>')",
142
+ ]
143
+ for sel in next_selectors:
 
 
 
 
 
 
 
 
144
  el = page.locator(sel).first
145
  if el and el.is_visible(timeout=500):
146
  href = el.get_attribute("href") or ""
147
+ if href and href not in ("#", "javascript:void(0);"):
148
  return urljoin(current_url, href)
149
+ except Exception:
150
+ pass
 
 
 
 
 
151
 
152
+ # Method 2: Fallback query parameter page increment (e.g., ?page=X)
153
  try:
154
+ parsed = urlparse(current_url)
155
+ qs = parse_qs(parsed.query)
156
+ current_page = int(qs.get("page", [1])[0])
157
+ next_page = current_page + 1
158
+ qs["page"] = [str(next_page)]
159
+ new_query = urlencode(qs, doseq=True)
160
+ candidate_url = urlunparse((parsed.scheme, parsed.netloc, parsed.path, parsed.params, new_query, parsed.fragment))
161
+ return candidate_url
 
 
 
 
 
 
 
162
  except Exception:
163
  pass
164
 
165
+ return None
 
166
 
167
  def iter_qualifying_products(self, studio_url: str):
168
  """Stream qualifying products page by page."""
169
  current_url = studio_url
170
  page_num = 1
171
  visited_urls = set()
 
 
172
  self.scan_start_time = time.time()
173
 
174
  while current_url and page_num <= self.max_pages:
 
180
  break
181
 
182
  visited_urls.add(current_url)
183
+ self.pages_scanned += 1
184
  logger.info(f"Scanning listing page {page_num}: {current_url}")
185
 
186
  if not self.session.goto(current_url):
 
188
  break
189
 
190
  products = self._get_products_on_page()
 
 
191
  next_url = self._resolve_next_page_url(current_url)
192
 
193
  for p in products:
 
199
  yield p
200
 
201
  if not next_url:
202
+ logger.info("No next page found or reached end of pagination.")
203
  break
204
 
205
  current_url = next_url