Hammedalmodel commited on
Commit
4bdf769
·
verified ·
1 Parent(s): 2ffe051

Update app/listing_scraper.py

Browse files
Files changed (1) hide show
  1. app/listing_scraper.py +12 -27
app/listing_scraper.py CHANGED
@@ -61,27 +61,18 @@ class ListingScraper:
61
  def _get_products_on_page(self) -> List[Dict[str, Any]]:
62
  page = self.page
63
  products: List[Dict[str, Any]] = []
64
- logger.debug("Extracting products from listing container structure")
65
 
66
  try:
67
- # Updated selector matching the layout seen in your video
68
- rows = page.locator("div.row, div.item, div.media, tr").all()
69
- logger.debug(f"Found {len(rows)} potential product containers")
70
 
71
  for idx, row in enumerate(rows):
72
  try:
73
- # Find title and link inside the product block
74
- title_elem = None
75
- for t_sel in ["h4 a", "h3 a", ".title a", "a[href*='dvd_view']"]:
76
- try:
77
- candidate = row.locator(t_sel).first
78
- if candidate and candidate.count() > 0:
79
- title_elem = candidate
80
- break
81
- except Exception:
82
- continue
83
-
84
- if not title_elem:
85
  continue
86
 
87
  title = (title_elem.inner_text(timeout=300) or "").strip()
@@ -90,21 +81,15 @@ class ListingScraper:
90
  if not title or not link or "dvd_view" not in link.lower():
91
  continue
92
 
93
- # Extract price text container
94
  price = None
95
  try:
96
  row_text = row.inner_text(timeout=300)
97
  p_match = re.search(r"Lowest\s+Price\s*:\s*\$?\s*([\d,.]+)", row_text, re.IGNORECASE)
98
  if p_match:
99
  price = float(p_match.group(1).replace(",", ""))
100
- else:
101
- # Fallback price element search
102
- price_elem = row.locator("div.price, span.price, strong").first
103
- if price_elem.count() > 0:
104
- price_text = price_elem.inner_text(timeout=300).strip()
105
- price = _parse_price(price_text)
106
  except Exception as e:
107
- logger.debug(f"Row {idx}: Could not extract price: {e}")
108
 
109
  if price is None:
110
  self.skipped_unknown_price += 1
@@ -117,7 +102,7 @@ class ListingScraper:
117
 
118
  products.append({"url": link, "title": title, "price": price})
119
  except Exception as e:
120
- logger.debug(f"Row {idx}: Error processing row: {e}")
121
  continue
122
 
123
  logger.info(f"Extracted {len(products)} qualifying product(s) from page")
@@ -129,7 +114,7 @@ class ListingScraper:
129
  def _resolve_next_page_url(self, current_url: str) -> Optional[str]:
130
  page = self.page
131
 
132
- # Method 1: Check pagination list/buttons (e.g., 1, 2, 3, 4, 5, Next)
133
  try:
134
  next_selectors = [
135
  "ul.pagination li.active + li a",
@@ -149,7 +134,7 @@ class ListingScraper:
149
  except Exception:
150
  pass
151
 
152
- # Method 2: Fallback query parameter page increment (e.g., ?page=X)
153
  try:
154
  parsed = urlparse(current_url)
155
  qs = parse_qs(parsed.query)
 
61
  def _get_products_on_page(self) -> List[Dict[str, Any]]:
62
  page = self.page
63
  products: List[Dict[str, Any]] = []
64
+ logger.debug("Extracting products from new listing card layout")
65
 
66
  try:
67
+ # Target the new Tailwind min-w-0 product container blocks
68
+ rows = page.locator("div.min-w-0").all()
69
+ logger.debug(f"Found {len(rows)} potential product cards")
70
 
71
  for idx, row in enumerate(rows):
72
  try:
73
+ # Find title and link inside h3 > a
74
+ title_elem = row.locator("h3 a, a[href*='dvd_view']").first
75
+ if not title_elem or title_elem.count() == 0:
 
 
 
 
 
 
 
 
 
76
  continue
77
 
78
  title = (title_elem.inner_text(timeout=300) or "").strip()
 
81
  if not title or not link or "dvd_view" not in link.lower():
82
  continue
83
 
84
+ # Extract price text from the container block
85
  price = None
86
  try:
87
  row_text = row.inner_text(timeout=300)
88
  p_match = re.search(r"Lowest\s+Price\s*:\s*\$?\s*([\d,.]+)", row_text, re.IGNORECASE)
89
  if p_match:
90
  price = float(p_match.group(1).replace(",", ""))
 
 
 
 
 
 
91
  except Exception as e:
92
+ logger.debug(f"Card {idx}: Could not extract price: {e}")
93
 
94
  if price is None:
95
  self.skipped_unknown_price += 1
 
102
 
103
  products.append({"url": link, "title": title, "price": price})
104
  except Exception as e:
105
+ logger.debug(f"Card {idx}: Error processing card: {e}")
106
  continue
107
 
108
  logger.info(f"Extracted {len(products)} qualifying product(s) from page")
 
114
  def _resolve_next_page_url(self, current_url: str) -> Optional[str]:
115
  page = self.page
116
 
117
+ # Method 1: Check pagination buttons/links
118
  try:
119
  next_selectors = [
120
  "ul.pagination li.active + li a",
 
134
  except Exception:
135
  pass
136
 
137
+ # Method 2: Fallback query parameter page increment
138
  try:
139
  parsed = urlparse(current_url)
140
  qs = parse_qs(parsed.query)