Spaces:
Sleeping
Sleeping
Update app/listing_scraper.py
Browse files- app/listing_scraper.py +65 -77
app/listing_scraper.py
CHANGED
|
@@ -3,12 +3,10 @@ import re
|
|
| 3 |
import time
|
| 4 |
from typing import Callable, Any, Dict, List, Optional
|
| 5 |
from urllib.parse import urljoin, urlparse, parse_qs, urlencode, urlunparse
|
| 6 |
-
|
| 7 |
from browser_session import BrowserSession
|
| 8 |
|
| 9 |
logger = logging.getLogger("bot")
|
| 10 |
|
| 11 |
-
|
| 12 |
def _parse_price(text: str) -> Optional[float]:
|
| 13 |
"""Extract price from 'Lowest Price: $X.XX' format."""
|
| 14 |
if not text:
|
|
@@ -27,7 +25,6 @@ def _parse_price(text: str) -> Optional[float]:
|
|
| 27 |
pass
|
| 28 |
return None
|
| 29 |
|
| 30 |
-
|
| 31 |
class ListingScraper:
|
| 32 |
def __init__(
|
| 33 |
self,
|
|
@@ -55,6 +52,7 @@ class ListingScraper:
|
|
| 55 |
self.total_timeout = total_timeout
|
| 56 |
self.scan_start_time = None
|
| 57 |
self.last_product_found_time = None
|
|
|
|
| 58 |
|
| 59 |
@property
|
| 60 |
def page(self):
|
|
@@ -63,32 +61,48 @@ class ListingScraper:
|
|
| 63 |
def _get_products_on_page(self) -> List[Dict[str, Any]]:
|
| 64 |
page = self.page
|
| 65 |
products: List[Dict[str, Any]] = []
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
try:
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
|
|
|
| 73 |
for idx, row in enumerate(rows):
|
| 74 |
try:
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
continue
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
|
|
|
|
|
|
| 85 |
continue
|
| 86 |
|
|
|
|
| 87 |
price = None
|
| 88 |
try:
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
except Exception as e:
|
| 93 |
logger.debug(f"Row {idx}: Could not extract price: {e}")
|
| 94 |
|
|
@@ -96,90 +110,65 @@ class ListingScraper:
|
|
| 96 |
self.skipped_unknown_price += 1
|
| 97 |
continue
|
| 98 |
|
|
|
|
| 99 |
if price < self.min_price:
|
| 100 |
self.skipped_below_threshold += 1
|
| 101 |
continue
|
| 102 |
|
| 103 |
products.append({"url": link, "title": title, "price": price})
|
| 104 |
-
|
| 105 |
except Exception as e:
|
| 106 |
logger.debug(f"Row {idx}: Error processing row: {e}")
|
| 107 |
continue
|
| 108 |
|
| 109 |
logger.info(f"Extracted {len(products)} qualifying product(s) from page")
|
| 110 |
return products
|
| 111 |
-
|
| 112 |
except Exception as e:
|
| 113 |
logger.error(f"Failed to extract products: {e}")
|
| 114 |
return []
|
| 115 |
|
| 116 |
-
def
|
| 117 |
-
"""Find next page link in pagination."""
|
| 118 |
page = self.page
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
"a[rel='next']",
|
| 133 |
-
"a:has-text('Next')",
|
| 134 |
-
"a:has-text('»')",
|
| 135 |
-
"a:has-text('>')",
|
| 136 |
-
]
|
| 137 |
-
|
| 138 |
-
for sel in next_selectors:
|
| 139 |
-
try:
|
| 140 |
el = page.locator(sel).first
|
| 141 |
if el and el.is_visible(timeout=500):
|
| 142 |
href = el.get_attribute("href") or ""
|
| 143 |
-
if
|
| 144 |
return urljoin(current_url, href)
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
return None
|
| 149 |
-
|
| 150 |
-
def _resolve_next_page_url(self, current_url: str) -> Optional[str]:
|
| 151 |
-
page = self.page
|
| 152 |
|
| 153 |
-
# Method
|
| 154 |
try:
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
qs = parse_qs(parsed.query)
|
| 164 |
-
qs["page"] = [str(cur + 1)]
|
| 165 |
-
new_query = urlencode(qs, doseq=True)
|
| 166 |
-
return urlunparse(
|
| 167 |
-
(parsed.scheme, parsed.netloc, parsed.path, parsed.params, new_query, parsed.fragment)
|
| 168 |
-
)
|
| 169 |
-
return None
|
| 170 |
except Exception:
|
| 171 |
pass
|
| 172 |
|
| 173 |
-
|
| 174 |
-
return self._next_page_url(current_url)
|
| 175 |
|
| 176 |
def iter_qualifying_products(self, studio_url: str):
|
| 177 |
"""Stream qualifying products page by page."""
|
| 178 |
current_url = studio_url
|
| 179 |
page_num = 1
|
| 180 |
visited_urls = set()
|
| 181 |
-
pages_without_products = 0
|
| 182 |
-
|
| 183 |
self.scan_start_time = time.time()
|
| 184 |
|
| 185 |
while current_url and page_num <= self.max_pages:
|
|
@@ -191,6 +180,7 @@ class ListingScraper:
|
|
| 191 |
break
|
| 192 |
|
| 193 |
visited_urls.add(current_url)
|
|
|
|
| 194 |
logger.info(f"Scanning listing page {page_num}: {current_url}")
|
| 195 |
|
| 196 |
if not self.session.goto(current_url):
|
|
@@ -198,8 +188,6 @@ class ListingScraper:
|
|
| 198 |
break
|
| 199 |
|
| 200 |
products = self._get_products_on_page()
|
| 201 |
-
|
| 202 |
-
# Resolve next URL BEFORE yielding items so we maintain pagination state
|
| 203 |
next_url = self._resolve_next_page_url(current_url)
|
| 204 |
|
| 205 |
for p in products:
|
|
@@ -211,7 +199,7 @@ class ListingScraper:
|
|
| 211 |
yield p
|
| 212 |
|
| 213 |
if not next_url:
|
| 214 |
-
logger.info("No next page found
|
| 215 |
break
|
| 216 |
|
| 217 |
current_url = next_url
|
|
|
|
| 3 |
import time
|
| 4 |
from typing import Callable, Any, Dict, List, Optional
|
| 5 |
from urllib.parse import urljoin, urlparse, parse_qs, urlencode, urlunparse
|
|
|
|
| 6 |
from browser_session import BrowserSession
|
| 7 |
|
| 8 |
logger = logging.getLogger("bot")
|
| 9 |
|
|
|
|
| 10 |
def _parse_price(text: str) -> Optional[float]:
|
| 11 |
"""Extract price from 'Lowest Price: $X.XX' format."""
|
| 12 |
if not text:
|
|
|
|
| 25 |
pass
|
| 26 |
return None
|
| 27 |
|
|
|
|
| 28 |
class ListingScraper:
|
| 29 |
def __init__(
|
| 30 |
self,
|
|
|
|
| 52 |
self.total_timeout = total_timeout
|
| 53 |
self.scan_start_time = None
|
| 54 |
self.last_product_found_time = None
|
| 55 |
+
self.sort_verified = True
|
| 56 |
|
| 57 |
@property
|
| 58 |
def page(self):
|
|
|
|
| 61 |
def _get_products_on_page(self) -> List[Dict[str, Any]]:
|
| 62 |
page = self.page
|
| 63 |
products: List[Dict[str, Any]] = []
|
| 64 |
+
logger.debug("Extracting products from listing container structure")
|
| 65 |
+
|
|
|
|
| 66 |
try:
|
| 67 |
+
# Updated selector matching the layout seen in your video
|
| 68 |
+
rows = page.locator("div.row, div.item, div.media, tr").all()
|
| 69 |
+
logger.debug(f"Found {len(rows)} potential product containers")
|
| 70 |
+
|
| 71 |
for idx, row in enumerate(rows):
|
| 72 |
try:
|
| 73 |
+
# Find title and link inside the product block
|
| 74 |
+
title_elem = None
|
| 75 |
+
for t_sel in ["h4 a", "h3 a", ".title a", "a[href*='dvd_view']"]:
|
| 76 |
+
try:
|
| 77 |
+
candidate = row.locator(t_sel).first
|
| 78 |
+
if candidate and candidate.count() > 0:
|
| 79 |
+
title_elem = candidate
|
| 80 |
+
break
|
| 81 |
+
except Exception:
|
| 82 |
+
continue
|
| 83 |
+
|
| 84 |
+
if not title_elem:
|
| 85 |
continue
|
| 86 |
+
|
| 87 |
+
title = (title_elem.inner_text(timeout=300) or "").strip()
|
| 88 |
+
link = title_elem.get_attribute("href") or ""
|
| 89 |
+
|
| 90 |
+
if not title or not link or "dvd_view" not in link.lower():
|
| 91 |
continue
|
| 92 |
|
| 93 |
+
# Extract price text container
|
| 94 |
price = None
|
| 95 |
try:
|
| 96 |
+
row_text = row.inner_text(timeout=300)
|
| 97 |
+
p_match = re.search(r"Lowest\s+Price\s*:\s*\$?\s*([\d,.]+)", row_text, re.IGNORECASE)
|
| 98 |
+
if p_match:
|
| 99 |
+
price = float(p_match.group(1).replace(",", ""))
|
| 100 |
+
else:
|
| 101 |
+
# Fallback price element search
|
| 102 |
+
price_elem = row.locator("div.price, span.price, strong").first
|
| 103 |
+
if price_elem.count() > 0:
|
| 104 |
+
price_text = price_elem.inner_text(timeout=300).strip()
|
| 105 |
+
price = _parse_price(price_text)
|
| 106 |
except Exception as e:
|
| 107 |
logger.debug(f"Row {idx}: Could not extract price: {e}")
|
| 108 |
|
|
|
|
| 110 |
self.skipped_unknown_price += 1
|
| 111 |
continue
|
| 112 |
|
| 113 |
+
# Filter by minimum price threshold
|
| 114 |
if price < self.min_price:
|
| 115 |
self.skipped_below_threshold += 1
|
| 116 |
continue
|
| 117 |
|
| 118 |
products.append({"url": link, "title": title, "price": price})
|
|
|
|
| 119 |
except Exception as e:
|
| 120 |
logger.debug(f"Row {idx}: Error processing row: {e}")
|
| 121 |
continue
|
| 122 |
|
| 123 |
logger.info(f"Extracted {len(products)} qualifying product(s) from page")
|
| 124 |
return products
|
|
|
|
| 125 |
except Exception as e:
|
| 126 |
logger.error(f"Failed to extract products: {e}")
|
| 127 |
return []
|
| 128 |
|
| 129 |
+
def _resolve_next_page_url(self, current_url: str) -> Optional[str]:
|
|
|
|
| 130 |
page = self.page
|
| 131 |
+
|
| 132 |
+
# Method 1: Check pagination list/buttons (e.g., 1, 2, 3, 4, 5, Next)
|
| 133 |
+
try:
|
| 134 |
+
next_selectors = [
|
| 135 |
+
"ul.pagination li.active + li a",
|
| 136 |
+
"nav ul.pager a:has-text('Next')",
|
| 137 |
+
"ul.pagination a:has-text('Next')",
|
| 138 |
+
"a[rel='next']",
|
| 139 |
+
"a:has-text('Next')",
|
| 140 |
+
"a:has-text('»')",
|
| 141 |
+
"a:has-text('>')",
|
| 142 |
+
]
|
| 143 |
+
for sel in next_selectors:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
el = page.locator(sel).first
|
| 145 |
if el and el.is_visible(timeout=500):
|
| 146 |
href = el.get_attribute("href") or ""
|
| 147 |
+
if href and href not in ("#", "javascript:void(0);"):
|
| 148 |
return urljoin(current_url, href)
|
| 149 |
+
except Exception:
|
| 150 |
+
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
|
| 152 |
+
# Method 2: Fallback query parameter page increment (e.g., ?page=X)
|
| 153 |
try:
|
| 154 |
+
parsed = urlparse(current_url)
|
| 155 |
+
qs = parse_qs(parsed.query)
|
| 156 |
+
current_page = int(qs.get("page", [1])[0])
|
| 157 |
+
next_page = current_page + 1
|
| 158 |
+
qs["page"] = [str(next_page)]
|
| 159 |
+
new_query = urlencode(qs, doseq=True)
|
| 160 |
+
candidate_url = urlunparse((parsed.scheme, parsed.netloc, parsed.path, parsed.params, new_query, parsed.fragment))
|
| 161 |
+
return candidate_url
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 162 |
except Exception:
|
| 163 |
pass
|
| 164 |
|
| 165 |
+
return None
|
|
|
|
| 166 |
|
| 167 |
def iter_qualifying_products(self, studio_url: str):
|
| 168 |
"""Stream qualifying products page by page."""
|
| 169 |
current_url = studio_url
|
| 170 |
page_num = 1
|
| 171 |
visited_urls = set()
|
|
|
|
|
|
|
| 172 |
self.scan_start_time = time.time()
|
| 173 |
|
| 174 |
while current_url and page_num <= self.max_pages:
|
|
|
|
| 180 |
break
|
| 181 |
|
| 182 |
visited_urls.add(current_url)
|
| 183 |
+
self.pages_scanned += 1
|
| 184 |
logger.info(f"Scanning listing page {page_num}: {current_url}")
|
| 185 |
|
| 186 |
if not self.session.goto(current_url):
|
|
|
|
| 188 |
break
|
| 189 |
|
| 190 |
products = self._get_products_on_page()
|
|
|
|
|
|
|
| 191 |
next_url = self._resolve_next_page_url(current_url)
|
| 192 |
|
| 193 |
for p in products:
|
|
|
|
| 199 |
yield p
|
| 200 |
|
| 201 |
if not next_url:
|
| 202 |
+
logger.info("No next page found or reached end of pagination.")
|
| 203 |
break
|
| 204 |
|
| 205 |
current_url = next_url
|