Spaces:
Running
Running
File size: 5,674 Bytes
48aff15 5441c2f 48aff15 5441c2f 48aff15 7ee4ebc 76ae335 48aff15 7ee4ebc 5441c2f 48aff15 5441c2f 7ee4ebc 5441c2f 7ee4ebc 76ae335 7ee4ebc 76ae335 7ee4ebc 48aff15 5441c2f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 | import logging
import re
import time
from datetime import datetime
from urllib.parse import urljoin, urlparse
import requests
from bs4 import BeautifulSoup
from .models import RedditPost
logger = logging.getLogger(__name__)
HTML_SITES = [
{
"name": "Webmanagercenter",
"url": "https://www.webmanagercenter.com/",
"selectors": [".entry-title a", "h3 a", "h2 a"],
"domain": "webmanagercenter.com",
},
{
"name": "Directinfo",
"url": "https://directinfo.webmanagercenter.com/",
"selectors": [".entry-title a", "h3 a", "h2 a"],
"domain": "directinfo.webmanagercenter.com",
},
{
"name": "Tuniscope",
"url": "https://www.tuniscope.com/",
"selectors": [".entry-title a", "h3 a", "h2 a", ".article-title a"],
"domain": "tuniscope.com",
},
]
class HTMLSiteScraper:
def __init__(self, config):
self.config = config
self.session = requests.Session()
self.session.headers.update({
"User-Agent": "Mozilla/5.0 (compatible; newsapp/1.0)",
})
def fetch_posts(self) -> list[RedditPost]:
seen_urls = set()
posts = []
for site in HTML_SITES:
logger.info("Scraping HTML: %s (%s)", site["name"], site["url"])
try:
resp = self.session.get(site["url"], timeout=15)
resp.raise_for_status()
soup = BeautifulSoup(resp.text, "html.parser")
links = self._find_article_links(soup, site)
logger.info(" %s → %d links found", site["name"], len(links))
for a_tag in links[: self.config.posts_per_subreddit]:
title = a_tag.get_text(strip=True)
href = a_tag.get("href", "")
if not title or not href:
continue
full_url = urljoin(site["url"], href)
if full_url in seen_urls:
continue
seen_urls.add(full_url)
domain = site.get("domain") or urlparse(full_url).netloc
published, published_iso = self._extract_date(a_tag)
image_url = self._extract_image(a_tag, site["url"])
post = RedditPost(
id=full_url,
title=title,
url=full_url,
subreddit=domain,
score=0,
num_comments=0,
source_domain=domain,
image_url=image_url,
published=published,
published_iso=published_iso,
)
posts.append(post)
except Exception as exc:
logger.warning("Failed HTML scrape %s: %s", site["name"], exc)
time.sleep(1.0)
return posts
@staticmethod
def _extract_date(a_tag) -> tuple[str, str]:
for parent_tag in ("div", "article", "li", "section"):
parent = a_tag.find_parent(parent_tag)
if not parent:
continue
time_tag = parent.find("time")
if time_tag:
raw = time_tag.get("datetime") or time_tag.get_text(strip=True)
if raw:
parsed = _parse_date_str(raw)
if parsed:
return parsed
date_el = parent.find(["span", "div"], class_=re.compile(r"date|time", re.I))
if date_el:
raw = date_el.get("datetime") or date_el.get_text(strip=True)
if raw:
parsed = _parse_date_str(raw)
if parsed:
return parsed
return ("", "")
@staticmethod
def _extract_image(a_tag, base_url: str) -> str:
for ancestor in a_tag.parents:
if ancestor.name not in ("div", "article", "li", "section"):
continue
img = ancestor.find("img")
if not img:
continue
for attr in ("data-src", "data-lazy-src", "src", "data-srcset", "srcset"):
val = img.get(attr, "")
if val:
if attr in ("data-srcset", "srcset"):
val = val.split(",")[0].strip().split(" ")[0]
return urljoin(base_url, val)
return ""
@staticmethod
def _find_article_links(soup: BeautifulSoup, site: dict) -> list:
for selector in site["selectors"]:
found = soup.select(selector)
if found:
return found
return []
_DATE_FORMATS = [
"%Y-%m-%dT%H:%M:%S%z",
"%Y-%m-%dT%H:%M:%S",
"%Y-%m-%d",
"%d %b %Y",
"%d %B %Y",
"%b %d, %Y",
"%B %d, %Y",
"%d/%m/%Y",
"%m/%d/%Y",
]
def _parse_date_str(raw: str) -> tuple[str, str] | None:
raw = raw.strip()
if not raw:
return None
for fmt in _DATE_FORMATS:
try:
dt = datetime.strptime(raw, fmt)
if dt.tzinfo:
display = dt.strftime("%d %b %Y")
iso = dt.strftime("%Y-%m-%d")
else:
display = dt.strftime("%d %b %Y")
iso = dt.strftime("%Y-%m-%d")
return (display, iso)
except ValueError:
continue
try:
dt = datetime.fromisoformat(raw)
display = dt.strftime("%d %b %Y")
iso = dt.strftime("%Y-%m-%d")
return (display, iso)
except (ValueError, TypeError):
pass
return None
|