Spaces:
Runtime error
Runtime error
Download webscraper.py from plutoa386/NMS-AI-DEV: direct link, hf CLI and curl.
- Browser
- Download file 4.15 kB
-
https://huggingface.co/spaces/plutoa386/NMS-AI-DEV/resolve/main/webscraper.py
- Command line
-
hf download hf://spaces/plutoa386/NMS-AI-DEV/webscraper.py
-
curl -L -o webscraper.py https://huggingface.co/spaces/plutoa386/NMS-AI-DEV/resolve/main/webscraper.py
4.15 kB
| import requests | |
| from bs4 import BeautifulSoup | |
| import time | |
| import os | |
| # ================================================================ | |
| # Helper Functions | |
| # ================================================================ | |
| def clean_text(text): | |
| return " ".join(text.split()) | |
| def scrape_article_links(base_url: str, max_links=10) -> list[dict]: | |
| """Scrape list of {title, url} from a news listing page.""" | |
| headers = { | |
| 'User-Agent': 'Mozilla/5.0 (compatible; DementiaNewsBot/1.0)' | |
| } | |
| try: | |
| response = requests.get(base_url, headers=headers, timeout=10) | |
| response.raise_for_status() | |
| soup = BeautifulSoup(response.content, 'lxml') | |
| items = [] | |
| # Based on caregiving.org structure (from your KB): | |
| # Each news item is in <article> or <div class="post-item"> | |
| posts = soup.find_all(['article', 'div'], class_=lambda x: x and ('post' in x or 'news' in x or 'item' in x)) | |
| for post in posts[:max_links]: | |
| # Try to get title + link | |
| a_tag = post.find('a', href=True) | |
| if not a_tag: | |
| continue | |
| href = a_tag['href'] | |
| if not href.startswith('http'): | |
| href = requests.compat.urljoin(base_url, href) | |
| # Title: from <h2>, <h3>, or img alt | |
| title_tag = post.find(['h2', 'h3', 'h4']) | |
| if not title_tag: | |
| img = post.find('img') | |
| title = img.get('alt', '') if img else "" | |
| else: | |
| title = clean_text(title_tag.get_text()) | |
| if not title or title in ["Read More", "Learn More"]: | |
| continue | |
| items.append({"title": title, "url": href}) | |
| return items | |
| except Exception as e: | |
| print(f"scrape_article_links({base_url}) failed: {e}") | |
| return [] | |
| # Attempts to get the full raw article text | |
| def scrape_full_text(url: str) -> str: | |
| """Return raw full article text (no summarization).""" | |
| headers = { | |
| 'User-Agent': 'Mozilla/5.0 (compatible; DementiaNewsBot/1.0)' | |
| } | |
| try: | |
| response = requests.get(url, headers=headers, timeout=15) | |
| soup = BeautifulSoup(response.content, 'lxml') | |
| for s in soup(["script", "style", "nav", "footer", "aside", "header"]): | |
| s.decompose() | |
| # Target main content | |
| main = soup.find('article') or soup.find('main') or soup.find('div', id='main-content') | |
| if not main: | |
| main = soup.body | |
| paras = main.find_all('p') if main else soup.find_all('p') | |
| text = "\n\n".join(clean_text(p.get_text()) for p in paras if len(p.get_text().strip()) > 50) | |
| return text[:8000] # Cap for token efficiency | |
| except Exception as e: | |
| print(f"scrape_full_text({url}) failed: {e}") | |
| return "" | |
| # ================================================================ | |
| # News webscraping | |
| # ================================================================ | |
| # Gets new articles for patients | |
| def get_patient_news_raw(count: int = 5) -> list[dict]: | |
| base_urls = [ | |
| "https://www.caregiving.org/?s=dementia", | |
| "https://www.alz.org/search?query=dementia&filters=news", | |
| "https://www.dementiaresearch.org/latest-news" | |
| ] | |
| articles = [] | |
| for url in base_urls: | |
| if len(articles) >= count: | |
| break | |
| links = scrape_article_links(url, max_links=count) | |
| for item in links: | |
| if len(articles) >= count: | |
| break | |
| articles.append(item) | |
| return articles[:count] | |
| # Gets new articles for doctors | |
| def get_doctor_news_raw(count: int = 5) -> list[dict]: | |
| base_urls = [ | |
| "https://www.nature.com/search?q=dementia&order=date", | |
| "https://www.nejm.org/search?q=dementia", | |
| "https://www.thelancet.com/search?q=dementia" | |
| ] | |
| articles = [] | |
| for url in base_urls: | |
| if len(articles) >= count: | |
| break | |
| links = scrape_article_links(url, max_links=count) | |
| for item in links: | |
| if len(articles) >= count: | |
| break | |
| articles.append(item) | |
| return articles[:count] |