NMS-AI-DEV / webscraper.py
plutoa386's picture
cleaned up
adcb5fd
Raw History Blame Contribute Delete
4.15 kB
import requests
from bs4 import BeautifulSoup
import time
import os
# ================================================================
# Helper Functions
# ================================================================
def clean_text(text):
return " ".join(text.split())
def scrape_article_links(base_url: str, max_links=10) -> list[dict]:
"""Scrape list of {title, url} from a news listing page."""
headers = {
'User-Agent': 'Mozilla/5.0 (compatible; DementiaNewsBot/1.0)'
}
try:
response = requests.get(base_url, headers=headers, timeout=10)
response.raise_for_status()
soup = BeautifulSoup(response.content, 'lxml')
items = []
# Based on caregiving.org structure (from your KB):
# Each news item is in <article> or <div class="post-item">
posts = soup.find_all(['article', 'div'], class_=lambda x: x and ('post' in x or 'news' in x or 'item' in x))
for post in posts[:max_links]:
# Try to get title + link
a_tag = post.find('a', href=True)
if not a_tag:
continue
href = a_tag['href']
if not href.startswith('http'):
href = requests.compat.urljoin(base_url, href)
# Title: from <h2>, <h3>, or img alt
title_tag = post.find(['h2', 'h3', 'h4'])
if not title_tag:
img = post.find('img')
title = img.get('alt', '') if img else ""
else:
title = clean_text(title_tag.get_text())
if not title or title in ["Read More", "Learn More"]:
continue
items.append({"title": title, "url": href})
return items
except Exception as e:
print(f"scrape_article_links({base_url}) failed: {e}")
return []
# Attempts to get the full raw article text
def scrape_full_text(url: str) -> str:
"""Return raw full article text (no summarization)."""
headers = {
'User-Agent': 'Mozilla/5.0 (compatible; DementiaNewsBot/1.0)'
}
try:
response = requests.get(url, headers=headers, timeout=15)
soup = BeautifulSoup(response.content, 'lxml')
for s in soup(["script", "style", "nav", "footer", "aside", "header"]):
s.decompose()
# Target main content
main = soup.find('article') or soup.find('main') or soup.find('div', id='main-content')
if not main:
main = soup.body
paras = main.find_all('p') if main else soup.find_all('p')
text = "\n\n".join(clean_text(p.get_text()) for p in paras if len(p.get_text().strip()) > 50)
return text[:8000] # Cap for token efficiency
except Exception as e:
print(f"scrape_full_text({url}) failed: {e}")
return ""
# ================================================================
# News webscraping
# ================================================================
# Gets new articles for patients
def get_patient_news_raw(count: int = 5) -> list[dict]:
base_urls = [
"https://www.caregiving.org/?s=dementia",
"https://www.alz.org/search?query=dementia&filters=news",
"https://www.dementiaresearch.org/latest-news"
]
articles = []
for url in base_urls:
if len(articles) >= count:
break
links = scrape_article_links(url, max_links=count)
for item in links:
if len(articles) >= count:
break
articles.append(item)
return articles[:count]
# Gets new articles for doctors
def get_doctor_news_raw(count: int = 5) -> list[dict]:
base_urls = [
"https://www.nature.com/search?q=dementia&order=date",
"https://www.nejm.org/search?q=dementia",
"https://www.thelancet.com/search?q=dementia"
]
articles = []
for url in base_urls:
if len(articles) >= count:
break
links = scrape_article_links(url, max_links=count)
for item in links:
if len(articles) >= count:
break
articles.append(item)
return articles[:count]