import requests from bs4 import BeautifulSoup import time import os # ================================================================ # Helper Functions # ================================================================ def clean_text(text): return " ".join(text.split()) def scrape_article_links(base_url: str, max_links=10) -> list[dict]: """Scrape list of {title, url} from a news listing page.""" headers = { 'User-Agent': 'Mozilla/5.0 (compatible; DementiaNewsBot/1.0)' } try: response = requests.get(base_url, headers=headers, timeout=10) response.raise_for_status() soup = BeautifulSoup(response.content, 'lxml') items = [] # Based on caregiving.org structure (from your KB): # Each news item is in
or
posts = soup.find_all(['article', 'div'], class_=lambda x: x and ('post' in x or 'news' in x or 'item' in x)) for post in posts[:max_links]: # Try to get title + link a_tag = post.find('a', href=True) if not a_tag: continue href = a_tag['href'] if not href.startswith('http'): href = requests.compat.urljoin(base_url, href) # Title: from

,

, or img alt title_tag = post.find(['h2', 'h3', 'h4']) if not title_tag: img = post.find('img') title = img.get('alt', '') if img else "" else: title = clean_text(title_tag.get_text()) if not title or title in ["Read More", "Learn More"]: continue items.append({"title": title, "url": href}) return items except Exception as e: print(f"scrape_article_links({base_url}) failed: {e}") return [] # Attempts to get the full raw article text def scrape_full_text(url: str) -> str: """Return raw full article text (no summarization).""" headers = { 'User-Agent': 'Mozilla/5.0 (compatible; DementiaNewsBot/1.0)' } try: response = requests.get(url, headers=headers, timeout=15) soup = BeautifulSoup(response.content, 'lxml') for s in soup(["script", "style", "nav", "footer", "aside", "header"]): s.decompose() # Target main content main = soup.find('article') or soup.find('main') or soup.find('div', id='main-content') if not main: main = soup.body paras = main.find_all('p') if main else soup.find_all('p') text = "\n\n".join(clean_text(p.get_text()) for p in paras if len(p.get_text().strip()) > 50) return text[:8000] # Cap for token efficiency except Exception as e: print(f"scrape_full_text({url}) failed: {e}") return "" # ================================================================ # News webscraping # ================================================================ # Gets new articles for patients def get_patient_news_raw(count: int = 5) -> list[dict]: base_urls = [ "https://www.caregiving.org/?s=dementia", "https://www.alz.org/search?query=dementia&filters=news", "https://www.dementiaresearch.org/latest-news" ] articles = [] for url in base_urls: if len(articles) >= count: break links = scrape_article_links(url, max_links=count) for item in links: if len(articles) >= count: break articles.append(item) return articles[:count] # Gets new articles for doctors def get_doctor_news_raw(count: int = 5) -> list[dict]: base_urls = [ "https://www.nature.com/search?q=dementia&order=date", "https://www.nejm.org/search?q=dementia", "https://www.thelancet.com/search?q=dementia" ] articles = [] for url in base_urls: if len(articles) >= count: break links = scrape_article_links(url, max_links=count) for item in links: if len(articles) >= count: break articles.append(item) return articles[:count]