File size: 4,149 Bytes
79aae02 e887f35 79aae02 adcb5fd e887f35 79aae02 e887f35 79aae02 e887f35 79aae02 e887f35 79aae02 e887f35 79aae02 e887f35 79aae02 e887f35 79aae02 e887f35 79aae02 e887f35 79aae02 e887f35 79aae02 adcb5fd e887f35 adcb5fd e887f35 adcb5fd e887f35 adcb5fd e887f35 ac799e8 e887f35 adcb5fd e887f35 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | import requests
from bs4 import BeautifulSoup
import time
import os
# ================================================================
# Helper Functions
# ================================================================
def clean_text(text):
return " ".join(text.split())
def scrape_article_links(base_url: str, max_links=10) -> list[dict]:
"""Scrape list of {title, url} from a news listing page."""
headers = {
'User-Agent': 'Mozilla/5.0 (compatible; DementiaNewsBot/1.0)'
}
try:
response = requests.get(base_url, headers=headers, timeout=10)
response.raise_for_status()
soup = BeautifulSoup(response.content, 'lxml')
items = []
# Based on caregiving.org structure (from your KB):
# Each news item is in <article> or <div class="post-item">
posts = soup.find_all(['article', 'div'], class_=lambda x: x and ('post' in x or 'news' in x or 'item' in x))
for post in posts[:max_links]:
# Try to get title + link
a_tag = post.find('a', href=True)
if not a_tag:
continue
href = a_tag['href']
if not href.startswith('http'):
href = requests.compat.urljoin(base_url, href)
# Title: from <h2>, <h3>, or img alt
title_tag = post.find(['h2', 'h3', 'h4'])
if not title_tag:
img = post.find('img')
title = img.get('alt', '') if img else ""
else:
title = clean_text(title_tag.get_text())
if not title or title in ["Read More", "Learn More"]:
continue
items.append({"title": title, "url": href})
return items
except Exception as e:
print(f"scrape_article_links({base_url}) failed: {e}")
return []
# Attempts to get the full raw article text
def scrape_full_text(url: str) -> str:
"""Return raw full article text (no summarization)."""
headers = {
'User-Agent': 'Mozilla/5.0 (compatible; DementiaNewsBot/1.0)'
}
try:
response = requests.get(url, headers=headers, timeout=15)
soup = BeautifulSoup(response.content, 'lxml')
for s in soup(["script", "style", "nav", "footer", "aside", "header"]):
s.decompose()
# Target main content
main = soup.find('article') or soup.find('main') or soup.find('div', id='main-content')
if not main:
main = soup.body
paras = main.find_all('p') if main else soup.find_all('p')
text = "\n\n".join(clean_text(p.get_text()) for p in paras if len(p.get_text().strip()) > 50)
return text[:8000] # Cap for token efficiency
except Exception as e:
print(f"scrape_full_text({url}) failed: {e}")
return ""
# ================================================================
# News webscraping
# ================================================================
# Gets new articles for patients
def get_patient_news_raw(count: int = 5) -> list[dict]:
base_urls = [
"https://www.caregiving.org/?s=dementia",
"https://www.alz.org/search?query=dementia&filters=news",
"https://www.dementiaresearch.org/latest-news"
]
articles = []
for url in base_urls:
if len(articles) >= count:
break
links = scrape_article_links(url, max_links=count)
for item in links:
if len(articles) >= count:
break
articles.append(item)
return articles[:count]
# Gets new articles for doctors
def get_doctor_news_raw(count: int = 5) -> list[dict]:
base_urls = [
"https://www.nature.com/search?q=dementia&order=date",
"https://www.nejm.org/search?q=dementia",
"https://www.thelancet.com/search?q=dementia"
]
articles = []
for url in base_urls:
if len(articles) >= count:
break
links = scrape_article_links(url, max_links=count)
for item in links:
if len(articles) >= count:
break
articles.append(item)
return articles[:count] |