File size: 4,149 Bytes
79aae02
 
 
e887f35
79aae02
adcb5fd
 
 
 
 
e887f35
 
79aae02
e887f35
 
 
 
 
79aae02
e887f35
79aae02
e887f35
79aae02
e887f35
79aae02
e887f35
 
 
79aae02
e887f35
 
 
 
79aae02
e887f35
 
 
 
 
 
 
 
 
 
 
 
 
79aae02
 
e887f35
79aae02
e887f35
79aae02
 
adcb5fd
e887f35
 
adcb5fd
e887f35
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
adcb5fd
e887f35
 
adcb5fd
 
 
 
 
 
e887f35
 
ac799e8
 
e887f35
 
 
 
 
 
 
 
 
 
 
 
 
adcb5fd
e887f35
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
import requests
from bs4 import BeautifulSoup
import time
import os


# ================================================================
# Helper Functions
# ================================================================

def clean_text(text):
    return " ".join(text.split())

def scrape_article_links(base_url: str, max_links=10) -> list[dict]:
    """Scrape list of {title, url} from a news listing page."""
    headers = {
        'User-Agent': 'Mozilla/5.0 (compatible; DementiaNewsBot/1.0)'
    }
    try:
        response = requests.get(base_url, headers=headers, timeout=10)
        response.raise_for_status()
        soup = BeautifulSoup(response.content, 'lxml')

        items = []

        # Based on caregiving.org structure (from your KB):
        # Each news item is in <article> or <div class="post-item">
        posts = soup.find_all(['article', 'div'], class_=lambda x: x and ('post' in x or 'news' in x or 'item' in x))

        for post in posts[:max_links]:
            # Try to get title + link
            a_tag = post.find('a', href=True)
            if not a_tag:
                continue
            href = a_tag['href']
            if not href.startswith('http'):
                href = requests.compat.urljoin(base_url, href)

            # Title: from <h2>, <h3>, or img alt
            title_tag = post.find(['h2', 'h3', 'h4'])
            if not title_tag:
                img = post.find('img')
                title = img.get('alt', '') if img else ""
            else:
                title = clean_text(title_tag.get_text())

            if not title or title in ["Read More", "Learn More"]:
                continue

            items.append({"title": title, "url": href})

        return items

    except Exception as e:
        print(f"scrape_article_links({base_url}) failed: {e}")
        return []

# Attempts to get the full raw article text
def scrape_full_text(url: str) -> str:
    """Return raw full article text (no summarization)."""
    headers = {
        'User-Agent': 'Mozilla/5.0 (compatible; DementiaNewsBot/1.0)'
    }
    try:
        response = requests.get(url, headers=headers, timeout=15)
        soup = BeautifulSoup(response.content, 'lxml')
        for s in soup(["script", "style", "nav", "footer", "aside", "header"]):
            s.decompose()

        # Target main content
        main = soup.find('article') or soup.find('main') or soup.find('div', id='main-content')
        if not main:
            main = soup.body

        paras = main.find_all('p') if main else soup.find_all('p')
        text = "\n\n".join(clean_text(p.get_text()) for p in paras if len(p.get_text().strip()) > 50)
        return text[:8000]  # Cap for token efficiency

    except Exception as e:
        print(f"scrape_full_text({url}) failed: {e}")
        return ""


# ================================================================
# News webscraping 
# ================================================================

# Gets new articles for patients
def get_patient_news_raw(count: int = 5) -> list[dict]:
    base_urls = [
        "https://www.caregiving.org/?s=dementia",
        "https://www.alz.org/search?query=dementia&filters=news",
        "https://www.dementiaresearch.org/latest-news"
    ]
    articles = []
    for url in base_urls:
        if len(articles) >= count:
            break
        links = scrape_article_links(url, max_links=count)
        for item in links:
            if len(articles) >= count:
                break
            articles.append(item)
    return articles[:count]

# Gets new articles for doctors
def get_doctor_news_raw(count: int = 5) -> list[dict]:
    base_urls = [
        "https://www.nature.com/search?q=dementia&order=date",
        "https://www.nejm.org/search?q=dementia",
        "https://www.thelancet.com/search?q=dementia"
    ]
    articles = []
    for url in base_urls:
        if len(articles) >= count:
            break
        links = scrape_article_links(url, max_links=count)
        for item in links:
            if len(articles) >= count:
                break
            articles.append(item)
    return articles[:count]