plutoa386 commited on
Commit
e887f35
·
1 Parent(s): f81db3a

revamped webscraping logic

Browse files
Files changed (3) hide show
  1. app.py +0 -12
  2. crew.py +138 -140
  3. webscraper.py +94 -83
app.py CHANGED
@@ -32,18 +32,6 @@ def news_doctor(count: int = 5):
32
  except Exception as e:
33
  return {"status": "error", "message": str(e)}
34
 
35
- def assess_risk(docID: str, mode: str):
36
- if mode == "audio":
37
- return assess_audio(docID)
38
- elif mode == "lifestyle":
39
- return news_doctor()
40
- elif mode == "doctornews":
41
- return news_doctor()
42
- elif mode == "patientnews":
43
- return news_patient()
44
- else:
45
- return {"status": "error", "message": f"option '{mode}' not found"}
46
-
47
  with gr.Blocks() as demo:
48
  gr.Markdown("# 🧠 NMS Dementia Risk Assessment API - Development Zone")
49
 
 
32
  except Exception as e:
33
  return {"status": "error", "message": str(e)}
34
 
 
 
 
 
 
 
 
 
 
 
 
 
35
  with gr.Blocks() as demo:
36
  gr.Markdown("# 🧠 NMS Dementia Risk Assessment API - Development Zone")
37
 
crew.py CHANGED
@@ -6,7 +6,7 @@ import numpy as np
6
  from crewai import Agent, Task, Crew, LLM
7
  from crewai.tools import tool
8
  from firebaseLoader import *
9
- from webscraper import search_dementia_news
10
  from typing import Union
11
  import shap
12
  import xgboost as xgb
@@ -34,6 +34,9 @@ lifestyle_explainer = shap.TreeExplainer(lifestyle_model)
34
 
35
  placeholder_lifestyle_ID = "umWSG13rLSfHqqJlq1WluPzjLB93"
36
 
 
 
 
37
 
38
  # Define tools for the agents
39
  @tool("calculate_dementia_risk_audio")
@@ -301,48 +304,98 @@ def explain_dementia_risk_lifestyle(docID: str) -> str:
301
  @tool("gather_news_patient")
302
  def gather_news_patient(count: int = 5) -> dict:
303
  """
304
- Gathers recent dementia news articles which are relevant patients. Returns structured dict.
 
 
305
  """
306
- from webscraper import search_dementia_news, scrape_article_content
307
- articles = search_dementia_news(audience="patient", max_results=min(count, 10))
308
- clean_articles = []
309
- for art in articles:
310
- if "error" not in art:
311
- try:
312
- content = scrape_article_content(art["url"])
313
- clean_articles.append({
314
- "title": art["title"],
315
- "url": art["url"],
316
- "content": content[:3000] # Keep full content for LLM to summarize
317
- })
318
- except Exception:
319
- continue
320
- return {"articles": clean_articles} # ← Return dict, not JSON string
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
321
 
322
  @tool("gather_news_doctor")
323
  def gather_news_doctor(count: int = 5) -> dict:
324
  """
325
- Gathers recent dementia news articles which are relevant doctors. Returns structured dict.
 
326
  """
327
- from webscraper import search_dementia_news, scrape_article_content
328
- articles = search_dementia_news(audience="doctor", max_results=min(count, 10))
329
- clean_articles = []
330
- for art in articles:
331
- if "error" not in art:
332
- try:
333
- content = scrape_article_content(art["url"])
334
- clean_articles.append({
335
- "title": art["title"],
336
- "url": art["url"],
337
- "content": content[:3000]
338
- })
339
- except Exception:
340
- continue
341
- return {"articles": clean_articles}
342
 
343
- # Load 2 different llms for different purposes.
344
- llm_predict = LLM(model="huggingface/openai/gpt-oss-120b")
345
- llm_news = LLM(model="huggingface/meta-llama/Llama-3.1-8B-Instruct")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
346
 
347
  # Define agents for Alzheimer's detection system
348
  risk_calculator_audio = Agent(
@@ -363,22 +416,22 @@ risk_calculator_lifestyle = Agent(
363
  tools=[calculate_dementia_risk_lifestyle, explain_dementia_risk_lifestyle]
364
  )
365
 
366
- patient_news = Agent(
367
  role="Patient News Agent",
368
- goal="Gather dementia-related news articles that are helpful and easy to understand for patients and families",
369
- backstory="You find and share clear, trustworthy news about dementia—like new treatments, daily care tips, and support resources—written for patients and caregivers.",
370
- verbose=True,
371
  llm=llm_news,
372
- tools=[gather_news_patient]
373
  )
374
 
375
- doctor_news = Agent(
376
  role="Doctor News Agent",
377
- goal="Gather dementia-related news and research updates that are useful for doctors and medical professionals",
378
- backstory="You find and summarize the latest clinical studies, guidelines, and expert insights on dementia—focused on diagnosis, treatment, and research for healthcare providers.",
379
- verbose=True,
380
  llm=llm_news,
381
- tools=[gather_news_doctor]
382
  )
383
 
384
  # Define tasks for agents
@@ -408,48 +461,20 @@ explanation_task_lifestyle = Task(
408
 
409
  get_news_task_patient = Task(
410
  description=(
411
- "1. Use the 'gather_news_patient' tool to fetch recent dementia news articles.\n"
412
- "2. You will receive a list of articles, each with 'title', 'url', and 'content'.\n"
413
- "3. For EACH article, read its 'content' field and generate a concise 2–3 line summary that highlights key takeaways for patients/families.\n"
414
- "4. Ensure summaries are empathetic, jargon-free, and actionable.\n"
415
- "5. Return a JSON object with keys: 'titles', 'snippets', 'urls' — matching the order of the fetched articles.\n"
416
- "6. DO NOT invent titles or URLs — use only what was returned by the tool.\n"
417
- "7. Return ONLY valid JSON — no markdown, no extra text."
418
  ),
419
- expected_output=(
420
- "A JSON object with keys: 'titles', 'snippets', 'urls'.\n"
421
- "- 'titles': list of original article titles (strings).\n"
422
- "- 'snippets': list of 2–3 line plain-language summaries (strings), one per article.\n"
423
- "- 'urls': list of original article URLs (strings).\n"
424
- "All lists must be same length and aligned by index.\n"
425
- "Return ONLY valid JSON — no extra text."
426
- ),
427
- agent=patient_news
428
  )
429
 
430
  get_news_task_doctor = Task(
431
  description=(
432
- "1. Use the 'gather_news_doctor' tool to fetch recent dementia-related articles for medical professionals.\n"
433
- "2. You will receive a list of articles, each with 'title', 'url', and 'content' (scraped full text).\n"
434
- "3. For EACH article, read its 'content' field and generate a concise 2–3 line summary.\n"
435
- "4. Summaries must be clinical, evidence-focused, and highlight:\n"
436
- " - Trial design/outcomes (if applicable)\n"
437
- " - Diagnostic/therapeutic implications\n"
438
- " - Guideline updates or expert consensus changes\n"
439
- " - Biomarker or mechanistic insights\n"
440
- "5. DO NOT invent titles, URLs, or content — use only what the tool returns.\n"
441
- "6. Return a JSON object with keys: 'titles', 'snippets', 'urls', in matching order.\n"
442
- "7. Return ONLY valid JSON — no markdown, no commentary."
443
- ),
444
- expected_output=(
445
- "A JSON object with keys: 'titles', 'snippets', 'urls'.\n"
446
- "- 'titles': list of article titles.\n"
447
- "- 'snippets': list of 2–3 line clinical summaries.\n"
448
- "- 'urls': list of original URLs.\n"
449
- "All lists same length, aligned by index.\n"
450
- "Return ONLY valid JSON — no extra text."
451
  ),
452
- agent=doctor_news
 
453
  )
454
 
455
  # Human-readable mapping for audio features
@@ -688,63 +713,36 @@ def run_crewai_audio(docID: str) -> dict:
688
  return {"status": "error", "message": str(e)}
689
 
690
  def run_crewai_news_patient(count: int = 5) -> dict:
691
- try:
692
- # Just use the pre-defined task — it already knows to use gather_news_patient
693
- crew = Crew(
694
- agents=[patient_news],
695
- tasks=[get_news_task_patient],
696
- verbose=True,
697
- function_calling=False
698
- )
699
- raw = str(crew.kickoff()).strip()
700
-
701
- result = _parse_json_output(raw)
702
- # Validate structure
703
- for key in ["titles", "snippets", "urls"]:
704
- if key not in result or not isinstance(result[key], list):
705
- raise ValueError(f"Missing or invalid '{key}' in result")
706
- if not (len(result["titles"]) == len(result["snippets"]) == len(result["urls"])):
707
- raise ValueError("Mismatched list lengths")
708
-
709
- return {"status": "success", **result}
710
-
711
- except Exception as e:
712
- return {"status": "error", "message": str(e)}
713
 
714
  def run_crewai_news_doctor(count: int = 5) -> dict:
715
- try:
716
- original_gather = doctor_news.tools[0]
717
-
718
- # Wrap tool to inject count
719
- def gather_with_count(query=None):
720
- return gather_news_doctor(count=count)
721
-
722
- # Replace tool temporarily
723
- doctor_news.tools = [gather_with_count]
724
-
725
- # Run crew with static task
726
- crew = Crew(
727
- agents=[doctor_news],
728
- tasks=[get_news_task_doctor],
729
- verbose=True,
730
- function_calling=False
731
- )
732
- raw_output = str(crew.kickoff()).strip()
733
-
734
- # Restore original tool
735
- doctor_news.tools = [original_gather]
736
-
737
- # Parse JSON
738
- result = _parse_json_output(raw_output)
739
-
740
- # Validate structure
741
- for key in ["titles", "snippets", "urls"]:
742
- if key not in result or not isinstance(result[key], list):
743
- raise ValueError(f"Missing or invalid '{key}' in result")
744
- if not (len(result["titles"]) == len(result["snippets"]) == len(result["urls"])):
745
- raise ValueError("Mismatched list lengths in doctor news result")
746
-
747
- return {"status": "success", **result}
748
-
749
- except Exception as e:
750
- return {"status": "error", "message": str(e)}
 
6
  from crewai import Agent, Task, Crew, LLM
7
  from crewai.tools import tool
8
  from firebaseLoader import *
9
+ from webscraper import *
10
  from typing import Union
11
  import shap
12
  import xgboost as xgb
 
34
 
35
  placeholder_lifestyle_ID = "umWSG13rLSfHqqJlq1WluPzjLB93"
36
 
37
+ # Load 2 different llms for different purposes.
38
+ llm_predict = LLM(model="huggingface/openai/gpt-oss-120b")
39
+ llm_news = LLM(model="huggingface/meta-llama/Llama-3.1-8B-Instruct")
40
 
41
  # Define tools for the agents
42
  @tool("calculate_dementia_risk_audio")
 
304
  @tool("gather_news_patient")
305
  def gather_news_patient(count: int = 5) -> dict:
306
  """
307
+ Fetches count recent dementia news items for patients, retrieves full text,
308
+ and generates patient-friendly summaries using the LLM.
309
+ Returns: {"status": "success", "articles": [{"title", "summary", "url"}, ...]}
310
  """
311
+ try:
312
+ raw_articles = get_patient_news_raw(count=count)
313
+ results = []
314
+
315
+ for item in raw_articles:
316
+ url = item["url"]
317
+ title = item["title"]
318
+ full_text = scrape_full_text(url)
319
+
320
+ # ✅ LLM summarization happens HERE (using agent's llm or shared llm_news)
321
+ if not full_text.strip():
322
+ summary = "Summary unavailable."
323
+ else:
324
+ # Use llm_news — ensure it's in scope
325
+ prompt = f"""You are a compassionate health educator.
326
+ Summarize the following article in 2–3 clear, empathetic sentences for patients/families affected by dementia.
327
+ Focus on: What does this mean? What can they do? Hope & support.
328
+ Avoid jargon.
329
+
330
+ Article:
331
+ {full_text[:3000]}"""
332
+
333
+ response = llm_news.invoke(prompt) # ← YOUR LLM (e.g., LiteLLM-wrapped Llama 3.1)
334
+ summary = str(response).strip()
335
+
336
+ results.append({
337
+ "title": title,
338
+ "summary": summary,
339
+ "url": url
340
+ })
341
+
342
+ return {
343
+ "status": "success",
344
+ "articles": results
345
+ }
346
+
347
+ except Exception as e:
348
+ return {
349
+ "status": "error",
350
+ "message": str(e),
351
+ "articles": []
352
+ }
353
 
354
  @tool("gather_news_doctor")
355
  def gather_news_doctor(count: int = 5) -> dict:
356
  """
357
+ Fetches count recent dementia news items for doctors, retrieves full text,
358
+ and generates clinical summaries using the LLM.
359
  """
360
+ try:
361
+ raw_articles = get_doctor_news_raw(count=count)
362
+ results = []
 
 
 
 
 
 
 
 
 
 
 
 
363
 
364
+ for item in raw_articles:
365
+ url = item["url"]
366
+ title = item["title"]
367
+ full_text = scrape_full_text(url)
368
+
369
+ if not full_text.strip():
370
+ summary = "Summary unavailable."
371
+ else:
372
+ prompt = f"""You are a clinical expert.
373
+ Summarize the following article in 2–3 precise, evidence-based sentences for physicians.
374
+ Include: key findings, methods (if applicable), clinical relevance.
375
+
376
+ Article:
377
+ {full_text[:3000]}"""
378
+
379
+ response = llm_news.invoke(prompt)
380
+ summary = str(response).strip()
381
+
382
+ results.append({
383
+ "title": title,
384
+ "summary": summary,
385
+ "url": url
386
+ })
387
+
388
+ return {
389
+ "status": "success",
390
+ "articles": results
391
+ }
392
+
393
+ except Exception as e:
394
+ return {
395
+ "status": "error",
396
+ "message": str(e),
397
+ "articles": []
398
+ }
399
 
400
  # Define agents for Alzheimer's detection system
401
  risk_calculator_audio = Agent(
 
416
  tools=[calculate_dementia_risk_lifestyle, explain_dementia_risk_lifestyle]
417
  )
418
 
419
+ patient_news_agent = Agent(
420
  role="Patient News Agent",
421
+ goal="Gather and summarize dementia/caregiving news for patients and families in plain, supportive language.",
422
+ backstory="You find real, current articles from trusted sites like caregiving.org, extract full content, and use your medical communication expertise to make it clear and kind for non-experts.",
423
+ tools=[gather_news_patient],
424
  llm=llm_news,
425
+ verbose=True
426
  )
427
 
428
+ doctor_news_agent = Agent(
429
  role="Doctor News Agent",
430
+ goal="Gather and summarize high-impact dementia research for clinicians and scientists.",
431
+ backstory="You retrieve peer-reviewed research (e.g., from Europe PMC), extract key findings, and distill them into clinically actionable summaries for time-pressed professionals.",
432
+ tools=[gather_news_doctor],
433
  llm=llm_news,
434
+ verbose=True
435
  )
436
 
437
  # Define tasks for agents
 
461
 
462
  get_news_task_patient = Task(
463
  description=(
464
+ "Use the `gather_news_patient` tool to fetch {count} recent dementia/caregiving news articles "
465
+ "and return the raw JSON result. Do not generate or modify content."
 
 
 
 
 
466
  ),
467
+ expected_output="A JSON object with 'status' and 'articles': list of {title, summary, url}.",
468
+ agent=patient_news_agent
 
 
 
 
 
 
 
469
  )
470
 
471
  get_news_task_doctor = Task(
472
  description=(
473
+ "Use the `gather_news_doctor` tool to fetch {count} recent dementia research articles "
474
+ "and return the raw JSON result."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
475
  ),
476
+ expected_output="A JSON object with 'status' and 'articles': list of {title, summary, url}.",
477
+ agent=doctor_news_agent
478
  )
479
 
480
  # Human-readable mapping for audio features
 
713
  return {"status": "error", "message": str(e)}
714
 
715
  def run_crewai_news_patient(count: int = 5) -> dict:
716
+ crew = Crew(
717
+ agents=[patient_news_agent],
718
+ tasks=[get_news_task_patient],
719
+ verbose=True,
720
+ function_calling=False
721
+ )
722
+ result = crew.kickoff(inputs={"count": count})
723
+ # CrewAI returns ToolOutput → .raw or str — parse if needed
724
+ if isinstance(result, dict):
725
+ return result
726
+ else:
727
+ try:
728
+ import json
729
+ return json.loads(str(result))
730
+ except:
731
+ return {"status": "error", "message": "Failed to parse crew output", "raw": str(result)}
 
 
 
 
 
 
732
 
733
  def run_crewai_news_doctor(count: int = 5) -> dict:
734
+ crew = Crew(
735
+ agents=[doctor_news_agent],
736
+ tasks=[get_news_task_doctor],
737
+ verbose=True,
738
+ function_calling=False
739
+ )
740
+ result = crew.kickoff(inputs={"count": count})
741
+ if isinstance(result, dict):
742
+ return result
743
+ else:
744
+ try:
745
+ import json
746
+ return json.loads(str(result))
747
+ except:
748
+ return {"status": "error", "message": "Failed to parse crew output", "raw": str(result)}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
webscraper.py CHANGED
@@ -1,99 +1,110 @@
 
1
  import requests
2
  from bs4 import BeautifulSoup
3
- from duckduckgo_search import DDGS
4
  import time
5
- import re
6
- from typing import List, Dict
7
 
8
- def clean_text(text: str) -> str:
9
- """Clean and normalize text: remove extra whitespace, fix line breaks."""
10
- if not text:
11
- return ""
12
- text = re.sub(r'\s+', ' ', text).strip()
13
- return text[:1000]
14
 
15
- def scrape_article_content(url: str, timeout: int = 8) -> str:
16
- """Scrape main article text using requests + BeautifulSoup."""
 
 
 
17
  try:
18
- headers = {
19
- "User-Agent": "Mozilla/5.0 (compatible; DementiaNewsBot/1.0; +https://huggingface.co/spaces/plutoa386/...)"
20
- }
21
- response = requests.get(url, headers=headers, timeout=timeout)
22
  response.raise_for_status()
 
23
 
24
- soup = BeautifulSoup(response.content, "lxml")
25
-
26
- # Remove noisy elements
27
- for elem in soup(["script", "style", "nav", "footer", "header", "aside", "form"]):
28
- elem.decompose()
29
-
30
- # Try to find main content
31
- article = (
32
- soup.find("article") or
33
- soup.find("div", class_=re.compile(r"article|post|content|story", re.I)) or
34
- soup.find("main") or
35
- soup.find("div", id=re.compile(r"content|article", re.I)) or
36
- soup
37
- )
38
-
39
- # Extract paragraphs
40
- paragraphs = article.find_all("p")
41
- text = " ".join([
42
- clean_text(p.get_text())
43
- for p in paragraphs
44
- if len(p.get_text().strip()) > 30 # skip tiny lines
45
- ])
46
- return text[:1500] # keep it concise for LLM
47
 
48
- except Exception as e:
49
- return f"[Content extraction failed: {type(e).__name__}]"
 
50
 
51
- def search_dementia_news(audience: str = "patient", max_results: int = 3) -> List[Dict]:
52
- """
53
- Returns list of news items: {title, url, snippet}
54
- Uses DuckDuckGo for ethical, free, no-API-key search.
55
- """
56
- try:
57
- # Tailored queries
58
- if audience == "patient":
59
- query = (
60
- '"dementia" OR "Alzheimer\'s disease" '
61
- '(treatment OR care OR support OR lifestyle OR symptoms OR prevention OR early warning) '
62
- 'site:alz.org OR site:mayoclinic.org OR site:nia.nih.gov OR site:webmd.com OR site:healthline.com'
63
- )
64
- else: # doctor
65
- query = (
66
- '"dementia" OR "Alzheimer\'s" '
67
- '(clinical trial OR biomarker OR diagnosis OR guideline OR research OR drug approval) '
68
- 'site:nejm.org OR site:the-lancet.com OR site:fda.gov OR site:medscape.com OR site:alzheimersresearchuk.org'
69
- )
70
-
71
- results = []
72
- with DDGS() as ddgs:
73
- raw_results = list(ddgs.text(
74
- query,
75
- region="wt-wt",
76
- safesearch="off",
77
- timelimit="m", # past month
78
- max_results=max_results + 2 # fetch a few extra for filtering
79
- ))
80
-
81
- for r in raw_results:
82
- url = r.get("href", "").strip()
83
- # Skip non-http or social media
84
- if not url.startswith("http") or any(domain in url for domain in ["facebook.com", "twitter.com", "youtube.com", "linkedin.com"]):
85
  continue
86
- title = clean_text(r.get("title", ""))
87
- snippet = clean_text(r.get("body", ""))
88
- if len(title) < 10 or len(snippet) < 20:
 
 
 
 
 
 
 
 
 
 
89
  continue
90
- results.append({"title": title, "url": url, "snippet": snippet})
91
- if len(results) >= max_results:
92
- break
93
 
94
- time.sleep(0.4) # be respectful
95
 
96
- return results
97
 
98
  except Exception as e:
99
- return [{"error": f"Search failed: {str(e)}"}]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # webscraper.py
2
  import requests
3
  from bs4 import BeautifulSoup
 
4
  import time
5
+ import os
 
6
 
7
+ def clean_text(text):
8
+ return " ".join(text.split())
 
 
 
 
9
 
10
+ def scrape_article_links(base_url: str, max_links=10) -> list[dict]:
11
+ """Scrape list of {title, url} from a news listing page."""
12
+ headers = {
13
+ 'User-Agent': 'Mozilla/5.0 (compatible; DementiaNewsBot/1.0)'
14
+ }
15
  try:
16
+ response = requests.get(base_url, headers=headers, timeout=10)
 
 
 
17
  response.raise_for_status()
18
+ soup = BeautifulSoup(response.content, 'lxml')
19
 
20
+ items = []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
21
 
22
+ # Based on caregiving.org structure (from your KB):
23
+ # Each news item is in <article> or <div class="post-item">
24
+ posts = soup.find_all(['article', 'div'], class_=lambda x: x and ('post' in x or 'news' in x or 'item' in x))
25
 
26
+ for post in posts[:max_links]:
27
+ # Try to get title + link
28
+ a_tag = post.find('a', href=True)
29
+ if not a_tag:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
  continue
31
+ href = a_tag['href']
32
+ if not href.startswith('http'):
33
+ href = requests.compat.urljoin(base_url, href)
34
+
35
+ # Title: from <h2>, <h3>, or img alt
36
+ title_tag = post.find(['h2', 'h3', 'h4'])
37
+ if not title_tag:
38
+ img = post.find('img')
39
+ title = img.get('alt', '') if img else ""
40
+ else:
41
+ title = clean_text(title_tag.get_text())
42
+
43
+ if not title or title in ["Read More", "Learn More"]:
44
  continue
 
 
 
45
 
46
+ items.append({"title": title, "url": href})
47
 
48
+ return items
49
 
50
  except Exception as e:
51
+ print(f"⚠️ scrape_article_links({base_url}) failed: {e}")
52
+ return []
53
+
54
+ def scrape_full_text(url: str) -> str:
55
+ """Return raw full article text (no summarization)."""
56
+ headers = {
57
+ 'User-Agent': 'Mozilla/5.0 (compatible; DementiaNewsBot/1.0)'
58
+ }
59
+ try:
60
+ response = requests.get(url, headers=headers, timeout=15)
61
+ soup = BeautifulSoup(response.content, 'lxml')
62
+ for s in soup(["script", "style", "nav", "footer", "aside", "header"]):
63
+ s.decompose()
64
+
65
+ # Target main content
66
+ main = soup.find('article') or soup.find('main') or soup.find('div', id='main-content')
67
+ if not main:
68
+ main = soup.body
69
+
70
+ paras = main.find_all('p') if main else soup.find_all('p')
71
+ text = "\n\n".join(clean_text(p.get_text()) for p in paras if len(p.get_text().strip()) > 50)
72
+ return text[:8000] # Cap for token efficiency
73
+
74
+ except Exception as e:
75
+ print(f"⚠️ scrape_full_text({url}) failed: {e}")
76
+ return ""
77
+
78
+ def get_patient_news_raw(count: int = 5) -> list[dict]:
79
+ base_urls = [
80
+ "https://www.caregiving.org/news/",
81
+ "https://www.alz.org/news",
82
+ "https://www.dementiaresearch.org/latest-news"
83
+ ]
84
+ articles = []
85
+ for url in base_urls:
86
+ if len(articles) >= count:
87
+ break
88
+ links = scrape_article_links(url, max_links=count)
89
+ for item in links:
90
+ if len(articles) >= count:
91
+ break
92
+ articles.append(item)
93
+ return articles[:count]
94
+
95
+ def get_doctor_news_raw(count: int = 5) -> list[dict]:
96
+ base_urls = [
97
+ "https://www.nature.com/search?q=dementia&order=date",
98
+ "https://www.nejm.org/search?q=dementia",
99
+ "https://www.thelancet.com/search?q=dementia"
100
+ ]
101
+ articles = []
102
+ for url in base_urls:
103
+ if len(articles) >= count:
104
+ break
105
+ links = scrape_article_links(url, max_links=count)
106
+ for item in links:
107
+ if len(articles) >= count:
108
+ break
109
+ articles.append(item)
110
+ return articles[:count]