Spaces:
Runtime error
Runtime error
revamped webscraping logic
Browse files- app.py +0 -12
- crew.py +138 -140
- webscraper.py +94 -83
app.py
CHANGED
|
@@ -32,18 +32,6 @@ def news_doctor(count: int = 5):
|
|
| 32 |
except Exception as e:
|
| 33 |
return {"status": "error", "message": str(e)}
|
| 34 |
|
| 35 |
-
def assess_risk(docID: str, mode: str):
|
| 36 |
-
if mode == "audio":
|
| 37 |
-
return assess_audio(docID)
|
| 38 |
-
elif mode == "lifestyle":
|
| 39 |
-
return news_doctor()
|
| 40 |
-
elif mode == "doctornews":
|
| 41 |
-
return news_doctor()
|
| 42 |
-
elif mode == "patientnews":
|
| 43 |
-
return news_patient()
|
| 44 |
-
else:
|
| 45 |
-
return {"status": "error", "message": f"option '{mode}' not found"}
|
| 46 |
-
|
| 47 |
with gr.Blocks() as demo:
|
| 48 |
gr.Markdown("# 🧠 NMS Dementia Risk Assessment API - Development Zone")
|
| 49 |
|
|
|
|
| 32 |
except Exception as e:
|
| 33 |
return {"status": "error", "message": str(e)}
|
| 34 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
with gr.Blocks() as demo:
|
| 36 |
gr.Markdown("# 🧠 NMS Dementia Risk Assessment API - Development Zone")
|
| 37 |
|
crew.py
CHANGED
|
@@ -6,7 +6,7 @@ import numpy as np
|
|
| 6 |
from crewai import Agent, Task, Crew, LLM
|
| 7 |
from crewai.tools import tool
|
| 8 |
from firebaseLoader import *
|
| 9 |
-
from webscraper import
|
| 10 |
from typing import Union
|
| 11 |
import shap
|
| 12 |
import xgboost as xgb
|
|
@@ -34,6 +34,9 @@ lifestyle_explainer = shap.TreeExplainer(lifestyle_model)
|
|
| 34 |
|
| 35 |
placeholder_lifestyle_ID = "umWSG13rLSfHqqJlq1WluPzjLB93"
|
| 36 |
|
|
|
|
|
|
|
|
|
|
| 37 |
|
| 38 |
# Define tools for the agents
|
| 39 |
@tool("calculate_dementia_risk_audio")
|
|
@@ -301,48 +304,98 @@ def explain_dementia_risk_lifestyle(docID: str) -> str:
|
|
| 301 |
@tool("gather_news_patient")
|
| 302 |
def gather_news_patient(count: int = 5) -> dict:
|
| 303 |
"""
|
| 304 |
-
|
|
|
|
|
|
|
| 305 |
"""
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 321 |
|
| 322 |
@tool("gather_news_doctor")
|
| 323 |
def gather_news_doctor(count: int = 5) -> dict:
|
| 324 |
"""
|
| 325 |
-
|
|
|
|
| 326 |
"""
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
for art in articles:
|
| 331 |
-
if "error" not in art:
|
| 332 |
-
try:
|
| 333 |
-
content = scrape_article_content(art["url"])
|
| 334 |
-
clean_articles.append({
|
| 335 |
-
"title": art["title"],
|
| 336 |
-
"url": art["url"],
|
| 337 |
-
"content": content[:3000]
|
| 338 |
-
})
|
| 339 |
-
except Exception:
|
| 340 |
-
continue
|
| 341 |
-
return {"articles": clean_articles}
|
| 342 |
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 346 |
|
| 347 |
# Define agents for Alzheimer's detection system
|
| 348 |
risk_calculator_audio = Agent(
|
|
@@ -363,22 +416,22 @@ risk_calculator_lifestyle = Agent(
|
|
| 363 |
tools=[calculate_dementia_risk_lifestyle, explain_dementia_risk_lifestyle]
|
| 364 |
)
|
| 365 |
|
| 366 |
-
|
| 367 |
role="Patient News Agent",
|
| 368 |
-
goal="Gather
|
| 369 |
-
backstory="You find
|
| 370 |
-
|
| 371 |
llm=llm_news,
|
| 372 |
-
|
| 373 |
)
|
| 374 |
|
| 375 |
-
|
| 376 |
role="Doctor News Agent",
|
| 377 |
-
goal="Gather
|
| 378 |
-
backstory="You
|
| 379 |
-
|
| 380 |
llm=llm_news,
|
| 381 |
-
|
| 382 |
)
|
| 383 |
|
| 384 |
# Define tasks for agents
|
|
@@ -408,48 +461,20 @@ explanation_task_lifestyle = Task(
|
|
| 408 |
|
| 409 |
get_news_task_patient = Task(
|
| 410 |
description=(
|
| 411 |
-
"
|
| 412 |
-
"
|
| 413 |
-
"3. For EACH article, read its 'content' field and generate a concise 2–3 line summary that highlights key takeaways for patients/families.\n"
|
| 414 |
-
"4. Ensure summaries are empathetic, jargon-free, and actionable.\n"
|
| 415 |
-
"5. Return a JSON object with keys: 'titles', 'snippets', 'urls' — matching the order of the fetched articles.\n"
|
| 416 |
-
"6. DO NOT invent titles or URLs — use only what was returned by the tool.\n"
|
| 417 |
-
"7. Return ONLY valid JSON — no markdown, no extra text."
|
| 418 |
),
|
| 419 |
-
expected_output=
|
| 420 |
-
|
| 421 |
-
"- 'titles': list of original article titles (strings).\n"
|
| 422 |
-
"- 'snippets': list of 2–3 line plain-language summaries (strings), one per article.\n"
|
| 423 |
-
"- 'urls': list of original article URLs (strings).\n"
|
| 424 |
-
"All lists must be same length and aligned by index.\n"
|
| 425 |
-
"Return ONLY valid JSON — no extra text."
|
| 426 |
-
),
|
| 427 |
-
agent=patient_news
|
| 428 |
)
|
| 429 |
|
| 430 |
get_news_task_doctor = Task(
|
| 431 |
description=(
|
| 432 |
-
"
|
| 433 |
-
"
|
| 434 |
-
"3. For EACH article, read its 'content' field and generate a concise 2–3 line summary.\n"
|
| 435 |
-
"4. Summaries must be clinical, evidence-focused, and highlight:\n"
|
| 436 |
-
" - Trial design/outcomes (if applicable)\n"
|
| 437 |
-
" - Diagnostic/therapeutic implications\n"
|
| 438 |
-
" - Guideline updates or expert consensus changes\n"
|
| 439 |
-
" - Biomarker or mechanistic insights\n"
|
| 440 |
-
"5. DO NOT invent titles, URLs, or content — use only what the tool returns.\n"
|
| 441 |
-
"6. Return a JSON object with keys: 'titles', 'snippets', 'urls', in matching order.\n"
|
| 442 |
-
"7. Return ONLY valid JSON — no markdown, no commentary."
|
| 443 |
-
),
|
| 444 |
-
expected_output=(
|
| 445 |
-
"A JSON object with keys: 'titles', 'snippets', 'urls'.\n"
|
| 446 |
-
"- 'titles': list of article titles.\n"
|
| 447 |
-
"- 'snippets': list of 2–3 line clinical summaries.\n"
|
| 448 |
-
"- 'urls': list of original URLs.\n"
|
| 449 |
-
"All lists same length, aligned by index.\n"
|
| 450 |
-
"Return ONLY valid JSON — no extra text."
|
| 451 |
),
|
| 452 |
-
|
|
|
|
| 453 |
)
|
| 454 |
|
| 455 |
# Human-readable mapping for audio features
|
|
@@ -688,63 +713,36 @@ def run_crewai_audio(docID: str) -> dict:
|
|
| 688 |
return {"status": "error", "message": str(e)}
|
| 689 |
|
| 690 |
def run_crewai_news_patient(count: int = 5) -> dict:
|
| 691 |
-
|
| 692 |
-
|
| 693 |
-
|
| 694 |
-
|
| 695 |
-
|
| 696 |
-
|
| 697 |
-
|
| 698 |
-
|
| 699 |
-
|
| 700 |
-
|
| 701 |
-
|
| 702 |
-
|
| 703 |
-
|
| 704 |
-
|
| 705 |
-
|
| 706 |
-
|
| 707 |
-
raise ValueError("Mismatched list lengths")
|
| 708 |
-
|
| 709 |
-
return {"status": "success", **result}
|
| 710 |
-
|
| 711 |
-
except Exception as e:
|
| 712 |
-
return {"status": "error", "message": str(e)}
|
| 713 |
|
| 714 |
def run_crewai_news_doctor(count: int = 5) -> dict:
|
| 715 |
-
|
| 716 |
-
|
| 717 |
-
|
| 718 |
-
|
| 719 |
-
|
| 720 |
-
|
| 721 |
-
|
| 722 |
-
|
| 723 |
-
|
| 724 |
-
|
| 725 |
-
|
| 726 |
-
|
| 727 |
-
|
| 728 |
-
|
| 729 |
-
|
| 730 |
-
function_calling=False
|
| 731 |
-
)
|
| 732 |
-
raw_output = str(crew.kickoff()).strip()
|
| 733 |
-
|
| 734 |
-
# Restore original tool
|
| 735 |
-
doctor_news.tools = [original_gather]
|
| 736 |
-
|
| 737 |
-
# Parse JSON
|
| 738 |
-
result = _parse_json_output(raw_output)
|
| 739 |
-
|
| 740 |
-
# Validate structure
|
| 741 |
-
for key in ["titles", "snippets", "urls"]:
|
| 742 |
-
if key not in result or not isinstance(result[key], list):
|
| 743 |
-
raise ValueError(f"Missing or invalid '{key}' in result")
|
| 744 |
-
if not (len(result["titles"]) == len(result["snippets"]) == len(result["urls"])):
|
| 745 |
-
raise ValueError("Mismatched list lengths in doctor news result")
|
| 746 |
-
|
| 747 |
-
return {"status": "success", **result}
|
| 748 |
-
|
| 749 |
-
except Exception as e:
|
| 750 |
-
return {"status": "error", "message": str(e)}
|
|
|
|
| 6 |
from crewai import Agent, Task, Crew, LLM
|
| 7 |
from crewai.tools import tool
|
| 8 |
from firebaseLoader import *
|
| 9 |
+
from webscraper import *
|
| 10 |
from typing import Union
|
| 11 |
import shap
|
| 12 |
import xgboost as xgb
|
|
|
|
| 34 |
|
| 35 |
placeholder_lifestyle_ID = "umWSG13rLSfHqqJlq1WluPzjLB93"
|
| 36 |
|
| 37 |
+
# Load 2 different llms for different purposes.
|
| 38 |
+
llm_predict = LLM(model="huggingface/openai/gpt-oss-120b")
|
| 39 |
+
llm_news = LLM(model="huggingface/meta-llama/Llama-3.1-8B-Instruct")
|
| 40 |
|
| 41 |
# Define tools for the agents
|
| 42 |
@tool("calculate_dementia_risk_audio")
|
|
|
|
| 304 |
@tool("gather_news_patient")
|
| 305 |
def gather_news_patient(count: int = 5) -> dict:
|
| 306 |
"""
|
| 307 |
+
Fetches count recent dementia news items for patients, retrieves full text,
|
| 308 |
+
and generates patient-friendly summaries using the LLM.
|
| 309 |
+
Returns: {"status": "success", "articles": [{"title", "summary", "url"}, ...]}
|
| 310 |
"""
|
| 311 |
+
try:
|
| 312 |
+
raw_articles = get_patient_news_raw(count=count)
|
| 313 |
+
results = []
|
| 314 |
+
|
| 315 |
+
for item in raw_articles:
|
| 316 |
+
url = item["url"]
|
| 317 |
+
title = item["title"]
|
| 318 |
+
full_text = scrape_full_text(url)
|
| 319 |
+
|
| 320 |
+
# ✅ LLM summarization happens HERE (using agent's llm or shared llm_news)
|
| 321 |
+
if not full_text.strip():
|
| 322 |
+
summary = "Summary unavailable."
|
| 323 |
+
else:
|
| 324 |
+
# Use llm_news — ensure it's in scope
|
| 325 |
+
prompt = f"""You are a compassionate health educator.
|
| 326 |
+
Summarize the following article in 2–3 clear, empathetic sentences for patients/families affected by dementia.
|
| 327 |
+
Focus on: What does this mean? What can they do? Hope & support.
|
| 328 |
+
Avoid jargon.
|
| 329 |
+
|
| 330 |
+
Article:
|
| 331 |
+
{full_text[:3000]}"""
|
| 332 |
+
|
| 333 |
+
response = llm_news.invoke(prompt) # ← YOUR LLM (e.g., LiteLLM-wrapped Llama 3.1)
|
| 334 |
+
summary = str(response).strip()
|
| 335 |
+
|
| 336 |
+
results.append({
|
| 337 |
+
"title": title,
|
| 338 |
+
"summary": summary,
|
| 339 |
+
"url": url
|
| 340 |
+
})
|
| 341 |
+
|
| 342 |
+
return {
|
| 343 |
+
"status": "success",
|
| 344 |
+
"articles": results
|
| 345 |
+
}
|
| 346 |
+
|
| 347 |
+
except Exception as e:
|
| 348 |
+
return {
|
| 349 |
+
"status": "error",
|
| 350 |
+
"message": str(e),
|
| 351 |
+
"articles": []
|
| 352 |
+
}
|
| 353 |
|
| 354 |
@tool("gather_news_doctor")
|
| 355 |
def gather_news_doctor(count: int = 5) -> dict:
|
| 356 |
"""
|
| 357 |
+
Fetches count recent dementia news items for doctors, retrieves full text,
|
| 358 |
+
and generates clinical summaries using the LLM.
|
| 359 |
"""
|
| 360 |
+
try:
|
| 361 |
+
raw_articles = get_doctor_news_raw(count=count)
|
| 362 |
+
results = []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 363 |
|
| 364 |
+
for item in raw_articles:
|
| 365 |
+
url = item["url"]
|
| 366 |
+
title = item["title"]
|
| 367 |
+
full_text = scrape_full_text(url)
|
| 368 |
+
|
| 369 |
+
if not full_text.strip():
|
| 370 |
+
summary = "Summary unavailable."
|
| 371 |
+
else:
|
| 372 |
+
prompt = f"""You are a clinical expert.
|
| 373 |
+
Summarize the following article in 2–3 precise, evidence-based sentences for physicians.
|
| 374 |
+
Include: key findings, methods (if applicable), clinical relevance.
|
| 375 |
+
|
| 376 |
+
Article:
|
| 377 |
+
{full_text[:3000]}"""
|
| 378 |
+
|
| 379 |
+
response = llm_news.invoke(prompt)
|
| 380 |
+
summary = str(response).strip()
|
| 381 |
+
|
| 382 |
+
results.append({
|
| 383 |
+
"title": title,
|
| 384 |
+
"summary": summary,
|
| 385 |
+
"url": url
|
| 386 |
+
})
|
| 387 |
+
|
| 388 |
+
return {
|
| 389 |
+
"status": "success",
|
| 390 |
+
"articles": results
|
| 391 |
+
}
|
| 392 |
+
|
| 393 |
+
except Exception as e:
|
| 394 |
+
return {
|
| 395 |
+
"status": "error",
|
| 396 |
+
"message": str(e),
|
| 397 |
+
"articles": []
|
| 398 |
+
}
|
| 399 |
|
| 400 |
# Define agents for Alzheimer's detection system
|
| 401 |
risk_calculator_audio = Agent(
|
|
|
|
| 416 |
tools=[calculate_dementia_risk_lifestyle, explain_dementia_risk_lifestyle]
|
| 417 |
)
|
| 418 |
|
| 419 |
+
patient_news_agent = Agent(
|
| 420 |
role="Patient News Agent",
|
| 421 |
+
goal="Gather and summarize dementia/caregiving news for patients and families in plain, supportive language.",
|
| 422 |
+
backstory="You find real, current articles from trusted sites like caregiving.org, extract full content, and use your medical communication expertise to make it clear and kind for non-experts.",
|
| 423 |
+
tools=[gather_news_patient],
|
| 424 |
llm=llm_news,
|
| 425 |
+
verbose=True
|
| 426 |
)
|
| 427 |
|
| 428 |
+
doctor_news_agent = Agent(
|
| 429 |
role="Doctor News Agent",
|
| 430 |
+
goal="Gather and summarize high-impact dementia research for clinicians and scientists.",
|
| 431 |
+
backstory="You retrieve peer-reviewed research (e.g., from Europe PMC), extract key findings, and distill them into clinically actionable summaries for time-pressed professionals.",
|
| 432 |
+
tools=[gather_news_doctor],
|
| 433 |
llm=llm_news,
|
| 434 |
+
verbose=True
|
| 435 |
)
|
| 436 |
|
| 437 |
# Define tasks for agents
|
|
|
|
| 461 |
|
| 462 |
get_news_task_patient = Task(
|
| 463 |
description=(
|
| 464 |
+
"Use the `gather_news_patient` tool to fetch {count} recent dementia/caregiving news articles "
|
| 465 |
+
"and return the raw JSON result. Do not generate or modify content."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 466 |
),
|
| 467 |
+
expected_output="A JSON object with 'status' and 'articles': list of {title, summary, url}.",
|
| 468 |
+
agent=patient_news_agent
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 469 |
)
|
| 470 |
|
| 471 |
get_news_task_doctor = Task(
|
| 472 |
description=(
|
| 473 |
+
"Use the `gather_news_doctor` tool to fetch {count} recent dementia research articles "
|
| 474 |
+
"and return the raw JSON result."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 475 |
),
|
| 476 |
+
expected_output="A JSON object with 'status' and 'articles': list of {title, summary, url}.",
|
| 477 |
+
agent=doctor_news_agent
|
| 478 |
)
|
| 479 |
|
| 480 |
# Human-readable mapping for audio features
|
|
|
|
| 713 |
return {"status": "error", "message": str(e)}
|
| 714 |
|
| 715 |
def run_crewai_news_patient(count: int = 5) -> dict:
|
| 716 |
+
crew = Crew(
|
| 717 |
+
agents=[patient_news_agent],
|
| 718 |
+
tasks=[get_news_task_patient],
|
| 719 |
+
verbose=True,
|
| 720 |
+
function_calling=False
|
| 721 |
+
)
|
| 722 |
+
result = crew.kickoff(inputs={"count": count})
|
| 723 |
+
# CrewAI returns ToolOutput → .raw or str — parse if needed
|
| 724 |
+
if isinstance(result, dict):
|
| 725 |
+
return result
|
| 726 |
+
else:
|
| 727 |
+
try:
|
| 728 |
+
import json
|
| 729 |
+
return json.loads(str(result))
|
| 730 |
+
except:
|
| 731 |
+
return {"status": "error", "message": "Failed to parse crew output", "raw": str(result)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 732 |
|
| 733 |
def run_crewai_news_doctor(count: int = 5) -> dict:
|
| 734 |
+
crew = Crew(
|
| 735 |
+
agents=[doctor_news_agent],
|
| 736 |
+
tasks=[get_news_task_doctor],
|
| 737 |
+
verbose=True,
|
| 738 |
+
function_calling=False
|
| 739 |
+
)
|
| 740 |
+
result = crew.kickoff(inputs={"count": count})
|
| 741 |
+
if isinstance(result, dict):
|
| 742 |
+
return result
|
| 743 |
+
else:
|
| 744 |
+
try:
|
| 745 |
+
import json
|
| 746 |
+
return json.loads(str(result))
|
| 747 |
+
except:
|
| 748 |
+
return {"status": "error", "message": "Failed to parse crew output", "raw": str(result)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
webscraper.py
CHANGED
|
@@ -1,99 +1,110 @@
|
|
|
|
|
| 1 |
import requests
|
| 2 |
from bs4 import BeautifulSoup
|
| 3 |
-
from duckduckgo_search import DDGS
|
| 4 |
import time
|
| 5 |
-
import
|
| 6 |
-
from typing import List, Dict
|
| 7 |
|
| 8 |
-
def clean_text(text
|
| 9 |
-
"
|
| 10 |
-
if not text:
|
| 11 |
-
return ""
|
| 12 |
-
text = re.sub(r'\s+', ' ', text).strip()
|
| 13 |
-
return text[:1000]
|
| 14 |
|
| 15 |
-
def
|
| 16 |
-
"""Scrape
|
|
|
|
|
|
|
|
|
|
| 17 |
try:
|
| 18 |
-
|
| 19 |
-
"User-Agent": "Mozilla/5.0 (compatible; DementiaNewsBot/1.0; +https://huggingface.co/spaces/plutoa386/...)"
|
| 20 |
-
}
|
| 21 |
-
response = requests.get(url, headers=headers, timeout=timeout)
|
| 22 |
response.raise_for_status()
|
|
|
|
| 23 |
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
# Remove noisy elements
|
| 27 |
-
for elem in soup(["script", "style", "nav", "footer", "header", "aside", "form"]):
|
| 28 |
-
elem.decompose()
|
| 29 |
-
|
| 30 |
-
# Try to find main content
|
| 31 |
-
article = (
|
| 32 |
-
soup.find("article") or
|
| 33 |
-
soup.find("div", class_=re.compile(r"article|post|content|story", re.I)) or
|
| 34 |
-
soup.find("main") or
|
| 35 |
-
soup.find("div", id=re.compile(r"content|article", re.I)) or
|
| 36 |
-
soup
|
| 37 |
-
)
|
| 38 |
-
|
| 39 |
-
# Extract paragraphs
|
| 40 |
-
paragraphs = article.find_all("p")
|
| 41 |
-
text = " ".join([
|
| 42 |
-
clean_text(p.get_text())
|
| 43 |
-
for p in paragraphs
|
| 44 |
-
if len(p.get_text().strip()) > 30 # skip tiny lines
|
| 45 |
-
])
|
| 46 |
-
return text[:1500] # keep it concise for LLM
|
| 47 |
|
| 48 |
-
|
| 49 |
-
|
|
|
|
| 50 |
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
"""
|
| 56 |
-
try:
|
| 57 |
-
# Tailored queries
|
| 58 |
-
if audience == "patient":
|
| 59 |
-
query = (
|
| 60 |
-
'"dementia" OR "Alzheimer\'s disease" '
|
| 61 |
-
'(treatment OR care OR support OR lifestyle OR symptoms OR prevention OR early warning) '
|
| 62 |
-
'site:alz.org OR site:mayoclinic.org OR site:nia.nih.gov OR site:webmd.com OR site:healthline.com'
|
| 63 |
-
)
|
| 64 |
-
else: # doctor
|
| 65 |
-
query = (
|
| 66 |
-
'"dementia" OR "Alzheimer\'s" '
|
| 67 |
-
'(clinical trial OR biomarker OR diagnosis OR guideline OR research OR drug approval) '
|
| 68 |
-
'site:nejm.org OR site:the-lancet.com OR site:fda.gov OR site:medscape.com OR site:alzheimersresearchuk.org'
|
| 69 |
-
)
|
| 70 |
-
|
| 71 |
-
results = []
|
| 72 |
-
with DDGS() as ddgs:
|
| 73 |
-
raw_results = list(ddgs.text(
|
| 74 |
-
query,
|
| 75 |
-
region="wt-wt",
|
| 76 |
-
safesearch="off",
|
| 77 |
-
timelimit="m", # past month
|
| 78 |
-
max_results=max_results + 2 # fetch a few extra for filtering
|
| 79 |
-
))
|
| 80 |
-
|
| 81 |
-
for r in raw_results:
|
| 82 |
-
url = r.get("href", "").strip()
|
| 83 |
-
# Skip non-http or social media
|
| 84 |
-
if not url.startswith("http") or any(domain in url for domain in ["facebook.com", "twitter.com", "youtube.com", "linkedin.com"]):
|
| 85 |
continue
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
continue
|
| 90 |
-
results.append({"title": title, "url": url, "snippet": snippet})
|
| 91 |
-
if len(results) >= max_results:
|
| 92 |
-
break
|
| 93 |
|
| 94 |
-
|
| 95 |
|
| 96 |
-
return
|
| 97 |
|
| 98 |
except Exception as e:
|
| 99 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# webscraper.py
|
| 2 |
import requests
|
| 3 |
from bs4 import BeautifulSoup
|
|
|
|
| 4 |
import time
|
| 5 |
+
import os
|
|
|
|
| 6 |
|
| 7 |
+
def clean_text(text):
|
| 8 |
+
return " ".join(text.split())
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
|
| 10 |
+
def scrape_article_links(base_url: str, max_links=10) -> list[dict]:
|
| 11 |
+
"""Scrape list of {title, url} from a news listing page."""
|
| 12 |
+
headers = {
|
| 13 |
+
'User-Agent': 'Mozilla/5.0 (compatible; DementiaNewsBot/1.0)'
|
| 14 |
+
}
|
| 15 |
try:
|
| 16 |
+
response = requests.get(base_url, headers=headers, timeout=10)
|
|
|
|
|
|
|
|
|
|
| 17 |
response.raise_for_status()
|
| 18 |
+
soup = BeautifulSoup(response.content, 'lxml')
|
| 19 |
|
| 20 |
+
items = []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
|
| 22 |
+
# Based on caregiving.org structure (from your KB):
|
| 23 |
+
# Each news item is in <article> or <div class="post-item">
|
| 24 |
+
posts = soup.find_all(['article', 'div'], class_=lambda x: x and ('post' in x or 'news' in x or 'item' in x))
|
| 25 |
|
| 26 |
+
for post in posts[:max_links]:
|
| 27 |
+
# Try to get title + link
|
| 28 |
+
a_tag = post.find('a', href=True)
|
| 29 |
+
if not a_tag:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
continue
|
| 31 |
+
href = a_tag['href']
|
| 32 |
+
if not href.startswith('http'):
|
| 33 |
+
href = requests.compat.urljoin(base_url, href)
|
| 34 |
+
|
| 35 |
+
# Title: from <h2>, <h3>, or img alt
|
| 36 |
+
title_tag = post.find(['h2', 'h3', 'h4'])
|
| 37 |
+
if not title_tag:
|
| 38 |
+
img = post.find('img')
|
| 39 |
+
title = img.get('alt', '') if img else ""
|
| 40 |
+
else:
|
| 41 |
+
title = clean_text(title_tag.get_text())
|
| 42 |
+
|
| 43 |
+
if not title or title in ["Read More", "Learn More"]:
|
| 44 |
continue
|
|
|
|
|
|
|
|
|
|
| 45 |
|
| 46 |
+
items.append({"title": title, "url": href})
|
| 47 |
|
| 48 |
+
return items
|
| 49 |
|
| 50 |
except Exception as e:
|
| 51 |
+
print(f"⚠️ scrape_article_links({base_url}) failed: {e}")
|
| 52 |
+
return []
|
| 53 |
+
|
| 54 |
+
def scrape_full_text(url: str) -> str:
|
| 55 |
+
"""Return raw full article text (no summarization)."""
|
| 56 |
+
headers = {
|
| 57 |
+
'User-Agent': 'Mozilla/5.0 (compatible; DementiaNewsBot/1.0)'
|
| 58 |
+
}
|
| 59 |
+
try:
|
| 60 |
+
response = requests.get(url, headers=headers, timeout=15)
|
| 61 |
+
soup = BeautifulSoup(response.content, 'lxml')
|
| 62 |
+
for s in soup(["script", "style", "nav", "footer", "aside", "header"]):
|
| 63 |
+
s.decompose()
|
| 64 |
+
|
| 65 |
+
# Target main content
|
| 66 |
+
main = soup.find('article') or soup.find('main') or soup.find('div', id='main-content')
|
| 67 |
+
if not main:
|
| 68 |
+
main = soup.body
|
| 69 |
+
|
| 70 |
+
paras = main.find_all('p') if main else soup.find_all('p')
|
| 71 |
+
text = "\n\n".join(clean_text(p.get_text()) for p in paras if len(p.get_text().strip()) > 50)
|
| 72 |
+
return text[:8000] # Cap for token efficiency
|
| 73 |
+
|
| 74 |
+
except Exception as e:
|
| 75 |
+
print(f"⚠️ scrape_full_text({url}) failed: {e}")
|
| 76 |
+
return ""
|
| 77 |
+
|
| 78 |
+
def get_patient_news_raw(count: int = 5) -> list[dict]:
|
| 79 |
+
base_urls = [
|
| 80 |
+
"https://www.caregiving.org/news/",
|
| 81 |
+
"https://www.alz.org/news",
|
| 82 |
+
"https://www.dementiaresearch.org/latest-news"
|
| 83 |
+
]
|
| 84 |
+
articles = []
|
| 85 |
+
for url in base_urls:
|
| 86 |
+
if len(articles) >= count:
|
| 87 |
+
break
|
| 88 |
+
links = scrape_article_links(url, max_links=count)
|
| 89 |
+
for item in links:
|
| 90 |
+
if len(articles) >= count:
|
| 91 |
+
break
|
| 92 |
+
articles.append(item)
|
| 93 |
+
return articles[:count]
|
| 94 |
+
|
| 95 |
+
def get_doctor_news_raw(count: int = 5) -> list[dict]:
|
| 96 |
+
base_urls = [
|
| 97 |
+
"https://www.nature.com/search?q=dementia&order=date",
|
| 98 |
+
"https://www.nejm.org/search?q=dementia",
|
| 99 |
+
"https://www.thelancet.com/search?q=dementia"
|
| 100 |
+
]
|
| 101 |
+
articles = []
|
| 102 |
+
for url in base_urls:
|
| 103 |
+
if len(articles) >= count:
|
| 104 |
+
break
|
| 105 |
+
links = scrape_article_links(url, max_links=count)
|
| 106 |
+
for item in links:
|
| 107 |
+
if len(articles) >= count:
|
| 108 |
+
break
|
| 109 |
+
articles.append(item)
|
| 110 |
+
return articles[:count]
|