plutoa386 commited on
Commit
79aae02
·
1 Parent(s): 390d754
crew.py CHANGED
@@ -5,7 +5,8 @@ import joblib
5
  import numpy as np
6
  from crewai import Agent, Task, Crew, LLM
7
  from crewai.tools import tool
8
- from dataloaderImp import *
 
9
  from typing import Union
10
  import shap
11
  import xgboost as xgb
@@ -36,10 +37,10 @@ placeholder_lifestyle_ID = "umWSG13rLSfHqqJlq1WluPzjLB93"
36
 
37
  # Define tools for the agents
38
  @tool("calculate_dementia_risk_audio")
39
- def calculate_dementia_risk_audio(docID: str, return_json: bool = False) -> Union[str, dict]:
40
  """
41
  Calculates dementia risk for a specific patient using audio and transcript data from Firebase.
42
- If return_json=True, returns a structured dict. Otherwise, returns a formatted report string.
43
  """
44
  try:
45
  # Handle JSON-wrapped input (from CrewAI)
@@ -58,9 +59,7 @@ def calculate_dementia_risk_audio(docID: str, return_json: bool = False) -> Unio
58
 
59
  if audio_data is None:
60
  error_msg = f"Error: Could not retrieve or process audio data for patient '{docID}'"
61
- if return_json:
62
- return {"status": "error", "message": error_msg}
63
- return error_msg
64
 
65
  # Extract values
66
  age = audio_data.get("age")
@@ -70,9 +69,7 @@ def calculate_dementia_risk_audio(docID: str, return_json: bool = False) -> Unio
70
 
71
  if combined_features is None:
72
  error_msg = f"Error: Feature extraction failed for patient '{docID}'"
73
- if return_json:
74
- return {"status": "error", "message": error_msg}
75
- return error_msg
76
 
77
  # Convert to DMatrix
78
  audio_array = np.array(combined_features).reshape(1, -1)
@@ -109,38 +106,11 @@ def calculate_dementia_risk_audio(docID: str, return_json: bool = False) -> Unio
109
  "docID": docID
110
  }
111
 
112
- if return_json:
113
- return {"status": "success", **result_dict}
114
-
115
- # Fallback: formatted string report (for CrewAI agents)
116
- explanation = (
117
- f"Based on speech analysis and linguistic patterns, "
118
- f"the AI model detected {'concerning patterns' if prediction == 1 else 'normal patterns'} "
119
- f"in speech fluency, vocabulary diversity, and cognitive response time."
120
- )
121
-
122
- mmse_note = "\n[Note: MMSE score not provided. Assessment based on speech and demographics only. Accuracy may be reduced.]"
123
-
124
- report = f"""
125
- DEMENTIA RISK ASSESSMENT (AUDIO)
126
- ================================
127
- Risk Level: {risk_level}
128
- Confidence: {confidence:.2%}
129
- Prediction: {'Positive' if prediction == 1 else 'Negative'}
130
- Age: {age}
131
- Gender: {gender_str}
132
- {mmse_note}
133
-
134
- EXPLANATION:
135
- {explanation}
136
- """
137
- return report
138
 
139
  except Exception as e:
140
  error_msg = f"Error calculating audio-based risk: {str(e)}"
141
- if return_json:
142
- return {"status": "error", "message": error_msg}
143
- return error_msg
144
 
145
  @tool("explain_dementia_risk_audio")
146
  def explain_dementia_risk_audio(docID: str) -> str:
@@ -178,10 +148,10 @@ def explain_dementia_risk_audio(docID: str) -> str:
178
  return f"SHAP explanation failed: {str(e)}"
179
 
180
  @tool("calculate_dementia_risk_lifestyle")
181
- def calculate_dementia_risk_lifestyle(docID: str, return_json: bool = False) -> Union[str, dict]:
182
  """
183
  Calculates dementia risk for a specific patient using lifestyle data.
184
- If return_json=True, returns a structured dict. Otherwise, returns a formatted report string.
185
  """
186
  try:
187
  # Handle JSON-wrapped input (from CrewAI)
@@ -199,9 +169,7 @@ def calculate_dementia_risk_lifestyle(docID: str, return_json: bool = False) ->
199
  lifestyle_data = pull_lifestyle(docID)
200
  if not lifestyle_data:
201
  error = f"Error: No lifestyle data found for document ID '{docID}'"
202
- if return_json:
203
- return {"status": "error", "message": error}
204
- return error
205
 
206
  FEATURE_MAPPING = {
207
  'Age': 'age',
@@ -227,15 +195,11 @@ def calculate_dementia_risk_lifestyle(docID: str, return_json: bool = False) ->
227
  firestore_key = FEATURE_MAPPING.get(feat_name)
228
  if firestore_key is None:
229
  error = f"Error: No mapping defined for feature '{feat_name}'"
230
- if return_json:
231
- return {"status": "error", "message": error}
232
- return error
233
  val = lifestyle_data.get(firestore_key)
234
  if val is None:
235
  error = f"Error: Missing required lifestyle feature '{feat_name}' (Firestore key: '{firestore_key}')"
236
- if return_json:
237
- return {"status": "error", "message": error}
238
- return error
239
  features.append(val)
240
 
241
  # Convert to DMatrix
@@ -272,35 +236,11 @@ def calculate_dementia_risk_lifestyle(docID: str, return_json: bool = False) ->
272
  "docID": docID
273
  }
274
 
275
- if return_json:
276
- return {"status": "success", **result_dict}
277
-
278
- # Original string report
279
- explanation = (
280
- f"Based on lifestyle factors including diet, physical activity, and medical history, "
281
- f"the AI model detected {'concerning patterns' if prediction == 1 else 'normal patterns'} "
282
- f"associated with increased dementia risk."
283
- )
284
-
285
- report = f"""
286
- DEMENTIA RISK ASSESSMENT (LIFESTYLE)
287
- ===================================
288
- Risk Level: {risk_level}
289
- Confidence: {confidence:.2%}
290
- Prediction: {'Positive' if prediction == 1 else 'Negative'}
291
- Age: {age}
292
- Gender: {gender_str}
293
-
294
- EXPLANATION:
295
- {explanation}
296
- """
297
- return report
298
 
299
  except Exception as e:
300
  error_msg = f"Error calculating risk: {str(e)}"
301
- if return_json:
302
- return {"status": "error", "message": error_msg}
303
- return error_msg
304
 
305
  @tool("explain_dementia_risk_lifestyle")
306
  def explain_dementia_risk_lifestyle(docID: str) -> str:
@@ -359,17 +299,18 @@ def explain_dementia_risk_lifestyle(docID: str) -> str:
359
  return f"SHAP explanation failed: {str(e)}"
360
 
361
  @tool("gather_news_patient")
362
- def gather_news_patient(return_json: bool = False):
363
  """
364
  Gathers dementia news articles, news articles that are related to and important to patients.
365
- If return_json=True, returns a structured dict. Otherwise, returns a formatted report string.
366
  """
367
  return "patient news"
 
368
  @tool("gather_news_doctor")
369
- def gather_news_doctor(return_json: bool = False):
370
  """
371
  Gathers dementia news articles, news articles that are related to and important to doctors.
372
- If return_json=True, returns a structured dict. Otherwise, returns a formatted report string.
373
  """
374
  return "doctor news"
375
 
@@ -479,7 +420,7 @@ def run_crewai_lifestyle(docID: str) -> dict:
479
  if not lifestyle_data:
480
  return {"status": "error", "message": f"No lifestyle data for {docID}"}
481
 
482
- risk_result = calculate_dementia_risk_lifestyle.run(docID, return_json=True)
483
  if risk_result["status"] != "success":
484
  return risk_result
485
  shap_explanation = explain_dementia_risk_lifestyle.run(docID)
@@ -514,7 +455,7 @@ def run_crewai_audio(docID: str) -> dict:
514
  raw_output = crew_audio.kickoff()
515
 
516
  # Get structured result
517
- risk_result = calculate_dementia_risk_audio.run(docID, return_json=True)
518
  if risk_result["status"] != "success":
519
  return risk_result
520
  shap_explanation = explain_dementia_risk_audio.run(docID)
 
5
  import numpy as np
6
  from crewai import Agent, Task, Crew, LLM
7
  from crewai.tools import tool
8
+ from firebaseLoader import *
9
+ from webscraper import search_dementia_news
10
  from typing import Union
11
  import shap
12
  import xgboost as xgb
 
37
 
38
  # Define tools for the agents
39
  @tool("calculate_dementia_risk_audio")
40
+ def calculate_dementia_risk_audio(docID: str) -> Union[str, dict]:
41
  """
42
  Calculates dementia risk for a specific patient using audio and transcript data from Firebase.
43
+ Returns a structured dict.
44
  """
45
  try:
46
  # Handle JSON-wrapped input (from CrewAI)
 
59
 
60
  if audio_data is None:
61
  error_msg = f"Error: Could not retrieve or process audio data for patient '{docID}'"
62
+ return {"status": "error", "message": error_msg}
 
 
63
 
64
  # Extract values
65
  age = audio_data.get("age")
 
69
 
70
  if combined_features is None:
71
  error_msg = f"Error: Feature extraction failed for patient '{docID}'"
72
+ return {"status": "error", "message": error_msg}
 
 
73
 
74
  # Convert to DMatrix
75
  audio_array = np.array(combined_features).reshape(1, -1)
 
106
  "docID": docID
107
  }
108
 
109
+ return {"status": "success", **result_dict}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
110
 
111
  except Exception as e:
112
  error_msg = f"Error calculating audio-based risk: {str(e)}"
113
+ return {"status": "error", "message": error_msg}
 
 
114
 
115
  @tool("explain_dementia_risk_audio")
116
  def explain_dementia_risk_audio(docID: str) -> str:
 
148
  return f"SHAP explanation failed: {str(e)}"
149
 
150
  @tool("calculate_dementia_risk_lifestyle")
151
+ def calculate_dementia_risk_lifestyle(docID: str) -> Union[str, dict]:
152
  """
153
  Calculates dementia risk for a specific patient using lifestyle data.
154
+ Returns a structured dict
155
  """
156
  try:
157
  # Handle JSON-wrapped input (from CrewAI)
 
169
  lifestyle_data = pull_lifestyle(docID)
170
  if not lifestyle_data:
171
  error = f"Error: No lifestyle data found for document ID '{docID}'"
172
+ return {"status": "error", "message": error}
 
 
173
 
174
  FEATURE_MAPPING = {
175
  'Age': 'age',
 
195
  firestore_key = FEATURE_MAPPING.get(feat_name)
196
  if firestore_key is None:
197
  error = f"Error: No mapping defined for feature '{feat_name}'"
198
+ return {"status": "error", "message": error}
 
 
199
  val = lifestyle_data.get(firestore_key)
200
  if val is None:
201
  error = f"Error: Missing required lifestyle feature '{feat_name}' (Firestore key: '{firestore_key}')"
202
+ return {"status": "error", "message": error}
 
 
203
  features.append(val)
204
 
205
  # Convert to DMatrix
 
236
  "docID": docID
237
  }
238
 
239
+ return {"status": "success", **result_dict}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
240
 
241
  except Exception as e:
242
  error_msg = f"Error calculating risk: {str(e)}"
243
+ return {"status": "error", "message": error_msg}
 
 
244
 
245
  @tool("explain_dementia_risk_lifestyle")
246
  def explain_dementia_risk_lifestyle(docID: str) -> str:
 
299
  return f"SHAP explanation failed: {str(e)}"
300
 
301
  @tool("gather_news_patient")
302
+ def gather_news_patient():
303
  """
304
  Gathers dementia news articles, news articles that are related to and important to patients.
305
+ Returns a structured dict.
306
  """
307
  return "patient news"
308
+
309
  @tool("gather_news_doctor")
310
+ def gather_news_doctor(max_count = 4):
311
  """
312
  Gathers dementia news articles, news articles that are related to and important to doctors.
313
+ Returns a structured dict.
314
  """
315
  return "doctor news"
316
 
 
420
  if not lifestyle_data:
421
  return {"status": "error", "message": f"No lifestyle data for {docID}"}
422
 
423
+ risk_result = calculate_dementia_risk_lifestyle.run(docID)
424
  if risk_result["status"] != "success":
425
  return risk_result
426
  shap_explanation = explain_dementia_risk_lifestyle.run(docID)
 
455
  raw_output = crew_audio.kickoff()
456
 
457
  # Get structured result
458
+ risk_result = calculate_dementia_risk_audio.run(docID)
459
  if risk_result["status"] != "success":
460
  return risk_result
461
  shap_explanation = explain_dementia_risk_audio.run(docID)
dataloaderImp.py → firebaseLoader.py RENAMED
File without changes
requirements.txt CHANGED
@@ -13,4 +13,5 @@ openai-whisper
13
  en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl
14
  shap
15
  xgboost==3.0.5
16
- litellm
 
 
13
  en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl
14
  shap
15
  xgboost==3.0.5
16
+ litellm
17
+ litellm[proxy]
webscraper.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import requests
2
+ from bs4 import BeautifulSoup
3
+ from duckduckgo_search import DDGS
4
+ import time
5
+ import re
6
+ from typing import List, Dict
7
+
8
+ def clean_text(text: str) -> str:
9
+ """Clean and normalize text: remove extra whitespace, fix line breaks."""
10
+ if not text:
11
+ return ""
12
+ text = re.sub(r'\s+', ' ', text).strip()
13
+ return text[:1000]
14
+
15
+ def scrape_article_content(url: str, timeout: int = 8) -> str:
16
+ """Scrape main article text using requests + BeautifulSoup."""
17
+ try:
18
+ headers = {
19
+ "User-Agent": "Mozilla/5.0 (compatible; DementiaNewsBot/1.0; +https://huggingface.co/spaces/plutoa386/...)"
20
+ }
21
+ response = requests.get(url, headers=headers, timeout=timeout)
22
+ response.raise_for_status()
23
+
24
+ soup = BeautifulSoup(response.content, "lxml")
25
+
26
+ # Remove noisy elements
27
+ for elem in soup(["script", "style", "nav", "footer", "header", "aside", "form"]):
28
+ elem.decompose()
29
+
30
+ # Try to find main content
31
+ article = (
32
+ soup.find("article") or
33
+ soup.find("div", class_=re.compile(r"article|post|content|story", re.I)) or
34
+ soup.find("main") or
35
+ soup.find("div", id=re.compile(r"content|article", re.I)) or
36
+ soup
37
+ )
38
+
39
+ # Extract paragraphs
40
+ paragraphs = article.find_all("p")
41
+ text = " ".join([
42
+ clean_text(p.get_text())
43
+ for p in paragraphs
44
+ if len(p.get_text().strip()) > 30 # skip tiny lines
45
+ ])
46
+ return text[:1500] # keep it concise for LLM
47
+
48
+ except Exception as e:
49
+ return f"[Content extraction failed: {type(e).__name__}]"
50
+
51
+ def search_dementia_news(audience: str = "patient", max_results: int = 3) -> List[Dict]:
52
+ """
53
+ Returns list of news items: {title, url, snippet}
54
+ Uses DuckDuckGo for ethical, free, no-API-key search.
55
+ """
56
+ try:
57
+ # Tailored queries
58
+ if audience == "patient":
59
+ query = (
60
+ '"dementia" OR "Alzheimer\'s disease" '
61
+ '(treatment OR care OR support OR lifestyle OR symptoms OR prevention OR early warning) '
62
+ 'site:alz.org OR site:mayoclinic.org OR site:nia.nih.gov OR site:webmd.com OR site:healthline.com'
63
+ )
64
+ else: # doctor
65
+ query = (
66
+ '"dementia" OR "Alzheimer\'s" '
67
+ '(clinical trial OR biomarker OR diagnosis OR guideline OR research OR drug approval) '
68
+ 'site:nejm.org OR site:the-lancet.com OR site:fda.gov OR site:medscape.com OR site:alzheimersresearchuk.org'
69
+ )
70
+
71
+ results = []
72
+ with DDGS() as ddgs:
73
+ raw_results = list(ddgs.text(
74
+ query,
75
+ region="wt-wt",
76
+ safesearch="off",
77
+ timelimit="m", # past month
78
+ max_results=max_results + 2 # fetch a few extra for filtering
79
+ ))
80
+
81
+ for r in raw_results:
82
+ url = r.get("href", "").strip()
83
+ # Skip non-http or social media
84
+ if not url.startswith("http") or any(domain in url for domain in ["facebook.com", "twitter.com", "youtube.com", "linkedin.com"]):
85
+ continue
86
+ title = clean_text(r.get("title", ""))
87
+ snippet = clean_text(r.get("body", ""))
88
+ if len(title) < 10 or len(snippet) < 20:
89
+ continue
90
+ results.append({"title": title, "url": url, "snippet": snippet})
91
+ if len(results) >= max_results:
92
+ break
93
+
94
+ time.sleep(0.4) # be respectful
95
+
96
+ return results
97
+
98
+ except Exception as e:
99
+ return [{"error": f"Search failed: {str(e)}"}]