File size: 11,094 Bytes
4745c27 2fdc3aa 6bc5f56 2546efa 4745c27 adcb5fd 4745c27 4534101 f6ff38a 4534101 6bc5f56 4ed5e34 6bc5f56 4745c27 6bc5f56 4745c27 2546efa adcb5fd 2546efa cf3d76d adcb5fd 6069fd3 4745c27 6bc5f56 4745c27 6bc5f56 4745c27 6bc5f56 6069fd3 4745c27 6069fd3 6bc5f56 4745c27 6bc5f56 4745c27 6bc5f56 4745c27 6bc5f56 4745c27 6bc5f56 4745c27 6bc5f56 4745c27 adcb5fd 4745c27 6bc5f56 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 | import os
import json
import tempfile
import spacy
import whisper
import pandas as pd
import librosa
import numpy as np
import firebase_admin
from firebase_admin import credentials, firestore, storage
# ================================================================
# Firebase Initialization
# ================================================================
if not firebase_admin._apps:
firebase_key = os.getenv("FIREBASE_SERVICE_ACCOUNT_JSON")
if firebase_key:
cred = credentials.Certificate(json.loads(firebase_key))
else:
cred_path = r"firebase/nms-database-3fd45-firebase-adminsdk-fbsvc-31e40a9937.json"
cred = credentials.Certificate(cred_path)
firebase_admin.initialize_app(cred, {
'storageBucket': 'nms-database-3fd45.firebasestorage.app'
})
db = firestore.client()
bucket = storage.bucket()
# Lazy-loaded model placeholders
_whisper_model = None
_nlp = None
# ================================================================
# Helper Fuctions for creating transcripts from audio files (.wav)
# ================================================================
def _load_whisper():
global _whisper_model
if _whisper_model is None:
print("Loading Whisper model...")
_whisper_model = whisper.load_model("medium")
return _whisper_model
def _load_spacy():
global _nlp
if _nlp is None:
print("Loading spaCy model...")
_nlp = spacy.load("en_core_web_sm")
return _nlp
def text_to_cha_lines(text):
text = text.replace("...", " ELLIPSIS_MARKER ")
sentences = []
current = ""
for char in text:
current += char
if char in ".!?":
sentences.append(current.strip())
current = ""
if current.strip():
sentences.append(current.strip())
sentences = [s.replace(" ELLIPSIS_MARKER ", " ... ") for s in sentences]
return sentences
def generate_mor_gra_line(doc):
mor_tokens = []
gra_tokens = []
for i, token in enumerate(doc):
if token.is_space or token.is_punct:
continue
pos = token.pos_.lower() if token.pos_ != 'PROPN' else 'n:prop'
lemma = token.lemma_.lower()
pos_map = {
'noun': 'n', 'verb': 'v', 'adj': 'adj', 'adv': 'adv',
'pron': 'pro', 'det': 'det', 'adp': 'prep', 'conj': 'conj',
'num': 'num', 'part': 'part', 'intj': 'co', 'propn': 'n:prop'
}
pos_tag = pos_map.get(pos, pos)
mor_tokens.append(f"{lemma}|{pos_tag}")
head_idx = token.head.i + 1
dep = token.dep_.lower()
gra_tokens.append(f"{head_idx}|{dep}")
return " ".join(mor_tokens), " ".join(gra_tokens)
# ================================================================
# Helper Functions for audio data pulling
# ================================================================
def extract_audio_features(wav_path):
"""Extract MFCC, ZCR, RMS from a single .wav file"""
try:
y, sr = librosa.load(wav_path, sr=None)
mfccs = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=13)
mfccs_mean = np.mean(mfccs, axis=1)
mfccs_std = np.std(mfccs, axis=1)
zcr = np.mean(librosa.feature.zero_crossing_rate(y))
rms = np.mean(librosa.feature.rms(y=y))
return np.concatenate([mfccs_mean, mfccs_std, [zcr, rms]])
except Exception as e:
print(f"Audio extraction failed for {wav_path}: {e}")
return None
def extract_text_features(cha_path):
"""Extract linguistic features from .cha file"""
try:
with open(cha_path, 'r', encoding='utf-8') as f:
lines = f.readlines()
utterances = [line.strip() for line in lines if line.startswith('*')]
if not utterances:
return None
word_count = sum(len(line.split()) for line in utterances)
sentence_count = len(utterances)
avg_words_per_sentence = word_count / sentence_count if sentence_count > 0 else 0
unique_words = len(set(word.lower().strip('.,!?;:') for line in utterances for word in line.split()))
lexical_diversity = unique_words / word_count if word_count > 0 else 0
pause_count = sum(line.count('<') + line.count('...') for line in utterances)
return np.array([word_count, sentence_count, avg_words_per_sentence, lexical_diversity, pause_count])
except Exception as e:
print(f"Text extraction failed for {cha_path}: {e}")
return None
# ================================================================
# Data loading functions
# ================================================================
def pull_audio(docID: str):
"""
Downloads .wav from Firebase Storage, fetches age/gender from Firestore,
checks for existing .cha and .txt in Firebase Storage.
If missing, generates them and uploads to storage.
Then extracts features and returns structured dict.
"""
try:
# 1. Fetch age and gender from Firestore
doc_ref = db.collection("Questionnaire").document(docID)
doc = doc_ref.get()
if not doc.exists:
print(f"Firestore document {docID} not found")
return None
data = doc.to_dict()
age = data.get("age")
gender_val = data.get("gender")
if age is None or gender_val is None:
print(f"Missing age or gender for {docID}")
return None
# Ensure gender is integer
try:
gender = int(gender_val)
except (ValueError, TypeError):
print(f"Invalid gender value '{gender_val}' for {docID} (expected 0 or 1)")
return None
if gender not in (0, 1):
print(f"Gender must be 0 or 1, got {gender} for {docID}")
return None
gender_str = "male" if gender == 1 else "female"
# 2. Ensure .wav exists
wav_blob = bucket.blob(f"audiostuff/wav/{docID}.wav")
if not wav_blob.exists():
print(f"WAV file not found in storage for {docID}")
return None
with tempfile.TemporaryDirectory() as tmpdir:
wav_path = os.path.join(tmpdir, f"{docID}.wav")
cha_path = os.path.join(tmpdir, f"{docID}.cha")
meta_path = os.path.join(tmpdir, f"{docID}.txt")
wav_blob.download_to_filename(wav_path)
# 3. Check for existing .txt and .cha in Firebase Storage
meta_blob = bucket.blob(f"audiostuff/meta/{docID}.txt")
cha_blob = bucket.blob(f"audiostuff/cha/{docID}.cha")
meta_exists = meta_blob.exists()
cha_exists = cha_blob.exists()
# 4. If meta missing → create and upload
if not meta_exists:
with open(meta_path, 'w') as f:
f.write(f"{age}\n{gender_str}\n")
meta_blob.upload_from_filename(meta_path)
print(f"Uploaded meta file for {docID}")
else:
meta_blob.download_to_filename(meta_path)
# Re-parse to ensure consistency (optional)
with open(meta_path, 'r') as f:
lines = f.read().strip().splitlines()
age = int(lines[0])
gender_str = lines[1].lower()
gender = 1 if gender_str == "male" else 0
# 5. If .cha missing → generate and upload
if not cha_exists:
print(f"Generating .cha for {docID}...")
y, sr = librosa.load(wav_path, sr=16000)
whisper_model = _load_whisper()
result = whisper_model.transcribe(
audio=y,
language="en",
suppress_tokens=[],
fp16=False
)
raw_text = result["text"].strip()
if not raw_text:
print(f"Empty transcription for {docID}")
return None
utterances = text_to_cha_lines(raw_text)
if not utterances:
return None
nlp = _load_spacy()
with open(cha_path, 'w', encoding='utf-8') as f:
f.write("@UTF8\n")
f.write("@PID:\t11312/t-00002184-1\n")
f.write("@Begin\n")
f.write("@Languages:\teng\n")
f.write("@Participants:\tPAR Participant, INV Investigator\n")
f.write(f"@ID:\teng|Pitt|PAR|??|?|Control||Participant|||\n")
f.write(f"@Media:\t{docID}, audio\n")
for idx, utt in enumerate(utterances):
if not utt.strip():
continue
clean_utt = utt.replace('"', "'").strip()
speaker = "INV" if idx == 0 else "PAR"
f.write(f'*{speaker}: {clean_utt}\n')
doc_spacy = nlp(clean_utt)
mor_line, gra_line = generate_mor_gra_line(doc_spacy)
if mor_line:
f.write(f"%mor: {mor_line}\n")
if gra_line:
f.write(f"%gra: {gra_line}\n")
f.write("@End\n")
cha_blob.upload_from_filename(cha_path)
print(f"Uploaded .cha file for {docID}")
else:
cha_blob.download_to_filename(cha_path)
# 6. Extract features
audio_features = extract_audio_features(wav_path)
text_features = extract_text_features(cha_path)
if audio_features is None or text_features is None:
return None
meta_features = np.array([age, gender, np.nan])
full_features = np.concatenate([audio_features, meta_features, text_features])
return {
'patient_id': docID,
'age': age,
'gender': gender,
'mmse': None,
'audio_features': audio_features,
'text_features': text_features,
'full_features': full_features
}
except Exception as e:
print(f"Error in pull_audio_features_from_firebase for {docID}: {e}")
return None
def pull_lifestyle(document_id: str):
"""Fetches lifestyle data from Firestore."""
# Just gets all the fields and there data from firebase
try:
doc_ref = db.collection("Questionnaire").document(document_id)
doc = doc_ref.get()
if doc.exists:
return doc.to_dict()
else:
print(f"Document {document_id} not found.")
return {}
except Exception as e:
print(f"Error fetching document {document_id}: {e}")
return {} |