Spaces:
Configuration error
Configuration error
Upload ml/model/pron_eval.py with huggingface_hub
Browse files- ml/model/pron_eval.py +72 -16
ml/model/pron_eval.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
| 2 |
ml/model/pron_eval.py - Acoustic-Phonetic Alignment & Phonation Analysis
|
| 3 |
========================================================================
|
| 4 |
Extracts objective acoustic correlates of phonation, LPC Burg formants,
|
| 5 |
-
spectral
|
| 6 |
and explicit phonological flaw heuristics ('R' Rhotacism, 'S' Sigmatism).
|
| 7 |
"""
|
| 8 |
from __future__ import annotations
|
|
@@ -20,10 +20,11 @@ import scipy.signal
|
|
| 20 |
|
| 21 |
SR = 16000
|
| 22 |
_CTC_MODEL = "facebook/wav2vec2-base-960h"
|
|
|
|
| 23 |
|
| 24 |
|
| 25 |
# --------------------------------------------------------------------------
|
| 26 |
-
# Signal conditioning &
|
| 27 |
# --------------------------------------------------------------------------
|
| 28 |
def _apply_acoustic_conditioning(arr: np.ndarray, sr: int = SR) -> np.ndarray:
|
| 29 |
"""Apply high-pass Butterworth 60Hz filter and peak gain standardization."""
|
|
@@ -43,15 +44,48 @@ def _apply_acoustic_conditioning(arr: np.ndarray, sr: int = SR) -> np.ndarray:
|
|
| 43 |
|
| 44 |
|
| 45 |
def is_silent_or_empty(arr: np.ndarray, sr: int = SR) -> bool:
|
| 46 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
if arr is None or len(arr) == 0:
|
| 48 |
return True
|
| 49 |
dur = len(arr) / sr
|
| 50 |
-
if dur < 0.
|
| 51 |
return True
|
|
|
|
| 52 |
rms = np.sqrt(np.mean(arr**2))
|
| 53 |
max_amp = np.max(np.abs(arr))
|
| 54 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
|
| 56 |
|
| 57 |
def _load_wave(path_or_arr: Any) -> np.ndarray:
|
|
@@ -198,15 +232,40 @@ def praat_metrics(wav_path: str) -> dict:
|
|
| 198 |
|
| 199 |
|
| 200 |
# --------------------------------------------------------------------------
|
| 201 |
-
# Dynamic Programming Word Alignment
|
| 202 |
# --------------------------------------------------------------------------
|
| 203 |
def _norm(s: str) -> str:
|
| 204 |
s = s.lower().translate(str.maketrans("", "", string.punctuation))
|
| 205 |
return " ".join(s.split())
|
| 206 |
|
| 207 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 208 |
def align_words(reference: str, hypothesis: str) -> List[Dict[str, Any]]:
|
| 209 |
-
"""Dynamic programming Levenshtein word alignment with
|
| 210 |
ref_words = _norm(reference).split()
|
| 211 |
hyp_words = _norm(hypothesis).split()
|
| 212 |
|
|
@@ -222,8 +281,7 @@ def align_words(reference: str, hypothesis: str) -> List[Dict[str, Any]]:
|
|
| 222 |
for j in range(1, m + 1):
|
| 223 |
w_ref = ref_words[i - 1]
|
| 224 |
w_hyp = hyp_words[j - 1]
|
| 225 |
-
|
| 226 |
-
cost = 0 if (w_ref == w_hyp or sim >= 0.78) else 1
|
| 227 |
dp[i][j] = min(
|
| 228 |
dp[i - 1][j - 1] + cost,
|
| 229 |
dp[i - 1][j] + 1,
|
|
@@ -236,10 +294,8 @@ def align_words(reference: str, hypothesis: str) -> List[Dict[str, Any]]:
|
|
| 236 |
if i > 0 and j > 0:
|
| 237 |
w_ref = ref_words[i - 1]
|
| 238 |
w_hyp = hyp_words[j - 1]
|
| 239 |
-
sim =
|
| 240 |
-
cost = 0 if (w_ref == w_hyp or sim >= 0.78) else 1
|
| 241 |
if dp[i][j] == dp[i - 1][j - 1] + cost:
|
| 242 |
-
status = "correct" if cost == 0 else "substitution"
|
| 243 |
alignment.append({
|
| 244 |
"expected": w_ref,
|
| 245 |
"spoken": w_hyp,
|
|
@@ -301,13 +357,13 @@ def analyze_speech_flaws(
|
|
| 301 |
word_omissions = []
|
| 302 |
word_insertions = []
|
| 303 |
|
| 304 |
-
# 1. R-Sound / Rhotacism Check (
|
| 305 |
f3_f2 = praat_dict.get("f3_f2_dist_hz", 900.0)
|
| 306 |
for item in alignment:
|
| 307 |
exp = item["expected"].lower()
|
| 308 |
spk = item["spoken"].lower()
|
| 309 |
if item["status"] == "substitution" and "r" in exp:
|
| 310 |
-
if
|
| 311 |
r_errors.append({
|
| 312 |
"expected": exp,
|
| 313 |
"spoken": spk,
|
|
@@ -315,14 +371,14 @@ def analyze_speech_flaws(
|
|
| 315 |
"message": f"Trouble with 'r' sound: said '{spk}' instead of '{exp}' (Acoustic F3-F2 gap: {f3_f2:.0f} Hz)"
|
| 316 |
})
|
| 317 |
|
| 318 |
-
# 2. S-Sound / Sigmatism / Lisping Check
|
| 319 |
centroid = praat_dict.get("spectral_centroid_hz", 1200.0)
|
| 320 |
s_markers = ["s", "z", "sh", "ch"]
|
| 321 |
for item in alignment:
|
| 322 |
exp = item["expected"].lower()
|
| 323 |
spk = item["spoken"].lower()
|
| 324 |
if item["status"] == "substitution" and any(m in exp for m in s_markers):
|
| 325 |
-
if
|
| 326 |
s_errors.append({
|
| 327 |
"expected": exp,
|
| 328 |
"spoken": spk,
|
|
|
|
| 2 |
ml/model/pron_eval.py - Acoustic-Phonetic Alignment & Phonation Analysis
|
| 3 |
========================================================================
|
| 4 |
Extracts objective acoustic correlates of phonation, LPC Burg formants,
|
| 5 |
+
spectral flatness VAD, dynamic programming Needleman-Wunsch word alignment,
|
| 6 |
and explicit phonological flaw heuristics ('R' Rhotacism, 'S' Sigmatism).
|
| 7 |
"""
|
| 8 |
from __future__ import annotations
|
|
|
|
| 20 |
|
| 21 |
SR = 16000
|
| 22 |
_CTC_MODEL = "facebook/wav2vec2-base-960h"
|
| 23 |
+
_WHISPER_MODEL = "openai/whisper-tiny"
|
| 24 |
|
| 25 |
|
| 26 |
# --------------------------------------------------------------------------
|
| 27 |
+
# Signal conditioning & Enhanced Multi-Factor VAD
|
| 28 |
# --------------------------------------------------------------------------
|
| 29 |
def _apply_acoustic_conditioning(arr: np.ndarray, sr: int = SR) -> np.ndarray:
|
| 30 |
"""Apply high-pass Butterworth 60Hz filter and peak gain standardization."""
|
|
|
|
| 44 |
|
| 45 |
|
| 46 |
def is_silent_or_empty(arr: np.ndarray, sr: int = SR) -> bool:
|
| 47 |
+
"""
|
| 48 |
+
Enhanced Multi-Factor Voice Activity Detection (VAD):
|
| 49 |
+
1. RMS Energy & Peak Amplitude Thresholding
|
| 50 |
+
2. Spectral Flatness (Wiener entropy) to reject fan hiss / white noise
|
| 51 |
+
3. Autocorrelation Voicing Peak to verify human harmonic pitch periodicity
|
| 52 |
+
"""
|
| 53 |
if arr is None or len(arr) == 0:
|
| 54 |
return True
|
| 55 |
dur = len(arr) / sr
|
| 56 |
+
if dur < 0.18: # Audio shorter than 180ms
|
| 57 |
return True
|
| 58 |
+
|
| 59 |
rms = np.sqrt(np.mean(arr**2))
|
| 60 |
max_amp = np.max(np.abs(arr))
|
| 61 |
+
if rms < 0.0035 or max_amp < 0.018:
|
| 62 |
+
return True
|
| 63 |
+
|
| 64 |
+
# Spectral Flatness check: white noise / fan hiss has flat PSD (flatness > 0.85)
|
| 65 |
+
try:
|
| 66 |
+
f, psd = scipy.signal.welch(arr, fs=sr, nperseg=min(512, len(arr)))
|
| 67 |
+
psd = psd + 1e-12
|
| 68 |
+
geom_mean = np.exp(np.mean(np.log(psd)))
|
| 69 |
+
arith_mean = np.mean(psd)
|
| 70 |
+
flatness = geom_mean / arith_mean
|
| 71 |
+
|
| 72 |
+
# Harmonic pitch autocorrelation (75 Hz to 500 Hz -> lags 32 to 213 samples)
|
| 73 |
+
head_len = min(len(arr), 4000)
|
| 74 |
+
corr = np.correlate(arr[:head_len], arr[:head_len], mode='full')
|
| 75 |
+
corr = corr[len(corr)//2:]
|
| 76 |
+
if corr[0] > 0:
|
| 77 |
+
norm_corr = corr / corr[0]
|
| 78 |
+
pitch_lags = norm_corr[32:min(213, len(norm_corr))]
|
| 79 |
+
harmonic_peak = np.max(pitch_lags) if len(pitch_lags) else 0.0
|
| 80 |
+
else:
|
| 81 |
+
harmonic_peak = 0.0
|
| 82 |
+
|
| 83 |
+
if flatness > 0.88 and harmonic_peak < 0.15:
|
| 84 |
+
return True # Pure noise / fan hiss rejected
|
| 85 |
+
except Exception:
|
| 86 |
+
pass
|
| 87 |
+
|
| 88 |
+
return False
|
| 89 |
|
| 90 |
|
| 91 |
def _load_wave(path_or_arr: Any) -> np.ndarray:
|
|
|
|
| 232 |
|
| 233 |
|
| 234 |
# --------------------------------------------------------------------------
|
| 235 |
+
# Dynamic Programming Word Alignment with Phonological Priority
|
| 236 |
# --------------------------------------------------------------------------
|
| 237 |
def _norm(s: str) -> str:
|
| 238 |
s = s.lower().translate(str.maketrans("", "", string.punctuation))
|
| 239 |
return " ".join(s.split())
|
| 240 |
|
| 241 |
|
| 242 |
+
def _classify_word_pair(w_ref: str, w_hyp: str) -> Tuple[str, int, float]:
|
| 243 |
+
"""
|
| 244 |
+
Evaluates word pair:
|
| 245 |
+
1. Exact match -> correct (cost 0)
|
| 246 |
+
2. Clinical phonological substitution -> substitution (cost 1, never swallowed)
|
| 247 |
+
3. Generic phonetic relaxation -> correct if ratio >= 0.82
|
| 248 |
+
"""
|
| 249 |
+
if w_ref == w_hyp:
|
| 250 |
+
return "correct", 0, 1.0
|
| 251 |
+
|
| 252 |
+
# Priority Clinical Sound Checks:
|
| 253 |
+
# Rhotacism: Expected 'r', spoken substitutes with 'w', 'l', 'y'
|
| 254 |
+
is_rhot = ("r" in w_ref) and any(sub in w_hyp for sub in ["w", "l", "y"])
|
| 255 |
+
# Sigmatism: Expected sibilant 's'/'z'/'sh', spoken substitutes with 'th'/'f'
|
| 256 |
+
is_sigm = any(s in w_ref for s in ["s", "z", "sh"]) and any(sub in w_hyp for sub in ["th", "f"])
|
| 257 |
+
|
| 258 |
+
sim = difflib.SequenceMatcher(None, w_ref, w_hyp).ratio()
|
| 259 |
+
if is_rhot or is_sigm:
|
| 260 |
+
return "substitution", 1, sim
|
| 261 |
+
|
| 262 |
+
if sim >= 0.82:
|
| 263 |
+
return "correct", 0, sim
|
| 264 |
+
return "substitution", 1, sim
|
| 265 |
+
|
| 266 |
+
|
| 267 |
def align_words(reference: str, hypothesis: str) -> List[Dict[str, Any]]:
|
| 268 |
+
"""Dynamic programming Levenshtein word alignment with diagnostic priority."""
|
| 269 |
ref_words = _norm(reference).split()
|
| 270 |
hyp_words = _norm(hypothesis).split()
|
| 271 |
|
|
|
|
| 281 |
for j in range(1, m + 1):
|
| 282 |
w_ref = ref_words[i - 1]
|
| 283 |
w_hyp = hyp_words[j - 1]
|
| 284 |
+
_, cost, _ = _classify_word_pair(w_ref, w_hyp)
|
|
|
|
| 285 |
dp[i][j] = min(
|
| 286 |
dp[i - 1][j - 1] + cost,
|
| 287 |
dp[i - 1][j] + 1,
|
|
|
|
| 294 |
if i > 0 and j > 0:
|
| 295 |
w_ref = ref_words[i - 1]
|
| 296 |
w_hyp = hyp_words[j - 1]
|
| 297 |
+
status, cost, sim = _classify_word_pair(w_ref, w_hyp)
|
|
|
|
| 298 |
if dp[i][j] == dp[i - 1][j - 1] + cost:
|
|
|
|
| 299 |
alignment.append({
|
| 300 |
"expected": w_ref,
|
| 301 |
"spoken": w_hyp,
|
|
|
|
| 357 |
word_omissions = []
|
| 358 |
word_insertions = []
|
| 359 |
|
| 360 |
+
# 1. R-Sound / Rhotacism Check (Target sound error tracking)
|
| 361 |
f3_f2 = praat_dict.get("f3_f2_dist_hz", 900.0)
|
| 362 |
for item in alignment:
|
| 363 |
exp = item["expected"].lower()
|
| 364 |
spk = item["spoken"].lower()
|
| 365 |
if item["status"] == "substitution" and "r" in exp:
|
| 366 |
+
if any(sub in spk for sub in ["w", "l", "y"]):
|
| 367 |
r_errors.append({
|
| 368 |
"expected": exp,
|
| 369 |
"spoken": spk,
|
|
|
|
| 371 |
"message": f"Trouble with 'r' sound: said '{spk}' instead of '{exp}' (Acoustic F3-F2 gap: {f3_f2:.0f} Hz)"
|
| 372 |
})
|
| 373 |
|
| 374 |
+
# 2. S-Sound / Sigmatism / Lisping Check
|
| 375 |
centroid = praat_dict.get("spectral_centroid_hz", 1200.0)
|
| 376 |
s_markers = ["s", "z", "sh", "ch"]
|
| 377 |
for item in alignment:
|
| 378 |
exp = item["expected"].lower()
|
| 379 |
spk = item["spoken"].lower()
|
| 380 |
if item["status"] == "substitution" and any(m in exp for m in s_markers):
|
| 381 |
+
if any(sub in spk for sub in ["th", "f"]):
|
| 382 |
s_errors.append({
|
| 383 |
"expected": exp,
|
| 384 |
"spoken": spk,
|