notUbaid commited on
Commit
98d87bd
·
verified ·
1 Parent(s): d11b479

Upload ml/model/pron_eval.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. ml/model/pron_eval.py +72 -16
ml/model/pron_eval.py CHANGED
@@ -2,7 +2,7 @@
2
  ml/model/pron_eval.py - Acoustic-Phonetic Alignment & Phonation Analysis
3
  ========================================================================
4
  Extracts objective acoustic correlates of phonation, LPC Burg formants,
5
- spectral moments, dynamic programming Needleman-Wunsch word alignment,
6
  and explicit phonological flaw heuristics ('R' Rhotacism, 'S' Sigmatism).
7
  """
8
  from __future__ import annotations
@@ -20,10 +20,11 @@ import scipy.signal
20
 
21
  SR = 16000
22
  _CTC_MODEL = "facebook/wav2vec2-base-960h"
 
23
 
24
 
25
  # --------------------------------------------------------------------------
26
- # Signal conditioning & Silence detection
27
  # --------------------------------------------------------------------------
28
  def _apply_acoustic_conditioning(arr: np.ndarray, sr: int = SR) -> np.ndarray:
29
  """Apply high-pass Butterworth 60Hz filter and peak gain standardization."""
@@ -43,15 +44,48 @@ def _apply_acoustic_conditioning(arr: np.ndarray, sr: int = SR) -> np.ndarray:
43
 
44
 
45
  def is_silent_or_empty(arr: np.ndarray, sr: int = SR) -> bool:
46
- """True if pure silence or empty audio buffer."""
 
 
 
 
 
47
  if arr is None or len(arr) == 0:
48
  return True
49
  dur = len(arr) / sr
50
- if dur < 0.15: # Audio shorter than 150ms
51
  return True
 
52
  rms = np.sqrt(np.mean(arr**2))
53
  max_amp = np.max(np.abs(arr))
54
- return bool(rms < 0.003 or max_amp < 0.015)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
55
 
56
 
57
  def _load_wave(path_or_arr: Any) -> np.ndarray:
@@ -198,15 +232,40 @@ def praat_metrics(wav_path: str) -> dict:
198
 
199
 
200
  # --------------------------------------------------------------------------
201
- # Dynamic Programming Word Alignment
202
  # --------------------------------------------------------------------------
203
  def _norm(s: str) -> str:
204
  s = s.lower().translate(str.maketrans("", "", string.punctuation))
205
  return " ".join(s.split())
206
 
207
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
208
  def align_words(reference: str, hypothesis: str) -> List[Dict[str, Any]]:
209
- """Dynamic programming Levenshtein word alignment with human phonetic tolerance."""
210
  ref_words = _norm(reference).split()
211
  hyp_words = _norm(hypothesis).split()
212
 
@@ -222,8 +281,7 @@ def align_words(reference: str, hypothesis: str) -> List[Dict[str, Any]]:
222
  for j in range(1, m + 1):
223
  w_ref = ref_words[i - 1]
224
  w_hyp = hyp_words[j - 1]
225
- sim = difflib.SequenceMatcher(None, w_ref, w_hyp).ratio()
226
- cost = 0 if (w_ref == w_hyp or sim >= 0.78) else 1
227
  dp[i][j] = min(
228
  dp[i - 1][j - 1] + cost,
229
  dp[i - 1][j] + 1,
@@ -236,10 +294,8 @@ def align_words(reference: str, hypothesis: str) -> List[Dict[str, Any]]:
236
  if i > 0 and j > 0:
237
  w_ref = ref_words[i - 1]
238
  w_hyp = hyp_words[j - 1]
239
- sim = difflib.SequenceMatcher(None, w_ref, w_hyp).ratio()
240
- cost = 0 if (w_ref == w_hyp or sim >= 0.78) else 1
241
  if dp[i][j] == dp[i - 1][j - 1] + cost:
242
- status = "correct" if cost == 0 else "substitution"
243
  alignment.append({
244
  "expected": w_ref,
245
  "spoken": w_hyp,
@@ -301,13 +357,13 @@ def analyze_speech_flaws(
301
  word_omissions = []
302
  word_insertions = []
303
 
304
- # 1. R-Sound / Rhotacism Check (ASR Alignment + Formant Dip Correlate)
305
  f3_f2 = praat_dict.get("f3_f2_dist_hz", 900.0)
306
  for item in alignment:
307
  exp = item["expected"].lower()
308
  spk = item["spoken"].lower()
309
  if item["status"] == "substitution" and "r" in exp:
310
- if "w" in spk or "l" in spk or "r" not in spk:
311
  r_errors.append({
312
  "expected": exp,
313
  "spoken": spk,
@@ -315,14 +371,14 @@ def analyze_speech_flaws(
315
  "message": f"Trouble with 'r' sound: said '{spk}' instead of '{exp}' (Acoustic F3-F2 gap: {f3_f2:.0f} Hz)"
316
  })
317
 
318
- # 2. S-Sound / Sigmatism / Lisping Check (ASR Alignment + Sibilant Spectral Centroid)
319
  centroid = praat_dict.get("spectral_centroid_hz", 1200.0)
320
  s_markers = ["s", "z", "sh", "ch"]
321
  for item in alignment:
322
  exp = item["expected"].lower()
323
  spk = item["spoken"].lower()
324
  if item["status"] == "substitution" and any(m in exp for m in s_markers):
325
- if "th" in spk or "f" in spk or not any(m in spk for m in s_markers):
326
  s_errors.append({
327
  "expected": exp,
328
  "spoken": spk,
 
2
  ml/model/pron_eval.py - Acoustic-Phonetic Alignment & Phonation Analysis
3
  ========================================================================
4
  Extracts objective acoustic correlates of phonation, LPC Burg formants,
5
+ spectral flatness VAD, dynamic programming Needleman-Wunsch word alignment,
6
  and explicit phonological flaw heuristics ('R' Rhotacism, 'S' Sigmatism).
7
  """
8
  from __future__ import annotations
 
20
 
21
  SR = 16000
22
  _CTC_MODEL = "facebook/wav2vec2-base-960h"
23
+ _WHISPER_MODEL = "openai/whisper-tiny"
24
 
25
 
26
  # --------------------------------------------------------------------------
27
+ # Signal conditioning & Enhanced Multi-Factor VAD
28
  # --------------------------------------------------------------------------
29
  def _apply_acoustic_conditioning(arr: np.ndarray, sr: int = SR) -> np.ndarray:
30
  """Apply high-pass Butterworth 60Hz filter and peak gain standardization."""
 
44
 
45
 
46
  def is_silent_or_empty(arr: np.ndarray, sr: int = SR) -> bool:
47
+ """
48
+ Enhanced Multi-Factor Voice Activity Detection (VAD):
49
+ 1. RMS Energy & Peak Amplitude Thresholding
50
+ 2. Spectral Flatness (Wiener entropy) to reject fan hiss / white noise
51
+ 3. Autocorrelation Voicing Peak to verify human harmonic pitch periodicity
52
+ """
53
  if arr is None or len(arr) == 0:
54
  return True
55
  dur = len(arr) / sr
56
+ if dur < 0.18: # Audio shorter than 180ms
57
  return True
58
+
59
  rms = np.sqrt(np.mean(arr**2))
60
  max_amp = np.max(np.abs(arr))
61
+ if rms < 0.0035 or max_amp < 0.018:
62
+ return True
63
+
64
+ # Spectral Flatness check: white noise / fan hiss has flat PSD (flatness > 0.85)
65
+ try:
66
+ f, psd = scipy.signal.welch(arr, fs=sr, nperseg=min(512, len(arr)))
67
+ psd = psd + 1e-12
68
+ geom_mean = np.exp(np.mean(np.log(psd)))
69
+ arith_mean = np.mean(psd)
70
+ flatness = geom_mean / arith_mean
71
+
72
+ # Harmonic pitch autocorrelation (75 Hz to 500 Hz -> lags 32 to 213 samples)
73
+ head_len = min(len(arr), 4000)
74
+ corr = np.correlate(arr[:head_len], arr[:head_len], mode='full')
75
+ corr = corr[len(corr)//2:]
76
+ if corr[0] > 0:
77
+ norm_corr = corr / corr[0]
78
+ pitch_lags = norm_corr[32:min(213, len(norm_corr))]
79
+ harmonic_peak = np.max(pitch_lags) if len(pitch_lags) else 0.0
80
+ else:
81
+ harmonic_peak = 0.0
82
+
83
+ if flatness > 0.88 and harmonic_peak < 0.15:
84
+ return True # Pure noise / fan hiss rejected
85
+ except Exception:
86
+ pass
87
+
88
+ return False
89
 
90
 
91
  def _load_wave(path_or_arr: Any) -> np.ndarray:
 
232
 
233
 
234
  # --------------------------------------------------------------------------
235
+ # Dynamic Programming Word Alignment with Phonological Priority
236
  # --------------------------------------------------------------------------
237
  def _norm(s: str) -> str:
238
  s = s.lower().translate(str.maketrans("", "", string.punctuation))
239
  return " ".join(s.split())
240
 
241
 
242
+ def _classify_word_pair(w_ref: str, w_hyp: str) -> Tuple[str, int, float]:
243
+ """
244
+ Evaluates word pair:
245
+ 1. Exact match -> correct (cost 0)
246
+ 2. Clinical phonological substitution -> substitution (cost 1, never swallowed)
247
+ 3. Generic phonetic relaxation -> correct if ratio >= 0.82
248
+ """
249
+ if w_ref == w_hyp:
250
+ return "correct", 0, 1.0
251
+
252
+ # Priority Clinical Sound Checks:
253
+ # Rhotacism: Expected 'r', spoken substitutes with 'w', 'l', 'y'
254
+ is_rhot = ("r" in w_ref) and any(sub in w_hyp for sub in ["w", "l", "y"])
255
+ # Sigmatism: Expected sibilant 's'/'z'/'sh', spoken substitutes with 'th'/'f'
256
+ is_sigm = any(s in w_ref for s in ["s", "z", "sh"]) and any(sub in w_hyp for sub in ["th", "f"])
257
+
258
+ sim = difflib.SequenceMatcher(None, w_ref, w_hyp).ratio()
259
+ if is_rhot or is_sigm:
260
+ return "substitution", 1, sim
261
+
262
+ if sim >= 0.82:
263
+ return "correct", 0, sim
264
+ return "substitution", 1, sim
265
+
266
+
267
  def align_words(reference: str, hypothesis: str) -> List[Dict[str, Any]]:
268
+ """Dynamic programming Levenshtein word alignment with diagnostic priority."""
269
  ref_words = _norm(reference).split()
270
  hyp_words = _norm(hypothesis).split()
271
 
 
281
  for j in range(1, m + 1):
282
  w_ref = ref_words[i - 1]
283
  w_hyp = hyp_words[j - 1]
284
+ _, cost, _ = _classify_word_pair(w_ref, w_hyp)
 
285
  dp[i][j] = min(
286
  dp[i - 1][j - 1] + cost,
287
  dp[i - 1][j] + 1,
 
294
  if i > 0 and j > 0:
295
  w_ref = ref_words[i - 1]
296
  w_hyp = hyp_words[j - 1]
297
+ status, cost, sim = _classify_word_pair(w_ref, w_hyp)
 
298
  if dp[i][j] == dp[i - 1][j - 1] + cost:
 
299
  alignment.append({
300
  "expected": w_ref,
301
  "spoken": w_hyp,
 
357
  word_omissions = []
358
  word_insertions = []
359
 
360
+ # 1. R-Sound / Rhotacism Check (Target sound error tracking)
361
  f3_f2 = praat_dict.get("f3_f2_dist_hz", 900.0)
362
  for item in alignment:
363
  exp = item["expected"].lower()
364
  spk = item["spoken"].lower()
365
  if item["status"] == "substitution" and "r" in exp:
366
+ if any(sub in spk for sub in ["w", "l", "y"]):
367
  r_errors.append({
368
  "expected": exp,
369
  "spoken": spk,
 
371
  "message": f"Trouble with 'r' sound: said '{spk}' instead of '{exp}' (Acoustic F3-F2 gap: {f3_f2:.0f} Hz)"
372
  })
373
 
374
+ # 2. S-Sound / Sigmatism / Lisping Check
375
  centroid = praat_dict.get("spectral_centroid_hz", 1200.0)
376
  s_markers = ["s", "z", "sh", "ch"]
377
  for item in alignment:
378
  exp = item["expected"].lower()
379
  spk = item["spoken"].lower()
380
  if item["status"] == "substitution" and any(m in exp for m in s_markers):
381
+ if any(sub in spk for sub in ["th", "f"]):
382
  s_errors.append({
383
  "expected": exp,
384
  "spoken": spk,