Pransfrance commited on
Commit
ffca4e5
·
verified ·
1 Parent(s): 247a5f0

Upload biased/eval_biased.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. biased/eval_biased.py +393 -0
biased/eval_biased.py ADDED
@@ -0,0 +1,393 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Evaluation / Inference for Biased Wav2Vec2 (single-neuron)
3
+ ============================================================
4
+ Runs inference on audio files and outputs JSON in the same format as the
5
+ old pipeline (segments, window_predictions, overall_label, etc.).
6
+
7
+ Supports both PyTorch and ONNX backends.
8
+
9
+ Usage:
10
+ # Single file
11
+ python eval_biased.py path/to/audio.wav
12
+
13
+ # Directory of files
14
+ python eval_biased.py path/to/folder/
15
+
16
+ # With ONNX (faster, no GPU needed)
17
+ python eval_biased.py path/to/audio.wav --backend onnx
18
+
19
+ # Custom threshold
20
+ python eval_biased.py path/to/audio.wav --threshold 0.70
21
+
22
+ # Evaluate on test split from manifest
23
+ python eval_biased.py --test-split
24
+ """
25
+
26
+ import argparse
27
+ import json
28
+ import os
29
+ import sys
30
+ import time
31
+ from pathlib import Path
32
+ from dataclasses import dataclass, asdict
33
+ from typing import List, Optional
34
+
35
+ import numpy as np
36
+
37
+ sys.path.insert(0, str(Path(__file__).parent / "old"))
38
+
39
+ SAMPLE_RATE = 16000
40
+ WINDOW_SEC = 5.0
41
+ HOP_SEC = 2.5
42
+ WINDOW_SAMPLES = int(WINDOW_SEC * SAMPLE_RATE)
43
+ HOP_SAMPLES = int(HOP_SEC * SAMPLE_RATE)
44
+ DEFAULT_THRESHOLD = 0.65
45
+ MIN_SPEECH_RATIO = 0.20
46
+
47
+ CKPT_DIR = Path("checkpoints_biased")
48
+ ONNX_PATH = CKPT_DIR / "biased_wav2vec2_quant.onnx"
49
+ PT_PATH = CKPT_DIR / "biased_wav2vec2_best.pt"
50
+ OUTPUT_DIR = Path("outputs_biased")
51
+
52
+
53
+ @dataclass
54
+ class WindowPrediction:
55
+ window_idx: int
56
+ start_sec: float
57
+ end_sec: float
58
+ label: str
59
+ confidence: float
60
+ speech_ratio: float
61
+
62
+
63
+ @dataclass
64
+ class Segment:
65
+ start_sec: float
66
+ end_sec: float
67
+ duration_sec: float
68
+ label: str
69
+ confidence: float
70
+
71
+
72
+ @dataclass
73
+ class PredictionResult:
74
+ filepath: str
75
+ filename: str
76
+ duration_sec: float
77
+ overall_label: str
78
+ overall_confidence: float
79
+ read_ratio: float
80
+ cheating_suspected: bool
81
+ segments: List[Segment]
82
+ window_predictions: List[WindowPrediction]
83
+
84
+
85
+ def load_audio(path: str) -> np.ndarray:
86
+ """Load audio file at 16kHz mono."""
87
+ import librosa
88
+ import warnings
89
+ with warnings.catch_warnings():
90
+ warnings.simplefilter("ignore")
91
+ audio, _ = librosa.load(path, sr=SAMPLE_RATE, mono=True, duration=120)
92
+ peak = np.max(np.abs(audio))
93
+ if peak > 1e-6:
94
+ audio = audio / peak * 0.95
95
+ return audio
96
+
97
+
98
+ def compute_speech_ratio(audio_chunk: np.ndarray) -> float:
99
+ """Simple energy-based speech ratio."""
100
+ import librosa
101
+ rms = librosa.feature.rms(y=audio_chunk, frame_length=512, hop_length=256)[0]
102
+ rms_max = float(np.max(rms))
103
+ if rms_max < 1e-8:
104
+ return 0.0
105
+ noise_floor = np.percentile(rms, 10)
106
+ signal_peak = np.percentile(rms, 95)
107
+ thresh = noise_floor + (signal_peak - noise_floor) * 0.15
108
+ thresh = max(thresh, rms_max * 0.05)
109
+ return float((rms > thresh).mean())
110
+
111
+
112
+ class ONNXBackend:
113
+ def __init__(self, model_path: str):
114
+ import onnxruntime as ort
115
+ self.session = ort.InferenceSession(
116
+ model_path, providers=["CPUExecutionProvider"]
117
+ )
118
+
119
+ def predict_batch(self, windows: np.ndarray) -> np.ndarray:
120
+ """Returns P(read) for each window."""
121
+ logits = self.session.run(["logit"], {"input_values": windows.astype(np.float32)})[0]
122
+ return 1.0 / (1.0 + np.exp(-logits)) # sigmoid
123
+
124
+
125
+ class PyTorchBackend:
126
+ def __init__(self, model_path: str):
127
+ import torch
128
+ self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
129
+ checkpoint = torch.load(model_path, map_location=self.device, weights_only=False)
130
+
131
+ from train_biased_wav2vec2 import BiasedSpeechClassifier
132
+ w2v_cfg = checkpoint["config"]
133
+ self.model = BiasedSpeechClassifier(
134
+ model_name=w2v_cfg["model_name"],
135
+ hidden_size=w2v_cfg["hidden_size"],
136
+ dropout=0.0,
137
+ freeze_layers=0,
138
+ ).to(self.device)
139
+ self.model.load_state_dict(checkpoint["model_state_dict"])
140
+ self.model.eval()
141
+
142
+ @torch.no_grad()
143
+ def predict_batch(self, windows: np.ndarray) -> np.ndarray:
144
+ import torch
145
+ x = torch.tensor(windows, dtype=torch.float32).to(self.device)
146
+ logits = self.model(x)
147
+ return torch.sigmoid(logits).cpu().numpy()
148
+
149
+
150
+ def predict_file(audio_path: str, backend, threshold: float) -> PredictionResult:
151
+ """Run inference on a single audio file."""
152
+ audio = load_audio(audio_path)
153
+ duration_sec = len(audio) / SAMPLE_RATE
154
+
155
+ # Create windows
156
+ windows = []
157
+ window_meta = []
158
+ idx = 0
159
+ start = 0
160
+ while start < len(audio):
161
+ end = start + WINDOW_SAMPLES
162
+ chunk = audio[start:end]
163
+ if len(chunk) < WINDOW_SAMPLES // 2:
164
+ break
165
+ if len(chunk) < WINDOW_SAMPLES:
166
+ chunk = np.pad(chunk, (0, WINDOW_SAMPLES - len(chunk)))
167
+
168
+ speech_ratio = compute_speech_ratio(chunk)
169
+ window_meta.append({
170
+ "idx": idx,
171
+ "start_sec": round(start / SAMPLE_RATE, 2),
172
+ "end_sec": round(min(end, len(audio)) / SAMPLE_RATE, 2),
173
+ "speech_ratio": round(speech_ratio, 3),
174
+ })
175
+ windows.append(chunk)
176
+ idx += 1
177
+ start += HOP_SAMPLES
178
+
179
+ if not windows:
180
+ return PredictionResult(
181
+ filepath=audio_path,
182
+ filename=Path(audio_path).name,
183
+ duration_sec=round(duration_sec, 2),
184
+ overall_label="silence",
185
+ overall_confidence=1.0,
186
+ read_ratio=0.0,
187
+ cheating_suspected=False,
188
+ segments=[Segment(0.0, round(duration_sec, 2), round(duration_sec, 2), "silence", 1.0)],
189
+ window_predictions=[],
190
+ )
191
+
192
+ # Batch inference
193
+ windows_arr = np.array(windows, dtype=np.float32)
194
+ probs = backend.predict_batch(windows_arr) # P(read) per window
195
+ if probs.ndim > 1:
196
+ probs = probs.squeeze(-1)
197
+
198
+ # Build window predictions
199
+ win_preds = []
200
+ for i, meta in enumerate(window_meta):
201
+ p_read = float(probs[i])
202
+ sr = meta["speech_ratio"]
203
+
204
+ if sr < MIN_SPEECH_RATIO:
205
+ label = "silence"
206
+ confidence = 1.0
207
+ elif p_read >= threshold:
208
+ label = "read"
209
+ confidence = round(p_read, 3)
210
+ else:
211
+ label = "spontaneous"
212
+ confidence = round(1.0 - p_read, 3)
213
+
214
+ win_preds.append(WindowPrediction(
215
+ window_idx=meta["idx"],
216
+ start_sec=meta["start_sec"],
217
+ end_sec=meta["end_sec"],
218
+ label=label,
219
+ confidence=confidence,
220
+ speech_ratio=sr,
221
+ ))
222
+
223
+ # Merge consecutive same-label windows into segments
224
+ segments = []
225
+ current_label = win_preds[0].label
226
+ seg_start = win_preds[0].start_sec
227
+ seg_confs = [win_preds[0].confidence]
228
+
229
+ for wp in win_preds[1:]:
230
+ if wp.label == current_label:
231
+ seg_confs.append(wp.confidence)
232
+ else:
233
+ seg_end = wp.start_sec
234
+ segments.append(Segment(
235
+ start_sec=seg_start,
236
+ end_sec=round(seg_end, 2),
237
+ duration_sec=round(seg_end - seg_start, 2),
238
+ label=current_label,
239
+ confidence=round(float(np.mean(seg_confs)), 3),
240
+ ))
241
+ current_label = wp.label
242
+ seg_start = wp.start_sec
243
+ seg_confs = [wp.confidence]
244
+
245
+ # Close last segment
246
+ seg_end = win_preds[-1].end_sec
247
+ segments.append(Segment(
248
+ start_sec=seg_start,
249
+ end_sec=round(seg_end, 2),
250
+ duration_sec=round(seg_end - seg_start, 2),
251
+ label=current_label,
252
+ confidence=round(float(np.mean(seg_confs)), 3),
253
+ ))
254
+
255
+ # Overall stats (only over speaking windows)
256
+ speaking = [wp for wp in win_preds if wp.label != "silence"]
257
+ if speaking:
258
+ read_count = sum(1 for wp in speaking if wp.label == "read")
259
+ read_ratio = round(read_count / len(speaking), 3)
260
+ if read_ratio > 0.5:
261
+ overall_label = "read"
262
+ overall_confidence = round(read_ratio, 3)
263
+ else:
264
+ overall_label = "spontaneous"
265
+ overall_confidence = round(1.0 - read_ratio, 3)
266
+ else:
267
+ overall_label = "silence"
268
+ overall_confidence = 1.0
269
+ read_ratio = 0.0
270
+
271
+ return PredictionResult(
272
+ filepath=audio_path,
273
+ filename=Path(audio_path).name,
274
+ duration_sec=round(duration_sec, 2),
275
+ overall_label=overall_label,
276
+ overall_confidence=overall_confidence,
277
+ read_ratio=read_ratio,
278
+ cheating_suspected=(overall_label == "read"),
279
+ segments=segments,
280
+ window_predictions=win_preds,
281
+ )
282
+
283
+
284
+ def eval_test_split(backend, threshold: float):
285
+ """Evaluate on the test split from the manifest."""
286
+ import pandas as pd
287
+ import yaml
288
+ from sklearn.metrics import accuracy_score, f1_score, classification_report
289
+
290
+ with open("old/configs/config.yaml") as f:
291
+ cfg = yaml.safe_load(f)
292
+
293
+ data_root = Path(cfg["paths"]["data_root"])
294
+ manifest_path = data_root / "old" / cfg["paths"]["manifest_csv"]
295
+ df = pd.read_csv(manifest_path)
296
+
297
+ # Filter to existing files + test split
298
+ df = df[df["filepath"].apply(os.path.exists)]
299
+ test_df = df[df["split"] == "test"].reset_index(drop=True)
300
+ print(f"Test set: {len(test_df)} files")
301
+
302
+ results = []
303
+ all_true, all_pred = [], []
304
+
305
+ for i, row in test_df.iterrows():
306
+ result = predict_file(row["filepath"], backend, threshold)
307
+ results.append(result)
308
+
309
+ true_label = "read" if row["label_int"] == 1 else "spontaneous"
310
+ all_true.append(row["label_int"])
311
+ pred_int = 1 if result.overall_label == "read" else 0
312
+ all_pred.append(pred_int)
313
+
314
+ if (i + 1) % 20 == 0:
315
+ print(f" Processed {i+1}/{len(test_df)} files...")
316
+
317
+ acc = accuracy_score(all_true, all_pred)
318
+ f1 = f1_score(all_true, all_pred, zero_division=0)
319
+ print(f"\nFile-level results (threshold={threshold}):")
320
+ print(f" Accuracy: {acc:.4f}")
321
+ print(f" F1: {f1:.4f}")
322
+ print(f"\n{classification_report(all_true, all_pred, target_names=['spontaneous', 'read'])}")
323
+
324
+ return results
325
+
326
+
327
+ def main():
328
+ parser = argparse.ArgumentParser(description="Eval biased wav2vec2")
329
+ parser.add_argument("input", nargs="?", help="Audio file or directory")
330
+ parser.add_argument("--backend", choices=["onnx", "pytorch"], default="onnx")
331
+ parser.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD)
332
+ parser.add_argument("--output", type=str, default=None, help="Output JSON path")
333
+ parser.add_argument("--test-split", action="store_true", help="Evaluate on manifest test split")
334
+ args = parser.parse_args()
335
+
336
+ # Load backend
337
+ if args.backend == "onnx":
338
+ if not ONNX_PATH.exists():
339
+ print(f"ONNX model not found at {ONNX_PATH}. Run export_biased_onnx.py first.")
340
+ sys.exit(1)
341
+ print(f"Backend: ONNX ({ONNX_PATH})")
342
+ backend = ONNXBackend(str(ONNX_PATH))
343
+ else:
344
+ print(f"Backend: PyTorch ({PT_PATH})")
345
+ import torch
346
+ backend = PyTorchBackend(str(PT_PATH))
347
+
348
+ print(f"Read threshold: {args.threshold}")
349
+
350
+ # Test split mode
351
+ if args.test_split:
352
+ results = eval_test_split(backend, args.threshold)
353
+ output_path = args.output or str(OUTPUT_DIR / "test_predictions.json")
354
+ else:
355
+ if not args.input:
356
+ parser.error("Provide an audio file/directory or use --test-split")
357
+
358
+ input_path = Path(args.input)
359
+ if input_path.is_file():
360
+ audio_files = [str(input_path)]
361
+ elif input_path.is_dir():
362
+ exts = {".wav", ".mp3", ".m4a", ".webm", ".ogg", ".flac", ".aac", ".wma", ".mp4"}
363
+ audio_files = sorted(
364
+ str(p) for p in input_path.iterdir() if p.suffix.lower() in exts
365
+ )
366
+ print(f"Found {len(audio_files)} audio files")
367
+ else:
368
+ print(f"Not found: {input_path}")
369
+ sys.exit(1)
370
+
371
+ results = []
372
+ for i, fpath in enumerate(audio_files):
373
+ t0 = time.perf_counter()
374
+ result = predict_file(fpath, backend, args.threshold)
375
+ elapsed = time.perf_counter() - t0
376
+ flag = " ** CHEATING" if result.cheating_suspected else ""
377
+ print(f" [{i+1}/{len(audio_files)}] {result.filename}: "
378
+ f"{result.overall_label} (conf={result.overall_confidence:.3f}, "
379
+ f"read_ratio={result.read_ratio:.3f}) [{elapsed:.1f}s]{flag}")
380
+ results.append(result)
381
+
382
+ output_path = args.output or str(OUTPUT_DIR / "predictions.json")
383
+
384
+ # Save JSON
385
+ OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
386
+ output_data = [asdict(r) for r in results]
387
+ with open(output_path, "w") as f:
388
+ json.dump(output_data, f, indent=2)
389
+ print(f"\nResults saved to: {output_path}")
390
+
391
+
392
+ if __name__ == "__main__":
393
+ main()