Spaces:
Running
Running
File size: 3,790 Bytes
d7013e6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 | """
Evaluation Service.
WER/CER evaluation on LibriSpeech (or other HF datasets).
"""
from __future__ import annotations
import time
import torch
import torchaudio
import torchaudio.transforms
from src.models.config import GrassmannConfig
from src.services.tokenizer_service import CharTokenizer
_mel_transform = None
def make_mel(waveform: torch.Tensor, sample_rate: int, config: GrassmannConfig) -> torch.Tensor:
"""Convert waveform to normalized mel spectrogram (matches training pipeline)."""
global _mel_transform
if _mel_transform is None:
_mel_transform = torchaudio.transforms.MelSpectrogram(
sample_rate=config.sample_rate, n_fft=config.n_fft,
hop_length=config.hop_length, n_mels=config.n_mels,
)
if sample_rate != config.sample_rate:
waveform = torchaudio.functional.resample(waveform, sample_rate, config.sample_rate)
if waveform.dim() == 1:
waveform = waveform.unsqueeze(0)
mel = _mel_transform(waveform)
mel = torch.log(mel.clamp(min=1e-9)).squeeze(0)
mean = mel.mean()
std = mel.std().clamp(min=1e-9)
mel = (mel - mean) / std
return mel.unsqueeze(0)
@torch.no_grad()
def greedy_decode(model, mel: torch.Tensor, tokenizer: CharTokenizer) -> str:
log_probs = model(mel)
preds = log_probs.argmax(dim=-1).squeeze(1)
return tokenizer.decode(preds)
def evaluate_wer(
model,
config: GrassmannConfig,
tokenizer: CharTokenizer,
split: str = "test.clean",
max_samples: int | None = None,
show_errors: int = 10,
) -> dict:
"""Run WER/CER evaluation on a LibriSpeech split.
Returns dict with wer, cer, num_samples, elapsed_seconds.
"""
from jiwer import wer as compute_wer, cer as compute_cer
from datasets import load_dataset
model.eval()
print(f" Loading LibriSpeech {split}...")
ds = load_dataset("openslr/librispeech_asr", split=split, streaming=True)
references, hypotheses, errors = [], [], []
start = time.time()
for i, sample in enumerate(ds):
if max_samples and i >= max_samples:
break
audio = sample["audio"]
waveform = torch.tensor(audio["array"], dtype=torch.float32).unsqueeze(0)
sr = audio["sampling_rate"]
ref = sample["text"].lower().strip()
mel = make_mel(waveform, sr, config)
hyp = greedy_decode(model, mel, tokenizer).strip()
references.append(ref)
hypotheses.append(hyp)
sample_wer = compute_wer(ref, hyp) if ref else 0.0
errors.append((sample_wer, ref, hyp))
if (i + 1) % 100 == 0:
elapsed = time.time() - start
current_wer = compute_wer(references, hypotheses)
print(f" {i+1} samples | WER: {current_wer:.2%} | {elapsed:.0f}s", flush=True)
total_time = time.time() - start
total_wer = compute_wer(references, hypotheses)
total_cer = compute_cer(references, hypotheses)
print(f"\n{'=' * 60}")
print(f" LibriSpeech {split} — Greedy CTC Decode")
print(f"{'=' * 60}")
print(f" Samples: {len(references)}")
print(f" WER: {total_wer:.2%}")
print(f" CER: {total_cer:.2%}")
print(f" Time: {total_time:.1f}s ({total_time/len(references):.2f}s/sample)")
print(f"{'=' * 60}")
if show_errors > 0:
errors.sort(key=lambda x: -x[0])
print(f"\n Top {show_errors} worst transcriptions:")
for j, (w, ref, hyp) in enumerate(errors[:show_errors]):
print(f"\n [{j+1}] WER: {w:.0%}")
print(f" GT: '{ref}'")
print(f" PRED: '{hyp}'")
return {
"wer": total_wer,
"cer": total_cer,
"num_samples": len(references),
"elapsed_seconds": total_time,
}
|