File size: 4,028 Bytes
38c1604 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | """
Step 4: measure accuracy on the test set (reciters never trained on + held-out Warsh surahs).
python evaluate.py --config config/pilot.yaml # PyTorch, best.pt
python evaluate.py --config config/pilot.yaml --onnx # the exported int8 model, via sherpa-onnx
python evaluate.py --config config/pilot.yaml --baseline # the app's current NVIDIA model, for comparison
Reports the letter error rate (CER), word error rate (WER) and skeleton error rate (letters
without alif/hamza, which is what tracking relies on) per riwaya and per reciter.
Both the model output and the reference are letters-only, so every model is scored the same way.
"""
import os
import sys
import torch
from quran_asr import audio, config as config_mod, data, features, model as model_mod, text
def main():
parser = config_mod.from_args(__doc__)
parser.add_argument("--onnx", action="store_true", help="evaluate runs/<name>/export/model.int8.onnx")
parser.add_argument("--baseline", action="store_true", help="evaluate the NVIDIA FastConformer instead")
parser.add_argument("--checkpoint", default="best.pt")
parser.add_argument("--limit", type=int, default=0, help="only the first N clips")
args = parser.parse_args()
cfg = config_mod.load(args.config)
test = data.read_manifest(os.path.join(config_mod.manifest_dir(cfg), "test.jsonl"))
if args.limit:
test = test[:: max(1, len(test) // args.limit)][: args.limit]
if args.onnx or args.baseline:
import sherpa_onnx
if args.baseline:
folder = os.path.join(cfg.paths.data_dir, "nvidia_fastconformer")
rec = sherpa_onnx.OfflineRecognizer.from_nemo_ctc(
model=os.path.join(folder, "model.onnx"), tokens=os.path.join(folder, "tokens.txt"), num_threads=8)
else:
out = os.path.join(cfg.paths.run_dir, "export")
rec = sherpa_onnx.OfflineRecognizer.from_zipformer_ctc(
model=os.path.join(out, "model.int8.onnx"), tokens=os.path.join(out, "tokens.txt"), num_threads=8)
def transcribe(wave):
s = rec.create_stream()
s.accept_waveform(features.SAMPLE_RATE, wave)
rec.decode_stream(s)
return text.normalize(s.result.text.replace(text.WORD, " "))
else:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
ckpt = torch.load(os.path.join(cfg.paths.run_dir, args.checkpoint), map_location="cpu", weights_only=False)
model = model_mod.build(config_mod._wrap(ckpt["model_config"]), len(text.VOCAB))
model.load_state_dict(ckpt["model"])
model.to(device).eval()
def transcribe(wave):
feats = features.fbank(torch.from_numpy(wave)).to(device)
with torch.no_grad():
log_probs, lens = model(feats[None], torch.tensor([feats.shape[0]], device=device))
return text.decode(model_mod.greedy_decode(log_probs, lens)[0])
totals = {}
for k, e in enumerate(test, 1):
hyp = transcribe(audio.load(e["path"]))
counts = text.error_counts(hyp, e["text"])
for key in ("all", e["riwaya"], f"{e['riwaya']}/{e['reciter']}"):
t = totals.setdefault(key, [0] * 6)
for j in range(6):
t[j] += counts[j]
if k % 200 == 0:
print(f" {k}/{len(test)}", flush=True)
name = "NVIDIA FastConformer (baseline)" if args.baseline else ("exported ONNX" if args.onnx else args.checkpoint)
print(f"\n{name} on {len(test)} test clips")
print("error rates; 'skeleton' ignores alif and hamza (spelling conventions), like the app's tracker")
print(f"{'':52s} {'letters':>8s} {'words':>8s} {'skeleton':>9s}")
for key, (ce, ct, we, wt, se, st) in sorted(totals.items(), key=lambda kv: (kv[0] != "all", kv[0])):
print(f"{key:52s} {ce / max(1, ct) * 100:7.2f}% {we / max(1, wt) * 100:7.2f}% {se / max(1, st) * 100:8.2f}%")
if __name__ == "__main__":
sys.exit(main())
|