Download scripts/score_devhard.py from Pricile/waxal2026-backup: direct link, hf CLI and curl.
- Browser
- Download file 1.53 kB
-
https://huggingface.co/Pricile/waxal2026-backup/resolve/main/scripts/score_devhard.py
- Command line
-
hf download hf://Pricile/waxal2026-backup/scripts/score_devhard.py
-
curl -L -o score_devhard.py https://huggingface.co/Pricile/waxal2026-backup/resolve/main/scripts/score_devhard.py
1.53 kB
| import argparse, json, collections | |
| import soundfile as sf, torch, jiwer | |
| from transformers import AutoModelForCTC, AutoProcessor | |
| SR=16000 | |
| def norm(s): return " ".join(str(s).replace("|"," ").split()) | |
| ap=argparse.ArgumentParser() | |
| ap.add_argument("--model",required=True) | |
| ap.add_argument("--manifest",default="/root/devhard/devhard_all.jsonl") | |
| a=ap.parse_args() | |
| proc=AutoProcessor.from_pretrained(a.model) | |
| model=AutoModelForCTC.from_pretrained(a.model,torch_dtype=torch.bfloat16).cuda().eval() | |
| rows=[json.loads(l) for l in open(a.manifest,encoding="utf-8")] | |
| by=collections.defaultdict(lambda:{"r":[],"h":[]}) | |
| with torch.inference_mode(): | |
| for i,r in enumerate(rows): | |
| au=sf.read(r["audio"],dtype="float32")[0] | |
| x=proc(au,sampling_rate=SR,return_tensors="pt") | |
| x={k:(v.to("cuda",dtype=torch.bfloat16) if v.dtype==torch.float32 else v.to("cuda")) for k,v in x.items()} | |
| ids=model(**x).logits.argmax(-1).cpu().numpy() | |
| hyp=norm(proc.batch_decode(ids)[0]); ref=norm(r["text"]) | |
| L=r.get("lang","?"); by[L]["r"].append(ref); by[L]["h"].append(hyp) | |
| if (i+1)%300==0: print(f"{i+1}/{len(rows)}",flush=True) | |
| scores=[] | |
| for L in sorted(by): | |
| R=[x for x in by[L]["r"] if x.strip()]; H=[by[L]["h"][j] for j,x in enumerate(by[L]["r"]) if x.strip()] | |
| wer=jiwer.wer(R,H); cer=jiwer.cer(R,H); comb=0.5*wer+0.5*cer; sc=1-comb | |
| scores.append(sc) | |
| print(f"{L}: n={len(R)} WER={wer:.4f} CER={cer:.4f} combine={comb:.4f} score={sc:.4f}") | |
| print(f"MACRO score={sum(scores)/len(scores):.4f}") | |