"""Word error rate of pingo-ptbr on the MiniMax `pt` sentences: synthesize from the .bin, transcribe with Qwen3-ASR, score. python evaluation/evaluate_wer.py synth --model weights/pingo-ptbr.bin --out runs/pingo python evaluation/evaluate_wer.py score --out runs/pingo --leaderboard /path/to/open-tts-leaderboard `--texts` defaults to the 100 sentences stored in `eval/minimax_pt_F_s4.jsonl` (field "text"). The scorer uses the normalizer of https://github.com/huggingface/open-tts-leaderboard (clone it and pass `--leaderboard`). Scoring needs `transformers`, `torch`, `kaldialign` and `num2words`, and downloads Qwen/Qwen3-ASR-1.7B-hf. """ import argparse, json, sys, time from pathlib import Path HERE = Path(__file__).resolve().parent.parent VOICES = ("F_s4", "M_sm4") def load_texts(path): rows = [json.loads(l) for l in Path(path).read_text(encoding="utf-8").splitlines() if l.strip()] return [{"id": f"{i:03d}", "text": r["text"]} for i, r in enumerate(rows)] def synth(args): sys.path.insert(0, str(HERE / "pingo")) import soundfile as sf, speak model = speak.Pingo(args.model); out = Path(args.out); manifest = [] rows = load_texts(args.texts)[: args.limit] for voice in VOICES: folder = out / "wav" / voice; folder.mkdir(parents=True, exist_ok=True); start = time.time(); seconds = 0.0 for row in rows: audio = speak.speak(model, row["text"], voice=voice) path = folder / f"{row['id']}.wav"; sf.write(path, audio, 16000); seconds += len(audio) / 16000 manifest.append({"voice": voice, "id": row["id"], "text": row["text"], "wav": str(path)}) print(f"{voice}: {len(rows)} sentences, {seconds:.0f} s of audio in {time.time() - start:.0f} s") (out / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8") def score(args): import soundfile as sf, torch sys.path.insert(0, args.leaderboard) from normalizer import eval_utils from transformers import AutoModelForMultimodalLM, AutoProcessor from transformers.audio_utils import load_audio out = Path(args.out); rows = json.loads((out / "manifest.json").read_text()) device = "mps" if torch.backends.mps.is_available() else "cuda" if torch.cuda.is_available() else "cpu" processor = AutoProcessor.from_pretrained("Qwen/Qwen3-ASR-1.7B-hf") asr = AutoModelForMultimodalLM.from_pretrained("Qwen/Qwen3-ASR-1.7B-hf", dtype=torch.bfloat16, device_map=device).eval() rate = processor.feature_extractor.sampling_rate def transcribe(paths): speech = [load_audio(p, sampling_rate=rate) for p in paths] inputs = processor.apply_transcription_request(speech, language=["pt"] * len(speech)).to(asr.device, asr.dtype) with torch.no_grad(): ids = asr.generate(**inputs, max_new_tokens=256) return processor.decode(ids[:, inputs["input_ids"].shape[1]:], return_format="transcription_only") for voice in sorted({r["voice"] for r in rows}): mine = [r for r in rows if r["voice"] == voice] for i in range(0, len(mine), 8): for r, text in zip(mine[i:i + 8], transcribe([r["wav"] for r in mine[i:i + 8]])): r["pred_text"] = text folder = out / "scores" / voice; folder.mkdir(parents=True, exist_ok=True) with open(folder / "minimax_pt.jsonl", "w", encoding="utf-8") as f: for r in mine: duration = sf.info(r["wav"]).duration f.write(json.dumps({"audio_filepath": r["wav"], "duration": duration, "time": duration / 150.0, "text": r["text"], "pred_text": r["pred_text"]}, ensure_ascii=False) + "\n") print(f"== {voice} ==") eval_utils.score_results(str(folder), language="pt") if __name__ == "__main__": ap = argparse.ArgumentParser(); sub = ap.add_subparsers(dest="cmd", required=True) a = sub.add_parser("synth"); a.add_argument("--model", default=str(HERE / "weights" / "pingo-ptbr.bin")) a.add_argument("--texts", default=str(HERE / "eval" / "minimax_pt_F_s4.jsonl")); a.add_argument("--out", required=True) a.add_argument("--limit", type=int, default=100) b = sub.add_parser("score"); b.add_argument("--out", required=True); b.add_argument("--leaderboard", required=True) args = ap.parse_args(); {"synth": synth, "score": score}[args.cmd](args)