pingo-ptbr / evaluation /evaluate_wer.py
itxtoledo's picture
Add pingo-ptbr: 0.87 MB pt-BR TTS (MLX + ONNX + web demo)
27d8618 verified
Raw History Blame Contribute Delete
4.4 kB
"""Word error rate of pingo-ptbr on the MiniMax `pt` sentences: synthesize from the .bin, transcribe with Qwen3-ASR, score.
python evaluation/evaluate_wer.py synth --model weights/pingo-ptbr.bin --out runs/pingo
python evaluation/evaluate_wer.py score --out runs/pingo --leaderboard /path/to/open-tts-leaderboard
`--texts` defaults to the 100 sentences stored in `eval/minimax_pt_F_s4.jsonl` (field "text"). The scorer uses the normalizer of
https://github.com/huggingface/open-tts-leaderboard (clone it and pass `--leaderboard`). Scoring needs `transformers`, `torch`,
`kaldialign` and `num2words`, and downloads Qwen/Qwen3-ASR-1.7B-hf.
"""
import argparse, json, sys, time
from pathlib import Path
HERE = Path(__file__).resolve().parent.parent
VOICES = ("F_s4", "M_sm4")
def load_texts(path):
rows = [json.loads(l) for l in Path(path).read_text(encoding="utf-8").splitlines() if l.strip()]
return [{"id": f"{i:03d}", "text": r["text"]} for i, r in enumerate(rows)]
def synth(args):
sys.path.insert(0, str(HERE / "pingo"))
import soundfile as sf, speak
model = speak.Pingo(args.model); out = Path(args.out); manifest = []
rows = load_texts(args.texts)[: args.limit]
for voice in VOICES:
folder = out / "wav" / voice; folder.mkdir(parents=True, exist_ok=True); start = time.time(); seconds = 0.0
for row in rows:
audio = speak.speak(model, row["text"], voice=voice)
path = folder / f"{row['id']}.wav"; sf.write(path, audio, 16000); seconds += len(audio) / 16000
manifest.append({"voice": voice, "id": row["id"], "text": row["text"], "wav": str(path)})
print(f"{voice}: {len(rows)} sentences, {seconds:.0f} s of audio in {time.time() - start:.0f} s")
(out / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8")
def score(args):
import soundfile as sf, torch
sys.path.insert(0, args.leaderboard)
from normalizer import eval_utils
from transformers import AutoModelForMultimodalLM, AutoProcessor
from transformers.audio_utils import load_audio
out = Path(args.out); rows = json.loads((out / "manifest.json").read_text())
device = "mps" if torch.backends.mps.is_available() else "cuda" if torch.cuda.is_available() else "cpu"
processor = AutoProcessor.from_pretrained("Qwen/Qwen3-ASR-1.7B-hf")
asr = AutoModelForMultimodalLM.from_pretrained("Qwen/Qwen3-ASR-1.7B-hf", dtype=torch.bfloat16, device_map=device).eval()
rate = processor.feature_extractor.sampling_rate
def transcribe(paths):
speech = [load_audio(p, sampling_rate=rate) for p in paths]
inputs = processor.apply_transcription_request(speech, language=["pt"] * len(speech)).to(asr.device, asr.dtype)
with torch.no_grad():
ids = asr.generate(**inputs, max_new_tokens=256)
return processor.decode(ids[:, inputs["input_ids"].shape[1]:], return_format="transcription_only")
for voice in sorted({r["voice"] for r in rows}):
mine = [r for r in rows if r["voice"] == voice]
for i in range(0, len(mine), 8):
for r, text in zip(mine[i:i + 8], transcribe([r["wav"] for r in mine[i:i + 8]])):
r["pred_text"] = text
folder = out / "scores" / voice; folder.mkdir(parents=True, exist_ok=True)
with open(folder / "minimax_pt.jsonl", "w", encoding="utf-8") as f:
for r in mine:
duration = sf.info(r["wav"]).duration
f.write(json.dumps({"audio_filepath": r["wav"], "duration": duration, "time": duration / 150.0,
"text": r["text"], "pred_text": r["pred_text"]}, ensure_ascii=False) + "\n")
print(f"== {voice} ==")
eval_utils.score_results(str(folder), language="pt")
if __name__ == "__main__":
ap = argparse.ArgumentParser(); sub = ap.add_subparsers(dest="cmd", required=True)
a = sub.add_parser("synth"); a.add_argument("--model", default=str(HERE / "weights" / "pingo-ptbr.bin"))
a.add_argument("--texts", default=str(HERE / "eval" / "minimax_pt_F_s4.jsonl")); a.add_argument("--out", required=True)
a.add_argument("--limit", type=int, default=100)
b = sub.add_parser("score"); b.add_argument("--out", required=True); b.add_argument("--leaderboard", required=True)
args = ap.parse_args(); {"synth": synth, "score": score}[args.cmd](args)