pingo-ptbr / evaluation.json
itxtoledo's picture
Duration limit for short utterances (fixes stretched 'Olá!')
99afc33 verified
Raw History Blame Contribute Delete
1.39 kB
{
"model": "pingo-ptbr",
"benchmark": "MiniMax pt (100 sentences per voice), Qwen3-ASR, open-tts-leaderboard normalizer",
"noise_note": "WER moves by about +/-0.2 points between runs; differences below that are noise",
"results": [
{
"model": "pingo-ptbr (width 64, quantized, with duration limit)",
"weights_mb": 0.871,
"wer_percent": {
"F_s4": 0.86,
"M_sm4": 0.81
},
"note": "measured after the duration limit of docs/EXPERIMENTS.md section 9; without the limit: F_s4 0.95, M_sm4 1.00"
},
{
"model": "pingo-ptbr-w96 (width 96, quantized)",
"weights_mb": 1.086,
"wer_percent": {
"F_s4": 0.81,
"M_sm4": 1.0
}
},
{
"model": "pivo-ptbr float32 (measured with MLX)",
"weights_mb": 5.61,
"wer_percent": {
"F_s4": 0.95,
"M_sm4": 0.95
}
},
{
"model": "pivo-ptbr float32 (ONNX, published)",
"weights_mb": 5.8,
"wer_percent": {
"F_s4": 0.95,
"M_sm4": 0.81
}
},
{
"model": "tiny-voca-ptbr",
"wer_percent": {
"F_s4": 0.9,
"M_sm4": 0.86
}
}
],
"quantization_study": "see docs/EXPERIMENTS.md",
"training_validation_loss": {
"2000": 0.2028,
"6000": 0.1841,
"10000": 0.1789,
"20000": 0.1736,
"30000": 0.1707,
"40000": 0.1699
}
}