| { |
| "model": "VoiceHub/DACVAE-Turkish", |
| "training_source_commit": "ef31744", |
| "github_report_commit": "d82893caf3b86ba58b8947caddf1a8ce205397f5", |
| "checkpoint_sha256": "9f0db3e4eb7b5f242222926df4ac64b72ed8e1c67c07e2f5f87e8f6b14ff4f96", |
| "weights_sha256": "9f2ca66004a7457f10e105c0eba3f8ce44961e2f9ef65bb2813ddbd2e464c7dd", |
| "evaluation": { |
| "split": "validation", |
| "crop_samples": 256, |
| "crop_seconds": 3, |
| "asr_full_utterances": 32, |
| "asr_duration_seconds": 224.24159963151928, |
| "sample_rate": 48000, |
| "codec_precision": "fp32", |
| "posterior": "mean", |
| "watermark": "bypass", |
| "loudness_db": -16, |
| "asr_model": "openai/whisper-large-v3-turbo", |
| "asr_revision": "41f01f3fe87f28c78e2fbf8b568835947dd65ed9" |
| }, |
| "models": { |
| "meta": { |
| "reconstruction": { |
| "mel": 0.2786256688996218, |
| "stft": 1.0640862321015447, |
| "si_sdr_db": 16.884010054171085 |
| }, |
| "asr": { |
| "wer": { |
| "errors": 23, |
| "substitutions": 17, |
| "deletions": 4, |
| "insertions": 2, |
| "reference_units": 461, |
| "hypothesis_units": 459, |
| "rate": 0.049891540130151846 |
| }, |
| "cer": { |
| "errors": 74, |
| "substitutions": 24, |
| "deletions": 32, |
| "insertions": 18, |
| "reference_units": 2886, |
| "hypothesis_units": 2872, |
| "rate": 0.02564102564102564 |
| } |
| } |
| }, |
| "japanese": { |
| "reconstruction": { |
| "mel": 0.26558971946360543, |
| "stft": 0.971278426470235, |
| "si_sdr_db": 17.442306242883205 |
| }, |
| "asr": { |
| "wer": { |
| "errors": 23, |
| "substitutions": 18, |
| "deletions": 4, |
| "insertions": 1, |
| "reference_units": 461, |
| "hypothesis_units": 458, |
| "rate": 0.049891540130151846 |
| }, |
| "cer": { |
| "errors": 73, |
| "substitutions": 22, |
| "deletions": 34, |
| "insertions": 17, |
| "reference_units": 2886, |
| "hypothesis_units": 2869, |
| "rate": 0.025294525294525295 |
| } |
| } |
| }, |
| "turkish": { |
| "reconstruction": { |
| "mel": 0.22691508114803582, |
| "stft": 0.8432678864337504, |
| "si_sdr_db": 17.419028524309397 |
| }, |
| "asr": { |
| "wer": { |
| "errors": 19, |
| "substitutions": 15, |
| "deletions": 3, |
| "insertions": 1, |
| "reference_units": 461, |
| "hypothesis_units": 459, |
| "rate": 0.04121475054229935 |
| }, |
| "cer": { |
| "errors": 66, |
| "substitutions": 18, |
| "deletions": 34, |
| "insertions": 14, |
| "reference_units": 2886, |
| "hypothesis_units": 2866, |
| "rate": 0.02286902286902287 |
| } |
| } |
| } |
| }, |
| "paired_episode_bootstrap": { |
| "meta": { |
| "method": "paired episode-cluster percentile bootstrap", |
| "estimand": "pooled sample mean; episodes resampled uniformly with all their paired rows", |
| "n_samples": 256, |
| "n_episodes": 18, |
| "resamples": 10000, |
| "confidence": 0.95, |
| "metrics": { |
| "mel": { |
| "lower_is_better": true, |
| "baseline_mean": 0.2786256688996218, |
| "candidate_mean": 0.22691508114803582, |
| "candidate_minus_baseline": -0.05171058775158599, |
| "delta_ci": [ |
| -0.0592170431373271, |
| -0.04379978859163707 |
| ], |
| "sample_improvement_fraction": 1.0, |
| "sample_improvement_fraction_ci": [ |
| 1.0, |
| 1.0 |
| ], |
| "sample_tie_fraction": 0.0, |
| "episode_improvement_fraction": 1.0, |
| "episode_improvement_fraction_ci": [ |
| 1.0, |
| 1.0 |
| ], |
| "relative_loss_reduction_percent": 18.559161456949372, |
| "relative_loss_reduction_percent_ci": [ |
| 15.841186723179938, |
| 21.090114311096205 |
| ] |
| }, |
| "stft": { |
| "lower_is_better": true, |
| "baseline_mean": 1.0640862321015447, |
| "candidate_mean": 0.8432678864337504, |
| "candidate_minus_baseline": -0.22081834566779435, |
| "delta_ci": [ |
| -0.2577764498334056, |
| -0.18161389496220354 |
| ], |
| "sample_improvement_fraction": 1.0, |
| "sample_improvement_fraction_ci": [ |
| 1.0, |
| 1.0 |
| ], |
| "sample_tie_fraction": 0.0, |
| "episode_improvement_fraction": 1.0, |
| "episode_improvement_fraction_ci": [ |
| 1.0, |
| 1.0 |
| ], |
| "relative_loss_reduction_percent": 20.751922072300797, |
| "relative_loss_reduction_percent_ci": [ |
| 17.674230372339018, |
| 23.3378993907297 |
| ] |
| }, |
| "si_sdr_db": { |
| "lower_is_better": false, |
| "baseline_mean": 16.884010054171085, |
| "candidate_mean": 17.419028524309397, |
| "candidate_minus_baseline": 0.5350184701383114, |
| "delta_ci": [ |
| 0.4437410553231082, |
| 0.6316349267443525 |
| ], |
| "sample_improvement_fraction": 0.91796875, |
| "sample_improvement_fraction_ci": [ |
| 0.854251012145749, |
| 0.9700854700854701 |
| ], |
| "sample_tie_fraction": 0.0, |
| "episode_improvement_fraction": 1.0, |
| "episode_improvement_fraction_ci": [ |
| 1.0, |
| 1.0 |
| ] |
| }, |
| "mel_plus_stft": { |
| "lower_is_better": true, |
| "baseline_mean": 1.3427119010011666, |
| "candidate_mean": 1.0701829675817862, |
| "candidate_minus_baseline": -0.27252893341938034, |
| "delta_ci": [ |
| -0.3139729544149179, |
| -0.22711216552550356 |
| ], |
| "sample_improvement_fraction": 1.0, |
| "sample_improvement_fraction_ci": [ |
| 1.0, |
| 1.0 |
| ], |
| "sample_tie_fraction": 0.0, |
| "episode_improvement_fraction": 1.0, |
| "episode_improvement_fraction_ci": [ |
| 1.0, |
| 1.0 |
| ], |
| "relative_loss_reduction_percent": 20.29690309709585, |
| "relative_loss_reduction_percent_ci": [ |
| 17.405983380223457, |
| 22.672448447772194 |
| ] |
| } |
| } |
| }, |
| "japanese": { |
| "method": "paired episode-cluster percentile bootstrap", |
| "estimand": "pooled sample mean; episodes resampled uniformly with all their paired rows", |
| "n_samples": 256, |
| "n_episodes": 18, |
| "resamples": 10000, |
| "confidence": 0.95, |
| "metrics": { |
| "mel": { |
| "lower_is_better": true, |
| "baseline_mean": 0.26558971946360543, |
| "candidate_mean": 0.22691508114803582, |
| "candidate_minus_baseline": -0.03867463831556961, |
| "delta_ci": [ |
| -0.043182247805592405, |
| -0.03414554283863557 |
| ], |
| "sample_improvement_fraction": 1.0, |
| "sample_improvement_fraction_ci": [ |
| 1.0, |
| 1.0 |
| ], |
| "sample_tie_fraction": 0.0, |
| "episode_improvement_fraction": 1.0, |
| "episode_improvement_fraction_ci": [ |
| 1.0, |
| 1.0 |
| ], |
| "relative_loss_reduction_percent": 14.561797946727118, |
| "relative_loss_reduction_percent_ci": [ |
| 12.812924720810779, |
| 16.33984431013747 |
| ] |
| }, |
| "stft": { |
| "lower_is_better": true, |
| "baseline_mean": 0.971278426470235, |
| "candidate_mean": 0.8432678864337504, |
| "candidate_minus_baseline": -0.1280105400364846, |
| "delta_ci": [ |
| -0.14295202443047406, |
| -0.1122004633242202 |
| ], |
| "sample_improvement_fraction": 1.0, |
| "sample_improvement_fraction_ci": [ |
| 1.0, |
| 1.0 |
| ], |
| "sample_tie_fraction": 0.0, |
| "episode_improvement_fraction": 1.0, |
| "episode_improvement_fraction_ci": [ |
| 1.0, |
| 1.0 |
| ], |
| "relative_loss_reduction_percent": 13.179592642831906, |
| "relative_loss_reduction_percent_ci": [ |
| 11.664869168390986, |
| 14.588833521267814 |
| ] |
| }, |
| "si_sdr_db": { |
| "lower_is_better": false, |
| "baseline_mean": 17.442306242883205, |
| "candidate_mean": 17.419028524309397, |
| "candidate_minus_baseline": -0.02327771857380867, |
| "delta_ci": [ |
| -0.0824172738416143, |
| 0.036898595112216714 |
| ], |
| "sample_improvement_fraction": 0.3828125, |
| "sample_improvement_fraction_ci": [ |
| 0.2766798418972332, |
| 0.4947791811846689 |
| ], |
| "sample_tie_fraction": 0.0, |
| "episode_improvement_fraction": 0.3888888888888889, |
| "episode_improvement_fraction_ci": [ |
| 0.16666666666666666, |
| 0.6111111111111112 |
| ] |
| }, |
| "mel_plus_stft": { |
| "lower_is_better": true, |
| "baseline_mean": 1.2368681459338404, |
| "candidate_mean": 1.0701829675817862, |
| "candidate_minus_baseline": -0.1666851783520542, |
| "delta_ci": [ |
| -0.18402446937180994, |
| -0.14767500246274123 |
| ], |
| "sample_improvement_fraction": 1.0, |
| "sample_improvement_fraction_ci": [ |
| 1.0, |
| 1.0 |
| ], |
| "sample_tie_fraction": 0.0, |
| "episode_improvement_fraction": 1.0, |
| "episode_improvement_fraction_ci": [ |
| 1.0, |
| 1.0 |
| ], |
| "relative_loss_reduction_percent": 13.476390260354407, |
| "relative_loss_reduction_percent_ci": [ |
| 11.990223045831089, |
| 14.795814532473681 |
| ] |
| } |
| } |
| } |
| }, |
| "limitations": [ |
| "Single training seed for waveform150", |
| "Development validation used for selection; not final test", |
| "Only two ASR utterances improve versus baselines", |
| "No listening/MOS/speaker/TTS superiority claim", |
| "Turkish model fine-tuned from Japanese, not directly from Meta" |
| ] |
| } |
|
|
