File size: 4,028 Bytes
b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 2d7d9d7 b1882a9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | {
"schema": "jyoti-public-evaluation-v1",
"arm": "D3-manifest50-extra-p050",
"inference_checkpoint_sha256": "417ed68677b85e644dfeea2415f4d0349322861f7201dcbd9d16435f18a04e00",
"training_checkpoint_sha256": "c4fe5a6cfe741da64ff74c928a05189c8a62ab6f3556d72042b404eb9e136b9d",
"step": 100000,
"quality_qualified": false,
"sealed_final_cohort_opened": false,
"english": {
"asr": "facebook/wav2vec2-base-960h @ 22aad52d435eb6dbaf354bdad9b0da84ce7d6156 (greedy CTC)",
"pools": {
"clean": "128 LibriSpeech dev-clean utterances",
"difficult": "128 LibriSpeech dev-other utterances",
"pink_10db": "the clean pool + synthetic pink noise at 10 dB SNR (codec input)",
"pink_5db": "the clean pool + synthetic pink noise at 5 dB SNR"
},
"mode": "packed bitstream (encode -> bytes -> decode)",
"results": {
"clean": {
"codec_wer": 0.06684611933602512,
"original_wer": 0.02826379542395693,
"codec_cer": 0.02497897392767031,
"original_cer": 0.007485281749369218,
"wer_increase_pp": 3.8582323912068186,
"mean_stoi": 0.8995658662229111,
"mean_pesq_wb": 2.111289313994348,
"utterances": 128
},
"difficult": {
"codec_wer": 0.20997732426303856,
"original_wer": 0.07936507936507936,
"codec_cer": 0.09342347879532882,
"original_cer": 0.029765563262797435,
"wer_increase_pp": 13.061224489795919,
"mean_stoi": 0.8848347751522835,
"mean_pesq_wb": 1.973732240498066,
"utterances": 128
},
"pink_10db": {
"codec_wer": 0.24136384028712426,
"original_wer": 0.16913414087034545,
"codec_cer": 0.11648444070647603,
"original_cer": 0.07846930193439866,
"wer_increase_pp": 7.222969941677881,
"mean_stoi": 0.7282501492237068,
"mean_pesq_wb": 1.2500946335494518,
"utterances": 128
},
"pink_5db": {
"codec_wer": 0.4064602960969044,
"original_wer": 0.5118887393449978,
"codec_cer": 0.21412952060555088,
"original_cer": 0.30538267451640033,
"wer_increase_pp": -10.542844324809336,
"mean_stoi": 0.6389370863985571,
"mean_pesq_wb": 1.1396433534100652,
"utterances": 128
}
}
},
"development_gates": {
"note": "Engineering targets registered before training; not values from any paper. Unchanged.",
"gates": [
{
"gate": "clean_wer_increase_pp",
"value": 3.8582323912068186,
"operator": "<=",
"threshold": 5.0,
"passed": true
},
{
"gate": "difficult_wer_increase_pp",
"value": 13.061224489795919,
"operator": "<=",
"threshold": 10.0,
"passed": false
},
{
"gate": "pink_10db_wer_increase_pp",
"value": 7.222969941677881,
"operator": "<=",
"threshold": 10.0,
"passed": true
},
{
"gate": "pink_5db_wer_increase_pp",
"value": -10.542844324809336,
"operator": "<=",
"threshold": 10.0,
"passed": true
},
{
"gate": "clean_mean_stoi",
"value": 0.8995658662229111,
"operator": ">=",
"threshold": 0.85,
"passed": true
}
],
"passed": 4,
"total": 5
},
"nepali_supplemental": {
"asr": "ampixa/nepali-conformer-offline @ a7830b7ab16fbc2659b590b4e5e4030737c4caf9",
"pool": "128 SLR54 utterances from 32 held-out development speakers (not in training)",
"codec": {
"word_errors": 242,
"reference_words": 378,
"character_errors": 596,
"reference_characters": 2106,
"wer": 0.6402116402116402,
"cer": 0.28300094966761635
},
"original_audio": {
"word_errors": 154,
"reference_words": 378,
"character_errors": 310,
"reference_characters": 2106,
"wer": 0.4074074074074074,
"cer": 0.14719848053181386
}
},
"sources": {
"english_summary_sha256": "15f08d1ee80caa41b4a95ae5a473d5196205263f99e1ec089b6256eff8fe7abe",
"nepali_summary_sha256": "996f741f1c5c816b618503a00bc3140bbb246cbe22ef1f5e73352cab00496b1c",
"nepali_original_summary_sha256": "80f7fef5e23d2e5932d0de600c8268407d81727fda3f6592923b6a664de49d38",
"qualification_targets_sha256": "23e14e40b027de87c217488ee2c7989176e2930277ecc991395de5ef845b5145"
}
}
|