Add v1 evaluation results (student, fine-tuned teacher, stock 1B, inference time)
Browse files- eval/inference_time.json +22 -0
- eval/large.json +185 -0
- eval/large_teacher.json +185 -0
- eval/stock_1b.json +126 -0
- eval/stock_1b_fleurs.json +68 -0
eval/inference_time.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"audio_seconds": 60.0,
|
| 3 |
+
"runs": 20,
|
| 4 |
+
"models": {
|
| 5 |
+
"teacher-300m": {
|
| 6 |
+
"ms": 249.481201171875,
|
| 7 |
+
"params": 325494996
|
| 8 |
+
},
|
| 9 |
+
"sota-base": {
|
| 10 |
+
"ms": 153.1131362915039,
|
| 11 |
+
"params": 72939491
|
| 12 |
+
},
|
| 13 |
+
"teacher-1b": {
|
| 14 |
+
"ms": 543.3133239746094,
|
| 15 |
+
"params": 975065300
|
| 16 |
+
},
|
| 17 |
+
"sota-large": {
|
| 18 |
+
"ms": 244.50640106201172,
|
| 19 |
+
"params": 203593119
|
| 20 |
+
}
|
| 21 |
+
}
|
| 22 |
+
}
|
eval/large.json
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"ckpt": "runs/ftfirst/stage3_kd16k/latest.pt",
|
| 3 |
+
"params_total": 203593119,
|
| 4 |
+
"params_encoder": 191023947.0,
|
| 5 |
+
"gmacs_per_audio_second": 12.274921696,
|
| 6 |
+
"dtype": "bf16",
|
| 7 |
+
"results": {
|
| 8 |
+
"ddd_dev": {
|
| 9 |
+
"cer": 0.02404184453722381,
|
| 10 |
+
"n": 400,
|
| 11 |
+
"rtf": 0.0025017910663208786,
|
| 12 |
+
"examples": [
|
| 13 |
+
{
|
| 14 |
+
"ref": "ααα
αΆααα’αΆααΆααα·ααααααααααΎαααΎαααααΏααααα αΆαα·ααααααΆααα»αααΆαααααα αα·αααΆααααααααΆααααααααα’αΆααααα»α",
|
| 15 |
+
"hyp": "ααα
αΆααα’αΆααΆαααααααααααααΎαααΎαααααΏααααα αΆαα·ααααααΆααα»αααΆαααααα αα·αααΆααααααααΆααααααααα’αΆααααα»α"
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"ref": "ααααΆααααααααα’αΆααΆαα·ααααΆααΆαααααααΎαααΎαααΆααα
ααααααα αα·αααΆαααααα ααΆααααααααα·ααααα½αα
ααα½ααααααααΎα±ααααΆααα
αααα",
|
| 19 |
+
"hyp": "ααααΆααααααααα’αΆααΆαα·ααααααΆαααααααΎαααΎαααΆααα
ααααααα αα·αααΆαααααα ααΆααααααααα·ααααα½αα
ααα½ααααααααΎα±ααααΆααα
αααα"
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"ref": "αα½αααααααΆαααααΏααΎαα·ααααΆαααααααΆαα· αα·αααΆααα·ααΈααΌααΆααααΈ αα·ααααα
α·ααα
αΉαααααΈ ααΆαααααΎα
ααααΆαααΆα ααα αα·αααααΆαααααα ααααΆααααααα½αααααΈαααΆααααααα½αααααα",
|
| 23 |
+
"hyp": "αα½αααααααΆαααααΏααΎαα·ααααΆαααααααΆαα· αα·αααΆααα·ααΈααΌααΆααααΈ αα·ααααα
α·ααα
αΉαααααΈ ααΆαααααΎα
ααααΆαααΆα ααα αα·αααααΆαααααα ααααΆααααααα½αααααΈαααΆααααααα½αααααα"
|
| 24 |
+
}
|
| 25 |
+
]
|
| 26 |
+
},
|
| 27 |
+
"ddd_test": {
|
| 28 |
+
"cer": 0.017692213963689533,
|
| 29 |
+
"n": 1993,
|
| 30 |
+
"rtf": 0.00235692812277266,
|
| 31 |
+
"examples": [
|
| 32 |
+
{
|
| 33 |
+
"ref": "αααα
αααααΌαααΆαα·αααα ααααα½ααααα»αα
αααΆααααααα²ααααΆαααααααααααααΆ ααΆαααΆαααΆααααααΆαα αα·ααααααΆααααααααααααΆαα’αΆα αΆαα²ααααΆααα»αααΆαααα’",
|
| 34 |
+
"hyp": "αααα
αααααΌαααΆαα·ααααααααα½ααααα»αα
αααΆααααααα±ααααΆαααααααααααααΆ ααΆαααΆαααΆααααααΆαα αα·ααααααΆααααααααααααΆαα’αΆα αΆαα±ααααΆααα»αααΆαααα’"
|
| 35 |
+
},
|
| 36 |
+
{
|
| 37 |
+
"ref": "ααΆααααααα αααα·αααΌααα½αα²αααααααααΆααΆααααΆααααααααΎα α αΎαααααΉαααΎααααα½αααααααααΌααααα·ααααα’αΌααααα αΎαααααα",
|
| 38 |
+
"hyp": "ααΆααααααααααα·αααΌααα½αα±αααααααααααΆαααΆαααΆααααααααΎα α αΎαααααΉαααΎααααα½αααααααααΌααααα·ααααα’αΌααααα αΎαααααα"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"ref": "ααΉααααα
ααΆαααααΆαααΆαααααααααααααα·αααα½α αααα»αα
ααααα’ααααααααα
α·ααααα»αααΆααααΎαα»αααΆα αα·ααααααααααααααααααααΆααα»αααααααααααααα",
|
| 42 |
+
"hyp": "ααΉααααα
ααΆαααααΆαααΆαααααααααααααα·αααα½ααααα»αα
ααααα’ααααααααα
α·ααααα»αααΆααααΎαα»αααΆα αα·ααααααααααααααααααααΆααα»αααααααααααααα"
|
| 43 |
+
}
|
| 44 |
+
]
|
| 45 |
+
},
|
| 46 |
+
"fleurs_km_dev": {
|
| 47 |
+
"cer": 0.10720711958371833,
|
| 48 |
+
"n": 326,
|
| 49 |
+
"rtf": 0.0027152539961343295,
|
| 50 |
+
"examples": [
|
| 51 |
+
{
|
| 52 |
+
"ref": "ααΆαααΎαα‘αΎαααααααααααΆααααΊαααααααα½αα₯αααα·ααααααααααααyzantine αα
ααΆαααΆαααΎα",
|
| 53 |
+
"hyp": "ααΆαααΎαααΎαααααααααααΆααααΊαααααααα½αα₯αααα·αααααααααααα αααα αααΆααααΈα n αα
ααΆαααΆαααΎα"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"ref": "ααΆααΆααααααααΆααααααααα·ααααααΆαααΆα
αααΎαααΌα
ααΆααΆαααααΎααααΆαααααααααααααΆααααΈα ααΆαααααΆααααααΌαααΈαααααααΆα
αααΆααα
ααΆαααααΆααΆααα ααΆαα·αα·αα αα·αααααΏααΆααααΆααααΆαααααααααααΆαααααααααΆααΆααααααα·ααααααΆααααα’ααααααααααααααααΆααααααα",
|
| 57 |
+
"hyp": "ααΆααΆααααααααΆααααααα·ααα αα·ααααααΆαααΆα
αααΎα ααΌα
ααΆααΆαααααΎααααΆαααααααααααααααΈα ααΆαααααΆααααααΌαααΈααααα
ααΆαα
ααΆααααααΆααΆαααααΆαα·αα·αα αα·αααααΏααΆααααΆααααΆααααααααα ααΆαααααααααΆααΆααααααα·ααααααΆααααα’ααααααααααααααααΆααααααα"
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"ref": "ααΆαααΆαααΎα α¬ααααα entinel ange ααΆααααααααααααΆαααααααααΈαα’ααααΆαααα·α ααΊαααα inson assif αααααΆαααααΌααααααα 4892 αααααα αααα ount inson",
|
| 61 |
+
"hyp": "ααΆαααΆαααΎαα¬ααααα entiner ing ααΆααααααααααααΆαααααααααΈαα’ααααΆαααα·α ααΊαααα αiαson ααΆααiα α αααααΆαααααΌααααααα 4892 αααααα ααααααΆα ααΈαααΉα "
|
| 62 |
+
}
|
| 63 |
+
]
|
| 64 |
+
},
|
| 65 |
+
"fleurs_km_test": {
|
| 66 |
+
"cer": 0.11397785977859778,
|
| 67 |
+
"n": 771,
|
| 68 |
+
"rtf": 0.002693553230811693,
|
| 69 |
+
"examples": [
|
| 70 |
+
{
|
| 71 |
+
"ref": "ααα»ααααα½αα
ααα½ααα·αααΆ ααΆαααα·ααΆαααααΌα ααα»ααααααα»αααααΆα
αααΎαααΏααΆααΆαααα·ααΆααα·αααααΌααα ααΆααααΆααααααααααααα’αΆαα·ααα ααααΎααααΎααα·ααα»ααα·ααααααΈ αα½αααΆααααααα’αΆαα·ααα αα·αααααΆαααααααααααα",
|
| 72 |
+
"hyp": "ααα»ααααα½αα
ααα½ααα·αααΆ αα»ααα»ααΈααααΌα ααα»ααααααα»αααααΆα
αααΎαααΏααΆ ααΌαα»ααΈαα·αααααΌααα ααΆααααΆααααααααααααα’αΆαα·αααααααΎααααΎααα·ααα»ααα·ααααααΈ αα½αααΆααααααα’αΆαα·ααα αα·αααααΆαααααααααααα"
|
| 73 |
+
},
|
| 74 |
+
{
|
| 75 |
+
"ref": "ααααΎααα
ααΎααααααα’αααααΉαααΆαααααα·ααααααΆαα½αααΉααααααΆααααΆααααα
αααααΉαα’αΆα
ααΉαααΆαα»αααΆα αα½α
αααααΎαααΆαααααα½ααααααα",
|
| 76 |
+
"hyp": "ααααΎααα
ααΎααααααα’αααααΉαααΆαααααΆααα·ααααααΆαα½αααΌααααααΆααααΆααααα
αααααΉαα’αΆα
ααΉαααΆαα»αααΆα αα½α
αααααΎαααΆαααααα½ααααααα"
|
| 77 |
+
},
|
| 78 |
+
{
|
| 79 |
+
"ref": "ααΈαααα»ααααααααΆααΌαααααΆαααΎααααΈα‘αΎαααααααααΎα yiragongo αα½αααΆαα½αααΉαααΆαααΆαountain orilla αααααΆααααααααααααα»ααα
α’αΆα ααααα·α",
|
| 80 |
+
"hyp": "ααΈαααα»ααααααααΆααΌαααααΆαααΎααααΈα‘αΎαααααααααΎα αααααΆααααΌ αα½αααΆαα½αααΉαααΆαααΆα ααααααα ααΌααΈα‘αΆ ls αααααΆαααααααααααα»ααα
α’αΆα ααααα·α"
|
| 81 |
+
}
|
| 82 |
+
]
|
| 83 |
+
},
|
| 84 |
+
"fleurs_en_dev": {
|
| 85 |
+
"cer": 0.08005514253360248,
|
| 86 |
+
"n": 394,
|
| 87 |
+
"wer": 0.2121286520991898,
|
| 88 |
+
"rtf": 0.0025477642716881673,
|
| 89 |
+
"examples": [
|
| 90 |
+
{
|
| 91 |
+
"ref": "when you call someone who is thousands of miles away you are using a satellite",
|
| 92 |
+
"hyp": "when you call someone who is thousands of miles away you are using a satellite"
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"ref": "now widely available throughout the archipelago javanese cuisine features an array of simply seasoned dishes the predominant flavorings the javanese favor being peanuts chillies sugar especially javanese coconut sugar and various aromatic spices",
|
| 96 |
+
"hyp": "now widely available throughout the archiplago japhanese cuisines features in a ray of simply season dishes the predominant flavorings the japanese favor being penuth chillis sugar especially japhanese coconut sugar and various aromatic spices"
|
| 97 |
+
},
|
| 98 |
+
{
|
| 99 |
+
"ref": "the un also hopes to finalize a fund to help countries affected by global warming to cope with the impacts",
|
| 100 |
+
"hyp": "the un also hopes to finalize a fund to help countries affected by global warming to cope with the impacts"
|
| 101 |
+
}
|
| 102 |
+
]
|
| 103 |
+
},
|
| 104 |
+
"fleurs_en_test": {
|
| 105 |
+
"cer": 0.0755862855513579,
|
| 106 |
+
"n": 647,
|
| 107 |
+
"wer": 0.19248466257668712,
|
| 108 |
+
"rtf": 0.0024945803598843365,
|
| 109 |
+
"examples": [
|
| 110 |
+
{
|
| 111 |
+
"ref": "however due to the slow communication channels styles in the west could lag behind by 25 to 30 year",
|
| 112 |
+
"hyp": "however due to the slow communication channels styles in the west could lag behind by 25 -30 years"
|
| 113 |
+
},
|
| 114 |
+
{
|
| 115 |
+
"ref": "all nouns alongside the word sie for you always begin with a capital letter even in the middle of a sentence",
|
| 116 |
+
"hyp": "all nowns alongside the world say for you always begin with a capital letter even in the middle of a sente"
|
| 117 |
+
},
|
| 118 |
+
{
|
| 119 |
+
"ref": "to the north and within easy reach is the romantic and fascinating town of sintra and which was made famous to foreigners after a glowing account of its splendours recorded by lord byron",
|
| 120 |
+
"hyp": "to the north and within easy reach as tha romantic and fastinating town of cintra and which was made famous to foreigners after a glowing account of hexplengureis recorded by lord byron"
|
| 121 |
+
}
|
| 122 |
+
]
|
| 123 |
+
},
|
| 124 |
+
"libri_dev_clean": {
|
| 125 |
+
"cer": 0.019471441182309288,
|
| 126 |
+
"n": 2703,
|
| 127 |
+
"wer": 0.059648542333002465,
|
| 128 |
+
"rtf": 0.0024097869451522305,
|
| 129 |
+
"examples": [
|
| 130 |
+
{
|
| 131 |
+
"ref": "mister quilter is the apostle of the middle classes and we are glad to welcome his gospel",
|
| 132 |
+
"hyp": "mister quilter is the apostle of the middle classes and we are glad to welcome his gospel"
|
| 133 |
+
},
|
| 134 |
+
{
|
| 135 |
+
"ref": "nor is mister quilter's manner less interesting than his matter",
|
| 136 |
+
"hyp": "nor is mister quilter's manner less interesting than his matter"
|
| 137 |
+
},
|
| 138 |
+
{
|
| 139 |
+
"ref": "he tells us that at this festive season of the year with christmas and roast beef looming before us similes drawn from eating and its results occur most readily to the mind",
|
| 140 |
+
"hyp": "he tells us that at this festive season of the year with christmus and rost beef looming before us symily is drawn from eating and its results occur most readily to the mind"
|
| 141 |
+
}
|
| 142 |
+
]
|
| 143 |
+
},
|
| 144 |
+
"libri_test_clean": {
|
| 145 |
+
"cer": 0.01943223332498467,
|
| 146 |
+
"n": 2620,
|
| 147 |
+
"wer": 0.0607311320754717,
|
| 148 |
+
"rtf": 0.0024275556567816365,
|
| 149 |
+
"examples": [
|
| 150 |
+
{
|
| 151 |
+
"ref": "he hoped there would be stew for dinner turnips and carrots and bruised potatoes and fat mutton pieces to be ladled out in thick peppered flour fattened sauce",
|
| 152 |
+
"hyp": "he hoped there would be stew for dinner turnips and carrots and bruised potatoes and fat mutton pieces to be ladled out in thick peppered flower fatten sauce"
|
| 153 |
+
},
|
| 154 |
+
{
|
| 155 |
+
"ref": "stuff it into you his belly counselled him",
|
| 156 |
+
"hyp": "stuffeit into you his belly councelled him"
|
| 157 |
+
},
|
| 158 |
+
{
|
| 159 |
+
"ref": "after early nightfall the yellow lamps would light up here and there the squalid quarter of the brothels",
|
| 160 |
+
"hyp": "after early night fall the yellow lamps would light up here and there the squalid quarter of the braffles"
|
| 161 |
+
}
|
| 162 |
+
]
|
| 163 |
+
},
|
| 164 |
+
"libri_test_other": {
|
| 165 |
+
"cer": 0.06964280917288251,
|
| 166 |
+
"n": 2939,
|
| 167 |
+
"wer": 0.1672812028351451,
|
| 168 |
+
"rtf": 0.0023894908624727414,
|
| 169 |
+
"examples": [
|
| 170 |
+
{
|
| 171 |
+
"ref": "there's iron they say in all our blood and a grain or two perhaps is good but his he makes me harshly feel has got a little too much of steel anon",
|
| 172 |
+
"hyp": "the's i and they say in all our blood and a gran too perhaps is good but his he makes me harshly feel has got a little too much of steel anon"
|
| 173 |
+
},
|
| 174 |
+
{
|
| 175 |
+
"ref": "margaret said mister hale as he returned from showing his guest downstairs i could not help watching your face with some anxiety when mister thornton made his confession of having been a shop boy",
|
| 176 |
+
"hyp": "margaret said mister hal as he returned from showing his guis downstairs i could not help watching your face with some anxiety when mister thawnton made his confission of having been a shop boy"
|
| 177 |
+
},
|
| 178 |
+
{
|
| 179 |
+
"ref": "you don't mean that you thought me so silly",
|
| 180 |
+
"hyp": "you don't mean that you thought me so silly"
|
| 181 |
+
}
|
| 182 |
+
]
|
| 183 |
+
}
|
| 184 |
+
}
|
| 185 |
+
}
|
eval/large_teacher.json
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"ckpt": "runs/ftfirst/stage1_finetune/teacher.pt",
|
| 3 |
+
"params_total": 975065300,
|
| 4 |
+
"params_encoder": 962496128.0,
|
| 5 |
+
"gmacs_per_audio_second": 50.277134336,
|
| 6 |
+
"dtype": "bf16",
|
| 7 |
+
"results": {
|
| 8 |
+
"ddd_dev": {
|
| 9 |
+
"cer": 0.033986958765304685,
|
| 10 |
+
"n": 400,
|
| 11 |
+
"rtf": 0.006292361957186866,
|
| 12 |
+
"examples": [
|
| 13 |
+
{
|
| 14 |
+
"ref": "ααα
αΆααα’αΆααΆααα·ααααααααααΎαααΎαααααΏααααα αΆαα·ααααααΆααα»αααΆαααααα αα·αααΆααααααααΆααααααααα’αΆααααα»α",
|
| 15 |
+
"hyp": "ααα
αΆααα’αΆααΆαααα·ααααααααααΎαααΎαααααΏααααα αΆααΉααααααΆααα»αααΆαααααα αα·αααΆααααααααΆααααααααα’αΆααααα»α"
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"ref": "ααααΆααααααααα’αΆααΆαα·ααααΆααΆαααααααΎαααΎαααΆααα
ααααααα αα·αααΆαααααα ααΆααααααααα·ααααα½αα
ααα½ααααααααΎα±ααααΆααα
αααα",
|
| 19 |
+
"hyp": "ααααΆααααααααα’αΆααΆαα·ααααΆααΆαααααααΎαααΎαααΆααα
ααααααα αα·αααΆαααααα ααΆααααααααα·ααααα½αα
ααα½ααααααααΎα±ααααΆααα
αααα"
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"ref": "αα½αααααααΆαααααΏααΎαα·ααααΆαααααααΆαα· αα·αααΆααα·ααΈααΌααΆααααΈ αα·ααααα
α·ααα
αΉαααααΈ ααΆαααααΎα
ααααΆαααΆα ααα αα·αααααΆαααααα ααααΆααααααα½αααααΈαααΆααααααα½αααααα",
|
| 23 |
+
"hyp": "αα½αααααααΆαααααΏααΎαα·ααααΆαααααααΆαα· αα·αααΆααα·ααΈααΌααΆααααΈ αα·ααααα
α·ααα
αΉαααααΈ ααΆαααααΎα
ααααΆαααΆα ααα αα·αααααΆαααααα ααααΆααααααα½αααααΈαααΆααααααα½αααααα"
|
| 24 |
+
}
|
| 25 |
+
]
|
| 26 |
+
},
|
| 27 |
+
"ddd_test": {
|
| 28 |
+
"cer": 0.024715486779578407,
|
| 29 |
+
"n": 1993,
|
| 30 |
+
"rtf": 0.0059981543289738416,
|
| 31 |
+
"examples": [
|
| 32 |
+
{
|
| 33 |
+
"ref": "αααα
αααααΌαααΆαα·αααα ααααα½ααααα»αα
αααΆααααααα²ααααΆαααααααααααααΆ ααΆαααΆαααΆααααααΆαα αα·ααααααΆααααααααααααΆαα’αΆα αΆαα²ααααΆααα»αααΆαααα’",
|
| 34 |
+
"hyp": "αααα
αααααΌαααΆαα·αααα ααααα½ααααα»αα
αααΆαααααα±ααααΆαααααααααααααΆ ααΆαααΆαααΆαααααΆαα αα·ααααααΆααααααααααααΆαα’αΆα αΆαα±ααααΆααα»αααΆαααα’"
|
| 35 |
+
},
|
| 36 |
+
{
|
| 37 |
+
"ref": "ααΆααααααα αααα·αααΌααα½αα²αααααααααΆααΆααααΆααααααααΎα α αΎαααααΉαααΎααααα½αααααααααΌααααα·ααααα’αΌααααα αΎαααααα",
|
| 38 |
+
"hyp": "ααΆααααααααααα·αααΌααα½αα±αααααααααααΆαααΆαααΆααααααααΎα α αΎαααααΉαααΎααααα½αααααααααΌααααα·ααααα’αΌααααα αΎαααααα"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"ref": "ααΉααααα
ααΆαααααΆαααΆαααααααααααααα·αααα½α αααα»αα
ααααα’ααααααααα
α·ααααα»αααΆααααΎαα»αααΆα αα·ααααααααααααααααααααΆααα»αααααααααααααα",
|
| 42 |
+
"hyp": "ααΉααααα
ααΆαααααΆαααΆαααααααααααααα·αααα½ααααα»αα
ααααα’ααααααααα
α·ααααα»αααΆααααΎαα»αααΆα αα·ααααααααααααααααααααΆααα»αααααααααααααα"
|
| 43 |
+
}
|
| 44 |
+
]
|
| 45 |
+
},
|
| 46 |
+
"fleurs_km_dev": {
|
| 47 |
+
"cer": 0.08833827748869329,
|
| 48 |
+
"n": 326,
|
| 49 |
+
"rtf": 0.006745861517385598,
|
| 50 |
+
"examples": [
|
| 51 |
+
{
|
| 52 |
+
"ref": "ααΆαααΎαα‘αΎαααααααααααΆααααΊαααααααα½αα₯αααα·ααααααααααααyzantine αα
ααΆαααΆαααΎα",
|
| 53 |
+
"hyp": "ααΆαααΎαα‘αΎαααααααααααΆααααΊαααααααα½αα₯αααα·ααααααααααααααααΆααΈα αα
ααΆαααΆαααΎα"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"ref": "ααΆααΆααααααααΆααααααααα·ααααααΆαααΆα
αααΎαααΌα
ααΆααΆαααααΎααααΆαααααααααααααΆααααΈα ααΆαααααΆααααααΌαααΈαααααααΆα
αααΆααα
ααΆαααααΆααΆααα ααΆαα·αα·αα αα·αααααΏααΆααααΆααααΆαααααααααααΆαααααααααΆααΆααααααα·ααααααΆααααα’ααααααααααααααααΆααααααα",
|
| 57 |
+
"hyp": "ααΆααΆααααααααΆααααααααα·ααααααΆαααΆα
αααΎα ααΌα
ααΆααΆαααααΎααααΆαααααααααααααΆααααΈα ααΆαααααΆααααααΌαα§ααΆαααααααα·αααΆαααααΆααααααΆααΆαααααΆαα·αα·αα αα·αααααΏααΆααααΆααααΆαααααααααααΆαααααααααΆααΆααααα αα·ααααααΆααααα’ααααααααααααααααΆααααααα"
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"ref": "ααΆαααΆαααΎα α¬ααααα entinel ange ααΆααααααααααααΆαααααααααΈαα’ααααΆαααα·α ααΊαααα inson assif αααααΆαααααΌααααααα 4892 αααααα αααα ount inson",
|
| 61 |
+
"hyp": "ααΆαααΆαααΎα α¬ααααα entinel ring ααΆααααααααααααΆαααααααααΈαα’ααααΆαααα·α ααΊαααα iαson αasααΈf αααααΆαααααΌααααααα 4892 αααααα ααααααΆn iαsα·α"
|
| 62 |
+
}
|
| 63 |
+
]
|
| 64 |
+
},
|
| 65 |
+
"fleurs_km_test": {
|
| 66 |
+
"cer": 0.09440590405904059,
|
| 67 |
+
"n": 771,
|
| 68 |
+
"rtf": 0.0066833982391532555,
|
| 69 |
+
"examples": [
|
| 70 |
+
{
|
| 71 |
+
"ref": "ααα»ααααα½αα
ααα½ααα·αααΆ ααΆαααα·ααΆαααααΌα ααα»ααααααα»αααααΆα
αααΎαααΏααΆααΆαααα·ααΆααα·αααααΌααα ααΆααααΆααααααααααααα’αΆαα·ααα ααααΎααααΎααα·ααα»ααα·ααααααΈ αα½αααΆααααααα’αΆαα·ααα αα·αααααΆαααααααααααα",
|
| 72 |
+
"hyp": "ααα»ααααα½αα
ααα½ααα·αααΆ ααα·ααΆαααααΌα ααα»ααααααα»αααααΆα
αααΎαααΏααΆααα·ααΆααα·αααααΌααα ααΆααααΆααααααααααααα’αΆαα·αααααααΎααααΎααα·ααα»ααα·ααααααΈ αα½αααΆααααααα’αΆαα·αααααΉαααααΆαααα’αααααααααα"
|
| 73 |
+
},
|
| 74 |
+
{
|
| 75 |
+
"ref": "ααααΎααα
ααΎααααααα’αααααΉαααΆαααααα·ααααααΆαα½αααΉααααααΆααααΆααααα
αααααΉαα’αΆα
ααΉαααΆαα»αααΆα αα½α
αααααΎαααΆαααααα½ααααααα",
|
| 76 |
+
"hyp": "ααααΎα αα
ααΎααααααα’αααααΉαααΆαααααΆαααααααΆαα½αααΉααααααΆααααΆααααα
αααααΉαα’αΆα
ααΉαααΆαα»αααΆα αα½α
αααααΎαααΆαααααα½ααααααα"
|
| 77 |
+
},
|
| 78 |
+
{
|
| 79 |
+
"ref": "ααΈαααα»ααααααααΆααΌαααααΆαααΎααααΈα‘αΎαααααααααΎα yiragongo αα½αααΆαα½αααΉαααΆαααΆαountain orilla αααααΆααααααααααααα»ααα
α’αΆα ααααα·α",
|
| 80 |
+
"hyp": "ααΈαααα»ααααααααΆααΌαααααΆαααΎααααΈα‘αΎαααααααααΎα ααΈαααΆαoαααΌ αα½αααΆαα½αααΉαααΆαααΆ ountan ααΌrαΈα‘αΆ αααααΆαααααααααααα»ααα
α’αΆα ααααα·α"
|
| 81 |
+
}
|
| 82 |
+
]
|
| 83 |
+
},
|
| 84 |
+
"fleurs_en_dev": {
|
| 85 |
+
"cer": 0.05019447589975875,
|
| 86 |
+
"n": 394,
|
| 87 |
+
"wer": 0.1367542352074638,
|
| 88 |
+
"rtf": 0.00638872020448837,
|
| 89 |
+
"examples": [
|
| 90 |
+
{
|
| 91 |
+
"ref": "when you call someone who is thousands of miles away you are using a satellite",
|
| 92 |
+
"hyp": "when you call someone who has thousands of miles away you are using a satellite"
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"ref": "now widely available throughout the archipelago javanese cuisine features an array of simply seasoned dishes the predominant flavorings the javanese favor being peanuts chillies sugar especially javanese coconut sugar and various aromatic spices",
|
| 96 |
+
"hyp": "now widely available throughout the archipelago japanese quisines features an array of simply season dishes the predominant flavorings the japanese favor being pinat chiles sugar especially japanese cocoanut sugar and various aromatic spices"
|
| 97 |
+
},
|
| 98 |
+
{
|
| 99 |
+
"ref": "the un also hopes to finalize a fund to help countries affected by global warming to cope with the impacts",
|
| 100 |
+
"hyp": "the un also hopes to finalize a fund to help countries affected by global warming to cope with the impacts"
|
| 101 |
+
}
|
| 102 |
+
]
|
| 103 |
+
},
|
| 104 |
+
"fleurs_en_test": {
|
| 105 |
+
"cer": 0.043481360039917316,
|
| 106 |
+
"n": 647,
|
| 107 |
+
"wer": 0.11942275515895148,
|
| 108 |
+
"rtf": 0.0062595013694743485,
|
| 109 |
+
"examples": [
|
| 110 |
+
{
|
| 111 |
+
"ref": "however due to the slow communication channels styles in the west could lag behind by 25 to 30 year",
|
| 112 |
+
"hyp": "however due to the slow communication channels styles in the west could lag behind by 25to 30 years"
|
| 113 |
+
},
|
| 114 |
+
{
|
| 115 |
+
"ref": "all nouns alongside the word sie for you always begin with a capital letter even in the middle of a sentence",
|
| 116 |
+
"hyp": "all nowns alongside the world say for you always begin with a capital letter even in the middle of a sentence"
|
| 117 |
+
},
|
| 118 |
+
{
|
| 119 |
+
"ref": "to the north and within easy reach is the romantic and fascinating town of sintra and which was made famous to foreigners after a glowing account of its splendours recorded by lord byron",
|
| 120 |
+
"hyp": "to the north and within easy reach is the romantic and fascinating town of cintra and which was made famous to foreigners after a glowing account of its splenduress recorded by lord byron"
|
| 121 |
+
}
|
| 122 |
+
]
|
| 123 |
+
},
|
| 124 |
+
"libri_dev_clean": {
|
| 125 |
+
"cer": 0.01169131218929113,
|
| 126 |
+
"n": 2703,
|
| 127 |
+
"wer": 0.03722289621705084,
|
| 128 |
+
"rtf": 0.006076880907681458,
|
| 129 |
+
"examples": [
|
| 130 |
+
{
|
| 131 |
+
"ref": "mister quilter is the apostle of the middle classes and we are glad to welcome his gospel",
|
| 132 |
+
"hyp": "mister quilter is the apostle of the middle classes and we are glad to welcome his gospel"
|
| 133 |
+
},
|
| 134 |
+
{
|
| 135 |
+
"ref": "nor is mister quilter's manner less interesting than his matter",
|
| 136 |
+
"hyp": "nor is mister quilter's manner less interesting than his matter"
|
| 137 |
+
},
|
| 138 |
+
{
|
| 139 |
+
"ref": "he tells us that at this festive season of the year with christmas and roast beef looming before us similes drawn from eating and its results occur most readily to the mind",
|
| 140 |
+
"hyp": "he tells us that at this festive season of the year with christmas and rose beef looming before us similees drawn from eating and its results occur most readily to the mind"
|
| 141 |
+
}
|
| 142 |
+
]
|
| 143 |
+
},
|
| 144 |
+
"libri_test_clean": {
|
| 145 |
+
"cer": 0.011085009543385699,
|
| 146 |
+
"n": 2620,
|
| 147 |
+
"wer": 0.03610012172854534,
|
| 148 |
+
"rtf": 0.006125383591282291,
|
| 149 |
+
"examples": [
|
| 150 |
+
{
|
| 151 |
+
"ref": "he hoped there would be stew for dinner turnips and carrots and bruised potatoes and fat mutton pieces to be ladled out in thick peppered flour fattened sauce",
|
| 152 |
+
"hyp": "he hoped there would be stew for dinner turnips and carrots and bruised potatoes and fat mutton pieces to be ladled out in thick peppered flower fattened sauce"
|
| 153 |
+
},
|
| 154 |
+
{
|
| 155 |
+
"ref": "stuff it into you his belly counselled him",
|
| 156 |
+
"hyp": "stuff it into you his belly counselled him"
|
| 157 |
+
},
|
| 158 |
+
{
|
| 159 |
+
"ref": "after early nightfall the yellow lamps would light up here and there the squalid quarter of the brothels",
|
| 160 |
+
"hyp": "after early nightfall the yellow lamps would light up here and there the squalid quarter of the broffles"
|
| 161 |
+
}
|
| 162 |
+
]
|
| 163 |
+
},
|
| 164 |
+
"libri_test_other": {
|
| 165 |
+
"cer": 0.030955344430813866,
|
| 166 |
+
"n": 2939,
|
| 167 |
+
"wer": 0.082093116558088,
|
| 168 |
+
"rtf": 0.006014662979128097,
|
| 169 |
+
"examples": [
|
| 170 |
+
{
|
| 171 |
+
"ref": "there's iron they say in all our blood and a grain or two perhaps is good but his he makes me harshly feel has got a little too much of steel anon",
|
| 172 |
+
"hyp": "there's iron they say in all our blood and a grain or two perhaps is good but his he makes me harshly feel has got a little too much of steel anon"
|
| 173 |
+
},
|
| 174 |
+
{
|
| 175 |
+
"ref": "margaret said mister hale as he returned from showing his guest downstairs i could not help watching your face with some anxiety when mister thornton made his confession of having been a shop boy",
|
| 176 |
+
"hyp": "margaret said mister hell as he returned from showing his guest downstairs i could not help watching your face with some anxiety when mister thornton made his confession of having been a shop boy"
|
| 177 |
+
},
|
| 178 |
+
{
|
| 179 |
+
"ref": "you don't mean that you thought me so silly",
|
| 180 |
+
"hyp": "you don't mean that you thought me so silly"
|
| 181 |
+
}
|
| 182 |
+
]
|
| 183 |
+
}
|
| 184 |
+
}
|
| 185 |
+
}
|
eval/stock_1b.json
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"ckpt": "/run/media/pc/disk1/ml-data/models/pretrained/omniASR-CTC-1B/omniASR-CTC-1B.omniprune.pt",
|
| 3 |
+
"params_total": 975065300,
|
| 4 |
+
"params_encoder": 962496128.0,
|
| 5 |
+
"gmacs_per_audio_second": 50.277134336,
|
| 6 |
+
"dtype": "bf16",
|
| 7 |
+
"results": {
|
| 8 |
+
"ddd_dev": {
|
| 9 |
+
"cer": 0.07939672561805132,
|
| 10 |
+
"n": 400,
|
| 11 |
+
"rtf": 0.006409313296584102,
|
| 12 |
+
"examples": [
|
| 13 |
+
{
|
| 14 |
+
"ref": "ααα
αΆααα’αΆααΆααα·ααααααααααΎαααΎαααααΏααααα αΆαα·ααααααΆααα»αααΆαααααα αα·αααΆααααααααΆααααααααα’αΆααααα»α",
|
| 15 |
+
"hyp": "ααα
αΆαα α’αΆααΆααα·α ααααα ααααΎαααΎα ααααΏα αααα αΆααΉα ααα ααΆα αα»αααΆα ααααα αα·α ααΆα ααααααααΆα ααααα ααα’αΆα αααα»α"
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"ref": "ααααΆααααααααα’αΆααΆαα·ααααΆααΆαααααααΎαααΎαααΆααα
ααααααα αα·αααΆαααααα ααΆααααααααα·ααααα½αα
ααα½ααααααααΎα±ααααΆααα
αααα",
|
| 19 |
+
"hyp": "ααααΆααα·ααααα α’αΆααΆαα·αα ααΆααΆαα ααα ααΎα ααΎα ααΆα αα
ααααααα αα·α ααΆαααααα ααΆα αααααα αα·ααα αα½α α
ααα½α ααα ααααΎ α²αα ααΆ ααα
ααααα"
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"ref": "αα½αααααααΆαααααΏααΎαα·ααααΆαααααααΆαα· αα·αααΆααα·ααΈααΌααΆααααΈ αα·ααααα
α·ααα
αΉαααααΈ ααΆαααααΎα
ααααΆαααΆα ααα αα·αααααΆαααααα ααααΆααααααα½αααααΈαααΆααααααα½αααααα",
|
| 23 |
+
"hyp": "αα½ααα αα ααΆα ααααΏ ααΎ αα·ααααΆα ααααααΆαα· αα·α ααΆα αα·ααΈ ααΌ ααΆαααααΈ αα·α ααα α
α·ααα
αΉα αααααΈ ααΆα ααααΎ α
αααΆα ααΆα ααα αα·α ααααΆα ααααααα αα ααΆ ααααα αα½α αα ααΈαααΆα αααα αα½ααα ααα"
|
| 24 |
+
}
|
| 25 |
+
]
|
| 26 |
+
},
|
| 27 |
+
"ddd_test": {
|
| 28 |
+
"cer": 0.05946630924820275,
|
| 29 |
+
"n": 1993,
|
| 30 |
+
"rtf": 0.0060830656866102135,
|
| 31 |
+
"examples": [
|
| 32 |
+
{
|
| 33 |
+
"ref": "αααα
αααααΌαααΆαα·αααα ααααα½ααααα»αα
αααΆααααααα²ααααΆαααααααααααααΆ ααΆαααΆαααΆααααααΆαα αα·ααααααΆααααααααααααΆαα’αΆα αΆαα²ααααΆααα»αααΆαααα’",
|
| 34 |
+
"hyp": "αααα
αααααΌα ααΆαα· αααα ααα αα½α αααα½α α
αααΆ αααα α±αα ααΆα αααααα ααααααΆ ααΆαααΆα ααΆααα»αααΈαα½α αα·α αααααΆ αααααααα ααααΆα α’αΆα αΆα α±αα ααΆα αα»αααΆα ααα’"
|
| 35 |
+
},
|
| 36 |
+
{
|
| 37 |
+
"ref": "ααΆααααααα αααα·αααΌααα½αα²αααααααααΆααΆααααΆααααααααΎα α αΎαααααΉαααΎααααα½αααααααααΌααααα·ααααα’αΌααααα αΎαααααα",
|
| 38 |
+
"hyp": "ααΆααααααα αααα·αααΌα αα½α α±αα αααααα
αΆααα ααΆα ααΆα ααα ααααΎα α αΎα ααααΉαααΎα αα αα½α αααααα ααΌα αααα·α αααα’αΌα αααα αΎα αα ααα"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"ref": "ααΉααααα
ααΆαααααΆαααΆαααααααααααααα·αααα½α αααα»αα
ααααα’ααααααααα
α·ααααα»αααΆααααΎαα»αααΆα αα·ααααααααααααααααααααΆααα»αααααααααααααα",
|
| 42 |
+
"hyp": "ααΉα αααα
ααΆα ααααΆα ααΆ ααααααα αα αα αα·αα αα½α αααα»α α
αααα α’ααα ααα αα α
α·ααα αα»αααΆαα ααΎ αα»αααΆα αα·α ααααααα ααααααα ααα ααΆα αα»α αααααααα ααααα"
|
| 43 |
+
}
|
| 44 |
+
]
|
| 45 |
+
},
|
| 46 |
+
"fleurs_km_test": {
|
| 47 |
+
"cer": 0.11262976629766297,
|
| 48 |
+
"n": 771,
|
| 49 |
+
"rtf": 0.006777807880311398,
|
| 50 |
+
"examples": [
|
| 51 |
+
{
|
| 52 |
+
"ref": "ααα»ααααα½αοΏ½οΏ½ααα½ααα·αααΆ ααΆαααα·ααΆαααααΌα ααα»ααααααα»αααααΆα
αααΎαααΏααΆααΆαααα·ααΆααα·αααααΌααα ααΆααααΆααααααααααααα’αΆαα·ααα ααααΎααααΎααα·ααα»ααα·ααααααΈ αα½αααΆααααααα’αΆαα·ααα αα·αααααΆαααααααααααα",
|
| 53 |
+
"hyp": "ααα»ααααα½αα
ααα½ααα·αααΆααΆαααα·ααΆαααααΌα ααα»ααααααα»αααααΆα
αααΎαααΏααΆααΆαααα·ααΆααα·αααααΌααα ααΆααααΆααααααααααααα’αΆαα·αααααααΎααααΎααα·ααα
αα·ααααααΈαα½αααΆααααααα’αΆαα·ααααα·αααααΆαααα’αααααααααα"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"ref": "ααααΎααα
ααΎααααααα’αααααΉαααΆαααααα·ααααααΆαα½αααΉααααααΆααααΆααααα
αααααΉαα’αΆα
ααΉαααΆαα»αααΆα αα½α
αααααΎαααΆαααααα½ααααααα",
|
| 57 |
+
"hyp": "ααααΎα αα
ααΎααααααα’αααααΉαααΆαααααΉαααα·ααααααΆαα½αααΉαααααααααΆααααα
αααααΉαα’αΆα
ααΉαααΆαα»αααΆα αα½α
αααααΎαααΆαααααα½ααααααα"
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"ref": "ααΈαααα»ααααααααΆααΌαααααΆαααΎααααΈα‘αΎαααααααααΎα yiragongo αα½αααΆαα½αααΉαααΆαααΆαountain orilla αααααΆααααααααααααα»ααα
α’αΆα ααααα·α",
|
| 61 |
+
"hyp": "ααΈαααα»ααααααααΆααΌαααααΆαααΎααααΈα‘αΎαααααααααΎα ααΈαααΆααoαααΌ αα½αααΆαα½αααΉαααΆαααΆαααααtαα ααΌααΈα‘αΆ αααααΆαααααααααααα»ααα
α’αΆα α ααααα·α"
|
| 62 |
+
}
|
| 63 |
+
]
|
| 64 |
+
},
|
| 65 |
+
"libri_dev_clean": {
|
| 66 |
+
"cer": 0.015860143522683594,
|
| 67 |
+
"n": 2703,
|
| 68 |
+
"wer": 0.04521892577478769,
|
| 69 |
+
"rtf": 0.006160100618348849,
|
| 70 |
+
"examples": [
|
| 71 |
+
{
|
| 72 |
+
"ref": "mister quilter is the apostle of the middle classes and we are glad to welcome his gospel",
|
| 73 |
+
"hyp": "mr quilter is the apostle of the middle classes and we are glad to welcome his gospel"
|
| 74 |
+
},
|
| 75 |
+
{
|
| 76 |
+
"ref": "nor is mister quilter's manner less interesting than his matter",
|
| 77 |
+
"hyp": "nor is mr quilter's manner less interesting than his matter"
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"ref": "he tells us that at this festive season of the year with christmas and roast beef looming before us similes drawn from eating and its results occur most readily to the mind",
|
| 81 |
+
"hyp": "he tells us that at this festive season of the year with christmas and rose beef looming before us similes drawn from eating and its results occur most readily to the mind"
|
| 82 |
+
}
|
| 83 |
+
]
|
| 84 |
+
},
|
| 85 |
+
"libri_test_clean": {
|
| 86 |
+
"cer": 0.01636625873370931,
|
| 87 |
+
"n": 2620,
|
| 88 |
+
"wer": 0.04397443700547778,
|
| 89 |
+
"rtf": 0.006210090116807237,
|
| 90 |
+
"examples": [
|
| 91 |
+
{
|
| 92 |
+
"ref": "he hoped there would be stew for dinner turnips and carrots and bruised potatoes and fat mutton pieces to be ladled out in thick peppered flour fattened sauce",
|
| 93 |
+
"hyp": "he hoped there would be stew for dinner turnips and carrots and bruised potatoes and fat mutton pieces to be ladled out in thick peppered flower-fattened sauce"
|
| 94 |
+
},
|
| 95 |
+
{
|
| 96 |
+
"ref": "stuff it into you his belly counselled him",
|
| 97 |
+
"hyp": "stuff it into you his belly counselled him"
|
| 98 |
+
},
|
| 99 |
+
{
|
| 100 |
+
"ref": "after early nightfall the yellow lamps would light up here and there the squalid quarter of the brothels",
|
| 101 |
+
"hyp": "after early nightfall the yellow lamps would light up here and there the squalid quarter of the brofles"
|
| 102 |
+
}
|
| 103 |
+
]
|
| 104 |
+
},
|
| 105 |
+
"libri_test_other": {
|
| 106 |
+
"cer": 0.036180234067892225,
|
| 107 |
+
"n": 2939,
|
| 108 |
+
"wer": 0.08870336052576276,
|
| 109 |
+
"rtf": 0.006097428758073933,
|
| 110 |
+
"examples": [
|
| 111 |
+
{
|
| 112 |
+
"ref": "there's iron they say in all our blood and a grain or two perhaps is good but his he makes me harshly feel has got a little too much of steel anon",
|
| 113 |
+
"hyp": "there's iron they say in all our blood and a grain or two perhaps is good but his he makes me harshly feel has got a little too much of steel anon"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"ref": "margaret said mister hale as he returned from showing his guest downstairs i could not help watching your face with some anxiety when mister thornton made his confession of having been a shop boy",
|
| 117 |
+
"hyp": "margaret said mr hal as he returned from showing his guest downstairs i could not help watching your face with some anxiety when mr thounton made his confession of having been a shop-boy"
|
| 118 |
+
},
|
| 119 |
+
{
|
| 120 |
+
"ref": "you don't mean that you thought me so silly",
|
| 121 |
+
"hyp": "you don't mean that you thought me so silly"
|
| 122 |
+
}
|
| 123 |
+
]
|
| 124 |
+
}
|
| 125 |
+
}
|
| 126 |
+
}
|
eval/stock_1b_fleurs.json
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"ckpt": "/run/media/pc/disk1/ml-data/models/pretrained/omniASR-CTC-1B/omniASR-CTC-1B.omniprune.pt",
|
| 3 |
+
"params_total": 975065300,
|
| 4 |
+
"params_encoder": 962496128.0,
|
| 5 |
+
"gmacs_per_audio_second": 50.277134336,
|
| 6 |
+
"dtype": "bf16",
|
| 7 |
+
"results": {
|
| 8 |
+
"fleurs_km_dev": {
|
| 9 |
+
"cer": 0.10689101784759034,
|
| 10 |
+
"n": 326,
|
| 11 |
+
"rtf": 0.006714766460513306,
|
| 12 |
+
"examples": [
|
| 13 |
+
{
|
| 14 |
+
"ref": "ααΆαααΎαα‘αΎαααααααααααΆααααΊαααααααα½αα₯αααα·ααααααααααααyzantine αα
ααΆαααΆαααΎα",
|
| 15 |
+
"hyp": "ααΆαααΎαα‘αΎαααααααααααΆααααΊαααααααα½αα₯αααα·ααααααααααααααα ααΆαtαΈα αα
ααΆαααΆαααΎα"
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"ref": "ααΆααΆααααααααΆααααααααα·ααααααΆαααΆα
αααΎαααΌα
ααΆααΆαααααΎααααΆαααααααααααααΆααααΈα ααΆαααααΆααααααΌαααΈαααααααΆα
αααΆααα
ααΆαααααΆααΆααα ααΆαα·αα·αα αα·αααααΏααΆααααΆααααΆαααααααααααΆαααααααααΆααΆααααααα·ααααααΆααααα’ααααααααααααααααΆααααααα",
|
| 19 |
+
"hyp": "ααΆααΆααααααααΆααααααααα·ααααααΆαααΆα
αααΎα ααΌα
ααΆααΆαααααΎααααΆαααααααααααααΆααααΈαααΆαααααΆααααααΌαααΈααααααααα
ααΆααα
ααΆααααααΆααΆαααααΆαα·αα·αα αα·αααααΏααΆααααΆααααΆαααααααααααΆαααααααααΆααΆαα αα·ααααααΆααααα’ααααααααααααααααΆααααααα"
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"ref": "ααΆαααΆαααΎα α¬ααααα entinel ange ααΆααααααααααααΆαααααααααΈαα’ααααΆαααα·α ααΊαααα inson assif αααααΆαααααΌααααααα 4892 αααααα αααα ount inson",
|
| 23 |
+
"hyp": "ααΆαααΆαααααΎαα¬αααααα αα·αtiαααriαg ααΆααααααααααααααΆαααααααααΈαα’ααααΆαααα·α ααΊαααααiααα·α αααΆαααΈα αα αααααΆαααααΌααααααααααααα ααααααααααΈααα·α"
|
| 24 |
+
}
|
| 25 |
+
]
|
| 26 |
+
},
|
| 27 |
+
"fleurs_en_dev": {
|
| 28 |
+
"cer": 0.054477869134951507,
|
| 29 |
+
"n": 394,
|
| 30 |
+
"wer": 0.1447336115885097,
|
| 31 |
+
"rtf": 0.006388914222975234,
|
| 32 |
+
"examples": [
|
| 33 |
+
{
|
| 34 |
+
"ref": "when you call someone who is thousands of miles away you are using a satellite",
|
| 35 |
+
"hyp": "when you call someone who has thousands of miles away you are using a satellite"
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"ref": "now widely available throughout the archipelago javanese cuisine features an array of simply seasoned dishes the predominant flavorings the javanese favor being peanuts chillies sugar especially javanese coconut sugar and various aromatic spices",
|
| 39 |
+
"hyp": "now widely available throughout the archipelago japanese cuisines features an array of simply season dishes the predominant flavorings the japanese favor being pinat chillis sugar especially japanese cocoanut sugar and various aromatic spices"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"ref": "the un also hopes to finalize a fund to help countries affected by global warming to cope with the impacts",
|
| 43 |
+
"hyp": "the un also hopes to finalize a fund to help countries affected by global warming to cope with the impacts"
|
| 44 |
+
}
|
| 45 |
+
]
|
| 46 |
+
},
|
| 47 |
+
"fleurs_en_test": {
|
| 48 |
+
"cer": 0.04931213914035213,
|
| 49 |
+
"n": 647,
|
| 50 |
+
"wer": 0.1238148354712772,
|
| 51 |
+
"rtf": 0.006276556365347645,
|
| 52 |
+
"examples": [
|
| 53 |
+
{
|
| 54 |
+
"ref": "however due to the slow communication channels styles in the west could lag behind by 25 to 30 year",
|
| 55 |
+
"hyp": "however due to the slow communication channels styles in the west could lag behind by to years"
|
| 56 |
+
},
|
| 57 |
+
{
|
| 58 |
+
"ref": "all nouns alongside the word sie for you always begin with a capital letter even in the middle of a sentence",
|
| 59 |
+
"hyp": "all nowns alongside the world's say for you always begin with a capital letter even in the middle of a sentence"
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"ref": "to the north and within easy reach is the romantic and fascinating town of sintra and which was made famous to foreigners after a glowing account of its splendours recorded by lord byron",
|
| 63 |
+
"hyp": "to the north and within easy reach is the romantic and fascinating town of cintra and which was made famous to foreigners after a glowing account of its splendouress recorded by lord byron"
|
| 64 |
+
}
|
| 65 |
+
]
|
| 66 |
+
}
|
| 67 |
+
}
|
| 68 |
+
}
|