Download evaluation.json from Lucasgitguy-co/Fleet-2: direct link, hf CLI and curl.
- Browser
- Download file 16.9 kB
-
https://huggingface.co/Lucasgitguy-co/Fleet-2/resolve/main/evaluation.json
- Command line
-
hf download hf://Lucasgitguy-co/Fleet-2/evaluation.json
-
curl -L -o evaluation.json https://huggingface.co/Lucasgitguy-co/Fleet-2/resolve/main/evaluation.json
16.9 kB
| { | |
| "selection": { | |
| "step": 2163, | |
| "macro_validation": 0.22121288067005115, | |
| "start_macro_validation": 0.22833260484316537, | |
| "reason": "Lowest macro validation error, better than the start model" | |
| }, | |
| "init": { | |
| "chosen": "fleet1", | |
| "macro_validation": { | |
| "sky": 0.2466074807442024, | |
| "fleet1": 0.22833260484316537 | |
| }, | |
| "rule": "lower 8-language mean validation error; tie keeps Sky-Kim", | |
| "adapter": { | |
| "repo": "Lucasgitguy-co/Fleet-1", | |
| "revision": "0cc581616f7e5493e2e3999e71c134cedc879a6d", | |
| "adapter": "fleet_lora.safetensors" | |
| }, | |
| "rebuilt_macro": 0.22833260484316537 | |
| }, | |
| "filter": { | |
| "fleurs/en": { | |
| "candidates": 200, | |
| "kept": 178, | |
| "rejected": { | |
| "disagrees_with_gold": 22 | |
| } | |
| }, | |
| "fleurs/de": { | |
| "candidates": 200, | |
| "kept": 164, | |
| "rejected": { | |
| "disagrees_with_gold": 36 | |
| } | |
| }, | |
| "fleurs/fr": { | |
| "candidates": 200, | |
| "kept": 165, | |
| "rejected": { | |
| "disagrees_with_gold": 35 | |
| } | |
| }, | |
| "fleurs/es": { | |
| "candidates": 200, | |
| "kept": 158, | |
| "rejected": { | |
| "disagrees_with_gold": 42 | |
| } | |
| }, | |
| "fleurs/it": { | |
| "candidates": 200, | |
| "kept": 169, | |
| "rejected": { | |
| "disagrees_with_gold": 31 | |
| } | |
| }, | |
| "fleurs/nl": { | |
| "candidates": 200, | |
| "kept": 137, | |
| "rejected": { | |
| "disagrees_with_gold": 63 | |
| } | |
| }, | |
| "fleurs/pl": { | |
| "candidates": 200, | |
| "kept": 163, | |
| "rejected": { | |
| "disagrees_with_gold": 37 | |
| } | |
| }, | |
| "fleurs/ko": { | |
| "candidates": 200, | |
| "kept": 178, | |
| "rejected": { | |
| "disagrees_with_gold": 22 | |
| } | |
| }, | |
| "voxpopuli/en": { | |
| "candidates": 600, | |
| "kept": 400, | |
| "rejected": { | |
| "disagrees_with_gold": 141, | |
| "over_quota": 59 | |
| } | |
| }, | |
| "voxpopuli/de": { | |
| "candidates": 600, | |
| "kept": 346, | |
| "rejected": { | |
| "disagrees_with_gold": 254 | |
| } | |
| }, | |
| "voxpopuli/fr": { | |
| "candidates": 600, | |
| "kept": 400, | |
| "rejected": { | |
| "disagrees_with_gold": 163, | |
| "over_quota": 37 | |
| } | |
| }, | |
| "voxpopuli/es": { | |
| "candidates": 600, | |
| "kept": 400, | |
| "rejected": { | |
| "disagrees_with_gold": 121, | |
| "over_quota": 79 | |
| } | |
| }, | |
| "voxpopuli/it": { | |
| "candidates": 600, | |
| "kept": 350, | |
| "rejected": { | |
| "disagrees_with_gold": 250 | |
| } | |
| }, | |
| "voxpopuli/nl": { | |
| "candidates": 600, | |
| "kept": 318, | |
| "rejected": { | |
| "disagrees_with_gold": 280, | |
| "avg_logprob": 2 | |
| } | |
| }, | |
| "voxpopuli/pl": { | |
| "candidates": 600, | |
| "kept": 400, | |
| "rejected": { | |
| "disagrees_with_gold": 165, | |
| "over_quota": 35 | |
| } | |
| }, | |
| "zeroth/ko": { | |
| "candidates": 600, | |
| "kept": 400, | |
| "rejected": { | |
| "disagrees_with_gold": 74, | |
| "over_quota": 126 | |
| } | |
| } | |
| }, | |
| "results": { | |
| "test": { | |
| "Fleet-2": { | |
| "en": { | |
| "metric": "WER", | |
| "error_rate": 0.14496573537163943, | |
| "edits": 275, | |
| "reference_units": 1897, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "de": { | |
| "metric": "WER", | |
| "error_rate": 0.22021276595744682, | |
| "edits": 414, | |
| "reference_units": 1880, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "fr": { | |
| "metric": "WER", | |
| "error_rate": 0.24926315789473685, | |
| "edits": 592, | |
| "reference_units": 2375, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "es": { | |
| "metric": "WER", | |
| "error_rate": 0.11927528938097634, | |
| "edits": 237, | |
| "reference_units": 1987, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "it": { | |
| "metric": "WER", | |
| "error_rate": 0.16657381615598885, | |
| "edits": 299, | |
| "reference_units": 1795, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "nl": { | |
| "metric": "WER", | |
| "error_rate": 0.30062530062530063, | |
| "edits": 625, | |
| "reference_units": 2079, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "pl": { | |
| "metric": "WER", | |
| "error_rate": 0.4161603888213852, | |
| "edits": 685, | |
| "reference_units": 1646, | |
| "clips": 100, | |
| "language_accuracy": 0.99 | |
| }, | |
| "ko": { | |
| "metric": "CER", | |
| "error_rate": 0.42800114057599087, | |
| "edits": 1501, | |
| "reference_units": 3507, | |
| "clips": 100, | |
| "language_accuracy": 0.97 | |
| } | |
| }, | |
| "Fleet-1": { | |
| "en": { | |
| "metric": "WER", | |
| "error_rate": 0.14391143911439114, | |
| "edits": 273, | |
| "reference_units": 1897, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "de": { | |
| "metric": "WER", | |
| "error_rate": 0.22180851063829787, | |
| "edits": 417, | |
| "reference_units": 1880, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "fr": { | |
| "metric": "WER", | |
| "error_rate": 0.2517894736842105, | |
| "edits": 598, | |
| "reference_units": 2375, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "es": { | |
| "metric": "WER", | |
| "error_rate": 0.11927528938097634, | |
| "edits": 237, | |
| "reference_units": 1987, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "it": { | |
| "metric": "WER", | |
| "error_rate": 0.16323119777158773, | |
| "edits": 293, | |
| "reference_units": 1795, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "nl": { | |
| "metric": "WER", | |
| "error_rate": 0.29533429533429534, | |
| "edits": 614, | |
| "reference_units": 2079, | |
| "clips": 100, | |
| "language_accuracy": 0.99 | |
| }, | |
| "pl": { | |
| "metric": "WER", | |
| "error_rate": 0.4313487241798299, | |
| "edits": 710, | |
| "reference_units": 1646, | |
| "clips": 100, | |
| "language_accuracy": 0.99 | |
| }, | |
| "ko": { | |
| "metric": "CER", | |
| "error_rate": 0.523524379811805, | |
| "edits": 1836, | |
| "reference_units": 3507, | |
| "clips": 100, | |
| "language_accuracy": 0.98 | |
| } | |
| }, | |
| "Sky-Kim": { | |
| "en": { | |
| "metric": "WER", | |
| "error_rate": 0.14865577227200844, | |
| "edits": 282, | |
| "reference_units": 1897, | |
| "clips": 100, | |
| "language_accuracy": 0.99 | |
| }, | |
| "de": { | |
| "metric": "WER", | |
| "error_rate": 0.2271276595744681, | |
| "edits": 427, | |
| "reference_units": 1880, | |
| "clips": 100, | |
| "language_accuracy": 0.99 | |
| }, | |
| "fr": { | |
| "metric": "WER", | |
| "error_rate": 0.25726315789473686, | |
| "edits": 611, | |
| "reference_units": 2375, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "es": { | |
| "metric": "WER", | |
| "error_rate": 0.1182687468545546, | |
| "edits": 235, | |
| "reference_units": 1987, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "it": { | |
| "metric": "WER", | |
| "error_rate": 0.166016713091922, | |
| "edits": 298, | |
| "reference_units": 1795, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "nl": { | |
| "metric": "WER", | |
| "error_rate": 0.29389129389129387, | |
| "edits": 611, | |
| "reference_units": 2079, | |
| "clips": 100, | |
| "language_accuracy": 0.99 | |
| }, | |
| "pl": { | |
| "metric": "WER", | |
| "error_rate": 0.43681652490886996, | |
| "edits": 719, | |
| "reference_units": 1646, | |
| "clips": 100, | |
| "language_accuracy": 0.99 | |
| }, | |
| "ko": { | |
| "metric": "CER", | |
| "error_rate": 0.3367550613059595, | |
| "edits": 1181, | |
| "reference_units": 3507, | |
| "clips": 100, | |
| "language_accuracy": 0.98 | |
| } | |
| }, | |
| "Whistle": { | |
| "en": { | |
| "metric": "WER", | |
| "error_rate": 0.13863995782814972, | |
| "edits": 263, | |
| "reference_units": 1897, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "de": { | |
| "metric": "WER", | |
| "error_rate": 0.22446808510638297, | |
| "edits": 422, | |
| "reference_units": 1880, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "fr": { | |
| "metric": "WER", | |
| "error_rate": 0.22147368421052632, | |
| "edits": 526, | |
| "reference_units": 2375, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "es": { | |
| "metric": "WER", | |
| "error_rate": 0.1182687468545546, | |
| "edits": 235, | |
| "reference_units": 1987, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "it": { | |
| "metric": "WER", | |
| "error_rate": 0.17437325905292478, | |
| "edits": 313, | |
| "reference_units": 1795, | |
| "clips": 100, | |
| "language_accuracy": 1.0 | |
| }, | |
| "nl": { | |
| "metric": "WER", | |
| "error_rate": 0.3294853294853295, | |
| "edits": 685, | |
| "reference_units": 2079, | |
| "clips": 100, | |
| "language_accuracy": 0.97 | |
| }, | |
| "pl": { | |
| "metric": "WER", | |
| "error_rate": 0.4538274605103281, | |
| "edits": 747, | |
| "reference_units": 1646, | |
| "clips": 100, | |
| "language_accuracy": 0.95 | |
| } | |
| } | |
| }, | |
| "ood": { | |
| "Fleet-2": { | |
| "en": { | |
| "metric": "WER", | |
| "error_rate": 0.11981020166073547, | |
| "edits": 101, | |
| "reference_units": 843, | |
| "clips": 50, | |
| "language_accuracy": 0.98 | |
| }, | |
| "de": { | |
| "metric": "WER", | |
| "error_rate": 0.18359375, | |
| "edits": 141, | |
| "reference_units": 768, | |
| "clips": 50, | |
| "language_accuracy": 1.0 | |
| }, | |
| "fr": { | |
| "metric": "WER", | |
| "error_rate": 0.1357142857142857, | |
| "edits": 133, | |
| "reference_units": 980, | |
| "clips": 50, | |
| "language_accuracy": 1.0 | |
| }, | |
| "es": { | |
| "metric": "WER", | |
| "error_rate": 0.11037107516650808, | |
| "edits": 116, | |
| "reference_units": 1051, | |
| "clips": 50, | |
| "language_accuracy": 0.98 | |
| }, | |
| "it": { | |
| "metric": "WER", | |
| "error_rate": 0.3319371727748691, | |
| "edits": 317, | |
| "reference_units": 955, | |
| "clips": 50, | |
| "language_accuracy": 0.98 | |
| }, | |
| "nl": { | |
| "metric": "WER", | |
| "error_rate": 0.22945205479452055, | |
| "edits": 201, | |
| "reference_units": 876, | |
| "clips": 50, | |
| "language_accuracy": 0.96 | |
| }, | |
| "pl": { | |
| "metric": "WER", | |
| "error_rate": 0.23529411764705882, | |
| "edits": 172, | |
| "reference_units": 731, | |
| "clips": 50, | |
| "language_accuracy": 0.98 | |
| }, | |
| "ko": { | |
| "metric": "CER", | |
| "error_rate": 0.1323529411764706, | |
| "edits": 279, | |
| "reference_units": 2108, | |
| "clips": 50, | |
| "language_accuracy": 1.0 | |
| } | |
| }, | |
| "Fleet-1": { | |
| "en": { | |
| "metric": "WER", | |
| "error_rate": 0.10913404507710557, | |
| "edits": 92, | |
| "reference_units": 843, | |
| "clips": 50, | |
| "language_accuracy": 0.96 | |
| }, | |
| "de": { | |
| "metric": "WER", | |
| "error_rate": 0.1953125, | |
| "edits": 150, | |
| "reference_units": 768, | |
| "clips": 50, | |
| "language_accuracy": 1.0 | |
| }, | |
| "fr": { | |
| "metric": "WER", | |
| "error_rate": 0.12040816326530612, | |
| "edits": 118, | |
| "reference_units": 980, | |
| "clips": 50, | |
| "language_accuracy": 1.0 | |
| }, | |
| "es": { | |
| "metric": "WER", | |
| "error_rate": 0.1170313986679353, | |
| "edits": 123, | |
| "reference_units": 1051, | |
| "clips": 50, | |
| "language_accuracy": 0.98 | |
| }, | |
| "it": { | |
| "metric": "WER", | |
| "error_rate": 0.38219895287958117, | |
| "edits": 365, | |
| "reference_units": 955, | |
| "clips": 50, | |
| "language_accuracy": 0.98 | |
| }, | |
| "nl": { | |
| "metric": "WER", | |
| "error_rate": 0.2317351598173516, | |
| "edits": 203, | |
| "reference_units": 876, | |
| "clips": 50, | |
| "language_accuracy": 0.96 | |
| }, | |
| "pl": { | |
| "metric": "WER", | |
| "error_rate": 0.22982216142270862, | |
| "edits": 168, | |
| "reference_units": 731, | |
| "clips": 50, | |
| "language_accuracy": 0.96 | |
| }, | |
| "ko": { | |
| "metric": "CER", | |
| "error_rate": 0.15654648956356737, | |
| "edits": 330, | |
| "reference_units": 2108, | |
| "clips": 50, | |
| "language_accuracy": 1.0 | |
| } | |
| }, | |
| "Sky-Kim": { | |
| "en": { | |
| "metric": "WER", | |
| "error_rate": 0.1269276393831554, | |
| "edits": 107, | |
| "reference_units": 843, | |
| "clips": 50, | |
| "language_accuracy": 0.94 | |
| }, | |
| "de": { | |
| "metric": "WER", | |
| "error_rate": 0.19661458333333334, | |
| "edits": 151, | |
| "reference_units": 768, | |
| "clips": 50, | |
| "language_accuracy": 1.0 | |
| }, | |
| "fr": { | |
| "metric": "WER", | |
| "error_rate": 0.12244897959183673, | |
| "edits": 120, | |
| "reference_units": 980, | |
| "clips": 50, | |
| "language_accuracy": 1.0 | |
| }, | |
| "es": { | |
| "metric": "WER", | |
| "error_rate": 0.11893434823977164, | |
| "edits": 125, | |
| "reference_units": 1051, | |
| "clips": 50, | |
| "language_accuracy": 0.98 | |
| }, | |
| "it": { | |
| "metric": "WER", | |
| "error_rate": 0.40209424083769635, | |
| "edits": 384, | |
| "reference_units": 955, | |
| "clips": 50, | |
| "language_accuracy": 0.98 | |
| }, | |
| "nl": { | |
| "metric": "WER", | |
| "error_rate": 0.2328767123287671, | |
| "edits": 204, | |
| "reference_units": 876, | |
| "clips": 50, | |
| "language_accuracy": 0.96 | |
| }, | |
| "pl": { | |
| "metric": "WER", | |
| "error_rate": 0.23529411764705882, | |
| "edits": 172, | |
| "reference_units": 731, | |
| "clips": 50, | |
| "language_accuracy": 0.98 | |
| }, | |
| "ko": { | |
| "metric": "CER", | |
| "error_rate": 0.2518975332068311, | |
| "edits": 531, | |
| "reference_units": 2108, | |
| "clips": 50, | |
| "language_accuracy": 1.0 | |
| } | |
| }, | |
| "Whistle": { | |
| "en": { | |
| "metric": "WER", | |
| "error_rate": 0.1055753262158956, | |
| "edits": 89, | |
| "reference_units": 843, | |
| "clips": 50, | |
| "language_accuracy": 1.0 | |
| }, | |
| "de": { | |
| "metric": "WER", | |
| "error_rate": 0.17317708333333334, | |
| "edits": 133, | |
| "reference_units": 768, | |
| "clips": 50, | |
| "language_accuracy": 1.0 | |
| }, | |
| "fr": { | |
| "metric": "WER", | |
| "error_rate": 0.1306122448979592, | |
| "edits": 128, | |
| "reference_units": 980, | |
| "clips": 50, | |
| "language_accuracy": 1.0 | |
| }, | |
| "es": { | |
| "metric": "WER", | |
| "error_rate": 0.09990485252140818, | |
| "edits": 105, | |
| "reference_units": 1051, | |
| "clips": 50, | |
| "language_accuracy": 1.0 | |
| }, | |
| "it": { | |
| "metric": "WER", | |
| "error_rate": 0.34450261780104713, | |
| "edits": 329, | |
| "reference_units": 955, | |
| "clips": 50, | |
| "language_accuracy": 0.98 | |
| }, | |
| "nl": { | |
| "metric": "WER", | |
| "error_rate": 0.23059360730593606, | |
| "edits": 202, | |
| "reference_units": 876, | |
| "clips": 50, | |
| "language_accuracy": 0.98 | |
| }, | |
| "pl": { | |
| "metric": "WER", | |
| "error_rate": 0.22024623803009577, | |
| "edits": 161, | |
| "reference_units": 731, | |
| "clips": 50, | |
| "language_accuracy": 0.96 | |
| } | |
| } | |
| } | |
| }, | |
| "gate": { | |
| "passed": false, | |
| "beats_fleet1_in": [ | |
| "de", | |
| "fr", | |
| "pl", | |
| "ko" | |
| ], | |
| "korean_not_worse_than_sky": false, | |
| "rule": ">=5 of 8 languages better than Fleet-1 and Korean CER <= Sky-Kim, on Fleet-1's FLEURS test clips" | |
| } | |
| } | |