{ "whisper": { "n_examples": 1978, "wer": 0.19286111707900203, "wer_zwnj_normalized": 0.18077200636819868, "exact_match": 0.006066734074823054, "hallucination_rate": 0.0, "rows_changed": 0.0, "fix_rate": 0.0, "preservation_rate": 1.0, "targeted_score": 0.5, "fix_rate_lenient": 0.0, "preservation_rate_lenient": 1.0, "targeted_score_lenient": 0.5, "fixed_lenient": 0, "broken_lenient": 0, "net_fixed_lenient": 0, "broken_per_100_correct": 0.0, "wer_clitic": 0.1839621023260069, "fixed_clitic": 0, "broken_clitic": 0, "net_fixed_clitic": 0, "broken_per_100_correct_clitic": 0.0, "fix_rate_recoverable": 0.0, "unrecoverable_targets": 966, "entity_fix_rate": 0.0, "entity_keep_rate": 1.0, "entity_spans_whisper_wrong": 57, "entity_spans_whisper_right": 217, "accepted": 0 }, "rules only": { "n_examples": 1978, "wer": 0.18624502140177493, "wer_zwnj_normalized": 0.17446440474419783, "exact_match": 0.006572295247724975, "hallucination_rate": 0.0, "rows_changed": 0.6875631951466128, "fix_rate": 0.05559820155718829, "preservation_rate": 0.9996089846635096, "targeted_score": 0.5276035931103489, "fix_rate_lenient": 0.05892479464856623, "preservation_rate_lenient": 0.9995903045402917, "targeted_score_lenient": 0.5292575495944289, "fixed_lenient": 3594, "broken_lenient": 123, "net_fixed_lenient": 3471, "broken_per_100_correct": 0.04096954597082835, "wer_clitic": 0.17667474784438397, "fixed_clitic": 3571, "broken_clitic": 137, "net_fixed_clitic": 3434, "broken_per_100_correct_clitic": 0.0468321630989902, "fix_rate_recoverable": 0.059573192063571395, "unrecoverable_targets": 966, "entity_fix_rate": 0.0, "entity_keep_rate": 1.0, "entity_spans_whisper_wrong": 57, "entity_spans_whisper_right": 217, "accepted": 0 }, "accept all": { "n_examples": 1978, "wer": 0.1833225911296942, "wer_zwnj_normalized": 0.17195238173935556, "exact_match": 0.006572295247724975, "hallucination_rate": 0.0, "rows_changed": 0.9408493427704753, "fix_rate": 0.12117556749643602, "preservation_rate": 0.9893423255720688, "targeted_score": 0.5552589465342525, "fix_rate_lenient": 0.12734248192415523, "preservation_rate_lenient": 0.9893679031919607, "targeted_score_lenient": 0.558355192558058, "fixed_lenient": 7767, "broken_lenient": 3192, "net_fixed_lenient": 4575, "broken_per_100_correct": 1.0632096808039357, "wer_clitic": 0.17458488667291813, "fixed_clitic": 7419, "broken_clitic": 3153, "net_fixed_clitic": 4266, "broken_per_100_correct_clitic": 1.0778234324898985, "fix_rate_recoverable": 0.12885868026054942, "unrecoverable_targets": 966, "entity_fix_rate": 0.19298245614035087, "entity_keep_rate": 0.8894009216589862, "entity_spans_whisper_wrong": 57, "entity_spans_whisper_right": 217, "accepted": 8781 }, "oracle": { "n_examples": 1978, "wer": 0.17497397081425903, "wer_zwnj_normalized": 0.16355142761131952, "exact_match": 0.007077856420626896, "hallucination_rate": 0.0, "rows_changed": 0.8624873609706775, "fix_rate": 0.11846537057634765, "preservation_rate": 0.999381727886746, "targeted_score": 0.5589235492315467, "fix_rate_lenient": 0.12511271785286837, "preservation_rate_lenient": 0.9994737245314317, "targeted_score_lenient": 0.5622932211921501, "fixed_lenient": 7631, "broken_lenient": 158, "net_fixed_lenient": 7473, "broken_per_100_correct": 0.05262754685683642, "wer_clitic": 0.16583948951352834, "fixed_clitic": 7318, "broken_clitic": 171, "net_fixed_clitic": 7147, "broken_per_100_correct_clitic": 0.05845474372209726, "fix_rate_recoverable": 0.1265930331350892, "unrecoverable_targets": 966, "entity_fix_rate": 0.19298245614035087, "entity_keep_rate": 1.0, "entity_spans_whisper_wrong": 57, "entity_spans_whisper_right": 217, "accepted": 3443 }, "verifier@0.3": { "n_examples": 1978, "wer": 0.1851102039916927, "wer_zwnj_normalized": 0.17342048290336975, "exact_match": 0.006572295247724975, "hallucination_rate": 0.0, "rows_changed": 0.8650151668351871, "fix_rate": 0.08191687685053185, "preservation_rate": 0.9955283887160327, "targeted_score": 0.5387226327832823, "fix_rate_lenient": 0.08650172970668765, "preservation_rate_lenient": 0.995533320231961, "targeted_score_lenient": 0.5410175249693243, "fixed_lenient": 5276, "broken_lenient": 1341, "net_fixed_lenient": 3935, "broken_per_100_correct": 0.4466679768039091, "wer_clitic": 0.1757055782997507, "fixed_clitic": 5170, "broken_clitic": 1335, "net_fixed_clitic": 3835, "broken_per_100_correct_clitic": 0.456357209760233, "fix_rate_recoverable": 0.08754393856098089, "unrecoverable_targets": 966, "entity_fix_rate": 0.17543859649122806, "entity_keep_rate": 0.9631336405529954, "entity_spans_whisper_wrong": 57, "entity_spans_whisper_right": 217, "accepted": 3830 }, "verifier@0.7": { "n_examples": 1978, "wer": 0.18586491265762117, "wer_zwnj_normalized": 0.17412010334550784, "exact_match": 0.006572295247724975, "hallucination_rate": 0.0, "rows_changed": 0.7639029322548029, "fix_rate": 0.06300816192251657, "preservation_rate": 0.9984559907225763, "targeted_score": 0.5307320763225465, "fix_rate_lenient": 0.06658141098158805, "preservation_rate_lenient": 0.9984444895960669, "targeted_score_lenient": 0.5325129502888275, "fixed_lenient": 4061, "broken_lenient": 467, "net_fixed_lenient": 3594, "broken_per_100_correct": 0.15555104039330764, "wer_clitic": 0.17632882007181575, "fixed_clitic": 4011, "broken_clitic": 473, "net_fixed_clitic": 3538, "broken_per_100_correct_clitic": 0.1616906069038129, "fix_rate_recoverable": 0.06733636530228064, "unrecoverable_targets": 966, "entity_fix_rate": 0.15789473684210525, "entity_keep_rate": 1.0, "entity_spans_whisper_wrong": 57, "entity_spans_whisper_right": 217, "accepted": 1215 }, "verifier@0.8": { "n_examples": 1978, "wer": 0.18587868471356878, "wer_zwnj_normalized": 0.17410908570074976, "exact_match": 0.006572295247724975, "hallucination_rate": 0.0, "rows_changed": 0.7320525783619818, "fix_rate": 0.059890652170507416, "preservation_rate": 0.9989773445045635, "targeted_score": 0.5294339983375355, "fix_rate_lenient": 0.06333513681897923, "preservation_rate_lenient": 0.9989641033498433, "targeted_score_lenient": 0.5311496200844112, "fixed_lenient": 3863, "broken_lenient": 311, "net_fixed_lenient": 3552, "broken_per_100_correct": 0.10358966501567168, "wer_clitic": 0.17634025570066097, "fixed_clitic": 3828, "broken_clitic": 317, "net_fixed_clitic": 3511, "broken_per_100_correct_clitic": 0.10836347228014521, "fix_rate_recoverable": 0.06403784963433122, "unrecoverable_targets": 966, "entity_fix_rate": 0.05263157894736842, "entity_keep_rate": 1.0, "entity_spans_whisper_wrong": 57, "entity_spans_whisper_right": 217, "accepted": 692 }, "verifier@0.9": { "n_examples": 1978, "wer": 0.18608251114159327, "wer_zwnj_normalized": 0.1743074033063952, "exact_match": 0.006572295247724975, "hallucination_rate": 0.0, "rows_changed": 0.7087967644084934, "fix_rate": 0.05729011639747466, "preservation_rate": 0.999338281738247, "targeted_score": 0.5283141990678608, "fix_rate_lenient": 0.0606626989982457, "preservation_rate_lenient": 0.9993304976634035, "targeted_score_lenient": 0.5299965983308246, "fixed_lenient": 3700, "broken_lenient": 201, "net_fixed_lenient": 3499, "broken_per_100_correct": 0.06695023365964634, "wer_clitic": 0.17650321341170552, "fixed_clitic": 3675, "broken_clitic": 211, "net_fixed_clitic": 3464, "broken_per_100_correct_clitic": 0.07212836798457616, "fix_rate_recoverable": 0.06133906408782715, "unrecoverable_targets": 966, "entity_fix_rate": 0.017543859649122806, "entity_keep_rate": 1.0, "entity_spans_whisper_wrong": 57, "entity_spans_whisper_right": 217, "accepted": 278 }, "verifier@0.95": { "n_examples": 1978, "wer": 0.1862229861122588, "wer_zwnj_normalized": 0.17444236945468167, "exact_match": 0.006572295247724975, "hallucination_rate": 0.0, "rows_changed": 0.6890798786653185, "fix_rate": 0.05578619209499788, "preservation_rate": 0.9995655385150106, "targeted_score": 0.5276758653050042, "fix_rate_lenient": 0.05912153853720919, "preservation_rate_lenient": 0.9995470033941437, "targeted_score_lenient": 0.5293342709656764, "fixed_lenient": 3606, "broken_lenient": 136, "net_fixed_lenient": 3470, "broken_per_100_correct": 0.04529966058563135, "wer_clitic": 0.17664615877227088, "fixed_clitic": 3585, "broken_clitic": 148, "net_fixed_clitic": 3437, "broken_per_100_correct_clitic": 0.050592409771171895, "fix_rate_recoverable": 0.05977310210405318, "unrecoverable_targets": 966, "entity_fix_rate": 0.0, "entity_keep_rate": 1.0, "entity_spans_whisper_wrong": 57, "entity_spans_whisper_right": 217, "accepted": 37 } }