| { |
| "whisper": { |
| "n_examples": 1010, |
| "wer": 0.18660667584675655, |
| "wer_zwnj_normalized": 0.17409613475517982, |
| "exact_match": 0.0049504950495049506, |
| "hallucination_rate": 0.0, |
| "rows_changed": 0.0, |
| "fix_rate": 0.0, |
| "preservation_rate": 1.0, |
| "targeted_score": 0.5, |
| "fix_rate_lenient": 0.0, |
| "preservation_rate_lenient": 1.0, |
| "targeted_score_lenient": 0.5, |
| "fixed_lenient": 0, |
| "broken_lenient": 0, |
| "net_fixed_lenient": 0, |
| "broken_per_100_correct": 0.0, |
| "fix_rate_recoverable": 0.0, |
| "unrecoverable_targets": 471, |
| "entity_fix_rate": 0.0, |
| "entity_keep_rate": 1.0, |
| "entity_spans_whisper_wrong": 57, |
| "entity_spans_whisper_right": 217, |
| "accepted": 0 |
| }, |
| "rules only": { |
| "n_examples": 1010, |
| "wer": 0.18000982056126644, |
| "wer_zwnj_normalized": 0.16776614254758168, |
| "exact_match": 0.0049504950495049506, |
| "hallucination_rate": 0.0, |
| "rows_changed": 0.7148514851485148, |
| "fix_rate": 0.0566740770892695, |
| "preservation_rate": 0.9996911693742963, |
| "targeted_score": 0.5281826232317829, |
| "fix_rate_lenient": 0.06019704433497537, |
| "preservation_rate_lenient": 0.9996856677315474, |
| "targeted_score_lenient": 0.5299413560332614, |
| "fixed_lenient": 1833, |
| "broken_lenient": 49, |
| "net_fixed_lenient": 1784, |
| "broken_per_100_correct": 0.03143322684525871, |
| "fix_rate_recoverable": 0.06077587644684613, |
| "unrecoverable_targets": 471, |
| "entity_fix_rate": 0.0, |
| "entity_keep_rate": 1.0, |
| "entity_spans_whisper_wrong": 57, |
| "entity_spans_whisper_right": 217, |
| "accepted": 0 |
| }, |
| "accept all": { |
| "n_examples": 1010, |
| "wer": 0.16964485861594134, |
| "wer_zwnj_normalized": 0.15781214974221028, |
| "exact_match": 0.0039603960396039604, |
| "hallucination_rate": 0.0, |
| "rows_changed": 0.9712871287128713, |
| "fix_rate": 0.18689920781538655, |
| "preservation_rate": 0.9872092649187711, |
| "targeted_score": 0.5870542363670789, |
| "fix_rate_lenient": 0.18627257799671593, |
| "preservation_rate_lenient": 0.9873625598193552, |
| "targeted_score_lenient": 0.5868175689080355, |
| "fixed_lenient": 5672, |
| "broken_lenient": 1970, |
| "net_fixed_lenient": 3702, |
| "broken_per_100_correct": 1.263744018064483, |
| "fix_rate_recoverable": 0.18859868574668934, |
| "unrecoverable_targets": 471, |
| "entity_fix_rate": 0.19298245614035087, |
| "entity_keep_rate": 0.8894009216589862, |
| "entity_spans_whisper_wrong": 57, |
| "entity_spans_whisper_right": 217, |
| "accepted": 6086 |
| }, |
| "oracle": { |
| "n_examples": 1010, |
| "wer": 0.15873549599171657, |
| "wer_zwnj_normalized": 0.14686542628708063, |
| "exact_match": 0.0049504950495049506, |
| "hallucination_rate": 0.0, |
| "rows_changed": 0.9247524752475248, |
| "fix_rate": 0.18217114945048063, |
| "preservation_rate": 0.9991700176934213, |
| "targeted_score": 0.590670583571951, |
| "fix_rate_lenient": 0.18269293924466337, |
| "preservation_rate_lenient": 0.9993713354630949, |
| "targeted_score_lenient": 0.5910321373538792, |
| "fixed_lenient": 5563, |
| "broken_lenient": 98, |
| "net_fixed_lenient": 5465, |
| "broken_per_100_correct": 0.06286645369051742, |
| "fix_rate_recoverable": 0.1849294506154308, |
| "unrecoverable_targets": 471, |
| "entity_fix_rate": 0.19298245614035087, |
| "entity_keep_rate": 1.0, |
| "entity_spans_whisper_wrong": 57, |
| "entity_spans_whisper_right": 217, |
| "accepted": 2996 |
| }, |
| "verifier@0.3": { |
| "n_examples": 1010, |
| "wer": 0.17786957867657263, |
| "wer_zwnj_normalized": 0.16573798315560254, |
| "exact_match": 0.0049504950495049506, |
| "hallucination_rate": 0.0, |
| "rows_changed": 0.8643564356435643, |
| "fix_rate": 0.08125371825782009, |
| "preservation_rate": 0.9966929387164227, |
| "targeted_score": 0.5389733284871214, |
| "fix_rate_lenient": 0.08597701149425287, |
| "preservation_rate_lenient": 0.9967412083188997, |
| "targeted_score_lenient": 0.5413591099065763, |
| "fixed_lenient": 2618, |
| "broken_lenient": 508, |
| "net_fixed_lenient": 2110, |
| "broken_per_100_correct": 0.32587916811002915, |
| "fix_rate_recoverable": 0.08686080256179325, |
| "unrecoverable_targets": 471, |
| "entity_fix_rate": 0.15789473684210525, |
| "entity_keep_rate": 1.0, |
| "entity_spans_whisper_wrong": 57, |
| "entity_spans_whisper_right": 217, |
| "accepted": 1510 |
| }, |
| "verifier@0.7": { |
| "n_examples": 1010, |
| "wer": 0.17922524311226395, |
| "wer_zwnj_normalized": 0.16699757688325273, |
| "exact_match": 0.0049504950495049506, |
| "hallucination_rate": 0.0, |
| "rows_changed": 0.7792079207920792, |
| "fix_rate": 0.06437674171024203, |
| "preservation_rate": 0.9986424320411774, |
| "targeted_score": 0.5315095868757097, |
| "fix_rate_lenient": 0.06821018062397373, |
| "preservation_rate_lenient": 0.9986721065393942, |
| "targeted_score_lenient": 0.5334411435816839, |
| "fixed_lenient": 2077, |
| "broken_lenient": 207, |
| "net_fixed_lenient": 1870, |
| "broken_per_100_correct": 0.13278934606058274, |
| "fix_rate_recoverable": 0.06888155041862637, |
| "unrecoverable_targets": 471, |
| "entity_fix_rate": 0.05263157894736842, |
| "entity_keep_rate": 1.0, |
| "entity_spans_whisper_wrong": 57, |
| "entity_spans_whisper_right": 217, |
| "accepted": 543 |
| }, |
| "verifier@0.8": { |
| "n_examples": 1010, |
| "wer": 0.1795508160672922, |
| "wer_zwnj_normalized": 0.16731781257672312, |
| "exact_match": 0.0049504950495049506, |
| "hallucination_rate": 0.0, |
| "rows_changed": 0.7504950495049505, |
| "fix_rate": 0.06052540939975577, |
| "preservation_rate": 0.9991828856361589, |
| "targeted_score": 0.5298541475179573, |
| "fix_rate_lenient": 0.06423645320197044, |
| "preservation_rate_lenient": 0.9991788871354708, |
| "targeted_score_lenient": 0.5317076701687206, |
| "fixed_lenient": 1956, |
| "broken_lenient": 128, |
| "net_fixed_lenient": 1828, |
| "broken_per_100_correct": 0.08211128645292072, |
| "fix_rate_recoverable": 0.06487874845725342, |
| "unrecoverable_targets": 471, |
| "entity_fix_rate": 0.03508771929824561, |
| "entity_keep_rate": 1.0, |
| "entity_spans_whisper_wrong": 57, |
| "entity_spans_whisper_right": 217, |
| "accepted": 255 |
| }, |
| "verifier@0.9": { |
| "n_examples": 1010, |
| "wer": 0.17997245973036155, |
| "wer_zwnj_normalized": 0.1677287817166768, |
| "exact_match": 0.0049504950495049506, |
| "hallucination_rate": 0.0, |
| "rows_changed": 0.7168316831683168, |
| "fix_rate": 0.05689325860287441, |
| "preservation_rate": 0.9996525655460833, |
| "targeted_score": 0.5282729120744788, |
| "fix_rate_lenient": 0.060426929392446635, |
| "preservation_rate_lenient": 0.9996471780660227, |
| "targeted_score_lenient": 0.5300370537292346, |
| "fixed_lenient": 1840, |
| "broken_lenient": 55, |
| "net_fixed_lenient": 1785, |
| "broken_per_100_correct": 0.03528219339773937, |
| "fix_rate_recoverable": 0.06100937322792622, |
| "unrecoverable_targets": 471, |
| "entity_fix_rate": 0.0, |
| "entity_keep_rate": 1.0, |
| "entity_spans_whisper_wrong": 57, |
| "entity_spans_whisper_right": 217, |
| "accepted": 11 |
| }, |
| "verifier@0.95": { |
| "n_examples": 1010, |
| "wer": 0.18000982056126644, |
| "wer_zwnj_normalized": 0.16776614254758168, |
| "exact_match": 0.0049504950495049506, |
| "hallucination_rate": 0.0, |
| "rows_changed": 0.7148514851485148, |
| "fix_rate": 0.0566740770892695, |
| "preservation_rate": 0.9996911693742963, |
| "targeted_score": 0.5281826232317829, |
| "fix_rate_lenient": 0.06019704433497537, |
| "preservation_rate_lenient": 0.9996856677315474, |
| "targeted_score_lenient": 0.5299413560332614, |
| "fixed_lenient": 1833, |
| "broken_lenient": 49, |
| "net_fixed_lenient": 1784, |
| "broken_per_100_correct": 0.03143322684525871, |
| "fix_rate_recoverable": 0.06077587644684613, |
| "unrecoverable_targets": 471, |
| "entity_fix_rate": 0.0, |
| "entity_keep_rate": 1.0, |
| "entity_spans_whisper_wrong": 57, |
| "entity_spans_whisper_right": 217, |
| "accepted": 0 |
| } |
| } |
