PedramR's picture
final
a7201d9 verified
Raw History Blame Contribute Delete
9.52 kB
{
"whisper": {
"n_examples": 1978,
"wer": 0.19286111707900203,
"wer_zwnj_normalized": 0.18077200636819868,
"exact_match": 0.006066734074823054,
"hallucination_rate": 0.0,
"rows_changed": 0.0,
"fix_rate": 0.0,
"preservation_rate": 1.0,
"targeted_score": 0.5,
"fix_rate_lenient": 0.0,
"preservation_rate_lenient": 1.0,
"targeted_score_lenient": 0.5,
"fixed_lenient": 0,
"broken_lenient": 0,
"net_fixed_lenient": 0,
"broken_per_100_correct": 0.0,
"wer_clitic": 0.1839621023260069,
"fixed_clitic": 0,
"broken_clitic": 0,
"net_fixed_clitic": 0,
"broken_per_100_correct_clitic": 0.0,
"fix_rate_recoverable": 0.0,
"unrecoverable_targets": 966,
"entity_fix_rate": 0.0,
"entity_keep_rate": 1.0,
"entity_spans_whisper_wrong": 57,
"entity_spans_whisper_right": 217,
"accepted": 0
},
"rules only": {
"n_examples": 1978,
"wer": 0.18624502140177493,
"wer_zwnj_normalized": 0.17446440474419783,
"exact_match": 0.006572295247724975,
"hallucination_rate": 0.0,
"rows_changed": 0.6875631951466128,
"fix_rate": 0.05559820155718829,
"preservation_rate": 0.9996089846635096,
"targeted_score": 0.5276035931103489,
"fix_rate_lenient": 0.05892479464856623,
"preservation_rate_lenient": 0.9995903045402917,
"targeted_score_lenient": 0.5292575495944289,
"fixed_lenient": 3594,
"broken_lenient": 123,
"net_fixed_lenient": 3471,
"broken_per_100_correct": 0.04096954597082835,
"wer_clitic": 0.17667474784438397,
"fixed_clitic": 3571,
"broken_clitic": 137,
"net_fixed_clitic": 3434,
"broken_per_100_correct_clitic": 0.0468321630989902,
"fix_rate_recoverable": 0.059573192063571395,
"unrecoverable_targets": 966,
"entity_fix_rate": 0.0,
"entity_keep_rate": 1.0,
"entity_spans_whisper_wrong": 57,
"entity_spans_whisper_right": 217,
"accepted": 0
},
"accept all": {
"n_examples": 1978,
"wer": 0.1833225911296942,
"wer_zwnj_normalized": 0.17195238173935556,
"exact_match": 0.006572295247724975,
"hallucination_rate": 0.0,
"rows_changed": 0.9408493427704753,
"fix_rate": 0.12117556749643602,
"preservation_rate": 0.9893423255720688,
"targeted_score": 0.5552589465342525,
"fix_rate_lenient": 0.12734248192415523,
"preservation_rate_lenient": 0.9893679031919607,
"targeted_score_lenient": 0.558355192558058,
"fixed_lenient": 7767,
"broken_lenient": 3192,
"net_fixed_lenient": 4575,
"broken_per_100_correct": 1.0632096808039357,
"wer_clitic": 0.17458488667291813,
"fixed_clitic": 7419,
"broken_clitic": 3153,
"net_fixed_clitic": 4266,
"broken_per_100_correct_clitic": 1.0778234324898985,
"fix_rate_recoverable": 0.12885868026054942,
"unrecoverable_targets": 966,
"entity_fix_rate": 0.19298245614035087,
"entity_keep_rate": 0.8894009216589862,
"entity_spans_whisper_wrong": 57,
"entity_spans_whisper_right": 217,
"accepted": 8781
},
"oracle": {
"n_examples": 1978,
"wer": 0.17497397081425903,
"wer_zwnj_normalized": 0.16355142761131952,
"exact_match": 0.007077856420626896,
"hallucination_rate": 0.0,
"rows_changed": 0.8624873609706775,
"fix_rate": 0.11846537057634765,
"preservation_rate": 0.999381727886746,
"targeted_score": 0.5589235492315467,
"fix_rate_lenient": 0.12511271785286837,
"preservation_rate_lenient": 0.9994737245314317,
"targeted_score_lenient": 0.5622932211921501,
"fixed_lenient": 7631,
"broken_lenient": 158,
"net_fixed_lenient": 7473,
"broken_per_100_correct": 0.05262754685683642,
"wer_clitic": 0.16583948951352834,
"fixed_clitic": 7318,
"broken_clitic": 171,
"net_fixed_clitic": 7147,
"broken_per_100_correct_clitic": 0.05845474372209726,
"fix_rate_recoverable": 0.1265930331350892,
"unrecoverable_targets": 966,
"entity_fix_rate": 0.19298245614035087,
"entity_keep_rate": 1.0,
"entity_spans_whisper_wrong": 57,
"entity_spans_whisper_right": 217,
"accepted": 3443
},
"verifier@0.3": {
"n_examples": 1978,
"wer": 0.1851102039916927,
"wer_zwnj_normalized": 0.17342048290336975,
"exact_match": 0.006572295247724975,
"hallucination_rate": 0.0,
"rows_changed": 0.8650151668351871,
"fix_rate": 0.08191687685053185,
"preservation_rate": 0.9955283887160327,
"targeted_score": 0.5387226327832823,
"fix_rate_lenient": 0.08650172970668765,
"preservation_rate_lenient": 0.995533320231961,
"targeted_score_lenient": 0.5410175249693243,
"fixed_lenient": 5276,
"broken_lenient": 1341,
"net_fixed_lenient": 3935,
"broken_per_100_correct": 0.4466679768039091,
"wer_clitic": 0.1757055782997507,
"fixed_clitic": 5170,
"broken_clitic": 1335,
"net_fixed_clitic": 3835,
"broken_per_100_correct_clitic": 0.456357209760233,
"fix_rate_recoverable": 0.08754393856098089,
"unrecoverable_targets": 966,
"entity_fix_rate": 0.17543859649122806,
"entity_keep_rate": 0.9631336405529954,
"entity_spans_whisper_wrong": 57,
"entity_spans_whisper_right": 217,
"accepted": 3830
},
"verifier@0.7": {
"n_examples": 1978,
"wer": 0.18586491265762117,
"wer_zwnj_normalized": 0.17412010334550784,
"exact_match": 0.006572295247724975,
"hallucination_rate": 0.0,
"rows_changed": 0.7639029322548029,
"fix_rate": 0.06300816192251657,
"preservation_rate": 0.9984559907225763,
"targeted_score": 0.5307320763225465,
"fix_rate_lenient": 0.06658141098158805,
"preservation_rate_lenient": 0.9984444895960669,
"targeted_score_lenient": 0.5325129502888275,
"fixed_lenient": 4061,
"broken_lenient": 467,
"net_fixed_lenient": 3594,
"broken_per_100_correct": 0.15555104039330764,
"wer_clitic": 0.17632882007181575,
"fixed_clitic": 4011,
"broken_clitic": 473,
"net_fixed_clitic": 3538,
"broken_per_100_correct_clitic": 0.1616906069038129,
"fix_rate_recoverable": 0.06733636530228064,
"unrecoverable_targets": 966,
"entity_fix_rate": 0.15789473684210525,
"entity_keep_rate": 1.0,
"entity_spans_whisper_wrong": 57,
"entity_spans_whisper_right": 217,
"accepted": 1215
},
"verifier@0.8": {
"n_examples": 1978,
"wer": 0.18587868471356878,
"wer_zwnj_normalized": 0.17410908570074976,
"exact_match": 0.006572295247724975,
"hallucination_rate": 0.0,
"rows_changed": 0.7320525783619818,
"fix_rate": 0.059890652170507416,
"preservation_rate": 0.9989773445045635,
"targeted_score": 0.5294339983375355,
"fix_rate_lenient": 0.06333513681897923,
"preservation_rate_lenient": 0.9989641033498433,
"targeted_score_lenient": 0.5311496200844112,
"fixed_lenient": 3863,
"broken_lenient": 311,
"net_fixed_lenient": 3552,
"broken_per_100_correct": 0.10358966501567168,
"wer_clitic": 0.17634025570066097,
"fixed_clitic": 3828,
"broken_clitic": 317,
"net_fixed_clitic": 3511,
"broken_per_100_correct_clitic": 0.10836347228014521,
"fix_rate_recoverable": 0.06403784963433122,
"unrecoverable_targets": 966,
"entity_fix_rate": 0.05263157894736842,
"entity_keep_rate": 1.0,
"entity_spans_whisper_wrong": 57,
"entity_spans_whisper_right": 217,
"accepted": 692
},
"verifier@0.9": {
"n_examples": 1978,
"wer": 0.18608251114159327,
"wer_zwnj_normalized": 0.1743074033063952,
"exact_match": 0.006572295247724975,
"hallucination_rate": 0.0,
"rows_changed": 0.7087967644084934,
"fix_rate": 0.05729011639747466,
"preservation_rate": 0.999338281738247,
"targeted_score": 0.5283141990678608,
"fix_rate_lenient": 0.0606626989982457,
"preservation_rate_lenient": 0.9993304976634035,
"targeted_score_lenient": 0.5299965983308246,
"fixed_lenient": 3700,
"broken_lenient": 201,
"net_fixed_lenient": 3499,
"broken_per_100_correct": 0.06695023365964634,
"wer_clitic": 0.17650321341170552,
"fixed_clitic": 3675,
"broken_clitic": 211,
"net_fixed_clitic": 3464,
"broken_per_100_correct_clitic": 0.07212836798457616,
"fix_rate_recoverable": 0.06133906408782715,
"unrecoverable_targets": 966,
"entity_fix_rate": 0.017543859649122806,
"entity_keep_rate": 1.0,
"entity_spans_whisper_wrong": 57,
"entity_spans_whisper_right": 217,
"accepted": 278
},
"verifier@0.95": {
"n_examples": 1978,
"wer": 0.1862229861122588,
"wer_zwnj_normalized": 0.17444236945468167,
"exact_match": 0.006572295247724975,
"hallucination_rate": 0.0,
"rows_changed": 0.6890798786653185,
"fix_rate": 0.05578619209499788,
"preservation_rate": 0.9995655385150106,
"targeted_score": 0.5276758653050042,
"fix_rate_lenient": 0.05912153853720919,
"preservation_rate_lenient": 0.9995470033941437,
"targeted_score_lenient": 0.5293342709656764,
"fixed_lenient": 3606,
"broken_lenient": 136,
"net_fixed_lenient": 3470,
"broken_per_100_correct": 0.04529966058563135,
"wer_clitic": 0.17664615877227088,
"fixed_clitic": 3585,
"broken_clitic": 148,
"net_fixed_clitic": 3437,
"broken_per_100_correct_clitic": 0.050592409771171895,
"fix_rate_recoverable": 0.05977310210405318,
"unrecoverable_targets": 966,
"entity_fix_rate": 0.0,
"entity_keep_rate": 1.0,
"entity_spans_whisper_wrong": 57,
"entity_spans_whisper_right": 217,
"accepted": 37
}
}