| { |
| "development": { |
| "baseline kev": { |
| "auroc": 0.9711489898989899, |
| "acc": 0.9464285714285714, |
| "brier": 0.04904470588256766, |
| "ece": 0.06521459834615231, |
| "cov@2%": 0.7611607142857143, |
| "cov@5%": 0.8236607142857143, |
| "cov@10%": 0.8928571428571429 |
| }, |
| "fine-tuned kev": { |
| "auroc": 0.9684027777777777, |
| "acc": 0.9464285714285714, |
| "brier": 0.04231278466552767, |
| "ece": 0.05751592743965165, |
| "cov@2%": 0.65625, |
| "cov@5%": 0.8415178571428571, |
| "cov@10%": 0.8928571428571429 |
| }, |
| "evidence score": { |
| "auroc": 0.9590277777777778, |
| "acc": 0.9553571428571429, |
| "brier": 0.04669189453125, |
| "ece": 0.07882254464285716, |
| "cov@2%": 0.6852678571428571, |
| "cov@5%": 0.8058035714285714, |
| "cov@10%": 0.8794642857142857 |
| }, |
| "mean(evidence, baseline)": { |
| "auroc": 0.9763257575757576, |
| "acc": 0.9464285714285714, |
| "brier": 0.039230741564232695, |
| "ece": 0.07375138990908783, |
| "cov@2%": 0.7834821428571429, |
| "cov@5%": 0.84375, |
| "cov@10%": 0.8928571428571429 |
| }, |
| "mean(evidence, fine-tuned)": { |
| "auroc": 0.9768623737373737, |
| "acc": 0.9575892857142857, |
| "brier": 0.03790862505206988, |
| "ece": 0.06543454258088069, |
| "cov@2%": 0.7321428571428571, |
| "cov@5%": 0.84375, |
| "cov@10%": 0.8928571428571429 |
| } |
| }, |
| "test_unseen_coder": { |
| "baseline kev": { |
| "auroc": 0.9638752052545156, |
| "acc": 0.9295774647887324, |
| "brier": 0.06336158381334493, |
| "ece": 0.039717450620457166, |
| "cov@2%": 0.28169014084507044, |
| "cov@5%": 0.4788732394366197, |
| "cov@10%": 0.647887323943662 |
| }, |
| "fine-tuned kev": { |
| "auroc": 0.9893267651888341, |
| "acc": 0.9154929577464789, |
| "brier": 0.055145570909923364, |
| "ece": 0.05822282925003647, |
| "cov@2%": 0.4507042253521127, |
| "cov@5%": 0.6197183098591549, |
| "cov@10%": 0.647887323943662 |
| }, |
| "evidence score": { |
| "auroc": 0.9433497536945813, |
| "acc": 0.9154929577464789, |
| "brier": 0.08483714788732394, |
| "ece": 0.1338028169014084, |
| "cov@2%": 0.15492957746478872, |
| "cov@5%": 0.15492957746478872, |
| "cov@10%": 0.5915492957746479 |
| }, |
| "mean(evidence, baseline)": { |
| "auroc": 0.9786535303776683, |
| "acc": 0.9154929577464789, |
| "brier": 0.06692693831242305, |
| "ece": 0.08200779432387027, |
| "cov@2%": 0.4084507042253521, |
| "cov@5%": 0.5774647887323944, |
| "cov@10%": 0.6197183098591549 |
| }, |
| "mean(evidence, fine-tuned)": { |
| "auroc": 0.9811165845648604, |
| "acc": 0.9154929577464789, |
| "brier": 0.0623680250089783, |
| "ece": 0.08442088475882523, |
| "cov@2%": 0.39436619718309857, |
| "cov@5%": 0.5774647887323944, |
| "cov@10%": 0.647887323943662 |
| } |
| }, |
| "test_unseen_coder_all_tasks": { |
| "baseline kev": { |
| "auroc": 0.9705357142857143, |
| "acc": 0.9262295081967213, |
| "brier": 0.0600707465937226, |
| "ece": 0.02946642500461688, |
| "cov@2%": 0.3114754098360656, |
| "cov@5%": 0.5122950819672131, |
| "cov@10%": 0.6229508196721312 |
| }, |
| "fine-tuned kev": { |
| "auroc": 0.9839285714285714, |
| "acc": 0.9467213114754098, |
| "brier": 0.03959559951764304, |
| "ece": 0.02761496864374707, |
| "cov@2%": 0.4713114754098361, |
| "cov@5%": 0.5881147540983607, |
| "cov@10%": 0.6352459016393442 |
| }, |
| "evidence score": { |
| "auroc": 0.943543956043956, |
| "acc": 0.9057377049180327, |
| "brier": 0.09111320255251247, |
| "ece": 0.09400301997638061, |
| "cov@2%": 0.0, |
| "cov@5%": 0.20901639344262296, |
| "cov@10%": 0.6188524590163934 |
| }, |
| "mean(evidence, baseline)": { |
| "auroc": 0.9703983516483516, |
| "acc": 0.9262295081967213, |
| "brier": 0.06636942416344632, |
| "ece": 0.06373308985502091, |
| "cov@2%": 0.3094262295081967, |
| "cov@5%": 0.5061475409836066, |
| "cov@10%": 0.6270491803278688 |
| }, |
| "mean(evidence, fine-tuned)": { |
| "auroc": 0.9801339285714286, |
| "acc": 0.9426229508196722, |
| "brier": 0.05355769253857334, |
| "ece": 0.08502821807951433, |
| "cov@2%": 0.3442622950819672, |
| "cov@5%": 0.5799180327868853, |
| "cov@10%": 0.6352459016393442 |
| } |
| }, |
| "test_unseen_qwen3_5_9b_defiant_iq2m": { |
| "baseline kev": { |
| "auroc": 0.9166666666666666, |
| "acc": 0.9166666666666666, |
| "brier": 0.07422605985253933, |
| "ece": 0.07673742820730627, |
| "cov@2%": 0.20833333333333334, |
| "cov@5%": 0.2777777777777778, |
| "cov@10%": 0.7083333333333334 |
| }, |
| "fine-tuned kev": { |
| "auroc": 0.9461805555555556, |
| "acc": 0.9027777777777778, |
| "brier": 0.06887600741202703, |
| "ece": 0.029364648847460262, |
| "cov@2%": 0.3472222222222222, |
| "cov@5%": 0.4444444444444444, |
| "cov@10%": 0.6944444444444444 |
| }, |
| "evidence score": { |
| "auroc": 0.9817708333333334, |
| "acc": 0.9166666666666666, |
| "brier": 0.0666232638888889, |
| "ece": 0.11284722222222222, |
| "cov@2%": 0.4861111111111111, |
| "cov@5%": 0.6111111111111112, |
| "cov@10%": 0.7083333333333334 |
| }, |
| "mean(evidence, baseline)": { |
| "auroc": 0.9800347222222222, |
| "acc": 0.9305555555555556, |
| "brier": 0.06047935935863502, |
| "ece": 0.10117383649315945, |
| "cov@2%": 0.5555555555555556, |
| "cov@5%": 0.6388888888888888, |
| "cov@10%": 0.7361111111111112 |
| }, |
| "mean(evidence, fine-tuned)": { |
| "auroc": 0.9782986111111112, |
| "acc": 0.9166666666666666, |
| "brier": 0.059680672299262584, |
| "ece": 0.1075121568818185, |
| "cov@2%": 0.5833333333333334, |
| "cov@5%": 0.625, |
| "cov@10%": 0.7222222222222222 |
| } |
| }, |
| "test_unseen_qwen3_5_9b_defiant_iq2m_all_tasks": { |
| "baseline kev": { |
| "auroc": 0.9662908930150309, |
| "acc": 0.9382716049382716, |
| "brier": 0.05267810391166952, |
| "ece": 0.03597538353446246, |
| "cov@2%": 0.345679012345679, |
| "cov@5%": 0.6172839506172839, |
| "cov@10%": 0.7119341563786008 |
| }, |
| "fine-tuned kev": { |
| "auroc": 0.9838822575891542, |
| "acc": 0.9567901234567902, |
| "brier": 0.035262343000345975, |
| "ece": 0.02702561098457988, |
| "cov@2%": 0.4794238683127572, |
| "cov@5%": 0.6666666666666666, |
| "cov@10%": 0.7119341563786008 |
| }, |
| "evidence score": { |
| "auroc": 0.9647159593280283, |
| "acc": 0.9238683127572016, |
| "brier": 0.06824839826404823, |
| "ece": 0.07486938929840165, |
| "cov@2%": 0.0, |
| "cov@5%": 0.49176954732510286, |
| "cov@10%": 0.6934156378600823 |
| }, |
| "mean(evidence, baseline)": { |
| "auroc": 0.9758325965222517, |
| "acc": 0.9403292181069959, |
| "brier": 0.05304097125217489, |
| "ece": 0.06640969399185599, |
| "cov@2%": 0.5185185185185185, |
| "cov@5%": 0.6213991769547325, |
| "cov@10%": 0.7119341563786008 |
| }, |
| "mean(evidence, fine-tuned)": { |
| "auroc": 0.9836243737105806, |
| "acc": 0.9567901234567902, |
| "brier": 0.04255867127685762, |
| "ece": 0.07530209304107607, |
| "cov@2%": 0.5246913580246914, |
| "cov@5%": 0.668724279835391, |
| "cov@10%": 0.7119341563786008 |
| } |
| } |
| } |
