| { |
| "train_key": "456965ab480be4a4", |
| "finetuned": { |
| "all": { |
| "n": 89838, |
| "accuracy": 0.6809, |
| "brier": 0.4125, |
| "nll": 0.8885, |
| "ece": 0.0701 |
| }, |
| "source/bev_tr": { |
| "n": 46320, |
| "accuracy": 0.771, |
| "brier": 0.3141, |
| "nll": 0.589, |
| "ece": 0.0314 |
| }, |
| "source/bev_tr/choice": { |
| "n": 18808, |
| "accuracy": 0.7402, |
| "brier": 0.3458, |
| "nll": 0.6765, |
| "ece": 0.0545 |
| }, |
| "source/bev_tr/noul": { |
| "n": 17572, |
| "accuracy": 0.8692, |
| "brier": 0.2012, |
| "nll": 0.3362, |
| "ece": 0.0296 |
| }, |
| "source/bev_tr/score": { |
| "n": 9940, |
| "accuracy": 0.6557, |
| "brier": 0.4537, |
| "nll": 0.87, |
| "ece": 0.0524 |
| }, |
| "source/bev_tr_en": { |
| "n": 5761, |
| "accuracy": 0.775, |
| "brier": 0.3109, |
| "nll": 0.5888, |
| "ece": 0.0363 |
| }, |
| "source/bev_tr_en/choice": { |
| "n": 2387, |
| "accuracy": 0.7357, |
| "brier": 0.3578, |
| "nll": 0.7025, |
| "ece": 0.0551 |
| }, |
| "source/bev_tr_en/noul": { |
| "n": 2112, |
| "accuracy": 0.8845, |
| "brier": 0.1782, |
| "nll": 0.3048, |
| "ece": 0.027 |
| }, |
| "source/bev_tr_en/score": { |
| "n": 1262, |
| "accuracy": 0.6664, |
| "brier": 0.4441, |
| "nll": 0.849, |
| "ece": 0.0586 |
| }, |
| "source/beyazperde_tr": { |
| "n": 5000, |
| "accuracy": 0.5312, |
| "brier": 0.5973, |
| "nll": 1.1068, |
| "ece": 0.0665 |
| }, |
| "source/beyazperde_tr/score": { |
| "n": 5000, |
| "accuracy": 0.5312, |
| "brier": 0.5973, |
| "nll": 1.1068, |
| "ece": 0.0665 |
| }, |
| "source/jev_bench_tr": { |
| "n": 22773, |
| "accuracy": 0.4902, |
| "brier": 0.6055, |
| "nll": 1.5258, |
| "ece": 0.1707 |
| }, |
| "source/jev_bench_tr/choice": { |
| "n": 9599, |
| "accuracy": 0.4262, |
| "brier": 0.6506, |
| "nll": 2.1598, |
| "ece": 0.1854 |
| }, |
| "source/jev_bench_tr/noul": { |
| "n": 7174, |
| "accuracy": 0.6968, |
| "brier": 0.4359, |
| "nll": 0.7049, |
| "ece": 0.1706 |
| }, |
| "source/jev_bench_tr/score": { |
| "n": 6000, |
| "accuracy": 0.3457, |
| "brier": 0.736, |
| "nll": 1.4929, |
| "ece": 0.1472 |
| }, |
| "source/jev_bench_tr_en": { |
| "n": 2000, |
| "accuracy": 0.5775, |
| "brier": 0.4968, |
| "nll": 1.2451, |
| "ece": 0.1054 |
| }, |
| "source/jev_bench_tr_en/choice": { |
| "n": 839, |
| "accuracy": 0.5459, |
| "brier": 0.5285, |
| "nll": 1.7378, |
| "ece": 0.1067 |
| }, |
| "source/jev_bench_tr_en/noul": { |
| "n": 635, |
| "accuracy": 0.7795, |
| "brier": 0.2983, |
| "nll": 0.5095, |
| "ece": 0.0858 |
| }, |
| "source/jev_bench_tr_en/score": { |
| "n": 526, |
| "accuracy": 0.384, |
| "brier": 0.6857, |
| "nll": 1.3473, |
| "ece": 0.1316 |
| }, |
| "source/massive_tr": { |
| "n": 2974, |
| "accuracy": 0.7811, |
| "brier": 0.3297, |
| "nll": 0.9699, |
| "ece": 0.0632 |
| }, |
| "source/massive_tr/choice": { |
| "n": 2974, |
| "accuracy": 0.7811, |
| "brier": 0.3297, |
| "nll": 0.9699, |
| "ece": 0.0632 |
| }, |
| "source/xnli_tr": { |
| "n": 5010, |
| "accuracy": 0.7375, |
| "brier": 0.3926, |
| "nll": 0.6972, |
| "ece": 0.0949 |
| }, |
| "source/xnli_tr/choice": { |
| "n": 5010, |
| "accuracy": 0.7375, |
| "brier": 0.3926, |
| "nll": 0.6972, |
| "ece": 0.0949 |
| }, |
| "type/choice": { |
| "n": 39617, |
| "accuracy": 0.6625, |
| "brier": 0.429, |
| "nll": 1.0846, |
| "ece": 0.0905 |
| }, |
| "type/noul": { |
| "n": 27493, |
| "accuracy": 0.8233, |
| "brier": 0.2629, |
| "nll": 0.434, |
| "ece": 0.061 |
| }, |
| "type/score": { |
| "n": 22728, |
| "accuracy": 0.5408, |
| "brier": 0.5647, |
| "nll": 1.0964, |
| "ece": 0.0713 |
| }, |
| "workflow/bev_tr/Browser interaction": { |
| "n": 286, |
| "accuracy": 0.7448, |
| "brier": 0.3899, |
| "nll": 0.7802, |
| "ece": 0.0895 |
| }, |
| "workflow/bev_tr/Civic and safety operations": { |
| "n": 261, |
| "accuracy": 0.7011, |
| "brier": 0.37, |
| "nll": 0.7452, |
| "ece": 0.0761 |
| }, |
| "workflow/bev_tr/Contract evidence": { |
| "n": 320, |
| "accuracy": 0.7, |
| "brier": 0.3975, |
| "nll": 0.6832, |
| "ece": 0.043 |
| }, |
| "workflow/bev_tr/Engagement and ranking": { |
| "n": 82, |
| "accuracy": 0.5732, |
| "brier": 0.4914, |
| "nll": 0.697, |
| "ece": 0.1592 |
| }, |
| "workflow/bev_tr/Financial reporting and banking": { |
| "n": 1740, |
| "accuracy": 0.5994, |
| "brier": 0.5204, |
| "nll": 0.9951, |
| "ece": 0.0424 |
| }, |
| "workflow/bev_tr/Game-state decisions": { |
| "n": 2039, |
| "accuracy": 0.8558, |
| "brier": 0.2074, |
| "nll": 0.3641, |
| "ece": 0.0701 |
| }, |
| "workflow/bev_tr/Reading comprehension": { |
| "n": 1422, |
| "accuracy": 0.7053, |
| "brier": 0.4404, |
| "nll": 0.6624, |
| "ece": 0.1453 |
| }, |
| "workflow/bev_tr/Response preference and quality": { |
| "n": 3176, |
| "accuracy": 0.5699, |
| "brier": 0.5603, |
| "nll": 1.0115, |
| "ece": 0.0386 |
| }, |
| "workflow/bev_tr/Retail, product, and shopping": { |
| "n": 6601, |
| "accuracy": 0.9382, |
| "brier": 0.1036, |
| "nll": 0.2381, |
| "ece": 0.0639 |
| }, |
| "workflow/bev_tr/Scientific and paper understanding": { |
| "n": 508, |
| "accuracy": 0.8268, |
| "brier": 0.2649, |
| "nll": 0.4787, |
| "ece": 0.0623 |
| }, |
| "workflow/bev_tr/Sensory quality rating": { |
| "n": 260, |
| "accuracy": 0.45, |
| "brier": 0.6741, |
| "nll": 1.2858, |
| "ece": 0.072 |
| }, |
| "workflow/bev_tr/Sentiment, emotion, and moderation": { |
| "n": 11526, |
| "accuracy": 0.8525, |
| "brier": 0.2205, |
| "nll": 0.4172, |
| "ece": 0.032 |
| }, |
| "workflow/bev_tr/Software engineering and code": { |
| "n": 2039, |
| "accuracy": 0.6106, |
| "brier": 0.5455, |
| "nll": 1.0456, |
| "ece": 0.1368 |
| }, |
| "workflow/bev_tr/Software security": { |
| "n": 2240, |
| "accuracy": 0.6531, |
| "brier": 0.4622, |
| "nll": 0.792, |
| "ece": 0.0884 |
| }, |
| "workflow/bev_tr/Spam detection": { |
| "n": 271, |
| "accuracy": 0.9557, |
| "brier": 0.0816, |
| "nll": 0.1752, |
| "ece": 0.0468 |
| }, |
| "workflow/bev_tr/Spatial and logical reasoning": { |
| "n": 8384, |
| "accuracy": 0.5816, |
| "brier": 0.5369, |
| "nll": 0.9921, |
| "ece": 0.0868 |
| }, |
| "workflow/bev_tr/Support and intent routing": { |
| "n": 2858, |
| "accuracy": 0.9699, |
| "brier": 0.0556, |
| "nll": 0.1613, |
| "ece": 0.0541 |
| }, |
| "workflow/bev_tr/Tool and workflow decisions": { |
| "n": 2307, |
| "accuracy": 0.9857, |
| "brier": 0.0322, |
| "nll": 0.1165, |
| "ece": 0.0651 |
| }, |
| "workflow/bev_tr_en/Browser interaction": { |
| "n": 39, |
| "accuracy": 0.7179, |
| "brier": 0.4923, |
| "nll": 0.9626, |
| "ece": 0.2261 |
| }, |
| "workflow/bev_tr_en/Civic and safety operations": { |
| "n": 31, |
| "accuracy": 0.4839, |
| "brier": 0.6397, |
| "nll": 1.2239, |
| "ece": 0.2769 |
| }, |
| "workflow/bev_tr_en/Contract evidence": { |
| "n": 49, |
| "accuracy": 0.7551, |
| "brier": 0.3823, |
| "nll": 0.7029, |
| "ece": 0.2028 |
| }, |
| "workflow/bev_tr_en/Engagement and ranking": { |
| "n": 6, |
| "accuracy": 0.8333, |
| "brier": 0.4349, |
| "nll": 0.6269, |
| "ece": 0.3409 |
| }, |
| "workflow/bev_tr_en/Financial reporting and banking": { |
| "n": 196, |
| "accuracy": 0.6735, |
| "brier": 0.466, |
| "nll": 0.94, |
| "ece": 0.0875 |
| }, |
| "workflow/bev_tr_en/Game-state decisions": { |
| "n": 303, |
| "accuracy": 0.8977, |
| "brier": 0.1682, |
| "nll": 0.3118, |
| "ece": 0.0653 |
| }, |
| "workflow/bev_tr_en/Reading comprehension": { |
| "n": 168, |
| "accuracy": 0.6786, |
| "brier": 0.4621, |
| "nll": 0.6826, |
| "ece": 0.1802 |
| }, |
| "workflow/bev_tr_en/Response preference and quality": { |
| "n": 389, |
| "accuracy": 0.5116, |
| "brier": 0.6053, |
| "nll": 1.1036, |
| "ece": 0.0985 |
| }, |
| "workflow/bev_tr_en/Retail, product, and shopping": { |
| "n": 902, |
| "accuracy": 0.9424, |
| "brier": 0.1003, |
| "nll": 0.2413, |
| "ece": 0.0704 |
| }, |
| "workflow/bev_tr_en/Scientific and paper understanding": { |
| "n": 66, |
| "accuracy": 0.8485, |
| "brier": 0.2585, |
| "nll": 0.5006, |
| "ece": 0.0904 |
| }, |
| "workflow/bev_tr_en/Sensory quality rating": { |
| "n": 37, |
| "accuracy": 0.4865, |
| "brier": 0.6809, |
| "nll": 1.2914, |
| "ece": 0.2337 |
| }, |
| "workflow/bev_tr_en/Sentiment, emotion, and moderation": { |
| "n": 1433, |
| "accuracy": 0.8946, |
| "brier": 0.1676, |
| "nll": 0.3238, |
| "ece": 0.0415 |
| }, |
| "workflow/bev_tr_en/Software engineering and code": { |
| "n": 260, |
| "accuracy": 0.6154, |
| "brier": 0.5456, |
| "nll": 1.0562, |
| "ece": 0.1593 |
| }, |
| "workflow/bev_tr_en/Software security": { |
| "n": 234, |
| "accuracy": 0.6154, |
| "brier": 0.512, |
| "nll": 0.8692, |
| "ece": 0.1382 |
| }, |
| "workflow/bev_tr_en/Spam detection": { |
| "n": 23, |
| "accuracy": 1.0, |
| "brier": 0.0103, |
| "nll": 0.0743, |
| "ece": 0.0716 |
| }, |
| "workflow/bev_tr_en/Spatial and logical reasoning": { |
| "n": 1042, |
| "accuracy": 0.5384, |
| "brier": 0.589, |
| "nll": 1.0846, |
| "ece": 0.1192 |
| }, |
| "workflow/bev_tr_en/Support and intent routing": { |
| "n": 310, |
| "accuracy": 0.9613, |
| "brier": 0.0693, |
| "nll": 0.1853, |
| "ece": 0.0497 |
| }, |
| "workflow/bev_tr_en/Tool and workflow decisions": { |
| "n": 273, |
| "accuracy": 0.9927, |
| "brier": 0.0185, |
| "nll": 0.0942, |
| "ece": 0.0709 |
| }, |
| "workflow/beyazperde_tr": { |
| "n": 5000, |
| "accuracy": 0.5312, |
| "brier": 0.5973, |
| "nll": 1.1068, |
| "ece": 0.0665 |
| }, |
| "workflow/jev_bench_tr/arc_challenge": { |
| "n": 1000, |
| "accuracy": 0.254, |
| "brier": 0.9528, |
| "nll": 1.7954, |
| "ece": 0.3298 |
| }, |
| "workflow/jev_bench_tr/banking77": { |
| "n": 1000, |
| "accuracy": 0.323, |
| "brier": 0.9502, |
| "nll": 3.3116, |
| "ece": 0.2924 |
| }, |
| "workflow/jev_bench_tr/boolq": { |
| "n": 1000, |
| "accuracy": 0.633, |
| "brier": 0.5397, |
| "nll": 0.79, |
| "ece": 0.2194 |
| }, |
| "workflow/jev_bench_tr/chaosnli": { |
| "n": 1599, |
| "accuracy": 0.5697, |
| "brier": 0.3207, |
| "nll": 1.3576, |
| "ece": 0.2328 |
| }, |
| "workflow/jev_bench_tr/civil_comments": { |
| "n": 2000, |
| "accuracy": 0.8305, |
| "brier": 0.1849, |
| "nll": 0.4636, |
| "ece": 0.0813 |
| }, |
| "workflow/jev_bench_tr/clinc150": { |
| "n": 1000, |
| "accuracy": 0.387, |
| "brier": 0.7844, |
| "nll": 2.8726, |
| "ece": 0.0485 |
| }, |
| "workflow/jev_bench_tr/fever_evidence": { |
| "n": 1000, |
| "accuracy": 0.752, |
| "brier": 0.3458, |
| "nll": 0.5243, |
| "ece": 0.0731 |
| }, |
| "workflow/jev_bench_tr/go_emotions": { |
| "n": 1000, |
| "accuracy": 0.308, |
| "brier": 0.4802, |
| "nll": 2.9553, |
| "ece": 0.2432 |
| }, |
| "workflow/jev_bench_tr/helpsteer2_helpfulness": { |
| "n": 1000, |
| "accuracy": 0.156, |
| "brier": 0.881, |
| "nll": 1.7749, |
| "ece": 0.2564 |
| }, |
| "workflow/jev_bench_tr/helpsteer2_verbosity": { |
| "n": 1000, |
| "accuracy": 0.299, |
| "brier": 0.7382, |
| "nll": 1.4194, |
| "ece": 0.0526 |
| }, |
| "workflow/jev_bench_tr/ledgar": { |
| "n": 1000, |
| "accuracy": 0.089, |
| "brier": 0.9987, |
| "nll": 4.359, |
| "ece": 0.1232 |
| }, |
| "workflow/jev_bench_tr/massive": { |
| "n": 1000, |
| "accuracy": 0.792, |
| "brier": 0.3181, |
| "nll": 0.9485, |
| "ece": 0.0652 |
| }, |
| "workflow/jev_bench_tr/measuring_hate_speech": { |
| "n": 1000, |
| "accuracy": 0.602, |
| "brier": 0.4646, |
| "nll": 1.1807, |
| "ece": 0.2498 |
| }, |
| "workflow/jev_bench_tr/mmlu": { |
| "n": 1000, |
| "accuracy": 0.258, |
| "brier": 0.8901, |
| "nll": 1.6694, |
| "ece": 0.269 |
| }, |
| "workflow/jev_bench_tr/mnli": { |
| "n": 1000, |
| "accuracy": 0.769, |
| "brier": 0.358, |
| "nll": 0.6495, |
| "ece": 0.078 |
| }, |
| "workflow/jev_bench_tr/paws": { |
| "n": 1000, |
| "accuracy": 0.461, |
| "brier": 0.8029, |
| "nll": 1.1797, |
| "ece": 0.3917 |
| }, |
| "workflow/jev_bench_tr/sms_spam": { |
| "n": 800, |
| "accuracy": 0.955, |
| "brier": 0.0716, |
| "nll": 0.161, |
| "ece": 0.0618 |
| }, |
| "workflow/jev_bench_tr/sst5": { |
| "n": 1000, |
| "accuracy": 0.374, |
| "brier": 0.7501, |
| "nll": 1.4187, |
| "ece": 0.1727 |
| }, |
| "workflow/jev_bench_tr/strategyqa_closed": { |
| "n": 687, |
| "accuracy": 0.5124, |
| "brier": 0.8002, |
| "nll": 1.2016, |
| "ece": 0.389 |
| }, |
| "workflow/jev_bench_tr/strategyqa_grounded": { |
| "n": 687, |
| "accuracy": 0.5473, |
| "brier": 0.6726, |
| "nll": 0.9917, |
| "ece": 0.2883 |
| }, |
| "workflow/jev_bench_tr/stsb": { |
| "n": 1000, |
| "accuracy": 0.207, |
| "brier": 0.8841, |
| "nll": 1.838, |
| "ece": 0.2022 |
| }, |
| "workflow/jev_bench_tr/yelp5": { |
| "n": 1000, |
| "accuracy": 0.436, |
| "brier": 0.6977, |
| "nll": 1.3257, |
| "ece": 0.1277 |
| }, |
| "workflow/jev_bench_tr_en/arc_challenge": { |
| "n": 91, |
| "accuracy": 0.2967, |
| "brier": 0.8757, |
| "nll": 1.6404, |
| "ece": 0.313 |
| }, |
| "workflow/jev_bench_tr_en/banking77": { |
| "n": 71, |
| "accuracy": 0.4648, |
| "brier": 0.7657, |
| "nll": 2.8336, |
| "ece": 0.1895 |
| }, |
| "workflow/jev_bench_tr_en/boolq": { |
| "n": 80, |
| "accuracy": 0.725, |
| "brier": 0.4328, |
| "nll": 0.6599, |
| "ece": 0.182 |
| }, |
| "workflow/jev_bench_tr_en/chaosnli": { |
| "n": 151, |
| "accuracy": 0.6821, |
| "brier": 0.2307, |
| "nll": 1.1925, |
| "ece": 0.1487 |
| }, |
| "workflow/jev_bench_tr_en/civil_comments": { |
| "n": 171, |
| "accuracy": 0.883, |
| "brier": 0.1086, |
| "nll": 0.3648, |
| "ece": 0.0575 |
| }, |
| "workflow/jev_bench_tr_en/clinc150": { |
| "n": 96, |
| "accuracy": 0.6042, |
| "brier": 0.5402, |
| "nll": 2.0147, |
| "ece": 0.1222 |
| }, |
| "workflow/jev_bench_tr_en/fever_evidence": { |
| "n": 91, |
| "accuracy": 0.8242, |
| "brier": 0.2424, |
| "nll": 0.3865, |
| "ece": 0.0749 |
| }, |
| "workflow/jev_bench_tr_en/go_emotions": { |
| "n": 93, |
| "accuracy": 0.4301, |
| "brier": 0.3831, |
| "nll": 2.703, |
| "ece": 0.1576 |
| }, |
| "workflow/jev_bench_tr_en/helpsteer2_helpfulness": { |
| "n": 89, |
| "accuracy": 0.2697, |
| "brier": 0.8004, |
| "nll": 1.568, |
| "ece": 0.1489 |
| }, |
| "workflow/jev_bench_tr_en/helpsteer2_verbosity": { |
| "n": 79, |
| "accuracy": 0.2532, |
| "brier": 0.7294, |
| "nll": 1.3758, |
| "ece": 0.0877 |
| }, |
| "workflow/jev_bench_tr_en/ledgar": { |
| "n": 70, |
| "accuracy": 0.3286, |
| "brier": 0.7949, |
| "nll": 2.7906, |
| "ece": 0.1533 |
| }, |
| "workflow/jev_bench_tr_en/massive": { |
| "n": 95, |
| "accuracy": 0.6947, |
| "brier": 0.4596, |
| "nll": 1.2966, |
| "ece": 0.1514 |
| }, |
| "workflow/jev_bench_tr_en/measuring_hate_speech": { |
| "n": 96, |
| "accuracy": 0.5208, |
| "brier": 0.4818, |
| "nll": 1.1402, |
| "ece": 0.1647 |
| }, |
| "workflow/jev_bench_tr_en/mmlu": { |
| "n": 80, |
| "accuracy": 0.3375, |
| "brier": 0.8825, |
| "nll": 1.6567, |
| "ece": 0.2195 |
| }, |
| "workflow/jev_bench_tr_en/mnli": { |
| "n": 92, |
| "accuracy": 0.8804, |
| "brier": 0.1863, |
| "nll": 0.3435, |
| "ece": 0.0482 |
| }, |
| "workflow/jev_bench_tr_en/paws": { |
| "n": 80, |
| "accuracy": 0.675, |
| "brier": 0.3821, |
| "nll": 0.5577, |
| "ece": 0.1829 |
| }, |
| "workflow/jev_bench_tr_en/sms_spam": { |
| "n": 82, |
| "accuracy": 0.9878, |
| "brier": 0.0332, |
| "nll": 0.1108, |
| "ece": 0.0644 |
| }, |
| "workflow/jev_bench_tr_en/sst5": { |
| "n": 73, |
| "accuracy": 0.4521, |
| "brier": 0.6665, |
| "nll": 1.1995, |
| "ece": 0.2019 |
| }, |
| "workflow/jev_bench_tr_en/strategyqa_closed": { |
| "n": 60, |
| "accuracy": 0.55, |
| "brier": 0.6608, |
| "nll": 0.9526, |
| "ece": 0.3191 |
| }, |
| "workflow/jev_bench_tr_en/strategyqa_grounded": { |
| "n": 71, |
| "accuracy": 0.6056, |
| "brier": 0.5811, |
| "nll": 0.8779, |
| "ece": 0.2428 |
| }, |
| "workflow/jev_bench_tr_en/stsb": { |
| "n": 90, |
| "accuracy": 0.2444, |
| "brier": 0.8279, |
| "nll": 1.7187, |
| "ece": 0.1456 |
| }, |
| "workflow/jev_bench_tr_en/yelp5": { |
| "n": 99, |
| "accuracy": 0.5354, |
| "brier": 0.6303, |
| "nll": 1.0981, |
| "ece": 0.1693 |
| }, |
| "workflow/massive_tr": { |
| "n": 2974, |
| "accuracy": 0.7811, |
| "brier": 0.3297, |
| "nll": 0.9699, |
| "ece": 0.0632 |
| }, |
| "workflow/xnli_tr": { |
| "n": 5010, |
| "accuracy": 0.7375, |
| "brier": 0.3926, |
| "nll": 0.6972, |
| "ece": 0.0949 |
| } |
| }, |
| "base": { |
| "all": { |
| "n": 89838, |
| "accuracy": 0.4325, |
| "brier": 0.7582, |
| "nll": 2.0922, |
| "ece": 0.277 |
| }, |
| "source/bev_tr": { |
| "n": 46320, |
| "accuracy": 0.4437, |
| "brier": 0.7623, |
| "nll": 2.0495, |
| "ece": 0.2749 |
| }, |
| "source/bev_tr/choice": { |
| "n": 18808, |
| "accuracy": 0.4499, |
| "brier": 0.7134, |
| "nll": 1.5746, |
| "ece": 0.2177 |
| }, |
| "source/bev_tr/noul": { |
| "n": 17572, |
| "accuracy": 0.5738, |
| "brier": 0.6909, |
| "nll": 2.5182, |
| "ece": 0.2981 |
| }, |
| "source/bev_tr/score": { |
| "n": 9940, |
| "accuracy": 0.2021, |
| "brier": 0.9807, |
| "nll": 2.1195, |
| "ece": 0.3422 |
| }, |
| "source/bev_tr_en": { |
| "n": 5761, |
| "accuracy": 0.4725, |
| "brier": 0.7085, |
| "nll": 1.8299, |
| "ece": 0.2428 |
| }, |
| "source/bev_tr_en/choice": { |
| "n": 2387, |
| "accuracy": 0.4604, |
| "brier": 0.6882, |
| "nll": 1.5258, |
| "ece": 0.2113 |
| }, |
| "source/bev_tr_en/noul": { |
| "n": 2112, |
| "accuracy": 0.6316, |
| "brier": 0.5714, |
| "nll": 1.8704, |
| "ece": 0.2377 |
| }, |
| "source/bev_tr_en/score": { |
| "n": 1262, |
| "accuracy": 0.229, |
| "brier": 0.9762, |
| "nll": 2.3374, |
| "ece": 0.3127 |
| }, |
| "source/beyazperde_tr": { |
| "n": 5000, |
| "accuracy": 0.1802, |
| "brier": 0.9535, |
| "nll": 1.9927, |
| "ece": 0.3291 |
| }, |
| "source/beyazperde_tr/score": { |
| "n": 5000, |
| "accuracy": 0.1802, |
| "brier": 0.9535, |
| "nll": 1.9927, |
| "ece": 0.3291 |
| }, |
| "source/jev_bench_tr": { |
| "n": 22773, |
| "accuracy": 0.402, |
| "brier": 0.769, |
| "nll": 2.3281, |
| "ece": 0.2949 |
| }, |
| "source/jev_bench_tr/choice": { |
| "n": 9599, |
| "accuracy": 0.3403, |
| "brier": 0.8114, |
| "nll": 3.1437, |
| "ece": 0.318 |
| }, |
| "source/jev_bench_tr/noul": { |
| "n": 7174, |
| "accuracy": 0.6391, |
| "brier": 0.5953, |
| "nll": 1.4894, |
| "ece": 0.2635 |
| }, |
| "source/jev_bench_tr/score": { |
| "n": 6000, |
| "accuracy": 0.2172, |
| "brier": 0.9089, |
| "nll": 2.0262, |
| "ece": 0.2957 |
| }, |
| "source/jev_bench_tr_en": { |
| "n": 2000, |
| "accuracy": 0.4995, |
| "brier": 0.6504, |
| "nll": 2.0143, |
| "ece": 0.2458 |
| }, |
| "source/jev_bench_tr_en/choice": { |
| "n": 839, |
| "accuracy": 0.4803, |
| "brier": 0.6591, |
| "nll": 2.6277, |
| "ece": 0.2565 |
| }, |
| "source/jev_bench_tr_en/noul": { |
| "n": 635, |
| "accuracy": 0.7213, |
| "brier": 0.4063, |
| "nll": 0.9013, |
| "ece": 0.1565 |
| }, |
| "source/jev_bench_tr_en/score": { |
| "n": 526, |
| "accuracy": 0.2624, |
| "brier": 0.9313, |
| "nll": 2.3795, |
| "ece": 0.3377 |
| }, |
| "source/massive_tr": { |
| "n": 2974, |
| "accuracy": 0.3547, |
| "brier": 0.8699, |
| "nll": 3.2285, |
| "ece": 0.2849 |
| }, |
| "source/massive_tr/choice": { |
| "n": 2974, |
| "accuracy": 0.3547, |
| "brier": 0.8699, |
| "nll": 3.2285, |
| "ece": 0.2849 |
| }, |
| "source/xnli_tr": { |
| "n": 5010, |
| "accuracy": 0.6926, |
| "brier": 0.511, |
| "nll": 1.1717, |
| "ece": 0.2111 |
| }, |
| "source/xnli_tr/choice": { |
| "n": 5010, |
| "accuracy": 0.6926, |
| "brier": 0.511, |
| "nll": 1.1717, |
| "ece": 0.2111 |
| }, |
| "type/choice": { |
| "n": 39617, |
| "accuracy": 0.4482, |
| "brier": 0.7206, |
| "nll": 2.0474, |
| "ece": 0.2465 |
| }, |
| "type/noul": { |
| "n": 27493, |
| "accuracy": 0.5987, |
| "brier": 0.6502, |
| "nll": 2.1626, |
| "ece": 0.2811 |
| }, |
| "type/score": { |
| "n": 22728, |
| "accuracy": 0.2042, |
| "brier": 0.9544, |
| "nll": 2.0851, |
| "ece": 0.3252 |
| }, |
| "workflow/bev_tr/Browser interaction": { |
| "n": 286, |
| "accuracy": 0.2762, |
| "brier": 0.8968, |
| "nll": 1.9321, |
| "ece": 0.2071 |
| }, |
| "workflow/bev_tr/Civic and safety operations": { |
| "n": 261, |
| "accuracy": 0.2529, |
| "brier": 0.9125, |
| "nll": 1.9973, |
| "ece": 0.2501 |
| }, |
| "workflow/bev_tr/Contract evidence": { |
| "n": 320, |
| "accuracy": 0.3344, |
| "brier": 0.9895, |
| "nll": 1.7097, |
| "ece": 0.4491 |
| }, |
| "workflow/bev_tr/Engagement and ranking": { |
| "n": 82, |
| "accuracy": 0.5488, |
| "brier": 0.5359, |
| "nll": 0.7447, |
| "ece": 0.1386 |
| }, |
| "workflow/bev_tr/Financial reporting and banking": { |
| "n": 1740, |
| "accuracy": 0.1661, |
| "brier": 1.2776, |
| "nll": 8.0822, |
| "ece": 0.5406 |
| }, |
| "workflow/bev_tr/Game-state decisions": { |
| "n": 2039, |
| "accuracy": 0.3448, |
| "brier": 0.9951, |
| "nll": 2.8801, |
| "ece": 0.4139 |
| }, |
| "workflow/bev_tr/Reading comprehension": { |
| "n": 1422, |
| "accuracy": 0.6287, |
| "brier": 0.5812, |
| "nll": 1.0839, |
| "ece": 0.2453 |
| }, |
| "workflow/bev_tr/Response preference and quality": { |
| "n": 3176, |
| "accuracy": 0.3237, |
| "brier": 0.8783, |
| "nll": 1.9154, |
| "ece": 0.267 |
| }, |
| "workflow/bev_tr/Retail, product, and shopping": { |
| "n": 6601, |
| "accuracy": 0.2874, |
| "brier": 1.0247, |
| "nll": 3.8548, |
| "ece": 0.4407 |
| }, |
| "workflow/bev_tr/Scientific and paper understanding": { |
| "n": 508, |
| "accuracy": 0.372, |
| "brier": 0.9247, |
| "nll": 1.8107, |
| "ece": 0.4028 |
| }, |
| "workflow/bev_tr/Sensory quality rating": { |
| "n": 260, |
| "accuracy": 0.0731, |
| "brier": 1.1058, |
| "nll": 2.5668, |
| "ece": 0.4822 |
| }, |
| "workflow/bev_tr/Sentiment, emotion, and moderation": { |
| "n": 11526, |
| "accuracy": 0.6164, |
| "brier": 0.496, |
| "nll": 0.896, |
| "ece": 0.1549 |
| }, |
| "workflow/bev_tr/Software engineering and code": { |
| "n": 2039, |
| "accuracy": 0.3217, |
| "brier": 0.7935, |
| "nll": 1.5021, |
| "ece": 0.1482 |
| }, |
| "workflow/bev_tr/Software security": { |
| "n": 2240, |
| "accuracy": 0.4147, |
| "brier": 0.8559, |
| "nll": 1.8811, |
| "ece": 0.3557 |
| }, |
| "workflow/bev_tr/Spam detection": { |
| "n": 271, |
| "accuracy": 0.5498, |
| "brier": 0.7614, |
| "nll": 2.08, |
| "ece": 0.3966 |
| }, |
| "workflow/bev_tr/Spatial and logical reasoning": { |
| "n": 8384, |
| "accuracy": 0.3204, |
| "brier": 0.8964, |
| "nll": 1.9159, |
| "ece": 0.3173 |
| }, |
| "workflow/bev_tr/Support and intent routing": { |
| "n": 2858, |
| "accuracy": 0.6767, |
| "brier": 0.4581, |
| "nll": 0.9638, |
| "ece": 0.1276 |
| }, |
| "workflow/bev_tr/Tool and workflow decisions": { |
| "n": 2307, |
| "accuracy": 0.7703, |
| "brier": 0.3383, |
| "nll": 0.7256, |
| "ece": 0.1085 |
| }, |
| "workflow/bev_tr_en/Browser interaction": { |
| "n": 39, |
| "accuracy": 0.2821, |
| "brier": 0.9354, |
| "nll": 2.1869, |
| "ece": 0.3387 |
| }, |
| "workflow/bev_tr_en/Civic and safety operations": { |
| "n": 31, |
| "accuracy": 0.0968, |
| "brier": 1.0532, |
| "nll": 2.4108, |
| "ece": 0.4439 |
| }, |
| "workflow/bev_tr_en/Contract evidence": { |
| "n": 49, |
| "accuracy": 0.4286, |
| "brier": 0.8485, |
| "nll": 1.4634, |
| "ece": 0.4012 |
| }, |
| "workflow/bev_tr_en/Engagement and ranking": { |
| "n": 6, |
| "accuracy": 0.6667, |
| "brier": 0.4465, |
| "nll": 0.6305, |
| "ece": 0.2653 |
| }, |
| "workflow/bev_tr_en/Financial reporting and banking": { |
| "n": 196, |
| "accuracy": 0.1429, |
| "brier": 1.297, |
| "nll": 7.8218, |
| "ece": 0.5676 |
| }, |
| "workflow/bev_tr_en/Game-state decisions": { |
| "n": 303, |
| "accuracy": 0.4983, |
| "brier": 0.6867, |
| "nll": 1.1958, |
| "ece": 0.196 |
| }, |
| "workflow/bev_tr_en/Reading comprehension": { |
| "n": 168, |
| "accuracy": 0.6369, |
| "brier": 0.5115, |
| "nll": 0.8706, |
| "ece": 0.2237 |
| }, |
| "workflow/bev_tr_en/Response preference and quality": { |
| "n": 389, |
| "accuracy": 0.3342, |
| "brier": 0.8502, |
| "nll": 1.7888, |
| "ece": 0.2471 |
| }, |
| "workflow/bev_tr_en/Retail, product, and shopping": { |
| "n": 902, |
| "accuracy": 0.2971, |
| "brier": 0.9608, |
| "nll": 3.1178, |
| "ece": 0.3787 |
| }, |
| "workflow/bev_tr_en/Scientific and paper understanding": { |
| "n": 66, |
| "accuracy": 0.3485, |
| "brier": 1.012, |
| "nll": 2.1604, |
| "ece": 0.4553 |
| }, |
| "workflow/bev_tr_en/Sensory quality rating": { |
| "n": 37, |
| "accuracy": 0.1081, |
| "brier": 0.8935, |
| "nll": 1.85, |
| "ece": 0.2321 |
| }, |
| "workflow/bev_tr_en/Sentiment, emotion, and moderation": { |
| "n": 1433, |
| "accuracy": 0.7139, |
| "brier": 0.4116, |
| "nll": 0.9539, |
| "ece": 0.1375 |
| }, |
| "workflow/bev_tr_en/Software engineering and code": { |
| "n": 260, |
| "accuracy": 0.3231, |
| "brier": 0.7653, |
| "nll": 1.4255, |
| "ece": 0.1383 |
| }, |
| "workflow/bev_tr_en/Software security": { |
| "n": 234, |
| "accuracy": 0.3718, |
| "brier": 0.8652, |
| "nll": 1.7216, |
| "ece": 0.3089 |
| }, |
| "workflow/bev_tr_en/Spam detection": { |
| "n": 23, |
| "accuracy": 0.5652, |
| "brier": 0.6446, |
| "nll": 1.6813, |
| "ece": 0.399 |
| }, |
| "workflow/bev_tr_en/Spatial and logical reasoning": { |
| "n": 1042, |
| "accuracy": 0.3081, |
| "brier": 0.8798, |
| "nll": 1.8558, |
| "ece": 0.286 |
| }, |
| "workflow/bev_tr_en/Support and intent routing": { |
| "n": 310, |
| "accuracy": 0.7258, |
| "brier": 0.3894, |
| "nll": 0.8425, |
| "ece": 0.1377 |
| }, |
| "workflow/bev_tr_en/Tool and workflow decisions": { |
| "n": 273, |
| "accuracy": 0.8022, |
| "brier": 0.2895, |
| "nll": 0.6295, |
| "ece": 0.1321 |
| }, |
| "workflow/beyazperde_tr": { |
| "n": 5000, |
| "accuracy": 0.1802, |
| "brier": 0.9535, |
| "nll": 1.9927, |
| "ece": 0.3291 |
| }, |
| "workflow/jev_bench_tr/arc_challenge": { |
| "n": 1000, |
| "accuracy": 0.249, |
| "brier": 0.8758, |
| "nll": 1.6658, |
| "ece": 0.2392 |
| }, |
| "workflow/jev_bench_tr/banking77": { |
| "n": 1000, |
| "accuracy": 0.27, |
| "brier": 1.1483, |
| "nll": 5.3527, |
| "ece": 0.4696 |
| }, |
| "workflow/jev_bench_tr/boolq": { |
| "n": 1000, |
| "accuracy": 0.687, |
| "brier": 0.5095, |
| "nll": 1.0117, |
| "ece": 0.2262 |
| }, |
| "workflow/jev_bench_tr/chaosnli": { |
| "n": 1599, |
| "accuracy": 0.4772, |
| "brier": 0.4824, |
| "nll": 2.3205, |
| "ece": 0.3991 |
| }, |
| "workflow/jev_bench_tr/civil_comments": { |
| "n": 2000, |
| "accuracy": 0.9125, |
| "brier": 0.0821, |
| "nll": 0.3281, |
| "ece": 0.0263 |
| }, |
| "workflow/jev_bench_tr/clinc150": { |
| "n": 1000, |
| "accuracy": 0.25, |
| "brier": 0.9822, |
| "nll": 4.0163, |
| "ece": 0.2849 |
| }, |
| "workflow/jev_bench_tr/fever_evidence": { |
| "n": 1000, |
| "accuracy": 0.551, |
| "brier": 0.7461, |
| "nll": 1.4558, |
| "ece": 0.3511 |
| }, |
| "workflow/jev_bench_tr/go_emotions": { |
| "n": 1000, |
| "accuracy": 0.278, |
| "brier": 0.73, |
| "nll": 4.5711, |
| "ece": 0.4802 |
| }, |
| "workflow/jev_bench_tr/helpsteer2_helpfulness": { |
| "n": 1000, |
| "accuracy": 0.15, |
| "brier": 0.9073, |
| "nll": 1.8959, |
| "ece": 0.2547 |
| }, |
| "workflow/jev_bench_tr/helpsteer2_verbosity": { |
| "n": 1000, |
| "accuracy": 0.211, |
| "brier": 0.8589, |
| "nll": 1.6432, |
| "ece": 0.2394 |
| }, |
| "workflow/jev_bench_tr/ledgar": { |
| "n": 1000, |
| "accuracy": 0.087, |
| "brier": 1.1279, |
| "nll": 5.1478, |
| "ece": 0.2658 |
| }, |
| "workflow/jev_bench_tr/massive": { |
| "n": 1000, |
| "accuracy": 0.375, |
| "brier": 0.8405, |
| "nll": 3.0528, |
| "ece": 0.2758 |
| }, |
| "workflow/jev_bench_tr/measuring_hate_speech": { |
| "n": 1000, |
| "accuracy": 0.313, |
| "brier": 0.5368, |
| "nll": 1.2331, |
| "ece": 0.1656 |
| }, |
| "workflow/jev_bench_tr/mmlu": { |
| "n": 1000, |
| "accuracy": 0.275, |
| "brier": 0.8531, |
| "nll": 1.6323, |
| "ece": 0.2134 |
| }, |
| "workflow/jev_bench_tr/mnli": { |
| "n": 1000, |
| "accuracy": 0.72, |
| "brier": 0.459, |
| "nll": 1.0265, |
| "ece": 0.1898 |
| }, |
| "workflow/jev_bench_tr/paws": { |
| "n": 1000, |
| "accuracy": 0.528, |
| "brier": 0.8257, |
| "nll": 1.7868, |
| "ece": 0.402 |
| }, |
| "workflow/jev_bench_tr/sms_spam": { |
| "n": 800, |
| "accuracy": 0.3275, |
| "brier": 1.2108, |
| "nll": 4.8477, |
| "ece": 0.6043 |
| }, |
| "workflow/jev_bench_tr/sst5": { |
| "n": 1000, |
| "accuracy": 0.286, |
| "brier": 0.8187, |
| "nll": 1.6116, |
| "ece": 0.1613 |
| }, |
| "workflow/jev_bench_tr/strategyqa_closed": { |
| "n": 687, |
| "accuracy": 0.5167, |
| "brier": 0.8117, |
| "nll": 1.4146, |
| "ece": 0.378 |
| }, |
| "workflow/jev_bench_tr/strategyqa_grounded": { |
| "n": 687, |
| "accuracy": 0.5488, |
| "brier": 0.726, |
| "nll": 1.3457, |
| "ece": 0.3305 |
| }, |
| "workflow/jev_bench_tr/stsb": { |
| "n": 1000, |
| "accuracy": 0.158, |
| "brier": 1.1593, |
| "nll": 2.5632, |
| "ece": 0.4602 |
| }, |
| "workflow/jev_bench_tr/yelp5": { |
| "n": 1000, |
| "accuracy": 0.185, |
| "brier": 1.1724, |
| "nll": 3.2104, |
| "ece": 0.4975 |
| }, |
| "workflow/jev_bench_tr_en/arc_challenge": { |
| "n": 91, |
| "accuracy": 0.3077, |
| "brier": 0.8521, |
| "nll": 1.618, |
| "ece": 0.2606 |
| }, |
| "workflow/jev_bench_tr_en/banking77": { |
| "n": 71, |
| "accuracy": 0.4366, |
| "brier": 0.9091, |
| "nll": 4.9136, |
| "ece": 0.4145 |
| }, |
| "workflow/jev_bench_tr_en/boolq": { |
| "n": 80, |
| "accuracy": 0.725, |
| "brier": 0.4529, |
| "nll": 0.886, |
| "ece": 0.219 |
| }, |
| "workflow/jev_bench_tr_en/chaosnli": { |
| "n": 151, |
| "accuracy": 0.6225, |
| "brier": 0.3674, |
| "nll": 1.9774, |
| "ece": 0.2756 |
| }, |
| "workflow/jev_bench_tr_en/civil_comments": { |
| "n": 171, |
| "accuracy": 0.9532, |
| "brier": 0.0364, |
| "nll": 0.253, |
| "ece": 0.0844 |
| }, |
| "workflow/jev_bench_tr_en/clinc150": { |
| "n": 96, |
| "accuracy": 0.4792, |
| "brier": 0.7581, |
| "nll": 2.9071, |
| "ece": 0.309 |
| }, |
| "workflow/jev_bench_tr_en/fever_evidence": { |
| "n": 91, |
| "accuracy": 0.7253, |
| "brier": 0.4523, |
| "nll": 0.7521, |
| "ece": 0.2241 |
| }, |
| "workflow/jev_bench_tr_en/go_emotions": { |
| "n": 93, |
| "accuracy": 0.3978, |
| "brier": 0.5766, |
| "nll": 4.0598, |
| "ece": 0.375 |
| }, |
| "workflow/jev_bench_tr_en/helpsteer2_helpfulness": { |
| "n": 89, |
| "accuracy": 0.1348, |
| "brier": 0.8934, |
| "nll": 1.7886, |
| "ece": 0.3137 |
| }, |
| "workflow/jev_bench_tr_en/helpsteer2_verbosity": { |
| "n": 79, |
| "accuracy": 0.1646, |
| "brier": 0.9505, |
| "nll": 1.6857, |
| "ece": 0.3926 |
| }, |
| "workflow/jev_bench_tr_en/ledgar": { |
| "n": 70, |
| "accuracy": 0.2714, |
| "brier": 0.9955, |
| "nll": 4.4382, |
| "ece": 0.32 |
| }, |
| "workflow/jev_bench_tr_en/massive": { |
| "n": 95, |
| "accuracy": 0.4105, |
| "brier": 0.8485, |
| "nll": 3.043, |
| "ece": 0.2822 |
| }, |
| "workflow/jev_bench_tr_en/measuring_hate_speech": { |
| "n": 96, |
| "accuracy": 0.4479, |
| "brier": 0.482, |
| "nll": 1.085, |
| "ece": 0.074 |
| }, |
| "workflow/jev_bench_tr_en/mmlu": { |
| "n": 80, |
| "accuracy": 0.3375, |
| "brier": 0.7891, |
| "nll": 1.4972, |
| "ece": 0.1576 |
| }, |
| "workflow/jev_bench_tr_en/mnli": { |
| "n": 92, |
| "accuracy": 0.8913, |
| "brier": 0.1693, |
| "nll": 0.3673, |
| "ece": 0.0821 |
| }, |
| "workflow/jev_bench_tr_en/paws": { |
| "n": 80, |
| "accuracy": 0.65, |
| "brier": 0.4374, |
| "nll": 0.6762, |
| "ece": 0.2444 |
| }, |
| "workflow/jev_bench_tr_en/sms_spam": { |
| "n": 82, |
| "accuracy": 0.5488, |
| "brier": 0.7829, |
| "nll": 2.661, |
| "ece": 0.3921 |
| }, |
| "workflow/jev_bench_tr_en/sst5": { |
| "n": 73, |
| "accuracy": 0.3014, |
| "brier": 1.0207, |
| "nll": 2.3759, |
| "ece": 0.4117 |
| }, |
| "workflow/jev_bench_tr_en/strategyqa_closed": { |
| "n": 60, |
| "accuracy": 0.5333, |
| "brier": 0.5838, |
| "nll": 0.8489, |
| "ece": 0.2998 |
| }, |
| "workflow/jev_bench_tr_en/strategyqa_grounded": { |
| "n": 71, |
| "accuracy": 0.5915, |
| "brier": 0.5658, |
| "nll": 0.9366, |
| "ece": 0.2513 |
| }, |
| "workflow/jev_bench_tr_en/stsb": { |
| "n": 90, |
| "accuracy": 0.2333, |
| "brier": 1.0749, |
| "nll": 2.5195, |
| "ece": 0.3886 |
| }, |
| "workflow/jev_bench_tr_en/yelp5": { |
| "n": 99, |
| "accuracy": 0.2727, |
| "brier": 1.1895, |
| "nll": 4.5951, |
| "ece": 0.5002 |
| }, |
| "workflow/massive_tr": { |
| "n": 2974, |
| "accuracy": 0.3547, |
| "brier": 0.8699, |
| "nll": 3.2285, |
| "ece": 0.2849 |
| }, |
| "workflow/xnli_tr": { |
| "n": 5010, |
| "accuracy": 0.6926, |
| "brier": 0.511, |
| "nll": 1.1717, |
| "ece": 0.2111 |
| } |
| } |
| } |
|
|
