Buckets:

Pranav2748's picture
download
raw
25.4 kB
{ "ours": { "base": { "model": "Qwen/Qwen3.5-4B", "temps": { "choice": 1.707683209388825, "noul": 2.4105066990845945, "score": 1.2847580560947693 }, "raw": { "pubmedqa": { "primitive": "noul", "ds": 48.05492263372704, "ci": [ 37.76904082341696, 57.150183634721955 ], "accuracy": 0.8066666666666666, "ece_10_bin": 0.06460862585758344, "T": 1.0, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": 2.049999831140492, "ci": [ -9.799345465028715, 13.108881184556378 ], "accuracy": 0.4033333333333333, "ece_10_bin": 0.15123322991920435, "T": 1.0, "n": 300 }, "banking77": { "primitive": "choice", "ds": 56.173306098208656, "ci": [ 49.27693100059662, 63.141876285290685 ], "accuracy": 0.7186666666666667, "ece_10_bin": 0.14011876588219976, "T": 1.0, "n": 300 } }, "calibrated": { "pubmedqa": { "primitive": "noul", "ds": 36.371473995104964, "ci": [ 29.35952403100504, 42.77706939107353 ], "accuracy": 0.8066666666666666, "ece_10_bin": 0.15700328478531872, "T": 2.4105066990845945, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": 4.103423043229137, "ci": [ -6.389453722683416, 13.956330811232187 ], "accuracy": 0.4033333333333333, "ece_10_bin": 0.08902712405177117, "T": 1.2847580560947693, "n": 300 }, "banking77": { "primitive": "choice", "ds": 59.3453919929658, "ci": [ 53.69828564607293, 64.93884389399817 ], "accuracy": 0.7186666666666667, "ece_10_bin": 0.04315345334218913, "T": 1.707683209388825, "n": 300 } } }, "base_2b": { "model": "openbmb/MiniCPM5-2B", "temps": { "choice": 4.78807442670338, "noul": 19.999999999982688, "score": 9.956643342471382 }, "raw": { "pubmedqa": { "primitive": "noul", "ds": 0.06537669220909592, "ci": [ -12.074739103400578, 12.86250789143286 ], "accuracy": 0.6833333333333333, "ece_10_bin": 0.2256417780118445, "T": 1.0, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": -26.542430292496523, "ci": [ -32.37154921739351, -19.847259796335507 ], "accuracy": 0.3233333333333333, "ece_10_bin": 0.4115217030994677, "T": 1.0, "n": 300 }, "banking77": { "primitive": "choice", "ds": 28.57680388837838, "ci": [ 20.866179600278087, 36.316383050601196 ], "accuracy": 0.57, "ece_10_bin": 0.3036123204111504, "T": 1.0, "n": 300 } }, "calibrated": { "pubmedqa": { "primitive": "noul", "ds": 7.871329959231909, "ci": [ 4.386042772766913, 11.112613249722312 ], "accuracy": 0.6833333333333333, "ece_10_bin": 0.125778638387935, "T": 19.999999999982688, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": -6.307747971961097, "ci": [ -12.458304188976395, -0.6936091297558415 ], "accuracy": 0.3233333333333333, "ece_10_bin": 0.010844527401254841, "T": 9.956643342471382, "n": 300 }, "banking77": { "primitive": "choice", "ds": 31.930029024290597, "ci": [ 28.786810587023467, 35.1970814222839 ], "accuracy": 0.57, "ece_10_bin": 0.2619731354805633, "T": 4.78807442670338, "n": 300 } } }, "base_2b_think": { "model": "openbmb/MiniCPM5-2B", "temps": { "choice": 11.20281383618261, "noul": 14.170122098039382, "score": 8.440970884106083 }, "raw": { "pubmedqa": { "primitive": "noul", "ds": 33.5031127255792, "ci": [ 14.994088405964881, 50.14401495430281 ], "accuracy": 0.8433333333333334, "ece_10_bin": 0.15666666652019579, "T": 1.0, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": -53.5046810509483, "ci": [ -70.32589555811374, -37.2137948447786 ], "accuracy": 0.33666666666666667, "ece_10_bin": 0.6633333118218515, "T": 1.0, "n": 300 }, "banking77": { "primitive": "choice", "ds": 36.503652709543644, "ci": [ 25.697439535484303, 47.31038520132706 ], "accuracy": 0.6866666666666666, "ece_10_bin": 0.3133331416314482, "T": 1.0, "n": 300 } }, "calibrated": { "pubmedqa": { "primitive": "noul", "ds": 43.57061048146679, "ci": [ 29.803221302295093, 55.62367362989394 ], "accuracy": 0.8433333333333334, "ece_10_bin": 0.025859414384324198, "T": 14.170122098039382, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": -24.399113849176544, "ci": [ -38.0616774021222, -11.746581566453877 ], "accuracy": 0.33666666666666667, "ece_10_bin": 0.4503987571107928, "T": 8.440970884106083, "n": 300 }, "banking77": { "primitive": "choice", "ds": 28.77450536506706, "ci": [ 25.907327767216973, 31.63217534210117 ], "accuracy": 0.6866666666666666, "ece_10_bin": 0.4068075231256754, "T": 11.20281383618261, "n": 300 } } }, "rl40": { "model": "runs/v2/checkpoints/rl40_merged", "temps": { "choice": 1.1597719952546015, "noul": 0.8079639789416141, "score": 0.7152072687033781 }, "raw": { "pubmedqa": { "primitive": "noul", "ds": 58.26569187854867, "ci": [ 47.682670427263815, 67.59957426059007 ], "accuracy": 0.86, "ece_10_bin": 0.02504456097376074, "T": 1.0, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": 6.046865704552173, "ci": [ -2.4892746013859646, 14.387750191722974 ], "accuracy": 0.42, "ece_10_bin": 0.09333381174925866, "T": 1.0, "n": 300 }, "banking77": { "primitive": "choice", "ds": 60.86873010019582, "ci": [ 54.64881897913666, 67.22493242614614 ], "accuracy": 0.732, "ece_10_bin": 0.05956597864906059, "T": 1.0, "n": 300 } }, "calibrated": { "pubmedqa": { "primitive": "noul", "ds": 58.07152293620064, "ci": [ 46.80979761941665, 68.33396374013778 ], "accuracy": 0.86, "ece_10_bin": 0.03793417592656142, "T": 0.8079639789416141, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": 2.0690909709641936, "ci": [ -7.552019439614597, 11.588130422238306 ], "accuracy": 0.42, "ece_10_bin": 0.15102980959657503, "T": 0.7152072687033781, "n": 300 }, "banking77": { "primitive": "choice", "ds": 61.28325240743173, "ci": [ 55.474958876375645, 67.25123138884102 ], "accuracy": 0.732, "ece_10_bin": 0.02114475384467073, "T": 1.1597719952546015, "n": 300 } } }, "rl60": { "model": "runs/v2/checkpoints/rl60_merged", "temps": { "choice": 1.1599743376333564, "noul": 0.7983784191901301, "score": 0.7276446912498098 }, "raw": { "pubmedqa": { "primitive": "noul", "ds": 58.72560050503446, "ci": [ 48.25519209655997, 67.9986456840194 ], "accuracy": 0.8633333333333333, "ece_10_bin": 0.026845709305701414, "T": 1.0, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": 6.037515788606118, "ci": [ -2.400834157155406, 14.35084224020631 ], "accuracy": 0.43666666666666665, "ece_10_bin": 0.095953934239622, "T": 1.0, "n": 300 }, "banking77": { "primitive": "choice", "ds": 60.955089258193006, "ci": [ 54.66931758063293, 67.32577013719438 ], "accuracy": 0.7313333333333333, "ece_10_bin": 0.05904332938166834, "T": 1.0, "n": 300 } }, "calibrated": { "pubmedqa": { "primitive": "noul", "ds": 58.614060949612615, "ci": [ 47.33820737643784, 68.84278523729319 ], "accuracy": 0.8633333333333333, "ece_10_bin": 0.03571861771195042, "T": 0.7983784191901301, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": 2.368980700043777, "ci": [ -7.2405524866967275, 11.746115459613257 ], "accuracy": 0.43666666666666665, "ece_10_bin": 0.1303236520061676, "T": 0.7276446912498098, "n": 300 }, "banking77": { "primitive": "choice", "ds": 61.33904164009218, "ci": [ 55.52009100804195, 67.34968363963038 ], "accuracy": 0.7313333333333333, "ece_10_bin": 0.027606077031200056, "T": 1.1599743376333564, "n": 300 } } }, "rl60_think": { "model": "runs/v2/checkpoints/rl60_merged", "temps": { "choice": 2.145082715273473, "noul": 1.9352338162597778, "score": 1.8069801445316314 }, "raw": { "pubmedqa": { "primitive": "noul", "ds": 58.936313009275274, "ci": [ 44.39233287434623, 72.34563008669913 ], "accuracy": 0.9, "ece_10_bin": 0.09364023899498322, "T": 1.0, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": -42.99011458744382, "ci": [ -61.37940857090847, -24.665619976798016 ], "accuracy": 0.41333333333333333, "ece_10_bin": 0.519184866453263, "T": 1.0, "n": 300 }, "banking77": { "primitive": "choice", "ds": 49.48519474863158, "ci": [ 40.343495405677444, 58.75819407518067 ], "accuracy": 0.7333333333333333, "ece_10_bin": 0.24588454845319418, "T": 1.0, "n": 300 } }, "calibrated": { "pubmedqa": { "primitive": "noul", "ds": 63.46531221234828, "ci": [ 51.2282704267052, 74.80118608216338 ], "accuracy": 0.9, "ece_10_bin": 0.045119675748778995, "T": 1.9352338162597778, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": -20.13990146885871, "ci": [ -35.33533731665648, -4.939007706671506 ], "accuracy": 0.41333333333333333, "ece_10_bin": 0.37806488700901164, "T": 1.8069801445316314, "n": 300 }, "banking77": { "primitive": "choice", "ds": 59.27752355436782, "ci": [ 53.606719794836316, 64.93246211545883 ], "accuracy": 0.7333333333333333, "ece_10_bin": 0.13413758897367453, "T": 2.145082715273473, "n": 300 } } }, "v1_joint": { "model": "checkpoints/joint_merged", "temps": { "choice": 1.2843740495491411, "noul": 1.1776347713530488, "score": 1.0535480782391846 }, "raw": { "pubmedqa": { "primitive": "noul", "ds": 49.206682779012525, "ci": [ 36.65442701300594, 59.903266381066885 ], "accuracy": 0.8233333333333334, "ece_10_bin": 0.06969906794374832, "T": 1.0, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": 6.7772824916263685, "ci": [ -2.2223595409748595, 16.175846550495578 ], "accuracy": 0.3933333333333333, "ece_10_bin": 0.17954795337837168, "T": 1.0, "n": 300 }, "banking77": { "primitive": "choice", "ds": 60.61334399242887, "ci": [ 54.24958965660347, 66.93369679868799 ], "accuracy": 0.7326666666666667, "ece_10_bin": 0.07895850726459176, "T": 1.0, "n": 300 } }, "calibrated": { "pubmedqa": { "primitive": "noul", "ds": 50.30267194300714, "ci": [ 38.78645963769695, 60.216646199918934 ], "accuracy": 0.8233333333333334, "ece_10_bin": 0.06609612462236357, "T": 1.1776347713530488, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": 7.5280862119676435, "ci": [ -1.2187682680666836, 16.69347049625058 ], "accuracy": 0.3933333333333333, "ece_10_bin": 0.16808161152147194, "T": 1.0535480782391846, "n": 300 }, "banking77": { "primitive": "choice", "ds": 61.483220852549046, "ci": [ 55.919259043009745, 67.11049272666077 ], "accuracy": 0.7326666666666667, "ece_10_bin": 0.05006183740765189, "T": 1.2843740495491411, "n": 300 } } }, "v2_2b": { "model": "runs/v2_2b/checkpoints/v2b_joint_merged", "temps": { "choice": 1.0924917038331285, "noul": 0.6133218472538287, "score": 0.7621787953970005 }, "raw": { "pubmedqa": { "primitive": "noul", "ds": 43.58234449799754, "ci": [ 32.467084766485215, 53.490310674231864 ], "accuracy": 0.79, "ece_10_bin": 0.06297236285262527, "T": 1.0, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": 0.6637233783827412, "ci": [ -6.537478300305655, 8.270746327898612 ], "accuracy": 0.35, "ece_10_bin": 0.17140852998786757, "T": 1.0, "n": 300 }, "banking77": { "primitive": "choice", "ds": 53.92308258456515, "ci": [ 47.8488111622375, 60.00593318759219 ], "accuracy": 0.668, "ece_10_bin": 0.048110853789338, "T": 1.0, "n": 300 } }, "calibrated": { "pubmedqa": { "primitive": "noul", "ds": 41.560019439645814, "ci": [ 27.87758590950197, 53.45290476044082 ], "accuracy": 0.79, "ece_10_bin": 0.0857771775727929, "T": 0.6133218472538287, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": -3.9949134233046912, "ci": [ -11.642008207818442, 4.443413141629088 ], "accuracy": 0.35, "ece_10_bin": 0.2241508614885324, "T": 0.7621787953970005, "n": 300 }, "banking77": { "primitive": "choice", "ds": 54.152173734130216, "ci": [ 48.384491001853675, 59.872845624000156 ], "accuracy": 0.668, "ece_10_bin": 0.03394477729754107, "T": 1.0924917038331285, "n": 300 } } }, "v2_2b_ro": { "model": "runs/v2_2b/checkpoints/v2b_ro_merged", "temps": { "choice": 1.0772703967295125, "noul": 0.6149937401183327, "score": 0.7973321069676444 }, "raw": { "pubmedqa": { "primitive": "noul", "ds": 47.04643260903809, "ci": [ 36.009090370403115, 56.80703000977964 ], "accuracy": 0.82, "ece_10_bin": 0.015477258953326474, "T": 1.0, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": 2.676392608952649, "ci": [ -4.610753531368588, 10.187235755087887 ], "accuracy": 0.4033333333333333, "ece_10_bin": 0.10693574766283558, "T": 1.0, "n": 300 }, "banking77": { "primitive": "choice", "ds": 53.88407802182469, "ci": [ 48.02925310834641, 60.06044789809394 ], "accuracy": 0.6793333333333333, "ece_10_bin": 0.026937718645406474, "T": 1.0, "n": 300 } }, "calibrated": { "pubmedqa": { "primitive": "noul", "ds": 44.575293822302086, "ci": [ 31.521025085187834, 56.53223179946575 ], "accuracy": 0.82, "ece_10_bin": 0.07335495778853046, "T": 0.6149937401183327, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": -0.3802936765010312, "ci": [ -8.192748138473686, 7.9471501430822675 ], "accuracy": 0.4033333333333333, "ece_10_bin": 0.13772908624584113, "T": 0.7973321069676444, "n": 300 }, "banking77": { "primitive": "choice", "ds": 53.90635910616057, "ci": [ 48.35518692964494, 59.81217350423332 ], "accuracy": 0.6793333333333333, "ece_10_bin": 0.045755905574880795, "T": 1.0772703967295125, "n": 300 } } }, "v2_joint": { "model": "runs/v2/checkpoints/v2_joint_merged", "temps": { "choice": 1.1592816013856364, "noul": 0.8031824332990728, "score": 0.716891860990727 }, "raw": { "pubmedqa": { "primitive": "noul", "ds": 57.9754923387608, "ci": [ 47.474841970587384, 67.33647236474492 ], "accuracy": 0.8566666666666667, "ece_10_bin": 0.024792356261037377, "T": 1.0, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": 6.072330512589563, "ci": [ -2.447039948281713, 14.421161550147108 ], "accuracy": 0.43, "ece_10_bin": 0.0758746614948931, "T": 1.0, "n": 300 }, "banking77": { "primitive": "choice", "ds": 60.90446514198684, "ci": [ 54.643973597052714, 67.26811195597078 ], "accuracy": 0.734, "ece_10_bin": 0.05840767711375198, "T": 1.0, "n": 300 } }, "calibrated": { "pubmedqa": { "primitive": "noul", "ds": 57.786338685636565, "ci": [ 46.39802375142668, 68.04438733775511 ], "accuracy": 0.8566666666666667, "ece_10_bin": 0.033081921946533914, "T": 0.8031824332990728, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": 2.314214972593698, "ci": [ -7.267135153938562, 11.84096549506603 ], "accuracy": 0.43, "ece_10_bin": 0.1364261756101774, "T": 0.716891860990727, "n": 300 }, "banking77": { "primitive": "choice", "ds": 61.31478198739936, "ci": [ 55.423784855833, 67.28099088659312 ], "accuracy": 0.734, "ece_10_bin": 0.02052732067327072, "T": 1.1592816013856364, "n": 300 } } }, "v2_joint_think": { "model": "runs/v2/checkpoints/v2_joint_merged", "temps": { "choice": 2.117644532766411, "noul": 1.8910262876636832, "score": 1.6148431811799775 }, "raw": { "pubmedqa": { "primitive": "noul", "ds": 53.8314296697503, "ci": [ 38.461630538462174, 67.87374086095278 ], "accuracy": 0.8833333333333333, "ece_10_bin": 0.1058673775031327, "T": 1.0, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": -26.198138966218607, "ci": [ -41.16874622377119, -10.352952534125588 ], "accuracy": 0.4066666666666667, "ece_10_bin": 0.43834127630132536, "T": 1.0, "n": 300 }, "banking77": { "primitive": "choice", "ds": 51.88605431545722, "ci": [ 43.24874709777629, 61.27065017189058 ], "accuracy": 0.7333333333333333, "ece_10_bin": 0.22046967225320246, "T": 1.0, "n": 300 } }, "calibrated": { "pubmedqa": { "primitive": "noul", "ds": 60.202782831455316, "ci": [ 47.716787071167964, 71.90589744680015 ], "accuracy": 0.8833333333333333, "ece_10_bin": 0.05377978866712399, "T": 1.8910262876636832, "n": 300 }, "helpsteer2": { "primitive": "score", "ds": -13.044833077924034, "ci": [ -26.61036521386943, 0.5771093535241466 ], "accuracy": 0.4066666666666667, "ece_10_bin": 0.344897159660971, "T": 1.6148431811799775, "n": 300 }, "banking77": { "primitive": "choice", "ds": 59.32358836134435, "ci": [ 53.92375680330441, 65.06160011264834 ], "accuracy": 0.7333333333333333, "ece_10_bin": 0.11909380664834994, "T": 2.117644532766411, "n": 300 } } } }, "board": [ { "board": "banking77", "name": "Gemini 3.8 Flash", "ds": 74.107, "ci": [ 68.4397, 79.7893 ], "accuracy": 0.846, "ece": 0.0319, "p95_ms": 3938, "cost_per_1k": 1.367931, "interface": "adapter" }, { "board": "banking77", "name": "Jev", "ds": 67.7806, "ci": [ 61.3393, 74.4909 ], "accuracy": 0.7967, "ece": 0.0981, "p95_ms": 693, "cost_per_1k": 0.043276, "interface": "native" }, { "board": "banking77", "name": "GLM-5.3", "ds": 66.8264, "ci": [ 61.9567, 71.7196 ], "accuracy": 0.7887, "ece": 0.0616, "p95_ms": 3186, "cost_per_1k": 0.718505, "interface": "adapter" }, { "board": "banking77", "name": "DeepSeek V4.1 Flash", "ds": 63.9292, "ci": [ 58.5065, 69.6119 ], "accuracy": 0.7587, "ece": 0.0276, "p95_ms": 1309, "cost_per_1k": 0.138384, "interface": "adapter" }, { "board": "banking77", "name": "Qwen3.8 Flash", "ds": 62.4389, "ci": [ 55.9178, 68.8961 ], "accuracy": 0.768, "ece": 0.0951, "p95_ms": 3105, "cost_per_1k": 0.122064, "interface": "adapter" }, { "board": "banking77", "name": "Mistral Medium 3.5", "ds": 59.4547, "ci": [ 54.2695, 64.8301 ], "accuracy": 0.7447, "ece": 0.0807, "p95_ms": 1444, "cost_per_1k": 1.515244, "interface": "adapter" }, { "board": "banking77", "name": "Mercury 2.5", "ds": 54.0395, "ci": [ 47.8178, 60.6156 ], "accuracy": 0.7033, "ece": 0.1044, "p95_ms": 1478, "cost_per_1k": 0.03606, "interface": "adapter" }, { "board": "banking77", "name": "Label prior", "ds": 0, "ci": [ 0, 0 ], "accuracy": 0.0133, "ece": 0, "p95_ms": null, "cost_per_1k": 0, "interface": "baseline" }, { "board": "helpsteer2", "name": "Jev", "ds": 9.2034, "ci": [ -4.365, 20.9655 ], "accuracy": 0.4127, "ece": 0.1966, "p95_ms": 670, "cost_per_1k": 0.036285, "interface": "native" }, { "board": "helpsteer2", "name": "GLM-5.3", "ds": 7.7828, "ci": [ -4.5884, 18.5647 ], "accuracy": 0.43, "ece": 0.1287, "p95_ms": 3631, "cost_per_1k": 0.784848, "interface": "adapter" }, { "board": "helpsteer2", "name": "Gemini 3.8 Flash", "ds": 4.5909, "ci": [ -12.2314, 18.9838 ], "accuracy": 0.424, "ece": 0.2259, "p95_ms": 3979, "cost_per_1k": 1.269273, "interface": "adapter" }, { "board": "helpsteer2", "name": "Label prior", "ds": 0, "ci": [ 0, 0 ], "accuracy": 0.4167, "ece": 0, "p95_ms": null, "cost_per_1k": 0, "interface": "baseline" }, { "board": "helpsteer2", "name": "Qwen3.8 Flash", "ds": -1.3701, "ci": [ -15.4058, 11.5108 ], "accuracy": 0.3607, "ece": 0.2577, "p95_ms": 2638, "cost_per_1k": 0.125698, "interface": "adapter" }, { "board": "helpsteer2", "name": "Mercury 2.5", "ds": -5.5457, "ci": [ -16.2223, 3.9934 ], "accuracy": 0.4173, "ece": 0.3148, "p95_ms": 1165, "cost_per_1k": 0.034331, "interface": "adapter" }, { "board": "helpsteer2", "name": "Mistral Medium 3.5", "ds": -13.7288, "ci": [ -28.2641, -0.1455 ], "accuracy": 0.4387, "ece": 0.3361, "p95_ms": 1178, "cost_per_1k": 1.325036, "interface": "adapter" }, { "board": "helpsteer2", "name": "DeepSeek V4.1 Flash", "ds": -18.9946, "ci": [ -36.0158, -3.3185 ], "accuracy": 0.3467, "ece": 0.3348, "p95_ms": 1203, "cost_per_1k": 0.125297, "interface": "adapter" }, { "board": "pubmedqa", "name": "Gemini 3.8 Flash", "ds": 72.983, "ci": [ 62.0659, 82.2398 ], "accuracy": 0.9253, "ece": 0.0198, "p95_ms": 5538, "cost_per_1k": 0.803439, "interface": "adapter" }, { "board": "pubmedqa", "name": "Jev", "ds": 69.0274, "ci": [ 59.7741, 76.5459 ], "accuracy": 0.9127, "ece": 0.0504, "p95_ms": 653, "cost_per_1k": 0.028535, "interface": "native" }, { "board": "pubmedqa", "name": "Qwen3.8 Flash", "ds": 62.4422, "ci": [ 50.905, 72.5479 ], "accuracy": 0.8967, "ece": 0.0255, "p95_ms": 3515, "cost_per_1k": 0.08613, "interface": "adapter" }, { "board": "pubmedqa", "name": "GLM-5.3", "ds": 60.6328, "ci": [ 49.9596, 69.8189 ], "accuracy": 0.8867, "ece": 0.0202, "p95_ms": 2879, "cost_per_1k": 0.512414, "interface": "adapter" }, { "board": "pubmedqa", "name": "Mistral Medium 3.5", "ds": 58.0327, "ci": [ 45.3414, 69.8002 ], "accuracy": 0.888, "ece": 0.0523, "p95_ms": 917, "cost_per_1k": 0.874716, "interface": "adapter" }, { "board": "pubmedqa", "name": "Mercury 2.5", "ds": 55.6583, "ci": [ 44.8114, 65.4167 ], "accuracy": 0.8713, "ece": 0.0468, "p95_ms": 1036, "cost_per_1k": 0.022595, "interface": "adapter" }, { "board": "pubmedqa", "name": "DeepSeek V4.1 Flash", "ds": 47.5001, "ci": [ 34.7497, 59.0847 ], "accuracy": 0.8367, "ece": 0.0616, "p95_ms": 1123, "cost_per_1k": 0.078411, "interface": "adapter" }, { "board": "pubmedqa", "name": "Label prior", "ds": 0, "ci": [ 0, 0 ], "accuracy": 0.62, "ece": 0, "p95_ms": null, "cost_per_1k": 0, "interface": "baseline" } ], "release": "2026-09-18", "suite": "0.1.0" }
{
"ours": {
"base": {
"model": "Qwen/Qwen3.5-4B",
"temps": {
"choice": 1.707683209388825,
"noul": 2.4105066990845945,
"score": 1.2847580560947693
},
"raw": {
"pubmedqa": {
"primitive": "noul",
"ds": 48.05492263372704,
"ci": [
37.76904082341696,
57.150183634721955
],
"accuracy": 0.8066666666666666,
"ece_10_bin": 0.06460862585758344,
"T": 1.0,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 2.049999831140492,
"ci": [
-9.799345465028715,
13.108881184556378
],
"accuracy": 0.4033333333333333,
"ece_10_bin": 0.15123322991920435,
"T": 1.0,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 56.173306098208656,
"ci": [
49.27693100059662,
63.141876285290685
],
"accuracy": 0.7186666666666667,
"ece_10_bin": 0.14011876588219976,
"T": 1.0,
"n": 300
}
},
"calibrated": {
"pubmedqa": {
"primitive": "noul",
"ds": 36.371473995104964,
"ci": [
29.35952403100504,
42.77706939107353
],
"accuracy": 0.8066666666666666,
"ece_10_bin": 0.15700328478531872,
"T": 2.4105066990845945,
"n": 300
},
"helpsteer2": {
"primitive": "score",
"ds": 4.103423043229137,
"ci": [
-6.389453722683416,
13.956330811232187
],
"accuracy": 0.4033333333333333,
"ece_10_bin": 0.08902712405177117,
"T": 1.2847580560947693,
"n": 300
},
"banking77": {
"primitive": "choice",
"ds": 59.3453919929658,
"ci": [
53.69828564607293,
64.93884389399817
],
"accuracy": 0.7186666666666667,
"ece_10_bin": 0.04315345334218913,
"T": 1.707683209388825,
"n": 300
}
}
},
"base_2b": {
"model": "openbmb/MiniCPM5-2B",
"temps": {
"choice": 4.78807442670338,
"noul": 19.999999999982688,
"score": 9.956643342471382
},
"raw": {
"pubmedqa": {
"primitive": "noul",
"ds": 0.06537669220909592,
"ci": [
-12.074739103400578,
12.86250789143286

Xet Storage Details

Size:
25.4 kB
·
Xet hash:
a96f519bf0a13d57db630d3aa29e8eb8fed6c84a1137c68ef25fffc26b0a2252

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.