| { |
| "model": "decider-2b-v11", |
| "temperature": 1.145, |
| "temperature_by_type": { |
| "choice": 1.164, |
| "noul": 1.624, |
| "score": 1.124 |
| }, |
| "temperature_fit": { |
| "global": "one scalar, pooled row NLL on the in-task half of the public regression set without banking77, clinc_oos, mmlu, arc, winogrande and hellaswag (61 tasks, 102,804 rows)", |
| "by_type": "decider.calibrate.fit_by_type (decider-ai 1.4.0), min_rows 50, on the same regression rows plus our own validation rows (val_jb, val_teacher, val_teacher2, cal_human without mmlu, arc and banking77, val_replay without six excluded families)", |
| "rows": { |
| "choice": 108910, |
| "noul": 1677, |
| "score": 535 |
| }, |
| "note": "no JevBench or Decision Index item was used for training, selection or temperature" |
| }, |
| "protocol": "Regression set: eval half of the mixture, 95 tasks (67 in-task / 28 held-out), plain state-first layout, large label sets sub-sampled to 10 options, ECE 15 bins, per-task mean. Own sets (heldout_jb, test_teacher2, cal_human, guard): stored T=1 logits read at the served temperatures, ECE 10 bins top label. Card lanes (fixtures, games, browser, probes, Bespoke, issue #9) through decider-ai 1.4.0 with the map; parents through 1.3.0 at their released temperatures, same rows and seeds; intervals are 95% paired bootstrap. JevBench public items read in process through 1.4.0 with the map; parents' files read earlier through 1.2.1.", |
| "regression": { |
| "map": { |
| "reg_in": 0.8016, |
| "reg_out": 0.7518, |
| "reg_in_nll": 0.4809, |
| "reg_out_nll": 0.6262, |
| "reg_in_ece": 0.0385, |
| "reg_out_ece": 0.0834 |
| }, |
| "global_T": { |
| "reg_in": 0.8016, |
| "reg_out": 0.7518, |
| "reg_in_nll": 0.4808, |
| "reg_out_nll": 0.628, |
| "reg_in_ece": 0.0383, |
| "reg_out_ece": 0.0847 |
| } |
| }, |
| "own_sets": { |
| "map": { |
| "heldout_jb": { |
| "all": { |
| "n": 5000, |
| "accuracy": 0.4288, |
| "nll": 1.4669, |
| "ece": 0.1562, |
| "mean_conf": 0.5832 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 3293, |
| "accuracy": 0.3702, |
| "nll": 1.743, |
| "ece": 0.172, |
| "mean_conf": 0.5395 |
| }, |
| "noul": { |
| "n": 1289, |
| "accuracy": 0.6005, |
| "nll": 0.7022, |
| "ece": 0.121, |
| "mean_conf": 0.7215 |
| }, |
| "score": { |
| "n": 418, |
| "accuracy": 0.3612, |
| "nll": 1.6498, |
| "ece": 0.1405, |
| "mean_conf": 0.5018 |
| } |
| } |
| }, |
| "test_teacher2": { |
| "all": { |
| "n": 449, |
| "accuracy": 0.7528, |
| "nll": 0.5668, |
| "ece": 0.0754, |
| "mean_conf": 0.8155 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 257, |
| "accuracy": 0.751, |
| "nll": 0.6501, |
| "ece": 0.0736, |
| "mean_conf": 0.8108 |
| }, |
| "noul": { |
| "n": 130, |
| "accuracy": 0.8077, |
| "nll": 0.325, |
| "ece": 0.0926, |
| "mean_conf": 0.8774 |
| }, |
| "score": { |
| "n": 62, |
| "accuracy": 0.6452, |
| "nll": 0.7286, |
| "ece": 0.1307, |
| "mean_conf": 0.7054 |
| } |
| } |
| }, |
| "guard": { |
| "all": { |
| "n": 2994, |
| "accuracy": 0.7151, |
| "nll": 0.7138, |
| "ece": 0.0463, |
| "mean_conf": 0.7614 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 2994, |
| "accuracy": 0.7151, |
| "nll": 0.7138, |
| "ece": 0.0463, |
| "mean_conf": 0.7614 |
| }, |
| "noul": null, |
| "score": null |
| } |
| }, |
| "val_jb": { |
| "all": { |
| "n": 5000, |
| "accuracy": 0.694, |
| "nll": 0.7801, |
| "ece": 0.0278, |
| "mean_conf": 0.7218 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 3374, |
| "accuracy": 0.6461, |
| "nll": 0.9402, |
| "ece": 0.0451, |
| "mean_conf": 0.6912 |
| }, |
| "noul": { |
| "n": 1273, |
| "accuracy": 0.8437, |
| "nll": 0.3275, |
| "ece": 0.0235, |
| "mean_conf": 0.8278 |
| }, |
| "score": { |
| "n": 353, |
| "accuracy": 0.6119, |
| "nll": 0.8828, |
| "ece": 0.0475, |
| "mean_conf": 0.6323 |
| } |
| } |
| }, |
| "val_teacher": { |
| "all": { |
| "n": 944, |
| "accuracy": 0.7744, |
| "nll": 0.5627, |
| "ece": 0.0586, |
| "mean_conf": 0.8252 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 540, |
| "accuracy": 0.7574, |
| "nll": 0.6695, |
| "ece": 0.057, |
| "mean_conf": 0.8081 |
| }, |
| "noul": { |
| "n": 284, |
| "accuracy": 0.8204, |
| "nll": 0.3607, |
| "ece": 0.0671, |
| "mean_conf": 0.8875 |
| }, |
| "score": { |
| "n": 120, |
| "accuracy": 0.7417, |
| "nll": 0.5603, |
| "ece": 0.0699, |
| "mean_conf": 0.7551 |
| } |
| } |
| }, |
| "val_teacher2": { |
| "all": { |
| "n": 447, |
| "accuracy": 0.7785, |
| "nll": 0.5757, |
| "ece": 0.0625, |
| "mean_conf": 0.8271 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 265, |
| "accuracy": 0.7736, |
| "nll": 0.6303, |
| "ece": 0.0694, |
| "mean_conf": 0.838 |
| }, |
| "noul": { |
| "n": 120, |
| "accuracy": 0.7833, |
| "nll": 0.4665, |
| "ece": 0.0835, |
| "mean_conf": 0.8657 |
| }, |
| "score": { |
| "n": 62, |
| "accuracy": 0.7903, |
| "nll": 0.5532, |
| "ece": 0.1172, |
| "mean_conf": 0.7053 |
| } |
| } |
| }, |
| "cal_human": { |
| "all": { |
| "n": 1595, |
| "accuracy": 0.7812, |
| "nll": 0.565, |
| "ece": 0.0298, |
| "mean_conf": 0.8051 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 1595, |
| "accuracy": 0.7812, |
| "nll": 0.565, |
| "ece": 0.0298, |
| "mean_conf": 0.8051 |
| }, |
| "noul": null, |
| "score": null |
| } |
| }, |
| "val_replay2": { |
| "all": { |
| "n": 1000, |
| "accuracy": 0.82, |
| "nll": 0.4696, |
| "ece": 0.0273, |
| "mean_conf": 0.8257 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 1000, |
| "accuracy": 0.82, |
| "nll": 0.4696, |
| "ece": 0.0273, |
| "mean_conf": 0.8257 |
| }, |
| "noul": null, |
| "score": null |
| } |
| } |
| }, |
| "global_T": { |
| "heldout_jb": { |
| "all": { |
| "n": 5000, |
| "accuracy": 0.4286, |
| "nll": 1.4935, |
| "ece": 0.1711, |
| "mean_conf": 0.5986 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 3293, |
| "accuracy": 0.3702, |
| "nll": 1.7521, |
| "ece": 0.1752, |
| "mean_conf": 0.5437 |
| }, |
| "noul": { |
| "n": 1289, |
| "accuracy": 0.6005, |
| "nll": 0.7852, |
| "ece": 0.1714, |
| "mean_conf": 0.7719 |
| }, |
| "score": { |
| "n": 418, |
| "accuracy": 0.3589, |
| "nll": 1.6405, |
| "ece": 0.1379, |
| "mean_conf": 0.4968 |
| } |
| } |
| }, |
| "test_teacher2": { |
| "all": { |
| "n": 449, |
| "accuracy": 0.7528, |
| "nll": 0.5784, |
| "ece": 0.0831, |
| "mean_conf": 0.826 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 257, |
| "accuracy": 0.751, |
| "nll": 0.6532, |
| "ece": 0.0795, |
| "mean_conf": 0.8131 |
| }, |
| "noul": { |
| "n": 130, |
| "accuracy": 0.8077, |
| "nll": 0.3594, |
| "ece": 0.1032, |
| "mean_conf": 0.9107 |
| }, |
| "score": { |
| "n": 62, |
| "accuracy": 0.6452, |
| "nll": 0.7276, |
| "ece": 0.1266, |
| "mean_conf": 0.7017 |
| } |
| } |
| }, |
| "guard": { |
| "all": { |
| "n": 2994, |
| "accuracy": 0.7151, |
| "nll": 0.7154, |
| "ece": 0.0497, |
| "mean_conf": 0.7648 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 2994, |
| "accuracy": 0.7151, |
| "nll": 0.7154, |
| "ece": 0.0497, |
| "mean_conf": 0.7648 |
| }, |
| "noul": null, |
| "score": null |
| } |
| }, |
| "val_jb": { |
| "all": { |
| "n": 5000, |
| "accuracy": 0.694, |
| "nll": 0.7815, |
| "ece": 0.0401, |
| "mean_conf": 0.7341 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 3374, |
| "accuracy": 0.6461, |
| "nll": 0.9423, |
| "ece": 0.0484, |
| "mean_conf": 0.6945 |
| }, |
| "noul": { |
| "n": 1273, |
| "accuracy": 0.8437, |
| "nll": 0.327, |
| "ece": 0.0273, |
| "mean_conf": 0.8683 |
| }, |
| "score": { |
| "n": 353, |
| "accuracy": 0.6119, |
| "nll": 0.883, |
| "ece": 0.0401, |
| "mean_conf": 0.6288 |
| } |
| } |
| }, |
| "val_teacher": { |
| "all": { |
| "n": 944, |
| "accuracy": 0.7754, |
| "nll": 0.5823, |
| "ece": 0.068, |
| "mean_conf": 0.8349 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 540, |
| "accuracy": 0.7574, |
| "nll": 0.6728, |
| "ece": 0.0606, |
| "mean_conf": 0.8104 |
| }, |
| "noul": { |
| "n": 284, |
| "accuracy": 0.8204, |
| "nll": 0.42, |
| "ece": 0.0958, |
| "mean_conf": 0.9162 |
| }, |
| "score": { |
| "n": 120, |
| "accuracy": 0.75, |
| "nll": 0.5594, |
| "ece": 0.0437, |
| "mean_conf": 0.7528 |
| } |
| } |
| }, |
| "val_teacher2": { |
| "all": { |
| "n": 447, |
| "accuracy": 0.7785, |
| "nll": 0.6006, |
| "ece": 0.0778, |
| "mean_conf": 0.8378 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 265, |
| "accuracy": 0.7736, |
| "nll": 0.6341, |
| "ece": 0.0744, |
| "mean_conf": 0.8402 |
| }, |
| "noul": { |
| "n": 120, |
| "accuracy": 0.7833, |
| "nll": 0.5502, |
| "ece": 0.1217, |
| "mean_conf": 0.9024 |
| }, |
| "score": { |
| "n": 62, |
| "accuracy": 0.7903, |
| "nll": 0.5546, |
| "ece": 0.129, |
| "mean_conf": 0.7028 |
| } |
| } |
| }, |
| "cal_human": { |
| "all": { |
| "n": 1595, |
| "accuracy": 0.7812, |
| "nll": 0.5651, |
| "ece": 0.0302, |
| "mean_conf": 0.8086 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 1595, |
| "accuracy": 0.7812, |
| "nll": 0.5651, |
| "ece": 0.0302, |
| "mean_conf": 0.8086 |
| }, |
| "noul": null, |
| "score": null |
| } |
| }, |
| "val_replay2": { |
| "all": { |
| "n": 1000, |
| "accuracy": 0.82, |
| "nll": 0.47, |
| "ece": 0.0307, |
| "mean_conf": 0.8284 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 1000, |
| "accuracy": 0.82, |
| "nll": 0.47, |
| "ece": 0.0307, |
| "mean_conf": 0.8284 |
| }, |
| "noul": null, |
| "score": null |
| } |
| } |
| } |
| }, |
| "fixtures": { |
| "general_validation": { |
| "this": { |
| "rows": 847, |
| "accuracy": 0.8193624557260921, |
| "macro_accuracy": 0.8264818843272046, |
| "groups": 65, |
| "nll": 0.4679994635746329, |
| "ece": 0.025221459893014928, |
| "brier": 0.24885000616245756, |
| "mean_confidence": 0.7965057105388945 |
| }, |
| "paired": { |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2": { |
| "accuracy": { |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "nll": { |
| "delta": 0.0008567327869000381, |
| "ci95": [ |
| 0.00012586933848329147, |
| 0.0014936147428319582 |
| ] |
| } |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130": { |
| "accuracy": { |
| "delta": -0.014167650531286895, |
| "ci95": [ |
| -0.0295159386068477, |
| 0.0011806375442739079 |
| ] |
| }, |
| "nll": { |
| "delta": 0.02396263800434385, |
| "ci95": [ |
| 0.007682289841923634, |
| 0.03881930597006884 |
| ] |
| } |
| } |
| } |
| }, |
| "typesafe": { |
| "this": { |
| "rows": 102, |
| "accuracy": 0.7549019607843137, |
| "macro_accuracy": 0.7613916256157636, |
| "groups": 4, |
| "nll": 0.6036617290590378, |
| "ece": 0.08752135903227565, |
| "brier": 0.35248420626905574, |
| "mean_confidence": 0.8094115233888813, |
| "mean_total_variation_to_reference": 0.2724384515080273 |
| }, |
| "paired": { |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2": { |
| "accuracy": { |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "nll": { |
| "delta": -0.0026735984139883862, |
| "ci95": [ |
| -0.005663155113504104, |
| -0.00015286414032123958 |
| ] |
| } |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130": { |
| "accuracy": { |
| "delta": -0.049019607843137254, |
| "ci95": [ |
| -0.10784313725490197, |
| 0.00980392156862745 |
| ] |
| }, |
| "nll": { |
| "delta": 0.01849389075312281, |
| "ci95": [ |
| -0.08772658186762586, |
| 0.13502854066724154 |
| ] |
| } |
| } |
| } |
| }, |
| "openjev_external": { |
| "this": { |
| "rows": 5252, |
| "accuracy": 0.6460396039603961, |
| "macro_accuracy": 0.8054024691358025, |
| "groups": 3, |
| "nll": 0.8595481612843723, |
| "ece": 0.124815208362153, |
| "brier": 0.5100238458753833, |
| "mean_confidence": 0.7668148385756631 |
| }, |
| "paired": { |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2": { |
| "accuracy": { |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "nll": { |
| "delta": -0.004652630797611012, |
| "ci95": [ |
| -0.005088558046264057, |
| -0.004202312508842571 |
| ] |
| } |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130": { |
| "accuracy": { |
| "delta": 0.013899466869763899, |
| "ci95": [ |
| 0.005902513328255902, |
| 0.022086824067022087 |
| ] |
| }, |
| "nll": { |
| "delta": -0.05702255018553582, |
| "ci95": [ |
| -0.0676423182247241, |
| -0.04578181130197777 |
| ] |
| } |
| } |
| } |
| }, |
| "mind2web": { |
| "this": { |
| "rows": 1770, |
| "accuracy": 0.8389830508474576, |
| "macro_accuracy": 0.8411027477788687, |
| "groups": 3, |
| "nll": 0.49544234456641184, |
| "ece": 0.039275789345051576, |
| "brier": 0.24172292854121424, |
| "mean_confidence": 0.8067077137151006 |
| }, |
| "paired": { |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2": { |
| "accuracy": { |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "nll": { |
| "delta": 0.0005301123790506686, |
| "ci95": [ |
| -0.0001095135512494298, |
| 0.001158587013081229 |
| ] |
| } |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130": { |
| "accuracy": { |
| "delta": 0.012994350282485875, |
| "ci95": [ |
| 0.0011299435028248588, |
| 0.025423728813559324 |
| ] |
| }, |
| "nll": { |
| "delta": -0.04784351744979385, |
| "ci95": [ |
| -0.07354020281814458, |
| -0.022660045517854176 |
| ] |
| } |
| } |
| } |
| } |
| }, |
| "games": { |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2|sampled|all": { |
| "n": 234, |
| "a": 0.2264957264957265, |
| "b": 0.2297008547008547, |
| "delta": -0.003205128205128205, |
| "ci95": [ |
| -0.008547008547008548, |
| 0.002136752136752137 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2|sampled|bag_game": { |
| "n": 64, |
| "a": 0.41796875, |
| "b": 0.4296875, |
| "delta": -0.01171875, |
| "ci95": [ |
| -0.03125, |
| 0.0078125 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2|sampled|stochastic_grid": { |
| "n": 64, |
| "a": 0.1484375, |
| "b": 0.1484375, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2|sampled|tic_tac_toe": { |
| "n": 74, |
| "a": 0.22635135135135134, |
| "b": 0.22635135135135134, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2|sampled|exact_minesweeper": { |
| "n": 32, |
| "a": 0.0, |
| "b": 0.0, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2|greedy|all": { |
| "n": 234, |
| "a": 0.24786324786324787, |
| "b": 0.24786324786324787, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2|greedy|bag_game": { |
| "n": 64, |
| "a": 0.46875, |
| "b": 0.46875, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2|greedy|stochastic_grid": { |
| "n": 64, |
| "a": 0.15625, |
| "b": 0.15625, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2|greedy|tic_tac_toe": { |
| "n": 74, |
| "a": 0.22972972972972974, |
| "b": 0.22972972972972974, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2|greedy|exact_minesweeper": { |
| "n": 32, |
| "a": 0.03125, |
| "b": 0.03125, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130|sampled|all": { |
| "n": 234, |
| "a": 0.2264957264957265, |
| "b": 0.23931623931623933, |
| "delta": -0.01282051282051282, |
| "ci95": [ |
| -0.03418803418803419, |
| 0.008547008547008548 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130|sampled|bag_game": { |
| "n": 64, |
| "a": 0.41796875, |
| "b": 0.41796875, |
| "delta": 0.0, |
| "ci95": [ |
| -0.0546875, |
| 0.0546875 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130|sampled|stochastic_grid": { |
| "n": 64, |
| "a": 0.1484375, |
| "b": 0.19140625, |
| "delta": -0.04296875, |
| "ci95": [ |
| -0.078125, |
| -0.01171875 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130|sampled|tic_tac_toe": { |
| "n": 74, |
| "a": 0.22635135135135134, |
| "b": 0.22972972972972974, |
| "delta": -0.0033783783783783786, |
| "ci95": [ |
| -0.037162162162162164, |
| 0.033783783783783786 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130|sampled|exact_minesweeper": { |
| "n": 32, |
| "a": 0.0, |
| "b": 0.0, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130|greedy|all": { |
| "n": 234, |
| "a": 0.24786324786324787, |
| "b": 0.2692307692307692, |
| "delta": -0.021367521367521368, |
| "ci95": [ |
| -0.05555555555555555, |
| 0.01282051282051282 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130|greedy|bag_game": { |
| "n": 64, |
| "a": 0.46875, |
| "b": 0.578125, |
| "delta": -0.109375, |
| "ci95": [ |
| -0.203125, |
| -0.03125 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130|greedy|stochastic_grid": { |
| "n": 64, |
| "a": 0.15625, |
| "b": 0.15625, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130|greedy|tic_tac_toe": { |
| "n": 74, |
| "a": 0.22972972972972974, |
| "b": 0.21621621621621623, |
| "delta": 0.013513513513513514, |
| "ci95": [ |
| -0.06756756756756757, |
| 0.0945945945945946 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130|greedy|exact_minesweeper": { |
| "n": 32, |
| "a": 0.03125, |
| "b": 0.0, |
| "delta": 0.03125, |
| "ci95": [ |
| 0.0, |
| 0.09375 |
| ] |
| } |
| }, |
| "browser": { |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2|sampled|all": { |
| "n": 176, |
| "a": 0.9034090909090909, |
| "b": 0.9034090909090909, |
| "delta": 0.0, |
| "ci95": [ |
| -0.017045454545454544, |
| 0.017045454545454544 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2|sampled|rewarded": { |
| "n": 128, |
| "a": 0.8984375, |
| "b": 0.90625, |
| "delta": -0.0078125, |
| "ci95": [ |
| -0.0234375, |
| 0.0 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2|sampled|held_out": { |
| "n": 48, |
| "a": 0.9166666666666666, |
| "b": 0.8958333333333334, |
| "delta": 0.020833333333333332, |
| "ci95": [ |
| 0.0, |
| 0.0625 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2|greedy|all": { |
| "n": 176, |
| "a": 0.9204545454545454, |
| "b": 0.9204545454545454, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2|greedy|rewarded": { |
| "n": 128, |
| "a": 0.921875, |
| "b": 0.921875, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-cand2b_C2B_e2|greedy|held_out": { |
| "n": 48, |
| "a": 0.9166666666666666, |
| "b": 0.9166666666666666, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130|sampled|all": { |
| "n": 176, |
| "a": 0.9034090909090909, |
| "b": 0.9318181818181818, |
| "delta": -0.028409090909090908, |
| "ci95": [ |
| -0.07386363636363637, |
| 0.011363636363636364 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130|sampled|rewarded": { |
| "n": 128, |
| "a": 0.8984375, |
| "b": 0.9375, |
| "delta": -0.0390625, |
| "ci95": [ |
| -0.09375, |
| 0.0078125 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130|sampled|held_out": { |
| "n": 48, |
| "a": 0.9166666666666666, |
| "b": 0.9166666666666666, |
| "delta": 0.0, |
| "ci95": [ |
| -0.08333333333333333, |
| 0.08333333333333333 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130|greedy|all": { |
| "n": 176, |
| "a": 0.9204545454545454, |
| "b": 0.9147727272727273, |
| "delta": 0.005681818181818182, |
| "ci95": [ |
| -0.0342329545454545, |
| 0.045454545454545456 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130|greedy|rewarded": { |
| "n": 128, |
| "a": 0.921875, |
| "b": 0.9140625, |
| "delta": 0.0078125, |
| "ci95": [ |
| -0.046875, |
| 0.0625 |
| ] |
| }, |
| "cand2b_C2B_e2_tt-v10_2b_p130|greedy|held_out": { |
| "n": 48, |
| "a": 0.9166666666666666, |
| "b": 0.9166666666666666, |
| "delta": 0.0, |
| "ci95": [ |
| -0.0625, |
| 0.0625 |
| ] |
| } |
| }, |
| "text_games_greedy": { |
| "pong": 8.0, |
| "freeway": 1.0, |
| "breakout": 22.0, |
| "frozenlake": 0.0, |
| "cliffwalking": -13.0, |
| "blackjack": -1.0, |
| "minigrid_empty": 0.0, |
| "minigrid_lavagap": 0.0, |
| "minigrid_doorkey": 0.0, |
| "babyai_goto": 0.1915625 |
| }, |
| "probes": { |
| "router_n": 31, |
| "router_tier": 0.935, |
| "needs_live_data": 0.774, |
| "commands_n": 45, |
| "command_risk": 0.822, |
| "destructive_called_safe": 0, |
| "outside_project": 0.556, |
| "browser_element": 0.938, |
| "browser_action": 0.875, |
| "batteries": { |
| "generic": 0.85, |
| "specific": 0.95, |
| "catchall": 0.95, |
| "noul_acc": 0.939, |
| "noul_brier": 0.048, |
| "n_choice": 60, |
| "n_noul": 49, |
| "abstain_battery": "8/8" |
| } |
| }, |
| "issue9": { |
| "c_1": { |
| "pred": 1, |
| "gold": 6, |
| "ok": false, |
| "p_gold": 0.371, |
| "p_pred": 0.436, |
| "T": 1.145 |
| }, |
| "c_2": { |
| "pred": 19, |
| "gold": 20, |
| "ok": false, |
| "p_gold": 0.385, |
| "p_pred": 0.477, |
| "T": 1.145 |
| }, |
| "b_10": { |
| "pred": 5, |
| "gold": 0, |
| "ok": false, |
| "p_gold": 0.35, |
| "p_pred": 0.411, |
| "T": 1.145 |
| }, |
| "a2_4": { |
| "pred": 3, |
| "gold": 4, |
| "ok": false, |
| "p_gold": 0.2, |
| "p_pred": 0.619, |
| "T": 1.145 |
| } |
| }, |
| "bespoke": { |
| "vitaminc-dev": { |
| "n": 599, |
| "type": "choice", |
| "acc": 0.6394, |
| "ece": 0.1965, |
| "brier": 0.5142, |
| "nll": 0.9165, |
| "trained": false, |
| "sec": 10.3 |
| }, |
| "massive-en-US": { |
| "n": 350, |
| "type": "choice", |
| "acc": 0.8143, |
| "ece": 0.0283, |
| "brier": 0.271, |
| "nll": 0.6137, |
| "trained": true, |
| "sec": 3.9 |
| }, |
| "massive-de-DE": { |
| "n": 350, |
| "type": "choice", |
| "acc": 0.7829, |
| "ece": 0.0479, |
| "brier": 0.31, |
| "nll": 0.7328, |
| "trained": false, |
| "sec": 2.9 |
| }, |
| "boolq": { |
| "n": 300, |
| "type": "noul", |
| "acc": 0.85, |
| "ece": 0.0213, |
| "brier": 0.2386, |
| "nll": 0.3897, |
| "trained": true, |
| "sec": 5.5 |
| }, |
| "squad2": { |
| "n": 299, |
| "type": "noul", |
| "acc": 0.7425, |
| "ece": 0.038, |
| "brier": 0.342, |
| "nll": 0.5098, |
| "trained": false, |
| "sec": 5.2 |
| }, |
| "paws": { |
| "n": 250, |
| "type": "noul", |
| "acc": 0.76, |
| "ece": 0.0636, |
| "brier": 0.3157, |
| "nll": 0.4813, |
| "trained": true, |
| "sec": 1.9 |
| }, |
| "multinli": { |
| "n": 299, |
| "type": "choice", |
| "acc": 0.8528, |
| "ece": 0.0371, |
| "brier": 0.2064, |
| "nll": 0.3756, |
| "trained": true, |
| "sec": 2.3 |
| }, |
| "civil_comments": { |
| "n": 300, |
| "type": "noul", |
| "acc": 0.87, |
| "ece": 0.0508, |
| "brier": 0.198, |
| "nll": 0.3286, |
| "trained": true, |
| "sec": 2.3 |
| }, |
| "aegis2": { |
| "n": 250, |
| "type": "noul", |
| "acc": 0.716, |
| "ece": 0.0566, |
| "brier": 0.3606, |
| "nll": 0.5352, |
| "trained": false, |
| "sec": 4.9 |
| }, |
| "helpsteer2": { |
| "n": 249, |
| "type": "score", |
| "acc": 0.4458, |
| "ece": 0.1666, |
| "brier": 0.6987, |
| "nll": 1.3297, |
| "trained": true, |
| "sec": 17.8, |
| "score_mae": 0.8578 |
| }, |
| "summeval-relevance": { |
| "n": 240, |
| "type": "score", |
| "acc": 0.2292, |
| "ece": 0.1768, |
| "brier": 0.7849, |
| "nll": 1.4652, |
| "trained": false, |
| "sec": 13.4, |
| "score_mae": 0.633 |
| }, |
| "summeval-consistency": { |
| "n": 144, |
| "type": "score", |
| "acc": 0.7569, |
| "ece": 0.3155, |
| "brier": 0.4993, |
| "nll": 0.9841, |
| "trained": false, |
| "sec": 8.2, |
| "score_mae": 1.0267 |
| }, |
| "pubmedqa": { |
| "n": 250, |
| "type": "choice", |
| "acc": 0.712, |
| "ece": 0.0543, |
| "brier": 0.4106, |
| "nll": 0.74, |
| "trained": true, |
| "sec": 2.1 |
| }, |
| "_macro": 0.7055, |
| "_micro": 0.7106, |
| "_macro_untrained": 0.6445 |
| }, |
| "jevbench_public": { |
| "easy": { |
| "n": 48, |
| "accuracy": 1.0, |
| "invalid": 0, |
| "ece": 0.015729166666666905, |
| "by_family": { |
| "extraction": { |
| "n": 12, |
| "accuracy": 1.0 |
| }, |
| "fact": { |
| "n": 12, |
| "accuracy": 1.0 |
| }, |
| "intent": { |
| "n": 12, |
| "accuracy": 1.0 |
| }, |
| "tool_selection": { |
| "n": 12, |
| "accuracy": 1.0 |
| } |
| } |
| }, |
| "standard": { |
| "n": 72, |
| "accuracy": 0.8888888888888888, |
| "invalid": 0, |
| "ece": 0.06715416666666665, |
| "by_family": { |
| "adequacy": { |
| "n": 12, |
| "accuracy": 0.6666666666666666 |
| }, |
| "extraction": { |
| "n": 12, |
| "accuracy": 1.0 |
| }, |
| "intent": { |
| "n": 12, |
| "accuracy": 1.0 |
| }, |
| "ordinal": { |
| "n": 12, |
| "accuracy": 1.0 |
| }, |
| "policy": { |
| "n": 12, |
| "accuracy": 0.8333333333333334 |
| }, |
| "routing": { |
| "n": 12, |
| "accuracy": 0.8333333333333334 |
| } |
| } |
| }, |
| "hard": { |
| "n": 111, |
| "accuracy": 0.5765765765765766, |
| "invalid": 0, |
| "ece": 0.17517657657657654, |
| "by_family": { |
| "adversarial": { |
| "n": 6, |
| "accuracy": 1.0 |
| }, |
| "ambiguous": { |
| "n": 7, |
| "accuracy": 0.5714285714285714 |
| }, |
| "judge_hard": { |
| "n": 17, |
| "accuracy": 0.5882352941176471 |
| }, |
| "long_policy": { |
| "n": 19, |
| "accuracy": 0.42105263157894735 |
| }, |
| "multi_hop": { |
| "n": 18, |
| "accuracy": 0.5555555555555556 |
| }, |
| "probability": { |
| "n": 10, |
| "accuracy": 0.7 |
| }, |
| "routing_hard": { |
| "n": 5, |
| "accuracy": 1.0 |
| }, |
| "temporal_numeric": { |
| "n": 15, |
| "accuracy": 0.2 |
| }, |
| "tradeoff": { |
| "n": 6, |
| "accuracy": 0.5 |
| }, |
| "trap": { |
| "n": 8, |
| "accuracy": 1.0 |
| } |
| } |
| } |
| }, |
| "references": { |
| "v10_2b": { |
| "reg_in": 0.8058, |
| "reg_out": 0.7549, |
| "reg_in_nll": 0.4736, |
| "reg_out_nll": 0.6224, |
| "reg_in_ece": 0.0378, |
| "reg_out_ece": 0.0837 |
| } |
| }, |
| "speed": "not measured again: the architecture and size equal the parent's, and the temperature map does not change the computation" |
| } |
