| { |
| "model": "decider-4b-v2.1", |
| "temperature": 1.099, |
| "temperature_by_type": { |
| "choice": 1.11, |
| "noul": 1.56, |
| "score": 1.287 |
| }, |
| "temperature_fit": { |
| "global": "one scalar, pooled row NLL on the in-task half of the public regression set without banking77, clinc_oos, mmlu, arc, winogrande and hellaswag (61 tasks, 102,804 rows)", |
| "by_type": "decider.calibrate.fit_by_type (decider-ai 1.4.0), min_rows 50, on the same regression rows plus our own validation rows (val_jb, val_teacher, val_teacher2, cal_human without mmlu, arc and banking77, val_replay without six excluded families)", |
| "rows": { |
| "choice": 108927, |
| "noul": 1677, |
| "score": 535 |
| }, |
| "note": "no JevBench or Decision Index item was used for training, selection or temperature" |
| }, |
| "protocol": "Regression set: eval half of the mixture, 95 tasks (67 in-task / 28 held-out), plain state-first layout, large label sets sub-sampled to 10 options, ECE 15 bins, per-task mean. Own sets (heldout_jb, test_teacher2, cal_human, guard): stored T=1 logits read at the served temperatures, ECE 10 bins top label. Card lanes (fixtures, games, browser, probes, Bespoke, issue #9) through decider-ai 1.4.0 with the map; parents through 1.3.0 at their released temperatures, same rows and seeds; intervals are 95% paired bootstrap. JevBench public items read in process through 1.4.0 with the map; parents' files read earlier through 1.2.1.", |
| "regression": { |
| "map": { |
| "reg_in": 0.8308, |
| "reg_out": 0.7838, |
| "reg_in_nll": 0.4145, |
| "reg_out_nll": 0.5689, |
| "reg_in_ece": 0.0309, |
| "reg_out_ece": 0.0775 |
| }, |
| "global_T": { |
| "reg_in": 0.8308, |
| "reg_out": 0.7838, |
| "reg_in_nll": 0.4145, |
| "reg_out_nll": 0.5703, |
| "reg_in_ece": 0.0308, |
| "reg_out_ece": 0.0781 |
| } |
| }, |
| "own_sets": { |
| "map": { |
| "heldout_jb": { |
| "all": { |
| "n": 5000, |
| "accuracy": 0.556, |
| "nll": 1.2571, |
| "ece": 0.1466, |
| "mean_conf": 0.7026 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 3293, |
| "accuracy": 0.5056, |
| "nll": 1.4995, |
| "ece": 0.1701, |
| "mean_conf": 0.6757 |
| }, |
| "noul": { |
| "n": 1289, |
| "accuracy": 0.7114, |
| "nll": 0.6074, |
| "ece": 0.1013, |
| "mean_conf": 0.8127 |
| }, |
| "score": { |
| "n": 418, |
| "accuracy": 0.4737, |
| "nll": 1.3504, |
| "ece": 0.1081, |
| "mean_conf": 0.5747 |
| } |
| } |
| }, |
| "test_teacher2": { |
| "all": { |
| "n": 449, |
| "accuracy": 0.8196, |
| "nll": 0.4304, |
| "ece": 0.043, |
| "mean_conf": 0.8559 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 257, |
| "accuracy": 0.8016, |
| "nll": 0.5041, |
| "ece": 0.0569, |
| "mean_conf": 0.8585 |
| }, |
| "noul": { |
| "n": 130, |
| "accuracy": 0.8692, |
| "nll": 0.2489, |
| "ece": 0.0479, |
| "mean_conf": 0.905 |
| }, |
| "score": { |
| "n": 62, |
| "accuracy": 0.7903, |
| "nll": 0.5051, |
| "ece": 0.0863, |
| "mean_conf": 0.7423 |
| } |
| } |
| }, |
| "guard": { |
| "all": { |
| "n": 2994, |
| "accuracy": 0.8186, |
| "nll": 0.4866, |
| "ece": 0.0168, |
| "mean_conf": 0.8344 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 2994, |
| "accuracy": 0.8186, |
| "nll": 0.4866, |
| "ece": 0.0168, |
| "mean_conf": 0.8344 |
| }, |
| "noul": null, |
| "score": null |
| } |
| }, |
| "val_jb": { |
| "all": { |
| "n": 5000, |
| "accuracy": 0.796, |
| "nll": 0.5264, |
| "ece": 0.0157, |
| "mean_conf": 0.8117 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 3374, |
| "accuracy": 0.7647, |
| "nll": 0.6258, |
| "ece": 0.0289, |
| "mean_conf": 0.7935 |
| }, |
| "noul": { |
| "n": 1273, |
| "accuracy": 0.901, |
| "nll": 0.2233, |
| "ece": 0.0161, |
| "mean_conf": 0.8849 |
| }, |
| "score": { |
| "n": 353, |
| "accuracy": 0.7167, |
| "nll": 0.6681, |
| "ece": 0.0317, |
| "mean_conf": 0.722 |
| } |
| } |
| }, |
| "val_teacher": { |
| "all": { |
| "n": 944, |
| "accuracy": 0.8379, |
| "nll": 0.4169, |
| "ece": 0.0369, |
| "mean_conf": 0.8619 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 540, |
| "accuracy": 0.8315, |
| "nll": 0.4643, |
| "ece": 0.0464, |
| "mean_conf": 0.8547 |
| }, |
| "noul": { |
| "n": 284, |
| "accuracy": 0.8873, |
| "nll": 0.2931, |
| "ece": 0.0453, |
| "mean_conf": 0.9088 |
| }, |
| "score": { |
| "n": 120, |
| "accuracy": 0.75, |
| "nll": 0.4969, |
| "ece": 0.0576, |
| "mean_conf": 0.7831 |
| } |
| } |
| }, |
| "val_teacher2": { |
| "all": { |
| "n": 447, |
| "accuracy": 0.8322, |
| "nll": 0.4435, |
| "ece": 0.0627, |
| "mean_conf": 0.8747 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 265, |
| "accuracy": 0.8377, |
| "nll": 0.4219, |
| "ece": 0.0503, |
| "mean_conf": 0.8858 |
| }, |
| "noul": { |
| "n": 120, |
| "accuracy": 0.85, |
| "nll": 0.3702, |
| "ece": 0.0827, |
| "mean_conf": 0.9034 |
| }, |
| "score": { |
| "n": 62, |
| "accuracy": 0.7742, |
| "nll": 0.6773, |
| "ece": 0.1441, |
| "mean_conf": 0.772 |
| } |
| } |
| }, |
| "cal_human": { |
| "all": { |
| "n": 1595, |
| "accuracy": 0.8564, |
| "nll": 0.3892, |
| "ece": 0.0205, |
| "mean_conf": 0.8717 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 1595, |
| "accuracy": 0.8564, |
| "nll": 0.3892, |
| "ece": 0.0205, |
| "mean_conf": 0.8717 |
| }, |
| "noul": null, |
| "score": null |
| } |
| }, |
| "val_replay4": { |
| "all": { |
| "n": 1000, |
| "accuracy": 0.889, |
| "nll": 0.2848, |
| "ece": 0.0297, |
| "mean_conf": 0.8702 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 1000, |
| "accuracy": 0.889, |
| "nll": 0.2848, |
| "ece": 0.0297, |
| "mean_conf": 0.8702 |
| }, |
| "noul": null, |
| "score": null |
| } |
| } |
| }, |
| "global_T": { |
| "heldout_jb": { |
| "all": { |
| "n": 5000, |
| "accuracy": 0.5562, |
| "nll": 1.2937, |
| "ece": 0.163, |
| "mean_conf": 0.7192 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 3293, |
| "accuracy": 0.5056, |
| "nll": 1.5059, |
| "ece": 0.1724, |
| "mean_conf": 0.678 |
| }, |
| "noul": { |
| "n": 1289, |
| "accuracy": 0.7114, |
| "nll": 0.7112, |
| "ece": 0.1463, |
| "mean_conf": 0.8577 |
| }, |
| "score": { |
| "n": 418, |
| "accuracy": 0.4761, |
| "nll": 1.4183, |
| "ece": 0.1402, |
| "mean_conf": 0.6163 |
| } |
| } |
| }, |
| "test_teacher2": { |
| "all": { |
| "n": 449, |
| "accuracy": 0.8196, |
| "nll": 0.4381, |
| "ece": 0.0497, |
| "mean_conf": 0.8664 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 257, |
| "accuracy": 0.8016, |
| "nll": 0.5058, |
| "ece": 0.058, |
| "mean_conf": 0.8596 |
| }, |
| "noul": { |
| "n": 130, |
| "accuracy": 0.8692, |
| "nll": 0.2755, |
| "ece": 0.0602, |
| "mean_conf": 0.9294 |
| }, |
| "score": { |
| "n": 62, |
| "accuracy": 0.7903, |
| "nll": 0.4984, |
| "ece": 0.0798, |
| "mean_conf": 0.7623 |
| } |
| } |
| }, |
| "guard": { |
| "all": { |
| "n": 2994, |
| "accuracy": 0.8186, |
| "nll": 0.4871, |
| "ece": 0.0194, |
| "mean_conf": 0.836 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 2994, |
| "accuracy": 0.8186, |
| "nll": 0.4871, |
| "ece": 0.0194, |
| "mean_conf": 0.836 |
| }, |
| "noul": null, |
| "score": null |
| } |
| }, |
| "val_jb": { |
| "all": { |
| "n": 5000, |
| "accuracy": 0.796, |
| "nll": 0.5269, |
| "ece": 0.0264, |
| "mean_conf": 0.8224 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 3374, |
| "accuracy": 0.7647, |
| "nll": 0.6267, |
| "ece": 0.0303, |
| "mean_conf": 0.795 |
| }, |
| "noul": { |
| "n": 1273, |
| "accuracy": 0.901, |
| "nll": 0.223, |
| "ece": 0.0172, |
| "mean_conf": 0.9146 |
| }, |
| "score": { |
| "n": 353, |
| "accuracy": 0.7167, |
| "nll": 0.6687, |
| "ece": 0.0363, |
| "mean_conf": 0.7514 |
| } |
| } |
| }, |
| "val_teacher": { |
| "all": { |
| "n": 944, |
| "accuracy": 0.8379, |
| "nll": 0.433, |
| "ece": 0.046, |
| "mean_conf": 0.8722 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 540, |
| "accuracy": 0.8315, |
| "nll": 0.4654, |
| "ece": 0.0435, |
| "mean_conf": 0.8558 |
| }, |
| "noul": { |
| "n": 284, |
| "accuracy": 0.8873, |
| "nll": 0.3405, |
| "ece": 0.0634, |
| "mean_conf": 0.9331 |
| }, |
| "score": { |
| "n": 120, |
| "accuracy": 0.75, |
| "nll": 0.5064, |
| "ece": 0.0603, |
| "mean_conf": 0.8016 |
| } |
| } |
| }, |
| "val_teacher2": { |
| "all": { |
| "n": 447, |
| "accuracy": 0.8277, |
| "nll": 0.4695, |
| "ece": 0.0806, |
| "mean_conf": 0.8852 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 265, |
| "accuracy": 0.8377, |
| "nll": 0.4233, |
| "ece": 0.0546, |
| "mean_conf": 0.8868 |
| }, |
| "noul": { |
| "n": 120, |
| "accuracy": 0.85, |
| "nll": 0.4441, |
| "ece": 0.1059, |
| "mean_conf": 0.9311 |
| }, |
| "score": { |
| "n": 62, |
| "accuracy": 0.7419, |
| "nll": 0.7165, |
| "ece": 0.1556, |
| "mean_conf": 0.7893 |
| } |
| } |
| }, |
| "cal_human": { |
| "all": { |
| "n": 1595, |
| "accuracy": 0.8564, |
| "nll": 0.3893, |
| "ece": 0.0195, |
| "mean_conf": 0.8733 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 1595, |
| "accuracy": 0.8564, |
| "nll": 0.3893, |
| "ece": 0.0195, |
| "mean_conf": 0.8733 |
| }, |
| "noul": null, |
| "score": null |
| } |
| }, |
| "val_replay4": { |
| "all": { |
| "n": 1000, |
| "accuracy": 0.889, |
| "nll": 0.2843, |
| "ece": 0.0287, |
| "mean_conf": 0.8713 |
| }, |
| "by_type": { |
| "choice": { |
| "n": 1000, |
| "accuracy": 0.889, |
| "nll": 0.2843, |
| "ece": 0.0287, |
| "mean_conf": 0.8713 |
| }, |
| "noul": null, |
| "score": null |
| } |
| } |
| } |
| }, |
| "fixtures": { |
| "general_validation": { |
| "this": { |
| "rows": 847, |
| "accuracy": 0.8500590318772137, |
| "macro_accuracy": 0.8552272255409907, |
| "groups": 65, |
| "nll": 0.43237405419192004, |
| "ece": 0.03505942904541031, |
| "brier": 0.22353081852050935, |
| "mean_confidence": 0.8244136475475226 |
| }, |
| "paired": { |
| "cand4b_B4_e2_tt-cand4b_B4_e2": { |
| "accuracy": { |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "nll": { |
| "delta": 0.00012970495542040443, |
| "ci95": [ |
| -0.00039863639304211823, |
| 0.0005942988193083918 |
| ] |
| } |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130": { |
| "accuracy": { |
| "delta": -0.010625737898465172, |
| "ci95": [ |
| -0.021251475796930343, |
| 0.0 |
| ] |
| }, |
| "nll": { |
| "delta": 0.015307099810435664, |
| "ci95": [ |
| 0.0030912070436396634, |
| 0.028028207832146323 |
| ] |
| } |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130": { |
| "accuracy": { |
| "delta": 0.0, |
| "ci95": [ |
| -0.015348288075560802, |
| 0.015348288075560802 |
| ] |
| }, |
| "nll": { |
| "delta": 0.013765296214476044, |
| "ci95": [ |
| -0.006225581100979938, |
| 0.0345539752628619 |
| ] |
| } |
| } |
| } |
| }, |
| "typesafe": { |
| "this": { |
| "rows": 102, |
| "accuracy": 0.8431372549019608, |
| "macro_accuracy": 0.8464696223316912, |
| "groups": 4, |
| "nll": 0.44128428424617644, |
| "ece": 0.06267209380280735, |
| "brier": 0.24132394423929263, |
| "mean_confidence": 0.8996653276331285, |
| "mean_total_variation_to_reference": 0.19246147475275616 |
| }, |
| "paired": { |
| "cand4b_B4_e2_tt-cand4b_B4_e2": { |
| "accuracy": { |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "nll": { |
| "delta": -0.0017070072924830098, |
| "ci95": [ |
| -0.003544773928864442, |
| -8.629601238631066e-05 |
| ] |
| } |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130": { |
| "accuracy": { |
| "delta": 0.029411764705882353, |
| "ci95": [ |
| -0.029411764705882353, |
| 0.08823529411764706 |
| ] |
| }, |
| "nll": { |
| "delta": -0.16956755666314244, |
| "ci95": [ |
| -0.3236270765220259, |
| -0.035752501984007506 |
| ] |
| } |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130": { |
| "accuracy": { |
| "delta": -0.0196078431372549, |
| "ci95": [ |
| -0.0784313725490196, |
| 0.0392156862745098 |
| ] |
| }, |
| "nll": { |
| "delta": 0.03116252024698791, |
| "ci95": [ |
| -0.06909971743191749, |
| 0.13434316002254298 |
| ] |
| } |
| } |
| } |
| }, |
| "openjev_external": { |
| "this": { |
| "rows": 5252, |
| "accuracy": 0.6597486671744097, |
| "macro_accuracy": 0.8063444444444444, |
| "groups": 3, |
| "nll": 0.8457289344223444, |
| "ece": 0.1539857582574297, |
| "brier": 0.5022800176356607, |
| "mean_confidence": 0.8111537299704116 |
| }, |
| "paired": { |
| "cand4b_B4_e2_tt-cand4b_B4_e2": { |
| "accuracy": { |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "nll": { |
| "delta": -0.003617502816960803, |
| "ci95": [ |
| -0.003919285763621306, |
| -0.00332506503930084 |
| ] |
| } |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130": { |
| "accuracy": { |
| "delta": 0.020944402132520943, |
| "ci95": [ |
| 0.012376237623762377, |
| 0.029322162985529324 |
| ] |
| }, |
| "nll": { |
| "delta": -0.047333755296254304, |
| "ci95": [ |
| -0.059559262901540976, |
| -0.035193633735333484 |
| ] |
| } |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130": { |
| "accuracy": { |
| "delta": -0.007425742574257425, |
| "ci95": [ |
| -0.015993907083015995, |
| 0.0017136329017517135 |
| ] |
| }, |
| "nll": { |
| "delta": 0.05671352997929895, |
| "ci95": [ |
| 0.04359541016972144, |
| 0.07033520922345497 |
| ] |
| } |
| } |
| } |
| }, |
| "mind2web": { |
| "this": { |
| "rows": 1770, |
| "accuracy": 0.8751412429378531, |
| "macro_accuracy": 0.8773196126201471, |
| "groups": 3, |
| "nll": 0.37523612083453944, |
| "ece": 0.016833950116135952, |
| "brier": 0.18402882486455008, |
| "mean_confidence": 0.8624463401776923 |
| }, |
| "paired": { |
| "cand4b_B4_e2_tt-cand4b_B4_e2": { |
| "accuracy": { |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "nll": { |
| "delta": 0.00026909733615400604, |
| "ci95": [ |
| -4.997823592689937e-05, |
| 0.0006061414472582659 |
| ] |
| } |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130": { |
| "accuracy": { |
| "delta": -0.00847457627118644, |
| "ci95": [ |
| -0.01694915254237288, |
| 0.0 |
| ] |
| }, |
| "nll": { |
| "delta": 0.00924747244030173, |
| "ci95": [ |
| -0.004092843962589085, |
| 0.022512164121517977 |
| ] |
| } |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130": { |
| "accuracy": { |
| "delta": 0.0011299435028248588, |
| "ci95": [ |
| -0.00847457627118644, |
| 0.011299435028248588 |
| ] |
| }, |
| "nll": { |
| "delta": -0.018244809747118285, |
| "ci95": [ |
| -0.032040489157299956, |
| -0.005027723688309381 |
| ] |
| } |
| } |
| } |
| } |
| }, |
| "games": { |
| "cand4b_B4_e2_tt-cand4b_B4_e2|sampled|all": { |
| "n": 234, |
| "a": 0.2692307692307692, |
| "b": 0.2692307692307692, |
| "delta": 0.0, |
| "ci95": [ |
| -0.004273504273504274, |
| 0.004273504273504274 |
| ] |
| }, |
| "cand4b_B4_e2_tt-cand4b_B4_e2|sampled|bag_game": { |
| "n": 64, |
| "a": 0.51953125, |
| "b": 0.51953125, |
| "delta": 0.0, |
| "ci95": [ |
| -0.01171875, |
| 0.01171875 |
| ] |
| }, |
| "cand4b_B4_e2_tt-cand4b_B4_e2|sampled|stochastic_grid": { |
| "n": 64, |
| "a": 0.16796875, |
| "b": 0.16796875, |
| "delta": 0.0, |
| "ci95": [ |
| -0.01171875, |
| 0.01171875 |
| ] |
| }, |
| "cand4b_B4_e2_tt-cand4b_B4_e2|sampled|tic_tac_toe": { |
| "n": 74, |
| "a": 0.2533783783783784, |
| "b": 0.2533783783783784, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand4b_B4_e2_tt-cand4b_B4_e2|sampled|exact_minesweeper": { |
| "n": 32, |
| "a": 0.0078125, |
| "b": 0.0078125, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand4b_B4_e2_tt-cand4b_B4_e2|greedy|all": { |
| "n": 234, |
| "a": 0.25213675213675213, |
| "b": 0.25213675213675213, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand4b_B4_e2_tt-cand4b_B4_e2|greedy|bag_game": { |
| "n": 64, |
| "a": 0.53125, |
| "b": 0.53125, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand4b_B4_e2_tt-cand4b_B4_e2|greedy|stochastic_grid": { |
| "n": 64, |
| "a": 0.109375, |
| "b": 0.109375, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand4b_B4_e2_tt-cand4b_B4_e2|greedy|tic_tac_toe": { |
| "n": 74, |
| "a": 0.24324324324324326, |
| "b": 0.24324324324324326, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand4b_B4_e2_tt-cand4b_B4_e2|greedy|exact_minesweeper": { |
| "n": 32, |
| "a": 0.0, |
| "b": 0.0, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130|sampled|all": { |
| "n": 234, |
| "a": 0.2692307692307692, |
| "b": 0.2777777777777778, |
| "delta": -0.008547008547008548, |
| "ci95": [ |
| -0.02564102564102564, |
| 0.009615384615384616 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130|sampled|bag_game": { |
| "n": 64, |
| "a": 0.51953125, |
| "b": 0.56640625, |
| "delta": -0.046875, |
| "ci95": [ |
| -0.08984375, |
| -0.00390625 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130|sampled|stochastic_grid": { |
| "n": 64, |
| "a": 0.16796875, |
| "b": 0.1640625, |
| "delta": 0.00390625, |
| "ci95": [ |
| -0.02734375, |
| 0.03515625 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130|sampled|tic_tac_toe": { |
| "n": 74, |
| "a": 0.2533783783783784, |
| "b": 0.24324324324324326, |
| "delta": 0.010135135135135136, |
| "ci95": [ |
| -0.013513513513513514, |
| 0.037162162162162164 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130|sampled|exact_minesweeper": { |
| "n": 32, |
| "a": 0.0078125, |
| "b": 0.0078125, |
| "delta": 0.0, |
| "ci95": [ |
| -0.0234375, |
| 0.0234375 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130|greedy|all": { |
| "n": 234, |
| "a": 0.25213675213675213, |
| "b": 0.2905982905982906, |
| "delta": -0.038461538461538464, |
| "ci95": [ |
| -0.07264957264957266, |
| -0.004273504273504274 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130|greedy|bag_game": { |
| "n": 64, |
| "a": 0.53125, |
| "b": 0.625, |
| "delta": -0.09375, |
| "ci95": [ |
| -0.171875, |
| -0.03125 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130|greedy|stochastic_grid": { |
| "n": 64, |
| "a": 0.109375, |
| "b": 0.125, |
| "delta": -0.015625, |
| "ci95": [ |
| -0.07851562499999987, |
| 0.046875 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130|greedy|tic_tac_toe": { |
| "n": 74, |
| "a": 0.24324324324324326, |
| "b": 0.2702702702702703, |
| "delta": -0.02702702702702703, |
| "ci95": [ |
| -0.0945945945945946, |
| 0.04054054054054054 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130|greedy|exact_minesweeper": { |
| "n": 32, |
| "a": 0.0, |
| "b": 0.0, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130|sampled|all": { |
| "n": 234, |
| "a": 0.2692307692307692, |
| "b": 0.22435897435897437, |
| "delta": 0.04487179487179487, |
| "ci95": [ |
| 0.024572649572649572, |
| 0.06623931623931624 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130|sampled|bag_game": { |
| "n": 64, |
| "a": 0.51953125, |
| "b": 0.37890625, |
| "delta": 0.140625, |
| "ci95": [ |
| 0.08203125, |
| 0.19921875 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130|sampled|stochastic_grid": { |
| "n": 64, |
| "a": 0.16796875, |
| "b": 0.16015625, |
| "delta": 0.0078125, |
| "ci95": [ |
| -0.0234375, |
| 0.0390625 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130|sampled|tic_tac_toe": { |
| "n": 74, |
| "a": 0.2533783783783784, |
| "b": 0.24324324324324326, |
| "delta": 0.010135135135135136, |
| "ci95": [ |
| -0.010135135135135136, |
| 0.033783783783783786 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130|sampled|exact_minesweeper": { |
| "n": 32, |
| "a": 0.0078125, |
| "b": 0.0, |
| "delta": 0.0078125, |
| "ci95": [ |
| 0.0, |
| 0.0234375 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130|greedy|all": { |
| "n": 234, |
| "a": 0.25213675213675213, |
| "b": 0.2777777777777778, |
| "delta": -0.02564102564102564, |
| "ci95": [ |
| -0.0641025641025641, |
| 0.01282051282051282 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130|greedy|bag_game": { |
| "n": 64, |
| "a": 0.53125, |
| "b": 0.484375, |
| "delta": 0.046875, |
| "ci95": [ |
| -0.03125, |
| 0.125 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130|greedy|stochastic_grid": { |
| "n": 64, |
| "a": 0.109375, |
| "b": 0.1875, |
| "delta": -0.078125, |
| "ci95": [ |
| -0.15625, |
| 0.0 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130|greedy|tic_tac_toe": { |
| "n": 74, |
| "a": 0.24324324324324326, |
| "b": 0.2972972972972973, |
| "delta": -0.05405405405405406, |
| "ci95": [ |
| -0.12162162162162163, |
| 0.02702702702702703 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130|greedy|exact_minesweeper": { |
| "n": 32, |
| "a": 0.0, |
| "b": 0.0, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| } |
| }, |
| "browser": { |
| "cand4b_B4_e2_tt-cand4b_B4_e2|sampled|all": { |
| "n": 176, |
| "a": 0.9318181818181818, |
| "b": 0.9318181818181818, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand4b_B4_e2_tt-cand4b_B4_e2|sampled|rewarded": { |
| "n": 128, |
| "a": 0.953125, |
| "b": 0.953125, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand4b_B4_e2_tt-cand4b_B4_e2|sampled|held_out": { |
| "n": 48, |
| "a": 0.875, |
| "b": 0.875, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand4b_B4_e2_tt-cand4b_B4_e2|greedy|all": { |
| "n": 176, |
| "a": 0.9375, |
| "b": 0.9375, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand4b_B4_e2_tt-cand4b_B4_e2|greedy|rewarded": { |
| "n": 128, |
| "a": 0.9609375, |
| "b": 0.9609375, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand4b_B4_e2_tt-cand4b_B4_e2|greedy|held_out": { |
| "n": 48, |
| "a": 0.875, |
| "b": 0.875, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130|sampled|all": { |
| "n": 176, |
| "a": 0.9318181818181818, |
| "b": 0.9090909090909091, |
| "delta": 0.022727272727272728, |
| "ci95": [ |
| -0.017045454545454544, |
| 0.0625 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130|sampled|rewarded": { |
| "n": 128, |
| "a": 0.953125, |
| "b": 0.9609375, |
| "delta": -0.0078125, |
| "ci95": [ |
| -0.0390625, |
| 0.0234375 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130|sampled|held_out": { |
| "n": 48, |
| "a": 0.875, |
| "b": 0.7708333333333334, |
| "delta": 0.10416666666666667, |
| "ci95": [ |
| 0.0, |
| 0.20833333333333334 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130|greedy|all": { |
| "n": 176, |
| "a": 0.9375, |
| "b": 0.9147727272727273, |
| "delta": 0.022727272727272728, |
| "ci95": [ |
| -0.005681818181818182, |
| 0.056818181818181816 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130|greedy|rewarded": { |
| "n": 128, |
| "a": 0.9609375, |
| "b": 0.9765625, |
| "delta": -0.015625, |
| "ci95": [ |
| -0.0390625, |
| 0.0 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v1_4b_p130|greedy|held_out": { |
| "n": 48, |
| "a": 0.875, |
| "b": 0.75, |
| "delta": 0.125, |
| "ci95": [ |
| 0.041666666666666664, |
| 0.22916666666666666 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130|sampled|all": { |
| "n": 176, |
| "a": 0.9318181818181818, |
| "b": 0.8806818181818182, |
| "delta": 0.05113636363636364, |
| "ci95": [ |
| 0.005681818181818182, |
| 0.09659090909090909 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130|sampled|rewarded": { |
| "n": 128, |
| "a": 0.953125, |
| "b": 0.9296875, |
| "delta": 0.0234375, |
| "ci95": [ |
| -0.0234375, |
| 0.0703125 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130|sampled|held_out": { |
| "n": 48, |
| "a": 0.875, |
| "b": 0.75, |
| "delta": 0.125, |
| "ci95": [ |
| 0.0, |
| 0.25 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130|greedy|all": { |
| "n": 176, |
| "a": 0.9375, |
| "b": 0.9261363636363636, |
| "delta": 0.011363636363636364, |
| "ci95": [ |
| -0.017045454545454544, |
| 0.045454545454545456 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130|greedy|rewarded": { |
| "n": 128, |
| "a": 0.9609375, |
| "b": 0.96875, |
| "delta": -0.0078125, |
| "ci95": [ |
| -0.0234375, |
| 0.0 |
| ] |
| }, |
| "cand4b_B4_e2_tt-v2_4b_p130|greedy|held_out": { |
| "n": 48, |
| "a": 0.875, |
| "b": 0.8125, |
| "delta": 0.0625, |
| "ci95": [ |
| -0.041666666666666664, |
| 0.16666666666666666 |
| ] |
| } |
| }, |
| "text_games_greedy": { |
| "pong": -5.0, |
| "freeway": 0.0, |
| "breakout": 35.0, |
| "frozenlake": 0.0, |
| "cliffwalking": -13.0, |
| "blackjack": -0.6, |
| "minigrid_empty": 0.0, |
| "minigrid_lavagap": 0.0, |
| "minigrid_doorkey": 0.0, |
| "babyai_goto": 0.1915625 |
| }, |
| "probes": { |
| "router_n": 31, |
| "router_tier": 0.968, |
| "needs_live_data": 0.839, |
| "commands_n": 45, |
| "command_risk": 0.889, |
| "destructive_called_safe": 0, |
| "outside_project": 0.933, |
| "browser_element": 0.938, |
| "browser_action": 0.75, |
| "batteries": { |
| "generic": 1.0, |
| "specific": 1.0, |
| "catchall": 0.95, |
| "noul_acc": 1.0, |
| "noul_brier": 0.01, |
| "n_choice": 60, |
| "n_noul": 49, |
| "abstain_battery": "8/8" |
| } |
| }, |
| "issue9": { |
| "c_1": { |
| "pred": 26, |
| "gold": 6, |
| "ok": false, |
| "p_gold": 0.198, |
| "p_pred": 0.764, |
| "T": 1.099 |
| }, |
| "c_2": { |
| "pred": 20, |
| "gold": 20, |
| "ok": true, |
| "p_gold": 0.409, |
| "p_pred": 0.409, |
| "T": 1.099 |
| }, |
| "b_10": { |
| "pred": 0, |
| "gold": 0, |
| "ok": true, |
| "p_gold": 0.732, |
| "p_pred": 0.732, |
| "T": 1.099 |
| }, |
| "a2_4": { |
| "pred": 4, |
| "gold": 4, |
| "ok": true, |
| "p_gold": 0.577, |
| "p_pred": 0.577, |
| "T": 1.099 |
| } |
| }, |
| "bespoke": { |
| "vitaminc-dev": { |
| "n": 599, |
| "type": "choice", |
| "acc": 0.7429, |
| "ece": 0.1456, |
| "brier": 0.3945, |
| "nll": 0.7789, |
| "trained": false, |
| "sec": 11.8 |
| }, |
| "massive-en-US": { |
| "n": 350, |
| "type": "choice", |
| "acc": 0.8629, |
| "ece": 0.0411, |
| "brier": 0.1936, |
| "nll": 0.4173, |
| "trained": true, |
| "sec": 6.2 |
| }, |
| "massive-de-DE": { |
| "n": 350, |
| "type": "choice", |
| "acc": 0.84, |
| "ece": 0.0297, |
| "brier": 0.2269, |
| "nll": 0.4947, |
| "trained": false, |
| "sec": 4.5 |
| }, |
| "boolq": { |
| "n": 300, |
| "type": "noul", |
| "acc": 0.8667, |
| "ece": 0.0319, |
| "brier": 0.1862, |
| "nll": 0.3007, |
| "trained": true, |
| "sec": 6.2 |
| }, |
| "squad2": { |
| "n": 299, |
| "type": "noul", |
| "acc": 0.7893, |
| "ece": 0.0441, |
| "brier": 0.2955, |
| "nll": 0.452, |
| "trained": false, |
| "sec": 7.2 |
| }, |
| "paws": { |
| "n": 250, |
| "type": "noul", |
| "acc": 0.764, |
| "ece": 0.1238, |
| "brier": 0.3398, |
| "nll": 0.5538, |
| "trained": true, |
| "sec": 3.0 |
| }, |
| "multinli": { |
| "n": 299, |
| "type": "choice", |
| "acc": 0.9331, |
| "ece": 0.0103, |
| "brier": 0.1089, |
| "nll": 0.2099, |
| "trained": true, |
| "sec": 3.1 |
| }, |
| "civil_comments": { |
| "n": 300, |
| "type": "noul", |
| "acc": 0.8733, |
| "ece": 0.0566, |
| "brier": 0.1917, |
| "nll": 0.3199, |
| "trained": true, |
| "sec": 3.2 |
| }, |
| "aegis2": { |
| "n": 250, |
| "type": "noul", |
| "acc": 0.804, |
| "ece": 0.0467, |
| "brier": 0.2797, |
| "nll": 0.4527, |
| "trained": false, |
| "sec": 6.1 |
| }, |
| "helpsteer2": { |
| "n": 249, |
| "type": "score", |
| "acc": 0.4498, |
| "ece": 0.142, |
| "brier": 0.6713, |
| "nll": 1.2502, |
| "trained": true, |
| "sec": 25.8, |
| "score_mae": 0.797 |
| }, |
| "summeval-relevance": { |
| "n": 240, |
| "type": "score", |
| "acc": 0.3958, |
| "ece": 0.0696, |
| "brier": 0.6844, |
| "nll": 1.2541, |
| "trained": false, |
| "sec": 20.1, |
| "score_mae": 0.6172 |
| }, |
| "summeval-consistency": { |
| "n": 144, |
| "type": "score", |
| "acc": 0.7986, |
| "ece": 0.3095, |
| "brier": 0.4741, |
| "nll": 0.9007, |
| "trained": false, |
| "sec": 12.2, |
| "score_mae": 0.7974 |
| }, |
| "pubmedqa": { |
| "n": 250, |
| "type": "choice", |
| "acc": 0.708, |
| "ece": 0.0863, |
| "brier": 0.4012, |
| "nll": 0.7205, |
| "trained": true, |
| "sec": 3.8 |
| }, |
| "_macro": 0.756, |
| "_micro": 0.7652, |
| "_macro_untrained": 0.7284 |
| }, |
| "jevbench_public": { |
| "easy": { |
| "n": 48, |
| "accuracy": 1.0, |
| "invalid": 0, |
| "ece": 0.0027979166666670663, |
| "by_family": { |
| "extraction": { |
| "n": 12, |
| "accuracy": 1.0 |
| }, |
| "fact": { |
| "n": 12, |
| "accuracy": 1.0 |
| }, |
| "intent": { |
| "n": 12, |
| "accuracy": 1.0 |
| }, |
| "tool_selection": { |
| "n": 12, |
| "accuracy": 1.0 |
| } |
| } |
| }, |
| "standard": { |
| "n": 72, |
| "accuracy": 0.9861111111111112, |
| "invalid": 0, |
| "ece": 0.0753152777777778, |
| "by_family": { |
| "adequacy": { |
| "n": 12, |
| "accuracy": 0.9166666666666666 |
| }, |
| "extraction": { |
| "n": 12, |
| "accuracy": 1.0 |
| }, |
| "intent": { |
| "n": 12, |
| "accuracy": 1.0 |
| }, |
| "ordinal": { |
| "n": 12, |
| "accuracy": 1.0 |
| }, |
| "policy": { |
| "n": 12, |
| "accuracy": 1.0 |
| }, |
| "routing": { |
| "n": 12, |
| "accuracy": 1.0 |
| } |
| } |
| }, |
| "hard": { |
| "n": 111, |
| "accuracy": 0.6486486486486487, |
| "invalid": 0, |
| "ece": 0.18370720720720732, |
| "by_family": { |
| "adversarial": { |
| "n": 6, |
| "accuracy": 1.0 |
| }, |
| "ambiguous": { |
| "n": 7, |
| "accuracy": 0.7142857142857143 |
| }, |
| "judge_hard": { |
| "n": 17, |
| "accuracy": 0.5294117647058824 |
| }, |
| "long_policy": { |
| "n": 19, |
| "accuracy": 0.47368421052631576 |
| }, |
| "multi_hop": { |
| "n": 18, |
| "accuracy": 0.8333333333333334 |
| }, |
| "probability": { |
| "n": 10, |
| "accuracy": 0.8 |
| }, |
| "routing_hard": { |
| "n": 5, |
| "accuracy": 1.0 |
| }, |
| "temporal_numeric": { |
| "n": 15, |
| "accuracy": 0.3333333333333333 |
| }, |
| "tradeoff": { |
| "n": 6, |
| "accuracy": 0.3333333333333333 |
| }, |
| "trap": { |
| "n": 8, |
| "accuracy": 1.0 |
| } |
| } |
| } |
| }, |
| "references": { |
| "v1_4b": { |
| "reg_in": 0.8338, |
| "reg_out": 0.7876, |
| "reg_in_nll": 0.4044, |
| "reg_out_nll": 0.5583, |
| "reg_in_ece": 0.0269, |
| "reg_out_ece": 0.0708 |
| }, |
| "v2_4b": { |
| "reg_in": 0.8241, |
| "reg_out": 0.7788, |
| "reg_in_nll": 0.4411, |
| "reg_out_nll": 0.5663, |
| "reg_in_ece": 0.0413, |
| "reg_out_ece": 0.0803 |
| } |
| }, |
| "speed": "not measured again: the architecture and size equal the parent's, and the temperature map does not change the computation" |
| } |
