{
 "model": "decider-2b-v11",
 "temperature": 1.145,
 "temperature_by_type": {
  "choice": 1.164,
  "noul": 1.624,
  "score": 1.124
 },
 "temperature_fit": {
  "global": "one scalar, pooled row NLL on the in-task half of the public regression set without banking77, clinc_oos, mmlu, arc, winogrande and hellaswag (61 tasks, 102,804 rows)",
  "by_type": "decider.calibrate.fit_by_type (decider-ai 1.4.0), min_rows 50, on the same regression rows plus our own validation rows (val_jb, val_teacher, val_teacher2, cal_human without mmlu, arc and banking77, val_replay without six excluded families)",
  "rows": {
   "choice": 108910,
   "noul": 1677,
   "score": 535
  },
  "note": "no JevBench or Decision Index item was used for training, selection or temperature"
 },
 "protocol": "Regression set: eval half of the mixture, 95 tasks (67 in-task / 28 held-out), plain state-first layout, large label sets sub-sampled to 10 options, ECE 15 bins, per-task mean. Own sets (heldout_jb, test_teacher2, cal_human, guard): stored T=1 logits read at the served temperatures, ECE 10 bins top label. Card lanes (fixtures, games, browser, probes, Bespoke, issue #9) through decider-ai 1.4.0 with the map; parents through 1.3.0 at their released temperatures, same rows and seeds; intervals are 95% paired bootstrap. JevBench public items read in process through 1.4.0 with the map; parents' files read earlier through 1.2.1.",
 "regression": {
  "map": {
   "reg_in": 0.8016,
   "reg_out": 0.7518,
   "reg_in_nll": 0.4809,
   "reg_out_nll": 0.6262,
   "reg_in_ece": 0.0385,
   "reg_out_ece": 0.0834
  },
  "global_T": {
   "reg_in": 0.8016,
   "reg_out": 0.7518,
   "reg_in_nll": 0.4808,
   "reg_out_nll": 0.628,
   "reg_in_ece": 0.0383,
   "reg_out_ece": 0.0847
  }
 },
 "own_sets": {
  "map": {
   "heldout_jb": {
    "all": {
     "n": 5000,
     "accuracy": 0.4288,
     "nll": 1.4669,
     "ece": 0.1562,
     "mean_conf": 0.5832
    },
    "by_type": {
     "choice": {
      "n": 3293,
      "accuracy": 0.3702,
      "nll": 1.743,
      "ece": 0.172,
      "mean_conf": 0.5395
     },
     "noul": {
      "n": 1289,
      "accuracy": 0.6005,
      "nll": 0.7022,
      "ece": 0.121,
      "mean_conf": 0.7215
     },
     "score": {
      "n": 418,
      "accuracy": 0.3612,
      "nll": 1.6498,
      "ece": 0.1405,
      "mean_conf": 0.5018
     }
    }
   },
   "test_teacher2": {
    "all": {
     "n": 449,
     "accuracy": 0.7528,
     "nll": 0.5668,
     "ece": 0.0754,
     "mean_conf": 0.8155
    },
    "by_type": {
     "choice": {
      "n": 257,
      "accuracy": 0.751,
      "nll": 0.6501,
      "ece": 0.0736,
      "mean_conf": 0.8108
     },
     "noul": {
      "n": 130,
      "accuracy": 0.8077,
      "nll": 0.325,
      "ece": 0.0926,
      "mean_conf": 0.8774
     },
     "score": {
      "n": 62,
      "accuracy": 0.6452,
      "nll": 0.7286,
      "ece": 0.1307,
      "mean_conf": 0.7054
     }
    }
   },
   "guard": {
    "all": {
     "n": 2994,
     "accuracy": 0.7151,
     "nll": 0.7138,
     "ece": 0.0463,
     "mean_conf": 0.7614
    },
    "by_type": {
     "choice": {
      "n": 2994,
      "accuracy": 0.7151,
      "nll": 0.7138,
      "ece": 0.0463,
      "mean_conf": 0.7614
     },
     "noul": null,
     "score": null
    }
   },
   "val_jb": {
    "all": {
     "n": 5000,
     "accuracy": 0.694,
     "nll": 0.7801,
     "ece": 0.0278,
     "mean_conf": 0.7218
    },
    "by_type": {
     "choice": {
      "n": 3374,
      "accuracy": 0.6461,
      "nll": 0.9402,
      "ece": 0.0451,
      "mean_conf": 0.6912
     },
     "noul": {
      "n": 1273,
      "accuracy": 0.8437,
      "nll": 0.3275,
      "ece": 0.0235,
      "mean_conf": 0.8278
     },
     "score": {
      "n": 353,
      "accuracy": 0.6119,
      "nll": 0.8828,
      "ece": 0.0475,
      "mean_conf": 0.6323
     }
    }
   },
   "val_teacher": {
    "all": {
     "n": 944,
     "accuracy": 0.7744,
     "nll": 0.5627,
     "ece": 0.0586,
     "mean_conf": 0.8252
    },
    "by_type": {
     "choice": {
      "n": 540,
      "accuracy": 0.7574,
      "nll": 0.6695,
      "ece": 0.057,
      "mean_conf": 0.8081
     },
     "noul": {
      "n": 284,
      "accuracy": 0.8204,
      "nll": 0.3607,
      "ece": 0.0671,
      "mean_conf": 0.8875
     },
     "score": {
      "n": 120,
      "accuracy": 0.7417,
      "nll": 0.5603,
      "ece": 0.0699,
      "mean_conf": 0.7551
     }
    }
   },
   "val_teacher2": {
    "all": {
     "n": 447,
     "accuracy": 0.7785,
     "nll": 0.5757,
     "ece": 0.0625,
     "mean_conf": 0.8271
    },
    "by_type": {
     "choice": {
      "n": 265,
      "accuracy": 0.7736,
      "nll": 0.6303,
      "ece": 0.0694,
      "mean_conf": 0.838
     },
     "noul": {
      "n": 120,
      "accuracy": 0.7833,
      "nll": 0.4665,
      "ece": 0.0835,
      "mean_conf": 0.8657
     },
     "score": {
      "n": 62,
      "accuracy": 0.7903,
      "nll": 0.5532,
      "ece": 0.1172,
      "mean_conf": 0.7053
     }
    }
   },
   "cal_human": {
    "all": {
     "n": 1595,
     "accuracy": 0.7812,
     "nll": 0.565,
     "ece": 0.0298,
     "mean_conf": 0.8051
    },
    "by_type": {
     "choice": {
      "n": 1595,
      "accuracy": 0.7812,
      "nll": 0.565,
      "ece": 0.0298,
      "mean_conf": 0.8051
     },
     "noul": null,
     "score": null
    }
   },
   "val_replay2": {
    "all": {
     "n": 1000,
     "accuracy": 0.82,
     "nll": 0.4696,
     "ece": 0.0273,
     "mean_conf": 0.8257
    },
    "by_type": {
     "choice": {
      "n": 1000,
      "accuracy": 0.82,
      "nll": 0.4696,
      "ece": 0.0273,
      "mean_conf": 0.8257
     },
     "noul": null,
     "score": null
    }
   }
  },
  "global_T": {
   "heldout_jb": {
    "all": {
     "n": 5000,
     "accuracy": 0.4286,
     "nll": 1.4935,
     "ece": 0.1711,
     "mean_conf": 0.5986
    },
    "by_type": {
     "choice": {
      "n": 3293,
      "accuracy": 0.3702,
      "nll": 1.7521,
      "ece": 0.1752,
      "mean_conf": 0.5437
     },
     "noul": {
      "n": 1289,
      "accuracy": 0.6005,
      "nll": 0.7852,
      "ece": 0.1714,
      "mean_conf": 0.7719
     },
     "score": {
      "n": 418,
      "accuracy": 0.3589,
      "nll": 1.6405,
      "ece": 0.1379,
      "mean_conf": 0.4968
     }
    }
   },
   "test_teacher2": {
    "all": {
     "n": 449,
     "accuracy": 0.7528,
     "nll": 0.5784,
     "ece": 0.0831,
     "mean_conf": 0.826
    },
    "by_type": {
     "choice": {
      "n": 257,
      "accuracy": 0.751,
      "nll": 0.6532,
      "ece": 0.0795,
      "mean_conf": 0.8131
     },
     "noul": {
      "n": 130,
      "accuracy": 0.8077,
      "nll": 0.3594,
      "ece": 0.1032,
      "mean_conf": 0.9107
     },
     "score": {
      "n": 62,
      "accuracy": 0.6452,
      "nll": 0.7276,
      "ece": 0.1266,
      "mean_conf": 0.7017
     }
    }
   },
   "guard": {
    "all": {
     "n": 2994,
     "accuracy": 0.7151,
     "nll": 0.7154,
     "ece": 0.0497,
     "mean_conf": 0.7648
    },
    "by_type": {
     "choice": {
      "n": 2994,
      "accuracy": 0.7151,
      "nll": 0.7154,
      "ece": 0.0497,
      "mean_conf": 0.7648
     },
     "noul": null,
     "score": null
    }
   },
   "val_jb": {
    "all": {
     "n": 5000,
     "accuracy": 0.694,
     "nll": 0.7815,
     "ece": 0.0401,
     "mean_conf": 0.7341
    },
    "by_type": {
     "choice": {
      "n": 3374,
      "accuracy": 0.6461,
      "nll": 0.9423,
      "ece": 0.0484,
      "mean_conf": 0.6945
     },
     "noul": {
      "n": 1273,
      "accuracy": 0.8437,
      "nll": 0.327,
      "ece": 0.0273,
      "mean_conf": 0.8683
     },
     "score": {
      "n": 353,
      "accuracy": 0.6119,
      "nll": 0.883,
      "ece": 0.0401,
      "mean_conf": 0.6288
     }
    }
   },
   "val_teacher": {
    "all": {
     "n": 944,
     "accuracy": 0.7754,
     "nll": 0.5823,
     "ece": 0.068,
     "mean_conf": 0.8349
    },
    "by_type": {
     "choice": {
      "n": 540,
      "accuracy": 0.7574,
      "nll": 0.6728,
      "ece": 0.0606,
      "mean_conf": 0.8104
     },
     "noul": {
      "n": 284,
      "accuracy": 0.8204,
      "nll": 0.42,
      "ece": 0.0958,
      "mean_conf": 0.9162
     },
     "score": {
      "n": 120,
      "accuracy": 0.75,
      "nll": 0.5594,
      "ece": 0.0437,
      "mean_conf": 0.7528
     }
    }
   },
   "val_teacher2": {
    "all": {
     "n": 447,
     "accuracy": 0.7785,
     "nll": 0.6006,
     "ece": 0.0778,
     "mean_conf": 0.8378
    },
    "by_type": {
     "choice": {
      "n": 265,
      "accuracy": 0.7736,
      "nll": 0.6341,
      "ece": 0.0744,
      "mean_conf": 0.8402
     },
     "noul": {
      "n": 120,
      "accuracy": 0.7833,
      "nll": 0.5502,
      "ece": 0.1217,
      "mean_conf": 0.9024
     },
     "score": {
      "n": 62,
      "accuracy": 0.7903,
      "nll": 0.5546,
      "ece": 0.129,
      "mean_conf": 0.7028
     }
    }
   },
   "cal_human": {
    "all": {
     "n": 1595,
     "accuracy": 0.7812,
     "nll": 0.5651,
     "ece": 0.0302,
     "mean_conf": 0.8086
    },
    "by_type": {
     "choice": {
      "n": 1595,
      "accuracy": 0.7812,
      "nll": 0.5651,
      "ece": 0.0302,
      "mean_conf": 0.8086
     },
     "noul": null,
     "score": null
    }
   },
   "val_replay2": {
    "all": {
     "n": 1000,
     "accuracy": 0.82,
     "nll": 0.47,
     "ece": 0.0307,
     "mean_conf": 0.8284
    },
    "by_type": {
     "choice": {
      "n": 1000,
      "accuracy": 0.82,
      "nll": 0.47,
      "ece": 0.0307,
      "mean_conf": 0.8284
     },
     "noul": null,
     "score": null
    }
   }
  }
 },
 "fixtures": {
  "general_validation": {
   "this": {
    "rows": 847,
    "accuracy": 0.8193624557260921,
    "macro_accuracy": 0.8264818843272046,
    "groups": 65,
    "nll": 0.4679994635746329,
    "ece": 0.025221459893014928,
    "brier": 0.24885000616245756,
    "mean_confidence": 0.7965057105388945
   },
   "paired": {
    "cand2b_C2B_e2_tt-cand2b_C2B_e2": {
     "accuracy": {
      "delta": 0.0,
      "ci95": [
       0.0,
       0.0
      ]
     },
     "nll": {
      "delta": 0.0008567327869000381,
      "ci95": [
       0.00012586933848329147,
       0.0014936147428319582
      ]
     }
    },
    "cand2b_C2B_e2_tt-v10_2b_p130": {
     "accuracy": {
      "delta": -0.014167650531286895,
      "ci95": [
       -0.0295159386068477,
       0.0011806375442739079
      ]
     },
     "nll": {
      "delta": 0.02396263800434385,
      "ci95": [
       0.007682289841923634,
       0.03881930597006884
      ]
     }
    }
   }
  },
  "typesafe": {
   "this": {
    "rows": 102,
    "accuracy": 0.7549019607843137,
    "macro_accuracy": 0.7613916256157636,
    "groups": 4,
    "nll": 0.6036617290590378,
    "ece": 0.08752135903227565,
    "brier": 0.35248420626905574,
    "mean_confidence": 0.8094115233888813,
    "mean_total_variation_to_reference": 0.2724384515080273
   },
   "paired": {
    "cand2b_C2B_e2_tt-cand2b_C2B_e2": {
     "accuracy": {
      "delta": 0.0,
      "ci95": [
       0.0,
       0.0
      ]
     },
     "nll": {
      "delta": -0.0026735984139883862,
      "ci95": [
       -0.005663155113504104,
       -0.00015286414032123958
      ]
     }
    },
    "cand2b_C2B_e2_tt-v10_2b_p130": {
     "accuracy": {
      "delta": -0.049019607843137254,
      "ci95": [
       -0.10784313725490197,
       0.00980392156862745
      ]
     },
     "nll": {
      "delta": 0.01849389075312281,
      "ci95": [
       -0.08772658186762586,
       0.13502854066724154
      ]
     }
    }
   }
  },
  "openjev_external": {
   "this": {
    "rows": 5252,
    "accuracy": 0.6460396039603961,
    "macro_accuracy": 0.8054024691358025,
    "groups": 3,
    "nll": 0.8595481612843723,
    "ece": 0.124815208362153,
    "brier": 0.5100238458753833,
    "mean_confidence": 0.7668148385756631
   },
   "paired": {
    "cand2b_C2B_e2_tt-cand2b_C2B_e2": {
     "accuracy": {
      "delta": 0.0,
      "ci95": [
       0.0,
       0.0
      ]
     },
     "nll": {
      "delta": -0.004652630797611012,
      "ci95": [
       -0.005088558046264057,
       -0.004202312508842571
      ]
     }
    },
    "cand2b_C2B_e2_tt-v10_2b_p130": {
     "accuracy": {
      "delta": 0.013899466869763899,
      "ci95": [
       0.005902513328255902,
       0.022086824067022087
      ]
     },
     "nll": {
      "delta": -0.05702255018553582,
      "ci95": [
       -0.0676423182247241,
       -0.04578181130197777
      ]
     }
    }
   }
  },
  "mind2web": {
   "this": {
    "rows": 1770,
    "accuracy": 0.8389830508474576,
    "macro_accuracy": 0.8411027477788687,
    "groups": 3,
    "nll": 0.49544234456641184,
    "ece": 0.039275789345051576,
    "brier": 0.24172292854121424,
    "mean_confidence": 0.8067077137151006
   },
   "paired": {
    "cand2b_C2B_e2_tt-cand2b_C2B_e2": {
     "accuracy": {
      "delta": 0.0,
      "ci95": [
       0.0,
       0.0
      ]
     },
     "nll": {
      "delta": 0.0005301123790506686,
      "ci95": [
       -0.0001095135512494298,
       0.001158587013081229
      ]
     }
    },
    "cand2b_C2B_e2_tt-v10_2b_p130": {
     "accuracy": {
      "delta": 0.012994350282485875,
      "ci95": [
       0.0011299435028248588,
       0.025423728813559324
      ]
     },
     "nll": {
      "delta": -0.04784351744979385,
      "ci95": [
       -0.07354020281814458,
       -0.022660045517854176
      ]
     }
    }
   }
  }
 },
 "games": {
  "cand2b_C2B_e2_tt-cand2b_C2B_e2|sampled|all": {
   "n": 234,
   "a": 0.2264957264957265,
   "b": 0.2297008547008547,
   "delta": -0.003205128205128205,
   "ci95": [
    -0.008547008547008548,
    0.002136752136752137
   ]
  },
  "cand2b_C2B_e2_tt-cand2b_C2B_e2|sampled|bag_game": {
   "n": 64,
   "a": 0.41796875,
   "b": 0.4296875,
   "delta": -0.01171875,
   "ci95": [
    -0.03125,
    0.0078125
   ]
  },
  "cand2b_C2B_e2_tt-cand2b_C2B_e2|sampled|stochastic_grid": {
   "n": 64,
   "a": 0.1484375,
   "b": 0.1484375,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand2b_C2B_e2_tt-cand2b_C2B_e2|sampled|tic_tac_toe": {
   "n": 74,
   "a": 0.22635135135135134,
   "b": 0.22635135135135134,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand2b_C2B_e2_tt-cand2b_C2B_e2|sampled|exact_minesweeper": {
   "n": 32,
   "a": 0.0,
   "b": 0.0,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand2b_C2B_e2_tt-cand2b_C2B_e2|greedy|all": {
   "n": 234,
   "a": 0.24786324786324787,
   "b": 0.24786324786324787,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand2b_C2B_e2_tt-cand2b_C2B_e2|greedy|bag_game": {
   "n": 64,
   "a": 0.46875,
   "b": 0.46875,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand2b_C2B_e2_tt-cand2b_C2B_e2|greedy|stochastic_grid": {
   "n": 64,
   "a": 0.15625,
   "b": 0.15625,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand2b_C2B_e2_tt-cand2b_C2B_e2|greedy|tic_tac_toe": {
   "n": 74,
   "a": 0.22972972972972974,
   "b": 0.22972972972972974,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand2b_C2B_e2_tt-cand2b_C2B_e2|greedy|exact_minesweeper": {
   "n": 32,
   "a": 0.03125,
   "b": 0.03125,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand2b_C2B_e2_tt-v10_2b_p130|sampled|all": {
   "n": 234,
   "a": 0.2264957264957265,
   "b": 0.23931623931623933,
   "delta": -0.01282051282051282,
   "ci95": [
    -0.03418803418803419,
    0.008547008547008548
   ]
  },
  "cand2b_C2B_e2_tt-v10_2b_p130|sampled|bag_game": {
   "n": 64,
   "a": 0.41796875,
   "b": 0.41796875,
   "delta": 0.0,
   "ci95": [
    -0.0546875,
    0.0546875
   ]
  },
  "cand2b_C2B_e2_tt-v10_2b_p130|sampled|stochastic_grid": {
   "n": 64,
   "a": 0.1484375,
   "b": 0.19140625,
   "delta": -0.04296875,
   "ci95": [
    -0.078125,
    -0.01171875
   ]
  },
  "cand2b_C2B_e2_tt-v10_2b_p130|sampled|tic_tac_toe": {
   "n": 74,
   "a": 0.22635135135135134,
   "b": 0.22972972972972974,
   "delta": -0.0033783783783783786,
   "ci95": [
    -0.037162162162162164,
    0.033783783783783786
   ]
  },
  "cand2b_C2B_e2_tt-v10_2b_p130|sampled|exact_minesweeper": {
   "n": 32,
   "a": 0.0,
   "b": 0.0,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand2b_C2B_e2_tt-v10_2b_p130|greedy|all": {
   "n": 234,
   "a": 0.24786324786324787,
   "b": 0.2692307692307692,
   "delta": -0.021367521367521368,
   "ci95": [
    -0.05555555555555555,
    0.01282051282051282
   ]
  },
  "cand2b_C2B_e2_tt-v10_2b_p130|greedy|bag_game": {
   "n": 64,
   "a": 0.46875,
   "b": 0.578125,
   "delta": -0.109375,
   "ci95": [
    -0.203125,
    -0.03125
   ]
  },
  "cand2b_C2B_e2_tt-v10_2b_p130|greedy|stochastic_grid": {
   "n": 64,
   "a": 0.15625,
   "b": 0.15625,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand2b_C2B_e2_tt-v10_2b_p130|greedy|tic_tac_toe": {
   "n": 74,
   "a": 0.22972972972972974,
   "b": 0.21621621621621623,
   "delta": 0.013513513513513514,
   "ci95": [
    -0.06756756756756757,
    0.0945945945945946
   ]
  },
  "cand2b_C2B_e2_tt-v10_2b_p130|greedy|exact_minesweeper": {
   "n": 32,
   "a": 0.03125,
   "b": 0.0,
   "delta": 0.03125,
   "ci95": [
    0.0,
    0.09375
   ]
  }
 },
 "browser": {
  "cand2b_C2B_e2_tt-cand2b_C2B_e2|sampled|all": {
   "n": 176,
   "a": 0.9034090909090909,
   "b": 0.9034090909090909,
   "delta": 0.0,
   "ci95": [
    -0.017045454545454544,
    0.017045454545454544
   ]
  },
  "cand2b_C2B_e2_tt-cand2b_C2B_e2|sampled|rewarded": {
   "n": 128,
   "a": 0.8984375,
   "b": 0.90625,
   "delta": -0.0078125,
   "ci95": [
    -0.0234375,
    0.0
   ]
  },
  "cand2b_C2B_e2_tt-cand2b_C2B_e2|sampled|held_out": {
   "n": 48,
   "a": 0.9166666666666666,
   "b": 0.8958333333333334,
   "delta": 0.020833333333333332,
   "ci95": [
    0.0,
    0.0625
   ]
  },
  "cand2b_C2B_e2_tt-cand2b_C2B_e2|greedy|all": {
   "n": 176,
   "a": 0.9204545454545454,
   "b": 0.9204545454545454,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand2b_C2B_e2_tt-cand2b_C2B_e2|greedy|rewarded": {
   "n": 128,
   "a": 0.921875,
   "b": 0.921875,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand2b_C2B_e2_tt-cand2b_C2B_e2|greedy|held_out": {
   "n": 48,
   "a": 0.9166666666666666,
   "b": 0.9166666666666666,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "cand2b_C2B_e2_tt-v10_2b_p130|sampled|all": {
   "n": 176,
   "a": 0.9034090909090909,
   "b": 0.9318181818181818,
   "delta": -0.028409090909090908,
   "ci95": [
    -0.07386363636363637,
    0.011363636363636364
   ]
  },
  "cand2b_C2B_e2_tt-v10_2b_p130|sampled|rewarded": {
   "n": 128,
   "a": 0.8984375,
   "b": 0.9375,
   "delta": -0.0390625,
   "ci95": [
    -0.09375,
    0.0078125
   ]
  },
  "cand2b_C2B_e2_tt-v10_2b_p130|sampled|held_out": {
   "n": 48,
   "a": 0.9166666666666666,
   "b": 0.9166666666666666,
   "delta": 0.0,
   "ci95": [
    -0.08333333333333333,
    0.08333333333333333
   ]
  },
  "cand2b_C2B_e2_tt-v10_2b_p130|greedy|all": {
   "n": 176,
   "a": 0.9204545454545454,
   "b": 0.9147727272727273,
   "delta": 0.005681818181818182,
   "ci95": [
    -0.0342329545454545,
    0.045454545454545456
   ]
  },
  "cand2b_C2B_e2_tt-v10_2b_p130|greedy|rewarded": {
   "n": 128,
   "a": 0.921875,
   "b": 0.9140625,
   "delta": 0.0078125,
   "ci95": [
    -0.046875,
    0.0625
   ]
  },
  "cand2b_C2B_e2_tt-v10_2b_p130|greedy|held_out": {
   "n": 48,
   "a": 0.9166666666666666,
   "b": 0.9166666666666666,
   "delta": 0.0,
   "ci95": [
    -0.0625,
    0.0625
   ]
  }
 },
 "text_games_greedy": {
  "pong": 8.0,
  "freeway": 1.0,
  "breakout": 22.0,
  "frozenlake": 0.0,
  "cliffwalking": -13.0,
  "blackjack": -1.0,
  "minigrid_empty": 0.0,
  "minigrid_lavagap": 0.0,
  "minigrid_doorkey": 0.0,
  "babyai_goto": 0.1915625
 },
 "probes": {
  "router_n": 31,
  "router_tier": 0.935,
  "needs_live_data": 0.774,
  "commands_n": 45,
  "command_risk": 0.822,
  "destructive_called_safe": 0,
  "outside_project": 0.556,
  "browser_element": 0.938,
  "browser_action": 0.875,
  "batteries": {
   "generic": 0.85,
   "specific": 0.95,
   "catchall": 0.95,
   "noul_acc": 0.939,
   "noul_brier": 0.048,
   "n_choice": 60,
   "n_noul": 49,
   "abstain_battery": "8/8"
  }
 },
 "issue9": {
  "c_1": {
   "pred": 1,
   "gold": 6,
   "ok": false,
   "p_gold": 0.371,
   "p_pred": 0.436,
   "T": 1.145
  },
  "c_2": {
   "pred": 19,
   "gold": 20,
   "ok": false,
   "p_gold": 0.385,
   "p_pred": 0.477,
   "T": 1.145
  },
  "b_10": {
   "pred": 5,
   "gold": 0,
   "ok": false,
   "p_gold": 0.35,
   "p_pred": 0.411,
   "T": 1.145
  },
  "a2_4": {
   "pred": 3,
   "gold": 4,
   "ok": false,
   "p_gold": 0.2,
   "p_pred": 0.619,
   "T": 1.145
  }
 },
 "bespoke": {
  "vitaminc-dev": {
   "n": 599,
   "type": "choice",
   "acc": 0.6394,
   "ece": 0.1965,
   "brier": 0.5142,
   "nll": 0.9165,
   "trained": false,
   "sec": 10.3
  },
  "massive-en-US": {
   "n": 350,
   "type": "choice",
   "acc": 0.8143,
   "ece": 0.0283,
   "brier": 0.271,
   "nll": 0.6137,
   "trained": true,
   "sec": 3.9
  },
  "massive-de-DE": {
   "n": 350,
   "type": "choice",
   "acc": 0.7829,
   "ece": 0.0479,
   "brier": 0.31,
   "nll": 0.7328,
   "trained": false,
   "sec": 2.9
  },
  "boolq": {
   "n": 300,
   "type": "noul",
   "acc": 0.85,
   "ece": 0.0213,
   "brier": 0.2386,
   "nll": 0.3897,
   "trained": true,
   "sec": 5.5
  },
  "squad2": {
   "n": 299,
   "type": "noul",
   "acc": 0.7425,
   "ece": 0.038,
   "brier": 0.342,
   "nll": 0.5098,
   "trained": false,
   "sec": 5.2
  },
  "paws": {
   "n": 250,
   "type": "noul",
   "acc": 0.76,
   "ece": 0.0636,
   "brier": 0.3157,
   "nll": 0.4813,
   "trained": true,
   "sec": 1.9
  },
  "multinli": {
   "n": 299,
   "type": "choice",
   "acc": 0.8528,
   "ece": 0.0371,
   "brier": 0.2064,
   "nll": 0.3756,
   "trained": true,
   "sec": 2.3
  },
  "civil_comments": {
   "n": 300,
   "type": "noul",
   "acc": 0.87,
   "ece": 0.0508,
   "brier": 0.198,
   "nll": 0.3286,
   "trained": true,
   "sec": 2.3
  },
  "aegis2": {
   "n": 250,
   "type": "noul",
   "acc": 0.716,
   "ece": 0.0566,
   "brier": 0.3606,
   "nll": 0.5352,
   "trained": false,
   "sec": 4.9
  },
  "helpsteer2": {
   "n": 249,
   "type": "score",
   "acc": 0.4458,
   "ece": 0.1666,
   "brier": 0.6987,
   "nll": 1.3297,
   "trained": true,
   "sec": 17.8,
   "score_mae": 0.8578
  },
  "summeval-relevance": {
   "n": 240,
   "type": "score",
   "acc": 0.2292,
   "ece": 0.1768,
   "brier": 0.7849,
   "nll": 1.4652,
   "trained": false,
   "sec": 13.4,
   "score_mae": 0.633
  },
  "summeval-consistency": {
   "n": 144,
   "type": "score",
   "acc": 0.7569,
   "ece": 0.3155,
   "brier": 0.4993,
   "nll": 0.9841,
   "trained": false,
   "sec": 8.2,
   "score_mae": 1.0267
  },
  "pubmedqa": {
   "n": 250,
   "type": "choice",
   "acc": 0.712,
   "ece": 0.0543,
   "brier": 0.4106,
   "nll": 0.74,
   "trained": true,
   "sec": 2.1
  },
  "_macro": 0.7055,
  "_micro": 0.7106,
  "_macro_untrained": 0.6445
 },
 "jevbench_public": {
  "easy": {
   "n": 48,
   "accuracy": 1.0,
   "invalid": 0,
   "ece": 0.015729166666666905,
   "by_family": {
    "extraction": {
     "n": 12,
     "accuracy": 1.0
    },
    "fact": {
     "n": 12,
     "accuracy": 1.0
    },
    "intent": {
     "n": 12,
     "accuracy": 1.0
    },
    "tool_selection": {
     "n": 12,
     "accuracy": 1.0
    }
   }
  },
  "standard": {
   "n": 72,
   "accuracy": 0.8888888888888888,
   "invalid": 0,
   "ece": 0.06715416666666665,
   "by_family": {
    "adequacy": {
     "n": 12,
     "accuracy": 0.6666666666666666
    },
    "extraction": {
     "n": 12,
     "accuracy": 1.0
    },
    "intent": {
     "n": 12,
     "accuracy": 1.0
    },
    "ordinal": {
     "n": 12,
     "accuracy": 1.0
    },
    "policy": {
     "n": 12,
     "accuracy": 0.8333333333333334
    },
    "routing": {
     "n": 12,
     "accuracy": 0.8333333333333334
    }
   }
  },
  "hard": {
   "n": 111,
   "accuracy": 0.5765765765765766,
   "invalid": 0,
   "ece": 0.17517657657657654,
   "by_family": {
    "adversarial": {
     "n": 6,
     "accuracy": 1.0
    },
    "ambiguous": {
     "n": 7,
     "accuracy": 0.5714285714285714
    },
    "judge_hard": {
     "n": 17,
     "accuracy": 0.5882352941176471
    },
    "long_policy": {
     "n": 19,
     "accuracy": 0.42105263157894735
    },
    "multi_hop": {
     "n": 18,
     "accuracy": 0.5555555555555556
    },
    "probability": {
     "n": 10,
     "accuracy": 0.7
    },
    "routing_hard": {
     "n": 5,
     "accuracy": 1.0
    },
    "temporal_numeric": {
     "n": 15,
     "accuracy": 0.2
    },
    "tradeoff": {
     "n": 6,
     "accuracy": 0.5
    },
    "trap": {
     "n": 8,
     "accuracy": 1.0
    }
   }
  }
 },
 "references": {
  "v10_2b": {
   "reg_in": 0.8058,
   "reg_out": 0.7549,
   "reg_in_nll": 0.4736,
   "reg_out_nll": 0.6224,
   "reg_in_ece": 0.0378,
   "reg_out_ece": 0.0837
  }
 },
 "speed": "not measured again: the architecture and size equal the parent's, and the temperature map does not change the computation"
}