{
 "prompt": 2,
 "orders": 1,
 "method": "temperature per question type and per option-count bucket (choice), by NLL, fitted on the out-of-distribution calibration split only; thresholds per type: the most permissive threshold whose accepted held-out calibration decisions (10 family folds, each refitted without its families) have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers",
 "temperature_per_prim": {
  "choice": 1.3739536474580891,
  "noul": 2.3443153776385026,
  "score": 2.3443153776385026
 },
 "temperature_per_bucket": {
  "choice": {
   "2-10": 1.657681300768087,
   "11-26": 1.2782470235604306,
   "27-255": 0.9038571383911009
  }
 },
 "thresholds": {
  "0.01": {
   "confidence": null,
   "method": "per type: the most permissive threshold whose accepted held-out calibration decisions (10 family folds, each refitted without its families) have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers",
   "per_type": {
    "choice": {
     "confidence": null,
     "heldout_coverage": 0.026191679299309416,
     "heldout_error": 0.09967845659163987
    },
    "noul": {
     "confidence": null,
     "heldout_coverage": 0.0,
     "heldout_error": null
    },
    "score": {
     "confidence": null,
     "heldout_coverage": 0.0,
     "heldout_error": null
    }
   },
   "heldout_coverage": 0.015444207180811442,
   "heldout_error": 0.09967845659163987
  },
  "0.05": {
   "confidence": null,
   "method": "per type: the most permissive threshold whose accepted held-out calibration decisions (10 family folds, each refitted without its families) have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers",
   "per_type": {
    "choice": {
     "confidence": 0.9497840170762596,
     "heldout_coverage": 0.46749199932625907,
     "heldout_error": 0.027202305890830503
    },
    "noul": {
     "confidence": 0.8899642791576896,
     "heldout_coverage": 0.46792809839167454,
     "heldout_error": 0.01657905378083302
    },
    "score": {
     "confidence": null,
     "heldout_coverage": 0.0,
     "heldout_error": null
    }
   },
   "heldout_coverage": 0.39847047723096785,
   "heldout_error": 0.023928215353938187
  }
 },
 "gates": {
  "ece_lofo": {
   "choice": {
    "pooled": 0.024225138602409248,
    "n": 11874,
    "gate": 0.05,
    "pass": true
   },
   "noul": {
    "pooled": 0.040490891273480684,
    "n": 5285,
    "gate": 0.05,
    "pass": true
   },
   "score": {
    "pooled": 0.045007389421837615,
    "n": 2978,
    "gate": 0.05,
    "pass": true
   }
  },
  "ece_worst_fold": 0.0913168098241169,
  "worst_fold": "agent",
  "automated_at": {
   "0.01": 0.015444207180811442,
   "0.05": 0.39847047723096785
  },
  "automated_error": {
   "0.01": 0.09967845659163987,
   "0.05": 0.023928215353938187
  },
  "note": "held-out numbers: every evaluation row predicted by the fit without its family"
 },
 "test": {
  "T": {
   "choice": {
    "acc": 0.8288697995620684,
    "ece": 0.025100915016266823,
    "n": 11874
   },
   "noul": {
    "acc": 0.867360454115421,
    "ece": 0.04020157397242457,
    "n": 5285
   },
   "score": {
    "acc": 0.6645399597044996,
    "ece": 0.04453554791809394,
    "n": 2978
   },
   "all": {
    "acc": 0.8146695138302626,
    "ece": 0.02605664299441066,
    "n": 20137
   }
  },
  "note": "evaluation pool (development and test splits), full fit"
 },
 "n": {
  "fit": 3560,
  "many": 1500,
  "eval": 20137
 },
 "engine": "fast",
 "fitted_on": {
  "split": "out-of-distribution calibration split (sealed-seed BH-, JDI- and agent-shaped families, many-option items)",
  "orders": 1,
  "readouts": "basal-serve --mode fast, ModelOpt FP8 package (recipe b2)"
 },
 "certified_on": "the same split (held-out folds)",
 "evaluated_on": "development and test splits, leave-one-family-out and leave-one-language-out"
}