| { |
| "prompt": 2, |
| "orders": 1, |
| "method": "temperature per question type and per option-count bucket (choice), by NLL, fitted on the out-of-distribution calibration split only; thresholds per type: the most permissive threshold whose accepted held-out calibration decisions (10 family folds, each refitted without its families) have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers", |
| "temperature_per_prim": { |
| "choice": 1.3739536474580891, |
| "noul": 2.3443153776385026, |
| "score": 2.3443153776385026 |
| }, |
| "temperature_per_bucket": { |
| "choice": { |
| "2-10": 1.657681300768087, |
| "11-26": 1.2782470235604306, |
| "27-255": 0.9038571383911009 |
| } |
| }, |
| "thresholds": { |
| "0.01": { |
| "confidence": null, |
| "method": "per type: the most permissive threshold whose accepted held-out calibration decisions (10 family folds, each refitted without its families) have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers", |
| "per_type": { |
| "choice": { |
| "confidence": null, |
| "heldout_coverage": 0.026191679299309416, |
| "heldout_error": 0.09967845659163987 |
| }, |
| "noul": { |
| "confidence": null, |
| "heldout_coverage": 0.0, |
| "heldout_error": null |
| }, |
| "score": { |
| "confidence": null, |
| "heldout_coverage": 0.0, |
| "heldout_error": null |
| } |
| }, |
| "heldout_coverage": 0.015444207180811442, |
| "heldout_error": 0.09967845659163987 |
| }, |
| "0.05": { |
| "confidence": null, |
| "method": "per type: the most permissive threshold whose accepted held-out calibration decisions (10 family folds, each refitted without its families) have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers", |
| "per_type": { |
| "choice": { |
| "confidence": 0.9497840170762596, |
| "heldout_coverage": 0.46749199932625907, |
| "heldout_error": 0.027202305890830503 |
| }, |
| "noul": { |
| "confidence": 0.8899642791576896, |
| "heldout_coverage": 0.46792809839167454, |
| "heldout_error": 0.01657905378083302 |
| }, |
| "score": { |
| "confidence": null, |
| "heldout_coverage": 0.0, |
| "heldout_error": null |
| } |
| }, |
| "heldout_coverage": 0.39847047723096785, |
| "heldout_error": 0.023928215353938187 |
| } |
| }, |
| "gates": { |
| "ece_lofo": { |
| "choice": { |
| "pooled": 0.024225138602409248, |
| "n": 11874, |
| "gate": 0.05, |
| "pass": true |
| }, |
| "noul": { |
| "pooled": 0.040490891273480684, |
| "n": 5285, |
| "gate": 0.05, |
| "pass": true |
| }, |
| "score": { |
| "pooled": 0.045007389421837615, |
| "n": 2978, |
| "gate": 0.05, |
| "pass": true |
| } |
| }, |
| "ece_worst_fold": 0.0913168098241169, |
| "worst_fold": "agent", |
| "automated_at": { |
| "0.01": 0.015444207180811442, |
| "0.05": 0.39847047723096785 |
| }, |
| "automated_error": { |
| "0.01": 0.09967845659163987, |
| "0.05": 0.023928215353938187 |
| }, |
| "note": "held-out numbers: every evaluation row predicted by the fit without its family" |
| }, |
| "test": { |
| "T": { |
| "choice": { |
| "acc": 0.8288697995620684, |
| "ece": 0.025100915016266823, |
| "n": 11874 |
| }, |
| "noul": { |
| "acc": 0.867360454115421, |
| "ece": 0.04020157397242457, |
| "n": 5285 |
| }, |
| "score": { |
| "acc": 0.6645399597044996, |
| "ece": 0.04453554791809394, |
| "n": 2978 |
| }, |
| "all": { |
| "acc": 0.8146695138302626, |
| "ece": 0.02605664299441066, |
| "n": 20137 |
| } |
| }, |
| "note": "evaluation pool (development and test splits), full fit" |
| }, |
| "n": { |
| "fit": 3560, |
| "many": 1500, |
| "eval": 20137 |
| }, |
| "engine": "fast", |
| "fitted_on": { |
| "split": "out-of-distribution calibration split (sealed-seed BH-, JDI- and agent-shaped families, many-option items)", |
| "orders": 1, |
| "readouts": "basal-serve --mode fast, ModelOpt FP8 package (recipe b2)" |
| }, |
| "certified_on": "the same split (held-out folds)", |
| "evaluated_on": "development and test splits, leave-one-family-out and leave-one-language-out" |
| } |
|
|
