| { |
| "prompt": 2, |
| "orders": 1, |
| "method": "temperature per question type and per option-count bucket (choice), by NLL, fitted on the out-of-distribution calibration split only; thresholds per type: the most permissive threshold whose accepted calibration decisions have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers", |
| "temperature_per_prim": { |
| "choice": 2.996614153753363, |
| "noul": 7.999999999999998, |
| "score": 4.756828460010884 |
| }, |
| "temperature_per_bucket": { |
| "choice": { |
| "2-10": 3.830413122794294, |
| "11-26": 2.747907294916178, |
| "27-255": 1.731073122012286 |
| } |
| }, |
| "thresholds": { |
| "0.01": { |
| "confidence": null, |
| "method": "per type: the most permissive threshold whose accepted calibration decisions have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers", |
| "per_type": { |
| "choice": { |
| "confidence": null, |
| "heldout_coverage": 0.0, |
| "heldout_error": null |
| }, |
| "noul": { |
| "confidence": null, |
| "heldout_coverage": 0.0, |
| "heldout_error": null |
| }, |
| "score": { |
| "confidence": null, |
| "heldout_coverage": 0.0, |
| "heldout_error": null |
| } |
| }, |
| "heldout_coverage": 0.0, |
| "heldout_error": null |
| }, |
| "0.05": { |
| "confidence": null, |
| "method": "per type: the most permissive threshold whose accepted calibration decisions have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers", |
| "per_type": { |
| "choice": { |
| "confidence": null, |
| "heldout_coverage": 0.0, |
| "heldout_error": null |
| }, |
| "noul": { |
| "confidence": null, |
| "heldout_coverage": 0.0, |
| "heldout_error": null |
| }, |
| "score": { |
| "confidence": null, |
| "heldout_coverage": 0.0, |
| "heldout_error": null |
| } |
| }, |
| "heldout_coverage": 0.0, |
| "heldout_error": null |
| } |
| }, |
| "gates": { |
| "ece_lofo": { |
| "choice": { |
| "pooled": 0.0606221052989346, |
| "n": 7229, |
| "gate": 0.05, |
| "pass": false |
| }, |
| "noul": { |
| "pooled": 0.05231366996904953, |
| "n": 3197, |
| "gate": 0.05, |
| "pass": false |
| }, |
| "score": { |
| "pooled": 0.0740364665334774, |
| "n": 2978, |
| "gate": 0.05, |
| "pass": false |
| } |
| }, |
| "ece_worst_fold": 0.12149008840247141, |
| "worst_fold": "agent", |
| "automated_at": { |
| "0.01": 0.0, |
| "0.05": 0.0 |
| }, |
| "automated_error": { |
| "0.01": null, |
| "0.05": null |
| }, |
| "note": "held-out numbers: every evaluation row predicted by the fit without its family" |
| }, |
| "test": { |
| "T": { |
| "choice": { |
| "acc": 0.6067229215659151, |
| "ece": 0.05329490346712401, |
| "n": 7229 |
| }, |
| "noul": { |
| "acc": 0.737566468564279, |
| "ece": 0.05231366996904953, |
| "n": 3197 |
| }, |
| "score": { |
| "acc": 0.5957018132975151, |
| "ece": 0.08010245549854482, |
| "n": 2978 |
| }, |
| "all": { |
| "acc": 0.6354819456878543, |
| "ece": 0.0323180305717361, |
| "n": 13404 |
| } |
| }, |
| "note": "evaluation pool (development and test splits), full fit" |
| }, |
| "n": { |
| "fit": 2614, |
| "many": 885, |
| "eval": 13404 |
| }, |
| "engine": "think", |
| "fitted_on": { |
| "split": "out-of-distribution calibration split (sealed-seed BH-, JDI- and agent-shaped families, many-option items)", |
| "orders": 1 |
| }, |
| "certified_on": "the same split", |
| "evaluated_on": "development and test splits, leave-one-family-out and leave-one-language-out" |
| } |
|
|
