| { |
| "eval_source": "calibration_slice", |
| "eval_mode": "in_sample_calibration", |
| "is_held_out": false, |
| "eval_items": 400, |
| "note": "Metrics reflect in-sample calibration fit, not independent held-out evaluation. Provide --eval for held-out validation.", |
| "before": { |
| "items": 400, |
| "loss": 1.6005, |
| "accuracy": 0.4, |
| "mean_confidence": 0.6787, |
| "ece": 0.2787, |
| "brier": 0.7818, |
| "brier_top1": 0.3007, |
| "by_type": { |
| "choice": { |
| "items": 197, |
| "loss": 2.5345, |
| "accuracy": 0.2081, |
| "mean_confidence": 0.6239, |
| "ece": 0.4157, |
| "brier": 1.079, |
| "brier_top1": 0.3564 |
| }, |
| "noul": { |
| "items": 203, |
| "loss": 0.6941, |
| "accuracy": 0.5862, |
| "mean_confidence": 0.732, |
| "ece": 0.1485, |
| "brier": 0.4934, |
| "brier_top1": 0.2467 |
| } |
| } |
| }, |
| "after": { |
| "items": 400, |
| "loss": 0.4219, |
| "accuracy": 0.8525, |
| "mean_confidence": 0.8694, |
| "ece": 0.0372, |
| "brier": 0.2124, |
| "brier_top1": 0.0912, |
| "by_type": { |
| "choice": { |
| "items": 197, |
| "loss": 0.8566, |
| "accuracy": 0.7005, |
| "mean_confidence": 0.7349, |
| "ece": 0.0755, |
| "brier": 0.4313, |
| "brier_top1": 0.1852 |
| }, |
| "noul": { |
| "items": 203, |
| "loss": 0.0, |
| "accuracy": 1.0, |
| "mean_confidence": 1.0, |
| "ece": 0.0, |
| "brier": 0.0, |
| "brier_top1": 0.0 |
| } |
| } |
| }, |
| "comparison": { |
| "delta_accuracy": 0.4525, |
| "delta_loss": -1.1786, |
| "delta_ece": -0.2415, |
| "delta_brier": -0.5694, |
| "delta_brier_top1": -0.2095, |
| "delta_mean_confidence": 0.1907 |
| }, |
| "calibration": { |
| "choice": { |
| "items": 197, |
| "temperature": 2.3200724124908447, |
| "issues": [], |
| "accuracy": 0.700507614213198, |
| "mean_confidence": 0.7349079253696408 |
| }, |
| "score": { |
| "items": 0, |
| "temperature": 1.0, |
| "issues": [] |
| }, |
| "noul": { |
| "items": 203, |
| "temperature": 1.0, |
| "issues": [ |
| "the fit came back unchanged at 1.0 (accuracy 1.00, mean confidence 1.000 on the calibration items), which usually means they gave it nothing to correct; this type's confidences are uncalibrated" |
| ], |
| "accuracy": 1.0, |
| "mean_confidence": 0.9999999999981326 |
| } |
| }, |
| "abstention_thresholds": { |
| "choice:6-10": 0.8826, |
| "noul:2": 1.0 |
| }, |
| "training": { |
| "epochs": 4, |
| "micro_batch": 8, |
| "grad_accum": 8, |
| "encoder_lr": 2.5e-05, |
| "head_lr": 0.0001, |
| "min_lr": 1e-06, |
| "weight_decay": 0.01, |
| "grad_clip": 1.0, |
| "loss": "rlcd", |
| "rl_samples": 4, |
| "sigma_start": 0.4, |
| "sigma_end": 0.1, |
| "w_sph": 0.75, |
| "w_rps": 1.0, |
| "shuffle_options": [], |
| "option_layout": null, |
| "freeze_encoder": false, |
| "calib_max": 400, |
| "calib_frac": 0.1, |
| "calib_seed": 20260922, |
| "seed": 0, |
| "max_len": null, |
| "head_max_len": null, |
| "amp": null, |
| "gradient_checkpointing": null, |
| "log_every": 100, |
| "label_smoothing": 0.0, |
| "text_column": "text", |
| "label_column": "label", |
| "question_id": "label", |
| "instructions": null, |
| "eval_data": null, |
| "target_error": 0.1, |
| "min_abstain_n": 10 |
| } |
| } |
