{ "eval_source": "val_v2.jsonl", "eval_mode": "held_out", "is_held_out": true, "eval_items": 2076, "note": "Metrics reflect independent held-out evaluation.", "before": { "items": 2076, "loss": 1.2961, "accuracy": 0.5766, "mean_confidence": 0.8606, "ece": 0.284, "brier": 0.6395, "brier_top1": 0.3198, "by_type": { "noul": { "items": 2076, "loss": 1.2961, "accuracy": 0.5766, "mean_confidence": 0.8606, "ece": 0.284, "brier": 0.6395, "brier_top1": 0.3198 } } }, "after": { "items": 2076, "loss": 0.194, "accuracy": 0.9475, "mean_confidence": 0.9749, "ece": 0.0278, "brier": 0.0972, "brier_top1": 0.0486, "by_type": { "noul": { "items": 2076, "loss": 0.194, "accuracy": 0.9475, "mean_confidence": 0.9749, "ece": 0.0278, "brier": 0.0972, "brier_top1": 0.0486 } } }, "comparison": { "delta_accuracy": 0.3709, "delta_loss": -1.1021, "delta_ece": -0.2562, "delta_brier": -0.5423, "delta_brier_top1": -0.2712, "delta_mean_confidence": 0.1143 }, "calibration": { "choice": { "items": 0, "temperature": 1.0, "issues": [] }, "score": { "items": 0, "temperature": 1.0, "issues": [] }, "noul": { "items": 400, "temperature": 5.0, "issues": [ "the fitted temperature 5 is on the [0.5, 5] clamp" ], "accuracy": 0.9375, "mean_confidence": 0.9739458855878135 } }, "abstention_thresholds": { "noul:2": 0.6732 }, "training": { "epochs": 2, "micro_batch": 8, "grad_accum": 4, "encoder_lr": 2.5e-05, "head_lr": 0.0001, "min_lr": 1e-06, "weight_decay": 0.01, "grad_clip": 1.0, "loss": "rlcd", "rl_samples": 4, "sigma_start": 0.4, "sigma_end": 0.1, "w_sph": 0.75, "w_rps": 1.0, "shuffle_options": [], "option_layout": null, "freeze_encoder": false, "calib_max": 400, "calib_frac": 0.1, "calib_seed": 20260922, "seed": 0, "max_len": null, "head_max_len": null, "amp": null, "gradient_checkpointing": null, "log_every": 100, "label_smoothing": 0.0, "text_column": "text", "label_column": "label", "question_id": "label", "instructions": null, "eval_data": "val_v2.jsonl", "target_error": 0.1, "min_abstain_n": 10 } }