| { | |
| "eval_source": "/kaggle/input/datasets/anirudhbharadwajpr/laya-router-training-data/eval.jsonl", | |
| "eval_mode": "held_out", | |
| "is_held_out": true, | |
| "eval_items": 39, | |
| "note": "Metrics reflect independent held-out evaluation.", | |
| "before": { | |
| "items": 39, | |
| "loss": 1.0705, | |
| "accuracy": 0.7436, | |
| "mean_confidence": 0.3639, | |
| "ece": 0.3797, | |
| "brier": 0.5708, | |
| "brier_top1": 0.3058, | |
| "by_type": { | |
| "choice": { | |
| "items": 39, | |
| "loss": 1.0705, | |
| "accuracy": 0.7436, | |
| "mean_confidence": 0.3639, | |
| "ece": 0.3797, | |
| "brier": 0.5708, | |
| "brier_top1": 0.3058 | |
| } | |
| } | |
| }, | |
| "after": { | |
| "items": 39, | |
| "loss": 0.6364, | |
| "accuracy": 0.9744, | |
| "mean_confidence": 0.9998, | |
| "ece": 0.0255, | |
| "brier": 0.0513, | |
| "brier_top1": 0.0256, | |
| "by_type": { | |
| "choice": { | |
| "items": 39, | |
| "loss": 0.6364, | |
| "accuracy": 0.9744, | |
| "mean_confidence": 0.9998, | |
| "ece": 0.0255, | |
| "brier": 0.0513, | |
| "brier_top1": 0.0256 | |
| } | |
| } | |
| }, | |
| "comparison": { | |
| "delta_accuracy": 0.2308, | |
| "delta_loss": -0.4341, | |
| "delta_ece": -0.3542, | |
| "delta_brier": -0.5195, | |
| "delta_brier_top1": -0.2802, | |
| "delta_mean_confidence": 0.6359 | |
| }, | |
| "calibration": { | |
| "choice": { | |
| "items": 16, | |
| "temperature": 1.0, | |
| "issues": [ | |
| "fitted on only 16 calibration items", | |
| "the fit came back unchanged at 1.0 (accuracy 1.00, mean confidence 1.000 on the calibration items), which usually means they gave it nothing to correct; this type's confidences are uncalibrated" | |
| ], | |
| "accuracy": 1.0, | |
| "mean_confidence": 0.999999883761271 | |
| }, | |
| "score": { | |
| "items": 0, | |
| "temperature": 1.0, | |
| "issues": [] | |
| }, | |
| "noul": { | |
| "items": 0, | |
| "temperature": 1.0, | |
| "issues": [] | |
| } | |
| }, | |
| "abstention_thresholds": { | |
| "choice:3-5": 1.0 | |
| }, | |
| "training": { | |
| "epochs": 4, | |
| "micro_batch": 4, | |
| "grad_accum": 4, | |
| "encoder_lr": 2.5e-05, | |
| "head_lr": 0.0001, | |
| "min_lr": 1e-06, | |
| "weight_decay": 0.01, | |
| "grad_clip": 1.0, | |
| "loss": "rlcd", | |
| "rl_samples": 4, | |
| "sigma_start": 0.4, | |
| "sigma_end": 0.1, | |
| "w_sph": 0.75, | |
| "w_rps": 1.0, | |
| "shuffle_options": [ | |
| "choice" | |
| ], | |
| "option_layout": null, | |
| "freeze_encoder": false, | |
| "calib_max": 400, | |
| "calib_frac": 0.1, | |
| "calib_seed": 20260922, | |
| "seed": 42, | |
| "max_len": null, | |
| "head_max_len": null, | |
| "amp": null, | |
| "gradient_checkpointing": null, | |
| "log_every": 100, | |
| "label_smoothing": 0.0, | |
| "text_column": "text", | |
| "label_column": "label", | |
| "question_id": "label", | |
| "instructions": null, | |
| "eval_data": "/kaggle/input/datasets/anirudhbharadwajpr/laya-router-training-data/eval.jsonl", | |
| "target_error": 0.1, | |
| "min_abstain_n": 10 | |
| } | |
| } |