File size: 2,490 Bytes
41bdb4a 226ce32 41bdb4a 226ce32 41bdb4a 226ce32 41bdb4a 226ce32 41bdb4a 226ce32 41bdb4a 226ce32 41bdb4a 226ce32 41bdb4a 226ce32 41bdb4a 226ce32 41bdb4a 226ce32 41bdb4a 226ce32 41bdb4a 226ce32 41bdb4a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 | {
"eval_source": "corpus_v2/eval.jsonl",
"eval_mode": "held_out",
"is_held_out": true,
"eval_items": 54,
"note": "Metrics reflect independent held-out evaluation.",
"before": {
"items": 54,
"loss": 1.8596,
"accuracy": 0.2407,
"mean_confidence": 0.3148,
"ece": 0.1869,
"brier": 0.7831,
"brier_top1": 0.1308,
"by_type": {
"choice": {
"items": 54,
"loss": 1.8596,
"accuracy": 0.2407,
"mean_confidence": 0.3148,
"ece": 0.1869,
"brier": 0.7831,
"brier_top1": 0.1308
}
}
},
"after": {
"items": 54,
"loss": 0.6661,
"accuracy": 0.8704,
"mean_confidence": 0.9589,
"ece": 0.0886,
"brier": 0.2397,
"brier_top1": 0.1096,
"by_type": {
"choice": {
"items": 54,
"loss": 0.6661,
"accuracy": 0.8704,
"mean_confidence": 0.9589,
"ece": 0.0886,
"brier": 0.2397,
"brier_top1": 0.1096
}
}
},
"comparison": {
"delta_accuracy": 0.6297,
"delta_loss": -1.1935,
"delta_ece": -0.0983,
"delta_brier": -0.5434,
"delta_brier_top1": -0.0212,
"delta_mean_confidence": 0.6441
},
"calibration": {
"choice": {
"items": 400,
"temperature": 3.9046413898468018,
"issues": [],
"accuracy": 0.9575,
"mean_confidence": 0.9617928987797445
},
"score": {
"items": 0,
"temperature": 1.0,
"issues": []
},
"noul": {
"items": 0,
"temperature": 1.0,
"issues": []
}
},
"abstention_thresholds": {
"choice:6-10": 0.5204
},
"training": {
"epochs": 4,
"micro_batch": 2,
"grad_accum": 32,
"encoder_lr": 2.5e-05,
"head_lr": 0.0001,
"min_lr": 1e-06,
"weight_decay": 0.01,
"grad_clip": 1.0,
"loss": "rlcd",
"rl_samples": 4,
"sigma_start": 0.4,
"sigma_end": 0.1,
"w_sph": 0.75,
"w_rps": 1.0,
"shuffle_options": [
"choice"
],
"option_layout": null,
"freeze_encoder": false,
"calib_max": 400,
"calib_frac": 0.1,
"calib_seed": 20260922,
"seed": 0,
"max_len": 256,
"head_max_len": null,
"amp": null,
"gradient_checkpointing": null,
"log_every": 100,
"label_smoothing": 0.0,
"text_column": "text",
"label_column": "label",
"question_id": "label",
"instructions": null,
"eval_data": "corpus_v2/eval.jsonl",
"target_error": 0.1,
"min_abstain_n": 10
}
} |