File size: 4,765 Bytes
6da389f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
{
  "encoder": "jhu-clsp/mmBERT-base",
  "head_layers": 2,
  "max_len": 1024,
  "head_max_len": 256,
  "max_prefixes": 6,
  "act_costs": {
    "escalate": 0.5
  },
  "cost_wrong_act": 3.0,
  "amp_dtype": "bf16",
  "model_name": "rl-agent",
  "temperature": [
    3.505923271179199,
    1.0,
    1.0
  ],
  "training": {
    "updates": 15987,
    "epochs_completed": 4,
    "hours": 4.97,
    "world_size": 1,
    "fine_tuned_from_checkpoint": false,
    "laya_train": {
      "epochs": 4,
      "micro_batch": 8,
      "grad_accum": 2,
      "encoder_lr": 2.5e-05,
      "head_lr": 0.0001,
      "min_lr": 1e-06,
      "weight_decay": 0.01,
      "grad_clip": 1.0,
      "loss": "soft-ce",
      "rl_samples": 4,
      "sigma_start": 0.4,
      "sigma_end": 0.1,
      "w_sph": 0.75,
      "w_rps": 1.0,
      "shuffle_options": [],
      "option_layout": null,
      "freeze_encoder": false,
      "calib_max": 0,
      "calib_frac": 0.0,
      "calib_seed": 20260922,
      "seed": 20261026,
      "max_len": null,
      "head_max_len": null,
      "amp": null,
      "gradient_checkpointing": null,
      "log_every": 100,
      "label_smoothing": 0.0,
      "text_column": "text",
      "label_column": "label",
      "question_id": "label",
      "instructions": null,
      "eval_data": null,
      "target_error": 0.1,
      "min_abstain_n": 10
    },
    "laya_train_calibration": {
      "choice": {
        "items": 600,
        "temperature": 3.505923271179199,
        "issues": [],
        "accuracy": 0.96,
        "mean_confidence": 0.9676679738041832
      },
      "score": {
        "items": 0,
        "temperature": 1.0,
        "issues": []
      },
      "noul": {
        "items": 0,
        "temperature": 1.0,
        "issues": []
      }
    },
    "train_report": {
      "eval_source": "calibration_v4.jsonl (independent family-disjoint split)",
      "eval_mode": "held_out",
      "is_held_out": true,
      "eval_items": 600,
      "note": "No evaluation performed (0 calibration items and no --eval set was provided).",
      "before": null,
      "after": null,
      "comparison": null,
      "calibration": {
        "choice": {
          "items": 600,
          "temperature": 3.505923271179199,
          "issues": [],
          "accuracy": 0.96,
          "mean_confidence": 0.9676679738041832
        },
        "score": {
          "items": 0,
          "temperature": 1.0,
          "issues": []
        },
        "noul": {
          "items": 0,
          "temperature": 1.0,
          "issues": []
        }
      },
      "abstention_thresholds": {
        "choice:6-10": 0.4057,
        "choice:3-5": 0.5571
      },
      "training": {
        "epochs": 4,
        "micro_batch": 8,
        "grad_accum": 2,
        "encoder_lr": 2.5e-05,
        "head_lr": 0.0001,
        "min_lr": 1e-06,
        "weight_decay": 0.01,
        "grad_clip": 1.0,
        "loss": "soft-ce",
        "rl_samples": 4,
        "sigma_start": 0.4,
        "sigma_end": 0.1,
        "w_sph": 0.75,
        "w_rps": 1.0,
        "shuffle_options": [],
        "option_layout": null,
        "freeze_encoder": false,
        "calib_max": 0,
        "calib_frac": 0.0,
        "calib_seed": 20260922,
        "seed": 20261026,
        "max_len": null,
        "head_max_len": null,
        "amp": null,
        "gradient_checkpointing": null,
        "log_every": 100,
        "label_smoothing": 0.0,
        "text_column": "text",
        "label_column": "label",
        "question_id": "label",
        "instructions": null,
        "eval_data": null,
        "target_error": 0.1,
        "min_abstain_n": 10
      },
      "before_calibration": {
        "items": 600,
        "loss": 0.3863,
        "accuracy": 0.96,
        "mean_confidence": 0.9971,
        "ece": 0.0399,
        "brier": 0.0781,
        "brier_top1": 0.0388,
        "by_type": {
          "choice": {
            "items": 600,
            "loss": 0.3863,
            "accuracy": 0.96,
            "mean_confidence": 0.9971,
            "ece": 0.0399,
            "brier": 0.0781,
            "brier_top1": 0.0388
          }
        }
      },
      "after_calibration": {
        "items": 600,
        "loss": 0.1454,
        "accuracy": 0.96,
        "mean_confidence": 0.9677,
        "ece": 0.015,
        "brier": 0.067,
        "brier_top1": 0.0311,
        "by_type": {
          "choice": {
            "items": 600,
            "loss": 0.1454,
            "accuracy": 0.96,
            "mean_confidence": 0.9677,
            "ece": 0.015,
            "brier": 0.067,
            "brier_top1": 0.0311
          }
        }
      }
    },
    "abstention_thresholds": {
      "choice:6-10": 0.4057,
      "choice:3-5": 0.5571
    }
  },
  "fine_tuned": true
}