suyeon918 commited on
Commit
c03caac
·
verified ·
1 Parent(s): 76a0f90

v13 (2026-11-18 BEST, checkpoint_latest 제외)

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ v13/tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text
v13/encoder/config.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "ModernBertForMaskedLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 2,
8
+ "classifier_activation": "gelu",
9
+ "classifier_bias": false,
10
+ "classifier_dropout": 0.0,
11
+ "classifier_pooling": "mean",
12
+ "cls_token_id": 1,
13
+ "decoder_bias": true,
14
+ "deterministic_flash_attn": false,
15
+ "dtype": "float32",
16
+ "embedding_dropout": 0.0,
17
+ "eos_token_id": 1,
18
+ "global_attn_every_n_layers": 3,
19
+ "gradient_checkpointing": false,
20
+ "hidden_activation": "gelu",
21
+ "hidden_size": 768,
22
+ "initializer_cutoff_factor": 2.0,
23
+ "initializer_range": 0.02,
24
+ "intermediate_size": 1152,
25
+ "layer_norm_eps": 1e-05,
26
+ "layer_types": [
27
+ "full_attention",
28
+ "sliding_attention",
29
+ "sliding_attention",
30
+ "full_attention",
31
+ "sliding_attention",
32
+ "sliding_attention",
33
+ "full_attention",
34
+ "sliding_attention",
35
+ "sliding_attention",
36
+ "full_attention",
37
+ "sliding_attention",
38
+ "sliding_attention",
39
+ "full_attention",
40
+ "sliding_attention",
41
+ "sliding_attention",
42
+ "full_attention",
43
+ "sliding_attention",
44
+ "sliding_attention",
45
+ "full_attention",
46
+ "sliding_attention",
47
+ "sliding_attention",
48
+ "full_attention"
49
+ ],
50
+ "local_attention": 128,
51
+ "mask_token_id": 4,
52
+ "max_position_embeddings": 8192,
53
+ "mlp_bias": false,
54
+ "mlp_dropout": 0.0,
55
+ "model_type": "modernbert",
56
+ "norm_bias": false,
57
+ "norm_eps": 1e-05,
58
+ "num_attention_heads": 12,
59
+ "num_hidden_layers": 22,
60
+ "pad_token_id": 0,
61
+ "position_embedding_type": "sans_pos",
62
+ "repad_logits_with_grad": false,
63
+ "rope_parameters": {
64
+ "full_attention": {
65
+ "rope_theta": 160000,
66
+ "rope_type": "default"
67
+ },
68
+ "sliding_attention": {
69
+ "rope_theta": 160000,
70
+ "rope_type": "default"
71
+ }
72
+ },
73
+ "sep_token_id": 1,
74
+ "sparse_pred_ignore_index": -100,
75
+ "sparse_prediction": false,
76
+ "tie_word_embeddings": true,
77
+ "transformers_version": "5.22.2",
78
+ "vocab_size": 256000
79
+ }
v13/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:726d99436b18ac487ac63d05e4f64e16e968fb88a2fd6a239f9189feb0b42a71
3
+ size 643835524
v13/questions.json ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "category": {
3
+ "type": "choice",
4
+ "instructions": "\uc774\uba54\uc77c\uc744 \uc5b4\ub290 \ub2f4\ub2f9 \ubd84\uc57c\uc5d0\uc11c \ucc98\ub9ac\ud574\uc57c \ud558\ub294\uac00?",
5
+ "criteria": {
6
+ "billing": "\uccad\uad6c\u00b7\uacb0\uc81c\u00b7\ud658\ubd88",
7
+ "technical": "\uae30\uc220 \ubb38\uc81c",
8
+ "sales": "\uc601\uc5c5\u00b7\uad6c\ub9e4",
9
+ "security": "\ubcf4\uc548",
10
+ "hr": "\uc778\uc0ac",
11
+ "other": "\uae30\ud0c0"
12
+ }
13
+ },
14
+ "is_spam": {
15
+ "type": "noul",
16
+ "instructions": "\uc6d0\ud558\uc9c0 \uc54a\ub294 \uc2a4\ud338 \ub610\ub294 \ub300\ub7c9 \ud64d\ubcf4 \uba54\uc77c\uc778\uac00?",
17
+ "criteria": {
18
+ "true": "\ucc38: \uc2a4\ud338\u00b7\ub300\ub7c9 \ud64d\ubcf4",
19
+ "false": "\uac70\uc9d3: \uc815\uc0c1 \uc774\uba54\uc77c"
20
+ }
21
+ },
22
+ "is_phishing": {
23
+ "type": "noul",
24
+ "instructions": "\uae08\uc804\u00b7\uc778\uc99d\uc815\ubcf4\u00b7\uac1c\uc778\uc815\ubcf4\ub97c \ud0c8\ucde8\ud558\ub824\ub294 \ud53c\uc2f1\uc774\ub098 \uc0ac\uae30\uc778\uac00?",
25
+ "criteria": {
26
+ "true": "\ucc38: \ud53c\uc2f1\u00b7\uc0ac\uae30\u00b7\ubd80\uc815\ud589\uc704",
27
+ "false": "\uac70\uc9d3: \uc815\uc0c1 \uc774\uba54\uc77c"
28
+ }
29
+ },
30
+ "urgency": {
31
+ "type": "score",
32
+ "instructions": "\ubcf8\ubb38 \uc694\uccad\uc744 \uc5bc\ub9c8\ub098 \uae34\uae09\ud558\uac8c \ucc98\ub9ac\ud574\uc57c \ud558\ub294\uac00?",
33
+ "criteria": [
34
+ "\ub0ae\uc74c: \uc2dc\uac04 \uc555\ubc15 \uc5c6\uc74c",
35
+ "\ubcf4\ud1b5: \ube60\ub978 \ud655\uc778 \ud544\uc694",
36
+ "\uae34\uae09: \uc5c5\ubb34 \ucc28\ub2e8 \ub610\ub294 \ud655\uc815 \uae30\ud55c"
37
+ ]
38
+ },
39
+ "needs_reply": {
40
+ "type": "noul",
41
+ "instructions": "\ubc1c\uc2e0\uc790\uac00 \ub2f5\uc7a5\uc744 \uae30\ub300\ud558\ub294\uac00?",
42
+ "criteria": {
43
+ "true": "\ucc38: \ub2f5\uc7a5\u00b7\ucc98\ub9ac\ub97c \uae30\ub300\ud568",
44
+ "false": "\uac70\uc9d3: \uc54c\ub9bc\u00b7\uc548\ub0b4\ub77c \ub2f5\uc7a5 \ubd88\ud544\uc694"
45
+ }
46
+ },
47
+ "action": {
48
+ "type": "choice",
49
+ "instructions": "\uc774 \uba54\uc77c\uc744 \ubc1b\uc740 \ub300\ud45c\uac00 \ub2e4\uc74c\uc5d0 \ud574\uc57c \ud560 \uc77c\uc740 \ubb34\uc5c7\uc778\uac00?",
50
+ "criteria": {
51
+ "\ud68c\uc2e0": "\ub2f5\uc7a5 \uba54\uc77c \ubcf8\ubb38\ub9cc\uc73c\ub85c \ub05d\ub0a8 (\uc9c8\ubb38 \ub2f5\ubcc0, \ud655\uc778, \uc218\ub77d\u00b7\uac70\uc808 \ud1b5\ubcf4)",
52
+ "\ubb38\uc11c\uc791\uc131": "\uc0c1\ub300\uac00 \uacac\uc801\u00b7\uc790\ub8cc\u00b7\uacc4\uc57d\uc11c\ub97c '\uc694\uccad'\ud574\uc11c \ub300\ud45c\uac00 \ubb38\uc11c\ub97c \ub9cc\ub4e4\uc5b4 \ubcf4\ub0b4\uc57c \ud568",
53
+ "\uc77c\uc815\ub4f1\ub85d": "\ubbf8\ud305\u00b7\ub0a9\uae30\u00b7\ub9c8\uac10 \uc77c\uc815\uc744 \ub4f1\ub85d\ud558\ub294 \uac83\uc774 \ud575\uc2ec",
54
+ "\uacb0\uc7ac\uc694\uccad": "\uc0c1\ub300\uac00 \uacac\uc801\u00b7\uccad\uad6c\uc11c\u00b7\uacc4\uc57d\uc11c\ub97c '\ubcf4\ub0b4\uc640\uc11c' \ub300\ud45c\uac00 \uc218\ub77d\u00b7\uc9c0\uae09\u00b7\ud658\ubd88\uc744 \uacb0\uc815\ud574\uc57c \ud568",
55
+ "\ucc98\ub9ac\ubd88\ud544\uc694": "\ub2e8\uc21c \uc548\ub0b4\u00b7\uad11\uace0\u00b7\uc774\ubbf8 \ud574\uacb0\ub41c \uac74"
56
+ }
57
+ },
58
+ "complexity": {
59
+ "type": "choice",
60
+ "instructions": "\ub300\ud45c\uac00 \ub9cc\ub4e4\uc5b4\uc57c \ud560 \uacb0\uacfc\ubb3c\uc774 \uba87 \uac1c\uc778\uac00? (\uc608\uc0b0\u00b7\ubc30\uc1a1\u00b7\uacb0\uc81c \uc870\uac74\uc774 \uc5ec\ub7ec \uac1c \uc801\ud600 \uc788\uc5b4\ub3c4 \uacac\uc801\uc11c \ud55c \uc7a5\uc774\uba74 \uacb0\uacfc\ubb3c\uc740 \ud558\ub098)",
61
+ "criteria": {
62
+ "\ub2e8\uc21c": "\uacb0\uacfc\ubb3c\uc774 \ud558\ub098\ub2e4 (\ud68c\uc2e0 1\ud1b5, \ub610\ub294 \ubb38\uc11c 1\uac1c)",
63
+ "\ubcf5\ud569": "\uc11c\ub85c \ub2e4\ub978 \uacb0\uacfc\ubb3c\uc774 2\uac1c \uc774\uc0c1\uc774\ub2e4 (\uc608: \uacac\uc801\uc11c+\uacc4\uc57d\uc11c+\uc77c\uc815), \ub610\ub294 \uc5ec\ub7ec \ubb38\uc11c\ub97c \uc885\ud569\ud574\uc57c \ud55c\ub2e4"
64
+ }
65
+ },
66
+ "sensitive": {
67
+ "type": "noul",
68
+ "instructions": "\uc774 \uba54\uc77c\uc5d0 \ud2b9\uc815 \uc0ac\ub78c\uc758 \uc778\uc801\uc0ac\ud56d(\uc8fc\ubbfc\ub4f1\ub85d\ubc88\ud638, \uc9d1 \uc8fc\uc18c, \uc0dd\ub144\uc6d4\uc77c, \uac1c\uc778 \uacc4\uc88c, \uac74\uac15 \uc815\ubcf4, \uc778\uc0ac \ud3c9\uac00, \uacc4\uc815 \ubcf4\uc548 \uc815\ubcf4\uc778 \ub85c\uadf8\uc778 IP\u00b7\uae30\uae30\u00b7\uc704\uce58)\uc774\ub098 \ud68c\uc0ac \ub0b4\ubd80 \uae30\ubc00(\uc6d0\uac00, \ub9c8\uc9c4, \uacf5\uac1c \uae08\uc9c0 \ub2e8\uac00, \uc7ac\ubb34 \uc0ac\uc815)\uc774 \uc2e4\uc81c\ub85c \uc801\ud600 \uc788\ub294\uac00?",
69
+ "criteria": {
70
+ "false": "\uc77c\ubc18 \uc5c5\ubb34 \ub0b4\uc6a9\ubfd0\uc774\ub2e4(\uacac\uc801 \uc694\uccad, \uc77c\uc815 \ubb38\uc758, \uc548\ub0b4, \uacf5\uc9c0). \uc778\uc801\uc0ac\ud56d\uc774\ub098 \uae30\ubc00\uc774 \uc2e4\uc81c\ub85c \uc801\ud600 \uc788\uc9c0 \uc54a\ub2e4.",
71
+ "true": "\ud2b9\uc815 \uc0ac\ub78c\uc758 \uc778\uc801\uc0ac\ud56d\uc774\ub098 \ud68c\uc0ac \ub0b4\ubd80 \uae30\ubc00\uc774 \uc2e4\uc81c\ub85c \uc801\ud600 \uc788\ub2e4."
72
+ }
73
+ }
74
+ }
v13/rl_agent_config.json ADDED
@@ -0,0 +1,257 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "encoder": "jhu-clsp/mmBERT-base",
3
+ "head_layers": 2,
4
+ "max_len": 2048,
5
+ "head_max_len": 256,
6
+ "max_prefixes": 6,
7
+ "act_costs": {
8
+ "escalate": 0.5
9
+ },
10
+ "cost_wrong_act": 3.0,
11
+ "amp_dtype": "bf16",
12
+ "model_name": "rl-agent",
13
+ "temperature": [
14
+ 4.242581367492676,
15
+ 4.761922359466553,
16
+ 4.510099411010742
17
+ ],
18
+ "training": {
19
+ "updates": 15987,
20
+ "epochs_completed": 4,
21
+ "hours": 4.97,
22
+ "world_size": 1,
23
+ "fine_tuned_from_checkpoint": false,
24
+ "laya_train": {
25
+ "epochs": 5,
26
+ "micro_batch": 2,
27
+ "grad_accum": 8,
28
+ "encoder_lr": 2.5e-05,
29
+ "head_lr": 0.0001,
30
+ "min_lr": 1e-06,
31
+ "weight_decay": 0.01,
32
+ "grad_clip": 1.0,
33
+ "loss": "rlcd",
34
+ "rl_samples": 4,
35
+ "sigma_start": 0.4,
36
+ "sigma_end": 0.1,
37
+ "w_sph": 0.75,
38
+ "w_rps": 1.0,
39
+ "shuffle_options": [
40
+ "choice"
41
+ ],
42
+ "option_layout": null,
43
+ "freeze_encoder": false,
44
+ "calib_max": 400,
45
+ "calib_frac": 0.1,
46
+ "calib_seed": 20260922,
47
+ "seed": 0,
48
+ "max_len": 2048,
49
+ "head_max_len": null,
50
+ "amp": null,
51
+ "gradient_checkpointing": null,
52
+ "log_every": 100,
53
+ "label_smoothing": 0.0,
54
+ "text_column": "text",
55
+ "label_column": "label",
56
+ "question_id": "label",
57
+ "instructions": null,
58
+ "eval_data": "data\\sets\\val.jsonl",
59
+ "target_error": 0.1,
60
+ "min_abstain_n": 10
61
+ },
62
+ "laya_train_calibration": {
63
+ "choice": {
64
+ "items": 126,
65
+ "temperature": 4.242581367492676,
66
+ "issues": [],
67
+ "accuracy": 0.9523809523809523,
68
+ "mean_confidence": 0.954239701485137
69
+ },
70
+ "score": {
71
+ "items": 29,
72
+ "temperature": 4.761922359466553,
73
+ "issues": [
74
+ "fitted on only 29 calibration items"
75
+ ],
76
+ "accuracy": 0.9655172413793104,
77
+ "mean_confidence": 0.9647458537741626
78
+ },
79
+ "noul": {
80
+ "items": 245,
81
+ "temperature": 4.510099411010742,
82
+ "issues": [],
83
+ "accuracy": 0.9836734693877551,
84
+ "mean_confidence": 0.9860408716061371
85
+ }
86
+ },
87
+ "train_report": {
88
+ "eval_source": "data\\sets\\val.jsonl",
89
+ "eval_mode": "overlapping_eval",
90
+ "is_held_out": false,
91
+ "eval_items": 2902,
92
+ "note": "Evaluation set contains inputs that overlap training data (7) and calibration data (1) (total 8/2902 items); metrics do not reflect strictly independent held-out evaluation.",
93
+ "before": {
94
+ "items": 2902,
95
+ "loss": 2.5109,
96
+ "accuracy": 0.4621,
97
+ "mean_confidence": 0.8259,
98
+ "ece": 0.3638,
99
+ "brier": 0.8539,
100
+ "brier_top1": 0.3883,
101
+ "by_type": {
102
+ "choice": {
103
+ "items": 1083,
104
+ "loss": 1.7383,
105
+ "accuracy": 0.506,
106
+ "mean_confidence": 0.7489,
107
+ "ece": 0.2429,
108
+ "brier": 0.7265,
109
+ "brier_top1": 0.2916
110
+ },
111
+ "score": {
112
+ "items": 347,
113
+ "loss": 2.1853,
114
+ "accuracy": 0.2478,
115
+ "mean_confidence": 0.7083,
116
+ "ece": 0.4605,
117
+ "brier": 1.0195,
118
+ "brier_top1": 0.4103
119
+ },
120
+ "noul": {
121
+ "items": 1472,
122
+ "loss": 3.1561,
123
+ "accuracy": 0.4803,
124
+ "mean_confidence": 0.9102,
125
+ "ece": 0.4302,
126
+ "brier": 0.9086,
127
+ "brier_top1": 0.4543
128
+ }
129
+ }
130
+ },
131
+ "after": {
132
+ "items": 2902,
133
+ "loss": 0.1931,
134
+ "accuracy": 0.9549,
135
+ "mean_confidence": 0.9724,
136
+ "ece": 0.0182,
137
+ "brier": 0.0806,
138
+ "brier_top1": 0.0389,
139
+ "by_type": {
140
+ "choice": {
141
+ "items": 1083,
142
+ "loss": 0.3187,
143
+ "accuracy": 0.9261,
144
+ "mean_confidence": 0.9599,
145
+ "ece": 0.0343,
146
+ "brier": 0.1304,
147
+ "brier_top1": 0.0618
148
+ },
149
+ "score": {
150
+ "items": 347,
151
+ "loss": 0.2915,
152
+ "accuracy": 0.9222,
153
+ "mean_confidence": 0.9595,
154
+ "ece": 0.0383,
155
+ "brier": 0.1325,
156
+ "brier_top1": 0.0651
157
+ },
158
+ "noul": {
159
+ "items": 1472,
160
+ "loss": 0.0776,
161
+ "accuracy": 0.9837,
162
+ "mean_confidence": 0.9847,
163
+ "ece": 0.0021,
164
+ "brier": 0.0316,
165
+ "brier_top1": 0.0158
166
+ }
167
+ }
168
+ },
169
+ "comparison": {
170
+ "delta_accuracy": 0.4928,
171
+ "delta_loss": -2.3178,
172
+ "delta_ece": -0.3456,
173
+ "delta_brier": -0.7733,
174
+ "delta_brier_top1": -0.3494,
175
+ "delta_mean_confidence": 0.1465
176
+ },
177
+ "calibration": {
178
+ "choice": {
179
+ "items": 126,
180
+ "temperature": 4.242581367492676,
181
+ "issues": [],
182
+ "accuracy": 0.9523809523809523,
183
+ "mean_confidence": 0.954239701485137
184
+ },
185
+ "score": {
186
+ "items": 29,
187
+ "temperature": 4.761922359466553,
188
+ "issues": [
189
+ "fitted on only 29 calibration items"
190
+ ],
191
+ "accuracy": 0.9655172413793104,
192
+ "mean_confidence": 0.9647458537741626
193
+ },
194
+ "noul": {
195
+ "items": 245,
196
+ "temperature": 4.510099411010742,
197
+ "issues": [],
198
+ "accuracy": 0.9836734693877551,
199
+ "mean_confidence": 0.9860408716061371
200
+ }
201
+ },
202
+ "abstention_thresholds": {
203
+ "noul:2": 0.946,
204
+ "choice:6-10": 0.9404,
205
+ "choice:2": 0.9841,
206
+ "choice:3-5": 0.9557,
207
+ "score:3-5": 0.9595
208
+ },
209
+ "training": {
210
+ "epochs": 5,
211
+ "micro_batch": 2,
212
+ "grad_accum": 8,
213
+ "encoder_lr": 2.5e-05,
214
+ "head_lr": 0.0001,
215
+ "min_lr": 1e-06,
216
+ "weight_decay": 0.01,
217
+ "grad_clip": 1.0,
218
+ "loss": "rlcd",
219
+ "rl_samples": 4,
220
+ "sigma_start": 0.4,
221
+ "sigma_end": 0.1,
222
+ "w_sph": 0.75,
223
+ "w_rps": 1.0,
224
+ "shuffle_options": [
225
+ "choice"
226
+ ],
227
+ "option_layout": null,
228
+ "freeze_encoder": false,
229
+ "calib_max": 400,
230
+ "calib_frac": 0.1,
231
+ "calib_seed": 20260922,
232
+ "seed": 0,
233
+ "max_len": 2048,
234
+ "head_max_len": null,
235
+ "amp": null,
236
+ "gradient_checkpointing": null,
237
+ "log_every": 100,
238
+ "label_smoothing": 0.0,
239
+ "text_column": "text",
240
+ "label_column": "label",
241
+ "question_id": "label",
242
+ "instructions": null,
243
+ "eval_data": "data\\sets\\val.jsonl",
244
+ "target_error": 0.1,
245
+ "min_abstain_n": 10
246
+ }
247
+ },
248
+ "abstention_thresholds": {
249
+ "noul:2": 0.946,
250
+ "choice:6-10": 0.9404,
251
+ "choice:2": 0.9841,
252
+ "choice:3-5": 0.9557,
253
+ "score:3-5": 0.9595
254
+ }
255
+ },
256
+ "fine_tuned": true
257
+ }
v13/thresholds.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "thresholds": {
3
+ "conf_min": {
4
+ "category": 0.3,
5
+ "action": 0.35,
6
+ "complexity": 0.3
7
+ },
8
+ "sens_lo": 0.05,
9
+ "sens_hi": 0.5,
10
+ "cloud_max": 0.01
11
+ },
12
+ "근거": {
13
+ "category": {
14
+ "자동처리 비율": 1.0,
15
+ "자동처리 정확도": 0.913,
16
+ "목표 달성": true
17
+ },
18
+ "action": {
19
+ "자동처리 비율": 0.991,
20
+ "자동처리 정확도": 0.901,
21
+ "목표 달성": true
22
+ },
23
+ "complexity": {
24
+ "자동처리 비율": 1.0,
25
+ "자동처리 정확도": 0.974,
26
+ "목표 달성": true
27
+ },
28
+ "sens_lo 탐색값(미사용)": 0.5,
29
+ "sensitive": {
30
+ "val 민감 건수": 87,
31
+ "sens_lo": 0.05,
32
+ "Laya 단독 재현율": 0.966
33
+ }
34
+ },
35
+ "목표 정확도": 0.9,
36
+ "보정 실행": "20261117-155830_v13_val",
37
+ "val 건수": 389,
38
+ "주의": "val 389건 기준이라 오차가 큼. test로 다시 확인할 것."
39
+ }
v13/tokenizer/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dbba919c6e5e492bc22beb80abb64e6fcc7e8744dc696e8388b8c977767afa4b
3
+ size 34363285
v13/tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<bos>",
3
+ "clean_up_tokenization_spaces": false,
4
+ "cls_token": "<bos>",
5
+ "eos_token": "<eos>",
6
+ "extra_0": "<start_of_turn>",
7
+ "extra_1": "<end_of_turn>",
8
+ "local_files_only": false,
9
+ "mask_token": "<mask>",
10
+ "model_input_names": [
11
+ "input_ids",
12
+ "attention_mask"
13
+ ],
14
+ "model_max_length": 8192,
15
+ "model_specific_special_tokens": {
16
+ "extra_0": "<start_of_turn>",
17
+ "extra_1": "<end_of_turn>"
18
+ },
19
+ "pad_token": "<pad>",
20
+ "padding_side": "right",
21
+ "sep_token": "<eos>",
22
+ "spaces_between_special_tokens": false,
23
+ "tokenizer_class": "PreTrainedTokenizerFast",
24
+ "unk_token": "<unk>"
25
+ }
v13/train_report.json ADDED
@@ -0,0 +1,161 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "eval_source": "data\\sets\\val.jsonl",
3
+ "eval_mode": "overlapping_eval",
4
+ "is_held_out": false,
5
+ "eval_items": 2902,
6
+ "note": "Evaluation set contains inputs that overlap training data (7) and calibration data (1) (total 8/2902 items); metrics do not reflect strictly independent held-out evaluation.",
7
+ "before": {
8
+ "items": 2902,
9
+ "loss": 2.5109,
10
+ "accuracy": 0.4621,
11
+ "mean_confidence": 0.8259,
12
+ "ece": 0.3638,
13
+ "brier": 0.8539,
14
+ "brier_top1": 0.3883,
15
+ "by_type": {
16
+ "choice": {
17
+ "items": 1083,
18
+ "loss": 1.7383,
19
+ "accuracy": 0.506,
20
+ "mean_confidence": 0.7489,
21
+ "ece": 0.2429,
22
+ "brier": 0.7265,
23
+ "brier_top1": 0.2916
24
+ },
25
+ "score": {
26
+ "items": 347,
27
+ "loss": 2.1853,
28
+ "accuracy": 0.2478,
29
+ "mean_confidence": 0.7083,
30
+ "ece": 0.4605,
31
+ "brier": 1.0195,
32
+ "brier_top1": 0.4103
33
+ },
34
+ "noul": {
35
+ "items": 1472,
36
+ "loss": 3.1561,
37
+ "accuracy": 0.4803,
38
+ "mean_confidence": 0.9102,
39
+ "ece": 0.4302,
40
+ "brier": 0.9086,
41
+ "brier_top1": 0.4543
42
+ }
43
+ }
44
+ },
45
+ "after": {
46
+ "items": 2902,
47
+ "loss": 0.1931,
48
+ "accuracy": 0.9549,
49
+ "mean_confidence": 0.9724,
50
+ "ece": 0.0182,
51
+ "brier": 0.0806,
52
+ "brier_top1": 0.0389,
53
+ "by_type": {
54
+ "choice": {
55
+ "items": 1083,
56
+ "loss": 0.3187,
57
+ "accuracy": 0.9261,
58
+ "mean_confidence": 0.9599,
59
+ "ece": 0.0343,
60
+ "brier": 0.1304,
61
+ "brier_top1": 0.0618
62
+ },
63
+ "score": {
64
+ "items": 347,
65
+ "loss": 0.2915,
66
+ "accuracy": 0.9222,
67
+ "mean_confidence": 0.9595,
68
+ "ece": 0.0383,
69
+ "brier": 0.1325,
70
+ "brier_top1": 0.0651
71
+ },
72
+ "noul": {
73
+ "items": 1472,
74
+ "loss": 0.0776,
75
+ "accuracy": 0.9837,
76
+ "mean_confidence": 0.9847,
77
+ "ece": 0.0021,
78
+ "brier": 0.0316,
79
+ "brier_top1": 0.0158
80
+ }
81
+ }
82
+ },
83
+ "comparison": {
84
+ "delta_accuracy": 0.4928,
85
+ "delta_loss": -2.3178,
86
+ "delta_ece": -0.3456,
87
+ "delta_brier": -0.7733,
88
+ "delta_brier_top1": -0.3494,
89
+ "delta_mean_confidence": 0.1465
90
+ },
91
+ "calibration": {
92
+ "choice": {
93
+ "items": 126,
94
+ "temperature": 4.242581367492676,
95
+ "issues": [],
96
+ "accuracy": 0.9523809523809523,
97
+ "mean_confidence": 0.954239701485137
98
+ },
99
+ "score": {
100
+ "items": 29,
101
+ "temperature": 4.761922359466553,
102
+ "issues": [
103
+ "fitted on only 29 calibration items"
104
+ ],
105
+ "accuracy": 0.9655172413793104,
106
+ "mean_confidence": 0.9647458537741626
107
+ },
108
+ "noul": {
109
+ "items": 245,
110
+ "temperature": 4.510099411010742,
111
+ "issues": [],
112
+ "accuracy": 0.9836734693877551,
113
+ "mean_confidence": 0.9860408716061371
114
+ }
115
+ },
116
+ "abstention_thresholds": {
117
+ "noul:2": 0.946,
118
+ "choice:6-10": 0.9404,
119
+ "choice:2": 0.9841,
120
+ "choice:3-5": 0.9557,
121
+ "score:3-5": 0.9595
122
+ },
123
+ "training": {
124
+ "epochs": 5,
125
+ "micro_batch": 2,
126
+ "grad_accum": 8,
127
+ "encoder_lr": 2.5e-05,
128
+ "head_lr": 0.0001,
129
+ "min_lr": 1e-06,
130
+ "weight_decay": 0.01,
131
+ "grad_clip": 1.0,
132
+ "loss": "rlcd",
133
+ "rl_samples": 4,
134
+ "sigma_start": 0.4,
135
+ "sigma_end": 0.1,
136
+ "w_sph": 0.75,
137
+ "w_rps": 1.0,
138
+ "shuffle_options": [
139
+ "choice"
140
+ ],
141
+ "option_layout": null,
142
+ "freeze_encoder": false,
143
+ "calib_max": 400,
144
+ "calib_frac": 0.1,
145
+ "calib_seed": 20260922,
146
+ "seed": 0,
147
+ "max_len": 2048,
148
+ "head_max_len": null,
149
+ "amp": null,
150
+ "gradient_checkpointing": null,
151
+ "log_every": 100,
152
+ "label_smoothing": 0.0,
153
+ "text_column": "text",
154
+ "label_column": "label",
155
+ "question_id": "label",
156
+ "instructions": null,
157
+ "eval_data": "data\\sets\\val.jsonl",
158
+ "target_error": 0.1,
159
+ "min_abstain_n": 10
160
+ }
161
+ }