Hugging Face's logo

T0KII
/
MASRIBERTV4

Fill-Mask
Transformers
Safetensors
bert
Generated from Trainer
Model card Files
xet
 Community
T0KII commited on
Commit
3f2a8c8
·
verified ·
1 Parent(s): f8a83d9

Training in progress, step 9500, checkpoint

Browse files
Files changed (10) hide show
  1. checkpoint-9500/config.json +36 -0
  2. checkpoint-9500/model.safetensors +3 -0
  3. checkpoint-9500/optimizer.pt +3 -0
  4. checkpoint-9500/rng_state.pth +3 -0
  5. checkpoint-9500/scaler.pt +3 -0
  6. checkpoint-9500/scheduler.pt +3 -0
  7. checkpoint-9500/tokenizer.json +0 -0
  8. checkpoint-9500/tokenizer_config.json +16 -0
  9. checkpoint-9500/trainer_state.json +461 -0
  10. checkpoint-9500/training_args.bin +3 -0
checkpoint-9500/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_cross_attention": false,
3
+ "architectures": [
4
+ "BertForMaskedLM"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "bos_token_id": null,
8
+ "classifier_dropout": null,
9
+ "directionality": "bidi",
10
+ "dtype": "float32",
11
+ "eos_token_id": null,
12
+ "gradient_checkpointing": false,
13
+ "hidden_act": "gelu",
14
+ "hidden_dropout_prob": 0.1,
15
+ "hidden_size": 768,
16
+ "initializer_range": 0.02,
17
+ "intermediate_size": 3072,
18
+ "is_decoder": false,
19
+ "layer_norm_eps": 1e-12,
20
+ "max_position_embeddings": 512,
21
+ "model_type": "bert",
22
+ "num_attention_heads": 12,
23
+ "num_hidden_layers": 12,
24
+ "pad_token_id": 0,
25
+ "pooler_fc_size": 768,
26
+ "pooler_num_attention_heads": 12,
27
+ "pooler_num_fc_layers": 3,
28
+ "pooler_size_per_head": 128,
29
+ "pooler_type": "first_token_transform",
30
+ "position_embedding_type": "absolute",
31
+ "tie_word_embeddings": false,
32
+ "transformers_version": "5.0.0",
33
+ "type_vocab_size": 2,
34
+ "use_cache": false,
35
+ "vocab_size": 100000
36
+ }
checkpoint-9500/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fa33acfa85451b8b002a5e93cf7a9f0ef6c1fee06e617701c463c6ca2f301ee7
3
+ size 959395416
checkpoint-9500/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c90204c38d97037edfef5de49500f7930df7bbb7e1d6b104244c31ed4314c1b3
3
+ size 1918914187
checkpoint-9500/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c12bb0ca033b8298e0707261b661fd8c208c5985470194499ed346c3d0a55589
3
+ size 14645
checkpoint-9500/scaler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f63fc3e8232e7916d42ea762fd11402b4822f5992aab17d87b27a8c87a477c0d
3
+ size 1383
checkpoint-9500/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cffd0ec2dae3e4000e1ccd03a6d1a27b53129fc0229bf44d28f48e23ee2a634f
3
+ size 1465
checkpoint-9500/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-9500/tokenizer_config.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "cls_token": "[CLS]",
4
+ "do_basic_tokenize": true,
5
+ "do_lower_case": true,
6
+ "is_local": false,
7
+ "mask_token": "[MASK]",
8
+ "model_max_length": 1000000000000000019884624838656,
9
+ "never_split": null,
10
+ "pad_token": "[PAD]",
11
+ "sep_token": "[SEP]",
12
+ "strip_accents": null,
13
+ "tokenize_chinese_chars": true,
14
+ "tokenizer_class": "BertTokenizer",
15
+ "unk_token": "[UNK]"
16
+ }
checkpoint-9500/trainer_state.json ADDED
@@ -0,0 +1,461 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 9500,
3
+ "best_metric": 1.6928783655166626,
4
+ "best_model_checkpoint": "/kaggle/working/MASRIBERTV4_ckpts/checkpoint-9500",
5
+ "epoch": 0.8115842979795822,
6
+ "eval_steps": 500,
7
+ "global_step": 9500,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.02135748152577848,
14
+ "grad_norm": 112.14698028564453,
15
+ "learning_rate": 1.245e-06,
16
+ "loss": 58.53020703125,
17
+ "step": 250
18
+ },
19
+ {
20
+ "epoch": 0.04271496305155696,
21
+ "grad_norm": 94.12735748291016,
22
+ "learning_rate": 2.4950000000000003e-06,
23
+ "loss": 49.33264453125,
24
+ "step": 500
25
+ },
26
+ {
27
+ "epoch": 0.04271496305155696,
28
+ "eval_loss": 5.498984336853027,
29
+ "eval_runtime": 1927.5234,
30
+ "eval_samples_per_second": 40.952,
31
+ "eval_steps_per_second": 2.56,
32
+ "step": 500
33
+ },
34
+ {
35
+ "epoch": 0.06407244457733544,
36
+ "grad_norm": 90.2774658203125,
37
+ "learning_rate": 3.7449999999999997e-06,
38
+ "loss": 44.8349609375,
39
+ "step": 750
40
+ },
41
+ {
42
+ "epoch": 0.08542992610311392,
43
+ "grad_norm": 95.37772369384766,
44
+ "learning_rate": 4.9950000000000005e-06,
45
+ "loss": 42.08175390625,
46
+ "step": 1000
47
+ },
48
+ {
49
+ "epoch": 0.08542992610311392,
50
+ "eval_loss": 4.845311641693115,
51
+ "eval_runtime": 1929.6401,
52
+ "eval_samples_per_second": 40.907,
53
+ "eval_steps_per_second": 2.557,
54
+ "step": 1000
55
+ },
56
+ {
57
+ "epoch": 0.1067874076288924,
58
+ "grad_norm": 86.53277587890625,
59
+ "learning_rate": 6.245e-06,
60
+ "loss": 40.29465234375,
61
+ "step": 1250
62
+ },
63
+ {
64
+ "epoch": 0.12814488915467087,
65
+ "grad_norm": 93.83633422851562,
66
+ "learning_rate": 7.495e-06,
67
+ "loss": 38.960453125,
68
+ "step": 1500
69
+ },
70
+ {
71
+ "epoch": 0.12814488915467087,
72
+ "eval_loss": 4.519171714782715,
73
+ "eval_runtime": 1928.7805,
74
+ "eval_samples_per_second": 40.925,
75
+ "eval_steps_per_second": 2.558,
76
+ "step": 1500
77
+ },
78
+ {
79
+ "epoch": 0.14950237068044936,
80
+ "grad_norm": 81.17627716064453,
81
+ "learning_rate": 8.745e-06,
82
+ "loss": 37.7989140625,
83
+ "step": 1750
84
+ },
85
+ {
86
+ "epoch": 0.17085985220622785,
87
+ "grad_norm": 84.88935089111328,
88
+ "learning_rate": 9.995e-06,
89
+ "loss": 36.94516015625,
90
+ "step": 2000
91
+ },
92
+ {
93
+ "epoch": 0.17085985220622785,
94
+ "eval_loss": 4.303403854370117,
95
+ "eval_runtime": 1928.1902,
96
+ "eval_samples_per_second": 40.938,
97
+ "eval_steps_per_second": 2.559,
98
+ "step": 2000
99
+ },
100
+ {
101
+ "epoch": 0.19221733373200633,
102
+ "grad_norm": 79.1358413696289,
103
+ "learning_rate": 1.1245e-05,
104
+ "loss": 36.1753828125,
105
+ "step": 2250
106
+ },
107
+ {
108
+ "epoch": 0.2135748152577848,
109
+ "grad_norm": 85.23380279541016,
110
+ "learning_rate": 1.2495000000000001e-05,
111
+ "loss": 35.4720703125,
112
+ "step": 2500
113
+ },
114
+ {
115
+ "epoch": 0.2135748152577848,
116
+ "eval_loss": 4.146875381469727,
117
+ "eval_runtime": 1927.7742,
118
+ "eval_samples_per_second": 40.947,
119
+ "eval_steps_per_second": 2.559,
120
+ "step": 2500
121
+ },
122
+ {
123
+ "epoch": 0.23493229678356328,
124
+ "grad_norm": 40.744110107421875,
125
+ "learning_rate": 1.3725000000000002e-05,
126
+ "loss": 17.465578125,
127
+ "step": 2750
128
+ },
129
+ {
130
+ "epoch": 0.25628977830934174,
131
+ "grad_norm": 41.063385009765625,
132
+ "learning_rate": 1.4975e-05,
133
+ "loss": 17.180029296875,
134
+ "step": 3000
135
+ },
136
+ {
137
+ "epoch": 0.25628977830934174,
138
+ "eval_loss": 2.0153398513793945,
139
+ "eval_runtime": 791.0335,
140
+ "eval_samples_per_second": 99.788,
141
+ "eval_steps_per_second": 12.474,
142
+ "step": 3000
143
+ },
144
+ {
145
+ "epoch": 0.27764725983512023,
146
+ "grad_norm": 43.8082275390625,
147
+ "learning_rate": 1.6225e-05,
148
+ "loss": 16.96137890625,
149
+ "step": 3250
150
+ },
151
+ {
152
+ "epoch": 0.2990047413608987,
153
+ "grad_norm": 41.902591705322266,
154
+ "learning_rate": 1.7475e-05,
155
+ "loss": 16.6866015625,
156
+ "step": 3500
157
+ },
158
+ {
159
+ "epoch": 0.2990047413608987,
160
+ "eval_loss": 1.964239239692688,
161
+ "eval_runtime": 796.0599,
162
+ "eval_samples_per_second": 99.158,
163
+ "eval_steps_per_second": 12.395,
164
+ "step": 3500
165
+ },
166
+ {
167
+ "epoch": 0.3203622228866772,
168
+ "grad_norm": 38.828636169433594,
169
+ "learning_rate": 1.8725e-05,
170
+ "loss": 16.456845703125,
171
+ "step": 3750
172
+ },
173
+ {
174
+ "epoch": 0.3417197044124557,
175
+ "grad_norm": 43.49476623535156,
176
+ "learning_rate": 1.9975e-05,
177
+ "loss": 16.399064453125,
178
+ "step": 4000
179
+ },
180
+ {
181
+ "epoch": 0.3417197044124557,
182
+ "eval_loss": 1.919270396232605,
183
+ "eval_runtime": 795.0187,
184
+ "eval_samples_per_second": 99.288,
185
+ "eval_steps_per_second": 12.411,
186
+ "step": 4000
187
+ },
188
+ {
189
+ "epoch": 0.3630771859382342,
190
+ "grad_norm": 41.28815460205078,
191
+ "learning_rate": 2.1225e-05,
192
+ "loss": 16.1804853515625,
193
+ "step": 4250
194
+ },
195
+ {
196
+ "epoch": 0.38443466746401267,
197
+ "grad_norm": 39.76205062866211,
198
+ "learning_rate": 2.2475e-05,
199
+ "loss": 16.03432421875,
200
+ "step": 4500
201
+ },
202
+ {
203
+ "epoch": 0.38443466746401267,
204
+ "eval_loss": 1.8889796733856201,
205
+ "eval_runtime": 793.8887,
206
+ "eval_samples_per_second": 99.43,
207
+ "eval_steps_per_second": 12.429,
208
+ "step": 4500
209
+ },
210
+ {
211
+ "epoch": 0.4057921489897911,
212
+ "grad_norm": 46.443695068359375,
213
+ "learning_rate": 2.372e-05,
214
+ "loss": 15.8476015625,
215
+ "step": 4750
216
+ },
217
+ {
218
+ "epoch": 0.4271496305155696,
219
+ "grad_norm": 37.068084716796875,
220
+ "learning_rate": 2.4970000000000003e-05,
221
+ "loss": 15.7752314453125,
222
+ "step": 5000
223
+ },
224
+ {
225
+ "epoch": 0.4271496305155696,
226
+ "eval_loss": 1.8540898561477661,
227
+ "eval_runtime": 762.2328,
228
+ "eval_samples_per_second": 103.559,
229
+ "eval_steps_per_second": 12.945,
230
+ "step": 5000
231
+ },
232
+ {
233
+ "epoch": 0.4485071120413481,
234
+ "grad_norm": 38.2252082824707,
235
+ "learning_rate": 2.622e-05,
236
+ "loss": 15.5765185546875,
237
+ "step": 5250
238
+ },
239
+ {
240
+ "epoch": 0.46986459356712656,
241
+ "grad_norm": 41.708736419677734,
242
+ "learning_rate": 2.7470000000000003e-05,
243
+ "loss": 15.5412373046875,
244
+ "step": 5500
245
+ },
246
+ {
247
+ "epoch": 0.46986459356712656,
248
+ "eval_loss": 1.8295842409133911,
249
+ "eval_runtime": 763.2977,
250
+ "eval_samples_per_second": 103.414,
251
+ "eval_steps_per_second": 12.927,
252
+ "step": 5500
253
+ },
254
+ {
255
+ "epoch": 0.49122207509290505,
256
+ "grad_norm": 41.928775787353516,
257
+ "learning_rate": 2.8720000000000003e-05,
258
+ "loss": 15.4088125,
259
+ "step": 5750
260
+ },
261
+ {
262
+ "epoch": 0.5125795566186835,
263
+ "grad_norm": 38.48265075683594,
264
+ "learning_rate": 2.9970000000000003e-05,
265
+ "loss": 15.3675712890625,
266
+ "step": 6000
267
+ },
268
+ {
269
+ "epoch": 0.5125795566186835,
270
+ "eval_loss": 1.803719401359558,
271
+ "eval_runtime": 761.5931,
272
+ "eval_samples_per_second": 103.646,
273
+ "eval_steps_per_second": 12.956,
274
+ "step": 6000
275
+ },
276
+ {
277
+ "epoch": 0.533937038144462,
278
+ "grad_norm": 40.2772331237793,
279
+ "learning_rate": 3.122e-05,
280
+ "loss": 15.1469111328125,
281
+ "step": 6250
282
+ },
283
+ {
284
+ "epoch": 0.5552945196702405,
285
+ "grad_norm": 39.967647552490234,
286
+ "learning_rate": 3.247e-05,
287
+ "loss": 15.100412109375,
288
+ "step": 6500
289
+ },
290
+ {
291
+ "epoch": 0.5552945196702405,
292
+ "eval_loss": 1.7818784713745117,
293
+ "eval_runtime": 761.6584,
294
+ "eval_samples_per_second": 103.637,
295
+ "eval_steps_per_second": 12.955,
296
+ "step": 6500
297
+ },
298
+ {
299
+ "epoch": 0.576652001196019,
300
+ "grad_norm": 38.99542999267578,
301
+ "learning_rate": 3.3715000000000005e-05,
302
+ "loss": 15.10202734375,
303
+ "step": 6750
304
+ },
305
+ {
306
+ "epoch": 0.5980094827217974,
307
+ "grad_norm": 38.9607048034668,
308
+ "learning_rate": 3.4965e-05,
309
+ "loss": 14.957875,
310
+ "step": 7000
311
+ },
312
+ {
313
+ "epoch": 0.5980094827217974,
314
+ "eval_loss": 1.7590579986572266,
315
+ "eval_runtime": 763.6639,
316
+ "eval_samples_per_second": 103.365,
317
+ "eval_steps_per_second": 12.921,
318
+ "step": 7000
319
+ },
320
+ {
321
+ "epoch": 0.619366964247576,
322
+ "grad_norm": 36.200218200683594,
323
+ "learning_rate": 3.6215000000000005e-05,
324
+ "loss": 14.8693701171875,
325
+ "step": 7250
326
+ },
327
+ {
328
+ "epoch": 0.6407244457733544,
329
+ "grad_norm": 38.16975402832031,
330
+ "learning_rate": 3.7465e-05,
331
+ "loss": 14.82056640625,
332
+ "step": 7500
333
+ },
334
+ {
335
+ "epoch": 0.6407244457733544,
336
+ "eval_loss": 1.7474839687347412,
337
+ "eval_runtime": 761.4056,
338
+ "eval_samples_per_second": 103.671,
339
+ "eval_steps_per_second": 12.959,
340
+ "step": 7500
341
+ },
342
+ {
343
+ "epoch": 0.6620819272991328,
344
+ "grad_norm": 35.05085372924805,
345
+ "learning_rate": 3.8715000000000005e-05,
346
+ "loss": 14.679634765625,
347
+ "step": 7750
348
+ },
349
+ {
350
+ "epoch": 0.6834394088249114,
351
+ "grad_norm": 39.50191116333008,
352
+ "learning_rate": 3.9965e-05,
353
+ "loss": 14.5730966796875,
354
+ "step": 8000
355
+ },
356
+ {
357
+ "epoch": 0.6834394088249114,
358
+ "eval_loss": 1.731168508529663,
359
+ "eval_runtime": 762.2289,
360
+ "eval_samples_per_second": 103.559,
361
+ "eval_steps_per_second": 12.945,
362
+ "step": 8000
363
+ },
364
+ {
365
+ "epoch": 0.7047968903506898,
366
+ "grad_norm": 39.16671371459961,
367
+ "learning_rate": 4.1215000000000004e-05,
368
+ "loss": 14.58540234375,
369
+ "step": 8250
370
+ },
371
+ {
372
+ "epoch": 0.7261543718764684,
373
+ "grad_norm": 35.04971694946289,
374
+ "learning_rate": 4.246500000000001e-05,
375
+ "loss": 14.4853076171875,
376
+ "step": 8500
377
+ },
378
+ {
379
+ "epoch": 0.7261543718764684,
380
+ "eval_loss": 1.7171440124511719,
381
+ "eval_runtime": 761.2683,
382
+ "eval_samples_per_second": 103.69,
383
+ "eval_steps_per_second": 12.961,
384
+ "step": 8500
385
+ },
386
+ {
387
+ "epoch": 0.7475118534022468,
388
+ "grad_norm": 38.985992431640625,
389
+ "learning_rate": 4.371e-05,
390
+ "loss": 14.4681435546875,
391
+ "step": 8750
392
+ },
393
+ {
394
+ "epoch": 0.7688693349280253,
395
+ "grad_norm": 33.91847610473633,
396
+ "learning_rate": 4.496e-05,
397
+ "loss": 14.435607421875,
398
+ "step": 9000
399
+ },
400
+ {
401
+ "epoch": 0.7688693349280253,
402
+ "eval_loss": 1.7089964151382446,
403
+ "eval_runtime": 763.3649,
404
+ "eval_samples_per_second": 103.405,
405
+ "eval_steps_per_second": 12.926,
406
+ "step": 9000
407
+ },
408
+ {
409
+ "epoch": 0.7902268164538038,
410
+ "grad_norm": 32.30923080444336,
411
+ "learning_rate": 4.6210000000000006e-05,
412
+ "loss": 14.4067099609375,
413
+ "step": 9250
414
+ },
415
+ {
416
+ "epoch": 0.8115842979795822,
417
+ "grad_norm": 33.230995178222656,
418
+ "learning_rate": 4.746e-05,
419
+ "loss": 14.2340126953125,
420
+ "step": 9500
421
+ },
422
+ {
423
+ "epoch": 0.8115842979795822,
424
+ "eval_loss": 1.6928783655166626,
425
+ "eval_runtime": 762.4223,
426
+ "eval_samples_per_second": 103.533,
427
+ "eval_steps_per_second": 12.942,
428
+ "step": 9500
429
+ }
430
+ ],
431
+ "logging_steps": 250,
432
+ "max_steps": 23412,
433
+ "num_input_tokens_seen": 0,
434
+ "num_train_epochs": 2,
435
+ "save_steps": 500,
436
+ "stateful_callbacks": {
437
+ "EarlyStoppingCallback": {
438
+ "args": {
439
+ "early_stopping_patience": 3,
440
+ "early_stopping_threshold": 0.0
441
+ },
442
+ "attributes": {
443
+ "early_stopping_patience_counter": 0
444
+ }
445
+ },
446
+ "TrainerControl": {
447
+ "args": {
448
+ "should_epoch_stop": false,
449
+ "should_evaluate": false,
450
+ "should_log": false,
451
+ "should_save": true,
452
+ "should_training_stop": false
453
+ },
454
+ "attributes": {}
455
+ }
456
+ },
457
+ "total_flos": 3.03790269923328e+17,
458
+ "train_batch_size": 16,
459
+ "trial_name": null,
460
+ "trial_params": null
461
+ }
checkpoint-9500/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:39853abe7c212a05cd37a6042b4d1c2f33b18beabbb723de8075009a188aad0f
3
+ size 5201