mphd1 commited on
Commit
f352630
·
verified ·
1 Parent(s): 9d89c48

Training in progress, epoch 17, checkpoint

Browse files
checkpoint-2108/config.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "openai-community/gpt2-large",
3
+ "activation_function": "gelu_new",
4
+ "architectures": [
5
+ "GPT2LMHeadModel"
6
+ ],
7
+ "attn_pdrop": 0.1,
8
+ "bos_token_id": 50256,
9
+ "embd_pdrop": 0.1,
10
+ "eos_token_id": 50256,
11
+ "initializer_range": 0.02,
12
+ "layer_norm_epsilon": 1e-05,
13
+ "model_type": "gpt2",
14
+ "n_ctx": 1024,
15
+ "n_embd": 1280,
16
+ "n_head": 20,
17
+ "n_inner": null,
18
+ "n_layer": 36,
19
+ "n_positions": 1024,
20
+ "reorder_and_upcast_attn": false,
21
+ "resid_pdrop": 0.1,
22
+ "scale_attn_by_inverse_layer_idx": false,
23
+ "scale_attn_weights": true,
24
+ "summary_activation": null,
25
+ "summary_first_dropout": 0.1,
26
+ "summary_proj_to_labels": true,
27
+ "summary_type": "cls_index",
28
+ "summary_use_proj": true,
29
+ "task_specific_params": {
30
+ "text-generation": {
31
+ "do_sample": true,
32
+ "max_length": 50
33
+ }
34
+ },
35
+ "torch_dtype": "float32",
36
+ "transformers_version": "4.47.1",
37
+ "use_cache": true,
38
+ "vocab_size": 50257
39
+ }
checkpoint-2108/generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 50256,
4
+ "eos_token_id": 50256,
5
+ "transformers_version": "4.47.1"
6
+ }
checkpoint-2108/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-2108/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4d782e8571381b5c8615dc5344489794273fb7a3e7ab8879e504ef272922f59d
3
+ size 3096165928
checkpoint-2108/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3865108b4e2da64e372c94e50001b3913fe0bcbe420abaa4352872f1d6baeee7
3
+ size 6192613990
checkpoint-2108/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6aeae495030bdd143b6d6d08905b7d4dd65c513cbf9aac0ae923ee7fd2ce9989
3
+ size 14244
checkpoint-2108/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ba7be8490bca2eb4734e5fcc2793a33d1223866c9936bb12a438c576884103d7
3
+ size 1064
checkpoint-2108/special_tokens_map.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
+ "pad_token": "<|endoftext|>",
5
+ "unk_token": "<|endoftext|>"
6
+ }
checkpoint-2108/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-2108/tokenizer_config.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "50256": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": true,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ }
12
+ },
13
+ "bos_token": "<|endoftext|>",
14
+ "clean_up_tokenization_spaces": false,
15
+ "eos_token": "<|endoftext|>",
16
+ "extra_special_tokens": {},
17
+ "model_max_length": 1024,
18
+ "pad_token": "<|endoftext|>",
19
+ "tokenizer_class": "GPT2Tokenizer",
20
+ "unk_token": "<|endoftext|>"
21
+ }
checkpoint-2108/trainer_state.json ADDED
@@ -0,0 +1,152 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 17.0,
5
+ "eval_steps": 500,
6
+ "global_step": 2108,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 1.0,
13
+ "grad_norm": 3.1197707653045654,
14
+ "learning_rate": 4.9692208514878444e-05,
15
+ "loss": 1.1128,
16
+ "step": 124
17
+ },
18
+ {
19
+ "epoch": 2.0,
20
+ "grad_norm": 1.8621318340301514,
21
+ "learning_rate": 4.877641290737884e-05,
22
+ "loss": 0.6276,
23
+ "step": 248
24
+ },
25
+ {
26
+ "epoch": 3.0,
27
+ "grad_norm": 1.429319143295288,
28
+ "learning_rate": 4.72751631047092e-05,
29
+ "loss": 0.4328,
30
+ "step": 372
31
+ },
32
+ {
33
+ "epoch": 4.0,
34
+ "grad_norm": 1.8360837697982788,
35
+ "learning_rate": 4.522542485937369e-05,
36
+ "loss": 0.3393,
37
+ "step": 496
38
+ },
39
+ {
40
+ "epoch": 5.0,
41
+ "grad_norm": 1.0911304950714111,
42
+ "learning_rate": 4.267766952966369e-05,
43
+ "loss": 0.2789,
44
+ "step": 620
45
+ },
46
+ {
47
+ "epoch": 6.0,
48
+ "grad_norm": 1.395412564277649,
49
+ "learning_rate": 3.969463130731183e-05,
50
+ "loss": 0.2392,
51
+ "step": 744
52
+ },
53
+ {
54
+ "epoch": 7.0,
55
+ "grad_norm": 0.8320960402488708,
56
+ "learning_rate": 3.634976249348867e-05,
57
+ "loss": 0.212,
58
+ "step": 868
59
+ },
60
+ {
61
+ "epoch": 8.0,
62
+ "grad_norm": 0.8032793402671814,
63
+ "learning_rate": 3.272542485937369e-05,
64
+ "loss": 0.1892,
65
+ "step": 992
66
+ },
67
+ {
68
+ "epoch": 9.0,
69
+ "grad_norm": 0.8925142288208008,
70
+ "learning_rate": 2.8910861626005776e-05,
71
+ "loss": 0.1727,
72
+ "step": 1116
73
+ },
74
+ {
75
+ "epoch": 10.0,
76
+ "grad_norm": 0.5616365671157837,
77
+ "learning_rate": 2.5e-05,
78
+ "loss": 0.1597,
79
+ "step": 1240
80
+ },
81
+ {
82
+ "epoch": 11.0,
83
+ "grad_norm": 0.6422319412231445,
84
+ "learning_rate": 2.1089138373994223e-05,
85
+ "loss": 0.1471,
86
+ "step": 1364
87
+ },
88
+ {
89
+ "epoch": 12.0,
90
+ "grad_norm": 0.9061075448989868,
91
+ "learning_rate": 1.7274575140626318e-05,
92
+ "loss": 0.1385,
93
+ "step": 1488
94
+ },
95
+ {
96
+ "epoch": 13.0,
97
+ "grad_norm": 0.6452886462211609,
98
+ "learning_rate": 1.3650237506511331e-05,
99
+ "loss": 0.1311,
100
+ "step": 1612
101
+ },
102
+ {
103
+ "epoch": 14.0,
104
+ "grad_norm": 0.6343705058097839,
105
+ "learning_rate": 1.0305368692688174e-05,
106
+ "loss": 0.1248,
107
+ "step": 1736
108
+ },
109
+ {
110
+ "epoch": 15.0,
111
+ "grad_norm": 0.5055845975875854,
112
+ "learning_rate": 7.3223304703363135e-06,
113
+ "loss": 0.1197,
114
+ "step": 1860
115
+ },
116
+ {
117
+ "epoch": 16.0,
118
+ "grad_norm": 0.5078000426292419,
119
+ "learning_rate": 4.7745751406263165e-06,
120
+ "loss": 0.1161,
121
+ "step": 1984
122
+ },
123
+ {
124
+ "epoch": 17.0,
125
+ "grad_norm": 0.48454469442367554,
126
+ "learning_rate": 2.7248368952908053e-06,
127
+ "loss": 0.1133,
128
+ "step": 2108
129
+ }
130
+ ],
131
+ "logging_steps": 500,
132
+ "max_steps": 2480,
133
+ "num_input_tokens_seen": 0,
134
+ "num_train_epochs": 20,
135
+ "save_steps": 500,
136
+ "stateful_callbacks": {
137
+ "TrainerControl": {
138
+ "args": {
139
+ "should_epoch_stop": false,
140
+ "should_evaluate": false,
141
+ "should_log": false,
142
+ "should_save": true,
143
+ "should_training_stop": false
144
+ },
145
+ "attributes": {}
146
+ }
147
+ },
148
+ "total_flos": 1.62565353474816e+16,
149
+ "train_batch_size": 8,
150
+ "trial_name": null,
151
+ "trial_params": null
152
+ }
checkpoint-2108/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7d641cc58aedf2c79ed6d2fe010d627feec4f355a064b510dc952c176c589fb1
3
+ size 5432
checkpoint-2108/vocab.json ADDED
The diff for this file is too large to render. See raw diff