T

DalusX64 commited on
Commit
a30420c
·
verified ·
1 Parent(s): b7377ca

Upload 3 files

Browse files
baseline_transformer_tinystories.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6102d768719877d214395643421f3765ba5a9a384db673da8e05e12ce39cdc5a
3
+ size 139852941
baseline_transformer_tinystories_tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
baseline_transformer_tinystories_training_config.json ADDED
@@ -0,0 +1,148 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "baseline_transformer_tinystories",
3
+ "checkpoint_tag": "final",
4
+ "epoch_value": 2.0,
5
+ "dataset": "roneneldan/TinyStories",
6
+ "num_train_texts": 100000,
7
+ "eval_texts": 2000,
8
+ "max_seq_len": 128,
9
+ "max_vocab_size": 48000,
10
+ "actual_vocab_size": 18604,
11
+ "param_count": 34949824,
12
+ "model_config": {
13
+ "d_model": 448,
14
+ "n_layers": 11,
15
+ "n_heads": 4,
16
+ "d_ff": 1792
17
+ },
18
+ "history": [
19
+ {
20
+ "epoch": 1,
21
+ "loss": 2.7925477427443095,
22
+ "ce": 2.7925477427443095,
23
+ "ppl": 16.322552536754753,
24
+ "seconds": 6709.080967903137,
25
+ "global_step": 13778
26
+ },
27
+ {
28
+ "epoch": 2,
29
+ "loss": 2.145891742479183,
30
+ "ce": 2.145891742479183,
31
+ "ppl": 8.54966193566955,
32
+ "seconds": 6647.170892953873,
33
+ "global_step": 27556
34
+ }
35
+ ],
36
+ "eval_history": [
37
+ {
38
+ "eval_ce": 3.019229908338063,
39
+ "eval_ppl": 20.475517585854988,
40
+ "eval_tokens": 418011,
41
+ "eval_batches": 4396,
42
+ "trigger_progress": 0.1,
43
+ "epoch_value": 0.2000027218661114,
44
+ "global_step": 2755,
45
+ "train_ce_running": 3.726730931783734,
46
+ "train_ppl_running": 41.54307878205579
47
+ },
48
+ {
49
+ "eval_ce": 2.6282812372172235,
50
+ "eval_ppl": 13.84994466359723,
51
+ "eval_tokens": 418011,
52
+ "eval_batches": 4396,
53
+ "trigger_progress": 0.2,
54
+ "epoch_value": 0.4000009072887038,
55
+ "global_step": 5510,
56
+ "train_ce_running": 3.298934572113122,
57
+ "train_ppl_running": 27.083767741875523
58
+ },
59
+ {
60
+ "eval_ce": 2.4289824040710375,
61
+ "eval_ppl": 11.34732920691451,
62
+ "eval_tokens": 418011,
63
+ "eval_batches": 4396,
64
+ "trigger_progress": 0.3,
65
+ "epoch_value": 0.6000036291548152,
66
+ "global_step": 8266,
67
+ "train_ce_running": 3.0639317816293103,
68
+ "train_ppl_running": 21.411577526098043
69
+ },
70
+ {
71
+ "eval_ce": 2.3028478895985414,
72
+ "eval_ppl": 10.002628311385484,
73
+ "eval_tokens": 418011,
74
+ "eval_batches": 4396,
75
+ "trigger_progress": 0.4,
76
+ "epoch_value": 0.8000018145774076,
77
+ "global_step": 11021,
78
+ "train_ce_running": 2.9075046430774236,
79
+ "train_ppl_running": 18.311048907242007
80
+ },
81
+ {
82
+ "eval_ce": 2.2137269049744646,
83
+ "eval_ppl": 9.14975318836517,
84
+ "eval_tokens": 418011,
85
+ "eval_batches": 4396,
86
+ "trigger_progress": 0.5,
87
+ "epoch_value": 1.0,
88
+ "global_step": 13777,
89
+ "train_ce_running": 2.7925477427443095,
90
+ "train_ppl_running": 16.322552536754753
91
+ },
92
+ {
93
+ "eval_ce": 2.1519488045059103,
94
+ "eval_ppl": 8.601604920454877,
95
+ "eval_tokens": 418011,
96
+ "eval_batches": 4396,
97
+ "trigger_progress": 0.6,
98
+ "epoch_value": 1.2000027218661113,
99
+ "global_step": 16533,
100
+ "train_ce_running": 2.2201592317098897,
101
+ "train_ppl_running": 9.20879708165047
102
+ },
103
+ {
104
+ "eval_ce": 2.1023585491979833,
105
+ "eval_ppl": 8.185452957131274,
106
+ "eval_tokens": 418011,
107
+ "eval_batches": 4396,
108
+ "trigger_progress": 0.7,
109
+ "epoch_value": 1.4000009072887039,
110
+ "global_step": 19288,
111
+ "train_ce_running": 2.199022379118235,
112
+ "train_ppl_running": 9.01619476916866
113
+ },
114
+ {
115
+ "eval_ce": 2.0675348904853807,
116
+ "eval_ppl": 7.905311619850248,
117
+ "eval_tokens": 418011,
118
+ "eval_batches": 4396,
119
+ "trigger_progress": 0.8,
120
+ "epoch_value": 1.6000036291548152,
121
+ "global_step": 22044,
122
+ "train_ce_running": 2.178444489460884,
123
+ "train_ppl_running": 8.83255643287514
124
+ },
125
+ {
126
+ "eval_ce": 2.045298832631953,
127
+ "eval_ppl": 7.731468607843494,
128
+ "eval_tokens": 418011,
129
+ "eval_batches": 4396,
130
+ "trigger_progress": 0.9,
131
+ "epoch_value": 1.8000018145774077,
132
+ "global_step": 24799,
133
+ "train_ce_running": 2.160078341528155,
134
+ "train_ppl_running": 8.671816995248255
135
+ },
136
+ {
137
+ "eval_ce": 2.0341344709381657,
138
+ "eval_ppl": 7.645631744354986,
139
+ "eval_tokens": 418011,
140
+ "eval_batches": 4396,
141
+ "trigger_progress": 1.0,
142
+ "epoch_value": 2.0,
143
+ "global_step": 27555,
144
+ "train_ce_running": 2.145891742479183,
145
+ "train_ppl_running": 8.54966193566955
146
+ }
147
+ ]
148
+ }