usrnotfound101 commited on
Commit
afe5429
·
verified ·
1 Parent(s): ded31bb

sweep lion lr=0.0001

Browse files
Files changed (1) hide show
  1. sweeps/lion_lr0.0001.json +124 -0
sweeps/lion_lr0.0001.json ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "optimizer": "lion",
3
+ "lr": 0.0001,
4
+ "hp": {
5
+ "lr": 0.0001,
6
+ "betas": [
7
+ 0.9,
8
+ 0.99
9
+ ],
10
+ "weight_decay": 1.0,
11
+ "grad_clip": 1.0
12
+ },
13
+ "data_fraction": 0.25,
14
+ "final_val_loss": 5.230143775939942,
15
+ "final_bleu": 1.465979894597459,
16
+ "eval": [
17
+ {
18
+ "step": 0,
19
+ "tokens": 0,
20
+ "frac_dataset": 0.0,
21
+ "frac_run": 0.0,
22
+ "val_loss": 9.790605514526368,
23
+ "val_ppl": 17865.12048319641,
24
+ "bleu": 0.6520166401004373,
25
+ "bleu_human_ref": 0.21961258356665092,
26
+ "n_refs": 7
27
+ },
28
+ {
29
+ "step": 153,
30
+ "tokens": 2506752,
31
+ "frac_dataset": 0.06244778480003571,
32
+ "frac_run": 0.25,
33
+ "val_loss": 5.965708309173584,
34
+ "val_ppl": 389.8290497970406,
35
+ "bleu": 0.2313014855748276,
36
+ "bleu_human_ref": 0.03892500139637253,
37
+ "n_refs": 7
38
+ },
39
+ {
40
+ "step": 306,
41
+ "tokens": 5013504,
42
+ "frac_dataset": 0.12489556960007142,
43
+ "frac_run": 0.5,
44
+ "val_loss": 5.501773933410645,
45
+ "val_ppl": 245.12638468908568,
46
+ "bleu": 1.3459528337228623,
47
+ "bleu_human_ref": 0.40431199324772477,
48
+ "n_refs": 7
49
+ },
50
+ {
51
+ "step": 459,
52
+ "tokens": 7520256,
53
+ "frac_dataset": 0.18734335440010713,
54
+ "frac_run": 0.75,
55
+ "val_loss": 5.310019428253174,
56
+ "val_ppl": 202.354159737804,
57
+ "bleu": 1.8153640703408127,
58
+ "bleu_human_ref": 0.47848006274424104,
59
+ "n_refs": 7
60
+ },
61
+ {
62
+ "step": 612,
63
+ "tokens": 10027008,
64
+ "frac_dataset": 0.24979113920014284,
65
+ "frac_run": 1.0,
66
+ "val_loss": 5.230143775939942,
67
+ "val_ppl": 186.81966176180674,
68
+ "bleu": 1.465979894597459,
69
+ "bleu_human_ref": 0.3609800553538616,
70
+ "n_refs": 7
71
+ }
72
+ ],
73
+ "train": [
74
+ {
75
+ "step": 100,
76
+ "tokens": 1638400,
77
+ "loss": 7.411983013153076,
78
+ "lr": 9.691462422148791e-05,
79
+ "grad_norm": 0.621061384677887,
80
+ "tok_per_s": 51248.75396118149
81
+ },
82
+ {
83
+ "step": 200,
84
+ "tokens": 3276800,
85
+ "loss": 5.95184326171875,
86
+ "lr": 8.253857251696876e-05,
87
+ "grad_norm": 0.5926935076713562,
88
+ "tok_per_s": 50256.48314739268
89
+ },
90
+ {
91
+ "step": 300,
92
+ "tokens": 4915200,
93
+ "loss": 5.566981792449951,
94
+ "lr": 6.0331394532304717e-05,
95
+ "grad_norm": 0.487734854221344,
96
+ "tok_per_s": 60771.569697115534
97
+ },
98
+ {
99
+ "step": 400,
100
+ "tokens": 6553600,
101
+ "loss": 5.370908260345459,
102
+ "lr": 3.660813128918998e-05,
103
+ "grad_norm": 0.48501840233802795,
104
+ "tok_per_s": 47082.613299241755
105
+ },
106
+ {
107
+ "step": 500,
108
+ "tokens": 8192000,
109
+ "loss": 5.262225151062012,
110
+ "lr": 1.8114952006071733e-05,
111
+ "grad_norm": 0.5202957987785339,
112
+ "tok_per_s": 48112.0654027739
113
+ },
114
+ {
115
+ "step": 600,
116
+ "tokens": 9830400,
117
+ "loss": 5.184643268585205,
118
+ "lr": 1.0110750265695487e-05,
119
+ "grad_norm": 0.4511629045009613,
120
+ "tok_per_s": 61308.83505938527
121
+ }
122
+ ],
123
+ "train_time_s": 198.26114654541016
124
+ }