usrnotfound101 commited on
Commit
ded31bb
·
verified ·
1 Parent(s): d4b84c3

sweep lion lr=0.0002

Browse files
Files changed (1) hide show
  1. sweeps/lion_lr0.0002.json +124 -0
sweeps/lion_lr0.0002.json ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "optimizer": "lion",
3
+ "lr": 0.0002,
4
+ "hp": {
5
+ "lr": 0.0002,
6
+ "betas": [
7
+ 0.9,
8
+ 0.99
9
+ ],
10
+ "weight_decay": 1.0,
11
+ "grad_clip": 1.0
12
+ },
13
+ "data_fraction": 0.25,
14
+ "final_val_loss": 5.14910791015625,
15
+ "final_bleu": 1.8625665064297257,
16
+ "eval": [
17
+ {
18
+ "step": 0,
19
+ "tokens": 0,
20
+ "frac_dataset": 0.0,
21
+ "frac_run": 0.0,
22
+ "val_loss": 9.790605514526368,
23
+ "val_ppl": 17865.12048319641,
24
+ "bleu": 0.6520166401004373,
25
+ "bleu_human_ref": 0.21961258356665092,
26
+ "n_refs": 7
27
+ },
28
+ {
29
+ "step": 153,
30
+ "tokens": 2506752,
31
+ "frac_dataset": 0.06244778480003571,
32
+ "frac_run": 0.25,
33
+ "val_loss": 5.799665901184082,
34
+ "val_ppl": 330.1892256500837,
35
+ "bleu": 0.43009932107187004,
36
+ "bleu_human_ref": 0.12890545830564296,
37
+ "n_refs": 7
38
+ },
39
+ {
40
+ "step": 306,
41
+ "tokens": 5013504,
42
+ "frac_dataset": 0.12489556960007142,
43
+ "frac_run": 0.5,
44
+ "val_loss": 5.427223346710205,
45
+ "val_ppl": 227.51663275060682,
46
+ "bleu": 1.203072436699661,
47
+ "bleu_human_ref": 0.22450970015812433,
48
+ "n_refs": 7
49
+ },
50
+ {
51
+ "step": 459,
52
+ "tokens": 7520256,
53
+ "frac_dataset": 0.18734335440010713,
54
+ "frac_run": 0.75,
55
+ "val_loss": 5.240798526763916,
56
+ "val_ppl": 188.82082070027494,
57
+ "bleu": 1.7548305675368272,
58
+ "bleu_human_ref": 0.443026766841217,
59
+ "n_refs": 7
60
+ },
61
+ {
62
+ "step": 612,
63
+ "tokens": 10027008,
64
+ "frac_dataset": 0.24979113920014284,
65
+ "frac_run": 1.0,
66
+ "val_loss": 5.14910791015625,
67
+ "val_ppl": 172.27773452778573,
68
+ "bleu": 1.8625665064297257,
69
+ "bleu_human_ref": 0.4140280146810212,
70
+ "n_refs": 7
71
+ }
72
+ ],
73
+ "train": [
74
+ {
75
+ "step": 100,
76
+ "tokens": 1638400,
77
+ "loss": 7.061666488647461,
78
+ "lr": 0.00019382924844297582,
79
+ "grad_norm": 0.7487174868583679,
80
+ "tok_per_s": 54309.326304510745
81
+ },
82
+ {
83
+ "step": 200,
84
+ "tokens": 3276800,
85
+ "loss": 5.778791904449463,
86
+ "lr": 0.00016507714503393752,
87
+ "grad_norm": 0.6075072884559631,
88
+ "tok_per_s": 52882.67791301201
89
+ },
90
+ {
91
+ "step": 300,
92
+ "tokens": 4915200,
93
+ "loss": 5.473942756652832,
94
+ "lr": 0.00012066278906460943,
95
+ "grad_norm": 0.4413580894470215,
96
+ "tok_per_s": 63088.77110841315
97
+ },
98
+ {
99
+ "step": 400,
100
+ "tokens": 6553600,
101
+ "loss": 5.301473140716553,
102
+ "lr": 7.321626257837996e-05,
103
+ "grad_norm": 0.46131232380867004,
104
+ "tok_per_s": 43662.67794138106
105
+ },
106
+ {
107
+ "step": 500,
108
+ "tokens": 8192000,
109
+ "loss": 5.187976360321045,
110
+ "lr": 3.6229904012143465e-05,
111
+ "grad_norm": 0.45139774680137634,
112
+ "tok_per_s": 42071.591256979234
113
+ },
114
+ {
115
+ "step": 600,
116
+ "tokens": 9830400,
117
+ "loss": 5.105480194091797,
118
+ "lr": 2.0221500531390974e-05,
119
+ "grad_norm": 0.432823121547699,
120
+ "tok_per_s": 55940.61199121058
121
+ }
122
+ ],
123
+ "train_time_s": 204.88745975494385
124
+ }