Upload folder using huggingface_hub
Browse files- runs/part1/c1_dense/metrics.json +230 -0
- runs/part1/c1_dense/model.pt +3 -0
- runs/part1/c2_moe_4e1a/expert_counts.npy +3 -0
- runs/part1/c2_moe_4e1a/metrics.json +280 -0
- runs/part1/c2_moe_4e1a/model.pt +3 -0
- runs/part1/c3_moe_4e2a/expert_counts.npy +3 -0
- runs/part1/c3_moe_4e2a/metrics.json +280 -0
- runs/part1/c3_moe_4e2a/model.pt +3 -0
- runs/part1/c4_moe_shared/expert_counts.npy +3 -0
- runs/part1/c4_moe_shared/metrics.json +274 -0
- runs/part1/c4_moe_shared/model.pt +3 -0
- runs/part1/c5_moe_4e2a_wide/expert_counts.npy +3 -0
- runs/part1/c5_moe_4e2a_wide/metrics.json +280 -0
- runs/part1/c5_moe_4e2a_wide/model.pt +3 -0
- runs/part1/cache/test_all_ids.npy +3 -0
- runs/part1/cache/test_all_lang.npy +3 -0
- runs/part1/cache/test_all_tgt.npy +3 -0
- runs/part1/cache/train_250000_ids.npy +3 -0
- runs/part1/cache/train_250000_lang.npy +3 -0
- runs/part1/cache/train_250000_tgt.npy +3 -0
- runs/part1/cache/train_446252_ids.npy +3 -0
- runs/part1/cache/train_446252_lang.npy +3 -0
- runs/part1/cache/train_446252_tgt.npy +3 -0
- runs/part1/cache/validation_all_ids.npy +3 -0
- runs/part1/cache/validation_all_lang.npy +3 -0
- runs/part1/cache/validation_all_tgt.npy +3 -0
- runs/part1/tokenizer_16000.json +0 -0
- runs/part2/adamw/metrics.json +153 -0
- runs/part2/adamw/model.pt +3 -0
- runs/part2/benchmark.json +52 -0
- runs/part2/cache/test.npy +3 -0
- runs/part2/cache/train.npy +3 -0
- runs/part2/cache/val.npy +3 -0
- runs/part2/lion/metrics.json +153 -0
- runs/part2/lion/model.pt +3 -0
- runs/part2/mars/metrics.json +154 -0
- runs/part2/mars/model.pt +3 -0
- runs/part2/muon/metrics.json +151 -0
- runs/part2/muon/model.pt +3 -0
- runs/part2/sophia/metrics.json +154 -0
- runs/part2/sophia/model.pt +3 -0
- runs/part2/sweep.json +157 -0
- runs/part2/tokenizer_16000.json +0 -0
- runs/part3/metrics.json +211 -0
runs/part1/c1_dense/metrics.json
ADDED
|
@@ -0,0 +1,230 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"config": {
|
| 3 |
+
"d_model": 512,
|
| 4 |
+
"n_ctx": 256,
|
| 5 |
+
"n_layers": 6,
|
| 6 |
+
"n_heads": 8,
|
| 7 |
+
"n_kv_heads": null,
|
| 8 |
+
"vocab_size": 16000,
|
| 9 |
+
"d_ff": 2048,
|
| 10 |
+
"dropout": 0.0,
|
| 11 |
+
"tie_embeddings": true,
|
| 12 |
+
"rope_base": 10000.0,
|
| 13 |
+
"use_moe": false,
|
| 14 |
+
"n_experts": 4,
|
| 15 |
+
"n_active": 2,
|
| 16 |
+
"n_shared": 0,
|
| 17 |
+
"expert_budget": "total",
|
| 18 |
+
"router_aux_coef": 0.01,
|
| 19 |
+
"router_z_coef": 0.001,
|
| 20 |
+
"router_noise": 0.0,
|
| 21 |
+
"name": "c1_dense"
|
| 22 |
+
},
|
| 23 |
+
"params_total": 27073024,
|
| 24 |
+
"params_active": 27073024,
|
| 25 |
+
"params_active_frac": 1.0,
|
| 26 |
+
"tokens_trained": 39993344,
|
| 27 |
+
"train_minutes": 5.779859657833973,
|
| 28 |
+
"history": {
|
| 29 |
+
"step": [
|
| 30 |
+
250,
|
| 31 |
+
500,
|
| 32 |
+
750,
|
| 33 |
+
1000,
|
| 34 |
+
1250,
|
| 35 |
+
1500,
|
| 36 |
+
1750,
|
| 37 |
+
2000,
|
| 38 |
+
2250,
|
| 39 |
+
2500,
|
| 40 |
+
2750,
|
| 41 |
+
3000,
|
| 42 |
+
3250,
|
| 43 |
+
3500,
|
| 44 |
+
3750,
|
| 45 |
+
4000,
|
| 46 |
+
4250,
|
| 47 |
+
4500,
|
| 48 |
+
4750,
|
| 49 |
+
4882
|
| 50 |
+
],
|
| 51 |
+
"tokens": [
|
| 52 |
+
2048000,
|
| 53 |
+
4096000,
|
| 54 |
+
6144000,
|
| 55 |
+
8192000,
|
| 56 |
+
10240000,
|
| 57 |
+
12288000,
|
| 58 |
+
14336000,
|
| 59 |
+
16384000,
|
| 60 |
+
18432000,
|
| 61 |
+
20480000,
|
| 62 |
+
22528000,
|
| 63 |
+
24576000,
|
| 64 |
+
26624000,
|
| 65 |
+
28672000,
|
| 66 |
+
30720000,
|
| 67 |
+
32768000,
|
| 68 |
+
34816000,
|
| 69 |
+
36864000,
|
| 70 |
+
38912000,
|
| 71 |
+
39993344
|
| 72 |
+
],
|
| 73 |
+
"train_loss": [
|
| 74 |
+
6.818709684371949,
|
| 75 |
+
5.115536183357238,
|
| 76 |
+
3.221318832397461,
|
| 77 |
+
2.5140862255096437,
|
| 78 |
+
2.3053058915138243,
|
| 79 |
+
2.1523772130012513,
|
| 80 |
+
2.057874378681183,
|
| 81 |
+
1.9787837824821473,
|
| 82 |
+
1.8937802448272705,
|
| 83 |
+
1.8302137441635131,
|
| 84 |
+
1.7867747120857238,
|
| 85 |
+
1.7311925644874573,
|
| 86 |
+
1.6877265725135804,
|
| 87 |
+
1.6492887783050536,
|
| 88 |
+
1.6061518397331238,
|
| 89 |
+
1.5972776565551758,
|
| 90 |
+
1.5594792408943177,
|
| 91 |
+
1.54410955286026,
|
| 92 |
+
1.5397181520462035,
|
| 93 |
+
1.5258417174671635
|
| 94 |
+
],
|
| 95 |
+
"val_loss": [
|
| 96 |
+
5.735805488648086,
|
| 97 |
+
4.201927793954861,
|
| 98 |
+
2.6709413744410644,
|
| 99 |
+
2.3541695244340435,
|
| 100 |
+
2.1962941569665633,
|
| 101 |
+
2.0811000020010773,
|
| 102 |
+
1.9930675109403617,
|
| 103 |
+
1.9096632413752879,
|
| 104 |
+
1.8506262599787517,
|
| 105 |
+
1.7941230297753494,
|
| 106 |
+
1.7442666527547392,
|
| 107 |
+
1.6907881975120802,
|
| 108 |
+
1.656655651435394,
|
| 109 |
+
1.6222454942804698,
|
| 110 |
+
1.5890829478698154,
|
| 111 |
+
1.5678247676120767,
|
| 112 |
+
1.5443462493662117,
|
| 113 |
+
1.5282856329913335,
|
| 114 |
+
1.5168391883117682,
|
| 115 |
+
1.5116203480413208
|
| 116 |
+
],
|
| 117 |
+
"val_ppl": [
|
| 118 |
+
309.76238037841244,
|
| 119 |
+
66.81501254263573,
|
| 120 |
+
14.453569010987033,
|
| 121 |
+
10.529380836178442,
|
| 122 |
+
8.991630111034493,
|
| 123 |
+
8.013278690617108,
|
| 124 |
+
7.33800869558851,
|
| 125 |
+
6.750815020513602,
|
| 126 |
+
6.3638036704668846,
|
| 127 |
+
6.014198135750792,
|
| 128 |
+
5.721703940661067,
|
| 129 |
+
5.423754010222743,
|
| 130 |
+
5.241751254834125,
|
| 131 |
+
5.064449750967814,
|
| 132 |
+
4.899253996284526,
|
| 133 |
+
4.796203980360552,
|
| 134 |
+
4.684907864177235,
|
| 135 |
+
4.610266354974985,
|
| 136 |
+
4.557796068193805,
|
| 137 |
+
4.534071619430077
|
| 138 |
+
],
|
| 139 |
+
"aux": [
|
| 140 |
+
0.0,
|
| 141 |
+
0.0,
|
| 142 |
+
0.0,
|
| 143 |
+
0.0,
|
| 144 |
+
0.0,
|
| 145 |
+
0.0,
|
| 146 |
+
0.0,
|
| 147 |
+
0.0,
|
| 148 |
+
0.0,
|
| 149 |
+
0.0,
|
| 150 |
+
0.0,
|
| 151 |
+
0.0,
|
| 152 |
+
0.0,
|
| 153 |
+
0.0,
|
| 154 |
+
0.0,
|
| 155 |
+
0.0,
|
| 156 |
+
0.0,
|
| 157 |
+
0.0,
|
| 158 |
+
0.0,
|
| 159 |
+
0.0
|
| 160 |
+
]
|
| 161 |
+
},
|
| 162 |
+
"test_loss": 1.5354485844556647,
|
| 163 |
+
"test_ppl": 4.643408023492571,
|
| 164 |
+
"bleu_all": 24.960734186719026,
|
| 165 |
+
"bleu_vi": 35.73287242901259,
|
| 166 |
+
"bleu_ja": 14.675578913345378,
|
| 167 |
+
"expert_usage": null,
|
| 168 |
+
"samples": [
|
| 169 |
+
{
|
| 170 |
+
"lang": "vi",
|
| 171 |
+
"hyp": "Require the key key to apply this topic",
|
| 172 |
+
"ref": "Require Main keyboard fancy to apply this theme."
|
| 173 |
+
},
|
| 174 |
+
{
|
| 175 |
+
"lang": "ja",
|
| 176 |
+
"hyp": "To apply this theme to the main keyboard fan",
|
| 177 |
+
"ref": "Require Main keyboard fancy to apply this theme."
|
| 178 |
+
},
|
| 179 |
+
{
|
| 180 |
+
"lang": "vi",
|
| 181 |
+
"hyp": "\u2022 Suitable for gerolor design. Higher efficiency than BMP motor.",
|
| 182 |
+
"ref": "\u2022 It adapts the gerolor design. Higher efficiency than BMP motor."
|
| 183 |
+
},
|
| 184 |
+
{
|
| 185 |
+
"lang": "ja",
|
| 186 |
+
"hyp": "\u2022 It is a high-speedolor design. BMP motor is high-speed.",
|
| 187 |
+
"ref": "\u2022 It adapts the gerolor design. Higher efficiency than BMP motor."
|
| 188 |
+
},
|
| 189 |
+
{
|
| 190 |
+
"lang": "vi",
|
| 191 |
+
"hyp": "Use token for the side payment in the application or sell it for the millions of global users.",
|
| 192 |
+
"ref": "Use token for in-app purchases or sell it to millions of users worldwide."
|
| 193 |
+
},
|
| 194 |
+
{
|
| 195 |
+
"lang": "ja",
|
| 196 |
+
"hyp": "You can use the TWO, use the world to sell it to the top of the world.",
|
| 197 |
+
"ref": "Use token for in-app purchases or sell it to millions of users worldwide."
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"lang": "vi",
|
| 201 |
+
"hyp": "Good Quality, Best Service",
|
| 202 |
+
"ref": "Good Quality, Best service"
|
| 203 |
+
},
|
| 204 |
+
{
|
| 205 |
+
"lang": "ja",
|
| 206 |
+
"hyp": "Good Quality Best Service",
|
| 207 |
+
"ref": "Good Quality, Best service"
|
| 208 |
+
},
|
| 209 |
+
{
|
| 210 |
+
"lang": "vi",
|
| 211 |
+
"hyp": "United States - Making the organization is a story. The organization is a different one! Can make live? It has more active? Read this report on the organization of the organization based on more than 30 years of experience of the Rodale Element. A fun document to close the frame",
|
| 212 |
+
"ref": "United States \u2013 Gardening organically is one thing. Farming organically is another one! Can one make a living? Is it more work? Read this report on farming organically based on more than 30 years of experience by the Rodale Institute. A document worth to be framed!"
|
| 213 |
+
},
|
| 214 |
+
{
|
| 215 |
+
"lang": "ja",
|
| 216 |
+
"hyp": "USA \u2013 The United States - The most popular is one of the most important thing. Is it possible to be a one thing! The older work? The more work? Rodale Institute of the 30 years experience in this reporting experience.",
|
| 217 |
+
"ref": "United States \u2013 Gardening organically is one thing. Farming organically is another one! Can one make a living? Is it more work? Read this report on farming organically based on more than 30 years of experience by the Rodale Institute. A document worth to be framed!"
|
| 218 |
+
},
|
| 219 |
+
{
|
| 220 |
+
"lang": "vi",
|
| 221 |
+
"hyp": "- Speed printing, higher efficiency",
|
| 222 |
+
"ref": "- Speed printing, higher efficiency"
|
| 223 |
+
},
|
| 224 |
+
{
|
| 225 |
+
"lang": "ja",
|
| 226 |
+
"hyp": "- High efficiency printing, high efficiency",
|
| 227 |
+
"ref": "- Speed printing, higher efficiency"
|
| 228 |
+
}
|
| 229 |
+
]
|
| 230 |
+
}
|
runs/part1/c1_dense/model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2e80496940fe2e83d72ea551fa5d6173364dbda31d1e04db204c66fe94d71179
|
| 3 |
+
size 108309012
|
runs/part1/c2_moe_4e1a/expert_counts.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3fa7432e1be8e1984240e0974cd961b4d4948ab66b3ac6417fbd19f628d52d17
|
| 3 |
+
size 896
|
runs/part1/c2_moe_4e1a/metrics.json
ADDED
|
@@ -0,0 +1,280 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"config": {
|
| 3 |
+
"d_model": 512,
|
| 4 |
+
"n_ctx": 256,
|
| 5 |
+
"n_layers": 6,
|
| 6 |
+
"n_heads": 8,
|
| 7 |
+
"n_kv_heads": null,
|
| 8 |
+
"vocab_size": 16000,
|
| 9 |
+
"d_ff": 2048,
|
| 10 |
+
"dropout": 0.0,
|
| 11 |
+
"tie_embeddings": true,
|
| 12 |
+
"rope_base": 10000.0,
|
| 13 |
+
"use_moe": true,
|
| 14 |
+
"n_experts": 4,
|
| 15 |
+
"n_active": 1,
|
| 16 |
+
"n_shared": 0,
|
| 17 |
+
"expert_budget": "total",
|
| 18 |
+
"router_aux_coef": 0.01,
|
| 19 |
+
"router_z_coef": 0.001,
|
| 20 |
+
"router_noise": 0.0,
|
| 21 |
+
"name": "c2_moe_4e1a"
|
| 22 |
+
},
|
| 23 |
+
"params_total": 27085312,
|
| 24 |
+
"params_active": 17648128,
|
| 25 |
+
"params_active_frac": 0.6515755845825221,
|
| 26 |
+
"tokens_trained": 39993344,
|
| 27 |
+
"train_minutes": 10.928626832117637,
|
| 28 |
+
"history": {
|
| 29 |
+
"step": [
|
| 30 |
+
250,
|
| 31 |
+
500,
|
| 32 |
+
750,
|
| 33 |
+
1000,
|
| 34 |
+
1250,
|
| 35 |
+
1500,
|
| 36 |
+
1750,
|
| 37 |
+
2000,
|
| 38 |
+
2250,
|
| 39 |
+
2500,
|
| 40 |
+
2750,
|
| 41 |
+
3000,
|
| 42 |
+
3250,
|
| 43 |
+
3500,
|
| 44 |
+
3750,
|
| 45 |
+
4000,
|
| 46 |
+
4250,
|
| 47 |
+
4500,
|
| 48 |
+
4750,
|
| 49 |
+
4882
|
| 50 |
+
],
|
| 51 |
+
"tokens": [
|
| 52 |
+
2048000,
|
| 53 |
+
4096000,
|
| 54 |
+
6144000,
|
| 55 |
+
8192000,
|
| 56 |
+
10240000,
|
| 57 |
+
12288000,
|
| 58 |
+
14336000,
|
| 59 |
+
16384000,
|
| 60 |
+
18432000,
|
| 61 |
+
20480000,
|
| 62 |
+
22528000,
|
| 63 |
+
24576000,
|
| 64 |
+
26624000,
|
| 65 |
+
28672000,
|
| 66 |
+
30720000,
|
| 67 |
+
32768000,
|
| 68 |
+
34816000,
|
| 69 |
+
36864000,
|
| 70 |
+
38912000,
|
| 71 |
+
39993344
|
| 72 |
+
],
|
| 73 |
+
"train_loss": [
|
| 74 |
+
7.005129949569702,
|
| 75 |
+
5.345415605545044,
|
| 76 |
+
3.5853098974227904,
|
| 77 |
+
2.8179011573791506,
|
| 78 |
+
2.5395464162826538,
|
| 79 |
+
2.3587241649627684,
|
| 80 |
+
2.2533317346572876,
|
| 81 |
+
2.1673336572647095,
|
| 82 |
+
2.0796625332832335,
|
| 83 |
+
2.0115441370010374,
|
| 84 |
+
1.970089602470398,
|
| 85 |
+
1.9127237586975097,
|
| 86 |
+
1.8687362909317016,
|
| 87 |
+
1.8266300711631775,
|
| 88 |
+
1.7815875220298767,
|
| 89 |
+
1.7733343586921693,
|
| 90 |
+
1.7347167477607728,
|
| 91 |
+
1.7206086363792419,
|
| 92 |
+
1.716024341583252,
|
| 93 |
+
1.7011918272032882
|
| 94 |
+
],
|
| 95 |
+
"val_loss": [
|
| 96 |
+
5.927063040837278,
|
| 97 |
+
4.319620455027253,
|
| 98 |
+
3.017888838372497,
|
| 99 |
+
2.614383635366264,
|
| 100 |
+
2.397865873959986,
|
| 101 |
+
2.2723451861828403,
|
| 102 |
+
2.1752542320226755,
|
| 103 |
+
2.0947302440897873,
|
| 104 |
+
2.035730676182507,
|
| 105 |
+
1.9744564385588637,
|
| 106 |
+
1.9187765526926717,
|
| 107 |
+
1.8723163728873267,
|
| 108 |
+
1.831244892301056,
|
| 109 |
+
1.7958330695394216,
|
| 110 |
+
1.7649989099079422,
|
| 111 |
+
1.736258307117853,
|
| 112 |
+
1.7141813690367893,
|
| 113 |
+
1.6987168479349155,
|
| 114 |
+
1.6859077647063483,
|
| 115 |
+
1.6807127555266617
|
| 116 |
+
],
|
| 117 |
+
"val_ppl": [
|
| 118 |
+
375.0513840951873,
|
| 119 |
+
75.16009624108786,
|
| 120 |
+
20.44807688825147,
|
| 121 |
+
13.658794984667923,
|
| 122 |
+
10.999676617531328,
|
| 123 |
+
9.702127449800042,
|
| 124 |
+
8.804423203999164,
|
| 125 |
+
8.12324938009831,
|
| 126 |
+
7.65784548706846,
|
| 127 |
+
7.202703477693308,
|
| 128 |
+
6.812618488821712,
|
| 129 |
+
6.503343133346493,
|
| 130 |
+
6.241652008540929,
|
| 131 |
+
6.024491452193805,
|
| 132 |
+
5.841565990237832,
|
| 133 |
+
5.6760655458050975,
|
| 134 |
+
5.552128507148048,
|
| 135 |
+
5.466927989270828,
|
| 136 |
+
5.397348231447952,
|
| 137 |
+
5.369381664023924
|
| 138 |
+
],
|
| 139 |
+
"aux": [
|
| 140 |
+
0.060628652572631836,
|
| 141 |
+
0.06066158786416054,
|
| 142 |
+
0.0603301040828228,
|
| 143 |
+
0.060409750789403915,
|
| 144 |
+
0.06014479324221611,
|
| 145 |
+
0.06049105152487755,
|
| 146 |
+
0.060275569558143616,
|
| 147 |
+
0.06024990230798721,
|
| 148 |
+
0.06026991084218025,
|
| 149 |
+
0.060117900371551514,
|
| 150 |
+
0.06015768647193909,
|
| 151 |
+
0.060124244540929794,
|
| 152 |
+
0.060057371854782104,
|
| 153 |
+
0.060130394995212555,
|
| 154 |
+
0.0600646436214447,
|
| 155 |
+
0.06001873314380646,
|
| 156 |
+
0.060023233294487,
|
| 157 |
+
0.060005780309438705,
|
| 158 |
+
0.06002525985240936,
|
| 159 |
+
0.060005396604537964
|
| 160 |
+
]
|
| 161 |
+
},
|
| 162 |
+
"test_loss": 1.7064422231434198,
|
| 163 |
+
"test_ppl": 5.509325617155147,
|
| 164 |
+
"bleu_all": 20.80958820937387,
|
| 165 |
+
"bleu_vi": 32.0872516005633,
|
| 166 |
+
"bleu_ja": 10.840698975162605,
|
| 167 |
+
"expert_usage": {
|
| 168 |
+
"share_per_layer": [
|
| 169 |
+
[
|
| 170 |
+
0.250146484375,
|
| 171 |
+
0.258355712890625,
|
| 172 |
+
0.24688313802083334,
|
| 173 |
+
0.24461466471354168
|
| 174 |
+
],
|
| 175 |
+
[
|
| 176 |
+
0.24344685872395833,
|
| 177 |
+
0.25264892578125,
|
| 178 |
+
0.24495035807291668,
|
| 179 |
+
0.258953857421875
|
| 180 |
+
],
|
| 181 |
+
[
|
| 182 |
+
0.25184733072916665,
|
| 183 |
+
0.24724527994791667,
|
| 184 |
+
0.2603068033854167,
|
| 185 |
+
0.2406005859375
|
| 186 |
+
],
|
| 187 |
+
[
|
| 188 |
+
0.269989013671875,
|
| 189 |
+
0.238604736328125,
|
| 190 |
+
0.24382120768229168,
|
| 191 |
+
0.24758504231770834
|
| 192 |
+
],
|
| 193 |
+
[
|
| 194 |
+
0.2714619954427083,
|
| 195 |
+
0.24654134114583334,
|
| 196 |
+
0.24258015950520834,
|
| 197 |
+
0.23941650390625
|
| 198 |
+
],
|
| 199 |
+
[
|
| 200 |
+
0.24441121419270834,
|
| 201 |
+
0.24997151692708333,
|
| 202 |
+
0.2598063151041667,
|
| 203 |
+
0.24581095377604167
|
| 204 |
+
]
|
| 205 |
+
],
|
| 206 |
+
"entropy_per_layer": [
|
| 207 |
+
0.9998441281882332,
|
| 208 |
+
0.9997760654968777,
|
| 209 |
+
0.9997038176065697,
|
| 210 |
+
0.9991841041120254,
|
| 211 |
+
0.9990922208410936,
|
| 212 |
+
0.9997921823231044
|
| 213 |
+
],
|
| 214 |
+
"mean_entropy": 0.9995654197613173,
|
| 215 |
+
"max_share": 0.2714619954427083,
|
| 216 |
+
"min_share": 0.238604736328125
|
| 217 |
+
},
|
| 218 |
+
"samples": [
|
| 219 |
+
{
|
| 220 |
+
"lang": "vi",
|
| 221 |
+
"hyp": "Encrypt Bossible Data Recent Apply This theme",
|
| 222 |
+
"ref": "Require Main keyboard fancy to apply this theme."
|
| 223 |
+
},
|
| 224 |
+
{
|
| 225 |
+
"lang": "ja",
|
| 226 |
+
"hyp": "To be aware of this topic, the main keyboard fan is not included",
|
| 227 |
+
"ref": "Require Main keyboard fancy to apply this theme."
|
| 228 |
+
},
|
| 229 |
+
{
|
| 230 |
+
"lang": "vi",
|
| 231 |
+
"hyp": "\u2022 Suitable for gerolor design. High efficiency of BMP.",
|
| 232 |
+
"ref": "\u2022 It adapts the gerolor design. Higher efficiency than BMP motor."
|
| 233 |
+
},
|
| 234 |
+
{
|
| 235 |
+
"lang": "ja",
|
| 236 |
+
"hyp": "\u2022 It is a high-speed cleaner. BMP motor more efficient.",
|
| 237 |
+
"ref": "\u2022 It adapts the gerolor design. Higher efficiency than BMP motor."
|
| 238 |
+
},
|
| 239 |
+
{
|
| 240 |
+
"lang": "vi",
|
| 241 |
+
"hyp": "Use token for the inner payment or sell it for the million global users.",
|
| 242 |
+
"ref": "Use token for in-app purchases or sell it to millions of users worldwide."
|
| 243 |
+
},
|
| 244 |
+
{
|
| 245 |
+
"lang": "ja",
|
| 246 |
+
"hyp": "Use to buy or get a million users can use it to sell users.",
|
| 247 |
+
"ref": "Use token for in-app purchases or sell it to millions of users worldwide."
|
| 248 |
+
},
|
| 249 |
+
{
|
| 250 |
+
"lang": "vi",
|
| 251 |
+
"hyp": "Good quality, best service",
|
| 252 |
+
"ref": "Good Quality, Best service"
|
| 253 |
+
},
|
| 254 |
+
{
|
| 255 |
+
"lang": "ja",
|
| 256 |
+
"hyp": "Good Quality Best Service",
|
| 257 |
+
"ref": "Good Quality, Best service"
|
| 258 |
+
},
|
| 259 |
+
{
|
| 260 |
+
"lang": "vi",
|
| 261 |
+
"hyp": "United States - Mouse owner is a story. The effect is a another! Can be live? It can work more than? Read this report on the owner helped on the more than 30 years of experience of VS. A greeting material is worth for frame!",
|
| 262 |
+
"ref": "United States \u2013 Gardening organically is one thing. Farming organically is another one! Can one make a living? Is it more work? Read this report on farming organically based on more than 30 years of experience by the Rodale Institute. A document worth to be framed!"
|
| 263 |
+
},
|
| 264 |
+
{
|
| 265 |
+
"lang": "ja",
|
| 266 |
+
"hyp": "The US - The battle is an unknown thing. The dynamic is a one thing! The dynamic is a one thing! It is a result? Could you work? Rodale Institute of 30 years experience in this report. Read more? Read more value of the rodale Institute",
|
| 267 |
+
"ref": "United States \u2013 Gardening organically is one thing. Farming organically is another one! Can one make a living? Is it more work? Read this report on farming organically based on more than 30 years of experience by the Rodale Institute. A document worth to be framed!"
|
| 268 |
+
},
|
| 269 |
+
{
|
| 270 |
+
"lang": "vi",
|
| 271 |
+
"hyp": "- Input speed, higher efficiency",
|
| 272 |
+
"ref": "- Speed printing, higher efficiency"
|
| 273 |
+
},
|
| 274 |
+
{
|
| 275 |
+
"lang": "ja",
|
| 276 |
+
"hyp": "- High power printing, high efficiency",
|
| 277 |
+
"ref": "- Speed printing, higher efficiency"
|
| 278 |
+
}
|
| 279 |
+
]
|
| 280 |
+
}
|
runs/part1/c2_moe_4e1a/model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:091d7b2be584e52974502ef0b713727bed2328bed1483d93f4949344fe7d2ce0
|
| 3 |
+
size 108373506
|
runs/part1/c3_moe_4e2a/expert_counts.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d3290d20f6e0d83fe86ffd5d19eb909b8b8b007dbbea94882a7b1052c85d19a8
|
| 3 |
+
size 896
|
runs/part1/c3_moe_4e2a/metrics.json
ADDED
|
@@ -0,0 +1,280 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"config": {
|
| 3 |
+
"d_model": 512,
|
| 4 |
+
"n_ctx": 256,
|
| 5 |
+
"n_layers": 6,
|
| 6 |
+
"n_heads": 8,
|
| 7 |
+
"n_kv_heads": null,
|
| 8 |
+
"vocab_size": 16000,
|
| 9 |
+
"d_ff": 2048,
|
| 10 |
+
"dropout": 0.0,
|
| 11 |
+
"tie_embeddings": true,
|
| 12 |
+
"rope_base": 10000.0,
|
| 13 |
+
"use_moe": true,
|
| 14 |
+
"n_experts": 4,
|
| 15 |
+
"n_active": 2,
|
| 16 |
+
"n_shared": 0,
|
| 17 |
+
"expert_budget": "total",
|
| 18 |
+
"router_aux_coef": 0.01,
|
| 19 |
+
"router_z_coef": 0.001,
|
| 20 |
+
"router_noise": 0.0,
|
| 21 |
+
"name": "c3_moe_4e2a"
|
| 22 |
+
},
|
| 23 |
+
"params_total": 27085312,
|
| 24 |
+
"params_active": 20793856,
|
| 25 |
+
"params_active_frac": 0.7677170563883481,
|
| 26 |
+
"tokens_trained": 39993344,
|
| 27 |
+
"train_minutes": 5.977490816513697,
|
| 28 |
+
"history": {
|
| 29 |
+
"step": [
|
| 30 |
+
250,
|
| 31 |
+
500,
|
| 32 |
+
750,
|
| 33 |
+
1000,
|
| 34 |
+
1250,
|
| 35 |
+
1500,
|
| 36 |
+
1750,
|
| 37 |
+
2000,
|
| 38 |
+
2250,
|
| 39 |
+
2500,
|
| 40 |
+
2750,
|
| 41 |
+
3000,
|
| 42 |
+
3250,
|
| 43 |
+
3500,
|
| 44 |
+
3750,
|
| 45 |
+
4000,
|
| 46 |
+
4250,
|
| 47 |
+
4500,
|
| 48 |
+
4750,
|
| 49 |
+
4882
|
| 50 |
+
],
|
| 51 |
+
"tokens": [
|
| 52 |
+
2048000,
|
| 53 |
+
4096000,
|
| 54 |
+
6144000,
|
| 55 |
+
8192000,
|
| 56 |
+
10240000,
|
| 57 |
+
12288000,
|
| 58 |
+
14336000,
|
| 59 |
+
16384000,
|
| 60 |
+
18432000,
|
| 61 |
+
20480000,
|
| 62 |
+
22528000,
|
| 63 |
+
24576000,
|
| 64 |
+
26624000,
|
| 65 |
+
28672000,
|
| 66 |
+
30720000,
|
| 67 |
+
32768000,
|
| 68 |
+
34816000,
|
| 69 |
+
36864000,
|
| 70 |
+
38912000,
|
| 71 |
+
39993344
|
| 72 |
+
],
|
| 73 |
+
"train_loss": [
|
| 74 |
+
6.932678838729858,
|
| 75 |
+
4.8501430549621585,
|
| 76 |
+
3.030063013076782,
|
| 77 |
+
2.5144922161102294,
|
| 78 |
+
2.312558856010437,
|
| 79 |
+
2.1567195405960082,
|
| 80 |
+
2.062794575691223,
|
| 81 |
+
1.9837687854766846,
|
| 82 |
+
1.8978928661346435,
|
| 83 |
+
1.8344840021133424,
|
| 84 |
+
1.7940024681091309,
|
| 85 |
+
1.7385272116661072,
|
| 86 |
+
1.6947979021072388,
|
| 87 |
+
1.655762710571289,
|
| 88 |
+
1.613321343421936,
|
| 89 |
+
1.6047331728935241,
|
| 90 |
+
1.5681046996116639,
|
| 91 |
+
1.5530167932510377,
|
| 92 |
+
1.5482245526313783,
|
| 93 |
+
1.5342231302550344
|
| 94 |
+
],
|
| 95 |
+
"val_loss": [
|
| 96 |
+
5.815957151536609,
|
| 97 |
+
3.5409681036793055,
|
| 98 |
+
2.631667544784523,
|
| 99 |
+
2.3553961554521643,
|
| 100 |
+
2.1983878397289955,
|
| 101 |
+
2.0893281588236725,
|
| 102 |
+
1.9944817877878547,
|
| 103 |
+
1.9137775086681483,
|
| 104 |
+
1.8551326133825503,
|
| 105 |
+
1.797981790468479,
|
| 106 |
+
1.7479540647892873,
|
| 107 |
+
1.6987387517934571,
|
| 108 |
+
1.662695400858155,
|
| 109 |
+
1.6283770956334715,
|
| 110 |
+
1.5972373789894518,
|
| 111 |
+
1.5728587451216054,
|
| 112 |
+
1.5533365294228045,
|
| 113 |
+
1.5368288293212071,
|
| 114 |
+
1.524390289034123,
|
| 115 |
+
1.5198453056157473
|
| 116 |
+
],
|
| 117 |
+
"val_ppl": [
|
| 118 |
+
335.61247681418155,
|
| 119 |
+
34.500302898978376,
|
| 120 |
+
13.896924334933233,
|
| 121 |
+
10.542304425931306,
|
| 122 |
+
9.010475453204101,
|
| 123 |
+
8.079485209628377,
|
| 124 |
+
7.348394013519694,
|
| 125 |
+
6.778646892573493,
|
| 126 |
+
6.3925459315518305,
|
| 127 |
+
6.0374503205739884,
|
| 128 |
+
5.742841167512068,
|
| 129 |
+
5.467047737399631,
|
| 130 |
+
5.273505917522247,
|
| 131 |
+
5.095598335675281,
|
| 132 |
+
4.939367956517269,
|
| 133 |
+
4.820408835562715,
|
| 134 |
+
4.7272163957235325,
|
| 135 |
+
4.649821488633603,
|
| 136 |
+
4.592342713825621,
|
| 137 |
+
4.571517952285592
|
| 138 |
+
],
|
| 139 |
+
"aux": [
|
| 140 |
+
0.06065455824136734,
|
| 141 |
+
0.061356235295534134,
|
| 142 |
+
0.06036873906850815,
|
| 143 |
+
0.06070352718234062,
|
| 144 |
+
0.060342706739902496,
|
| 145 |
+
0.060202568769454956,
|
| 146 |
+
0.06036487594246864,
|
| 147 |
+
0.06063545495271683,
|
| 148 |
+
0.06022658199071884,
|
| 149 |
+
0.060059718787670135,
|
| 150 |
+
0.06018009036779404,
|
| 151 |
+
0.06004098430275917,
|
| 152 |
+
0.0602945052087307,
|
| 153 |
+
0.06041521579027176,
|
| 154 |
+
0.06001526862382889,
|
| 155 |
+
0.060082633048295975,
|
| 156 |
+
0.06011537089943886,
|
| 157 |
+
0.06018105149269104,
|
| 158 |
+
0.05996623635292053,
|
| 159 |
+
0.060060061514377594
|
| 160 |
+
]
|
| 161 |
+
},
|
| 162 |
+
"test_loss": 1.5429499578047194,
|
| 163 |
+
"test_ppl": 4.678370931654026,
|
| 164 |
+
"bleu_all": 23.806592958015127,
|
| 165 |
+
"bleu_vi": 35.89816689251285,
|
| 166 |
+
"bleu_ja": 13.13796311190048,
|
| 167 |
+
"expert_usage": {
|
| 168 |
+
"share_per_layer": [
|
| 169 |
+
[
|
| 170 |
+
0.2514241536458333,
|
| 171 |
+
0.22482096354166667,
|
| 172 |
+
0.2512908935546875,
|
| 173 |
+
0.2724639892578125
|
| 174 |
+
],
|
| 175 |
+
[
|
| 176 |
+
0.2506174723307292,
|
| 177 |
+
0.2559611002604167,
|
| 178 |
+
0.24141642252604167,
|
| 179 |
+
0.2520050048828125
|
| 180 |
+
],
|
| 181 |
+
[
|
| 182 |
+
0.25352071126302084,
|
| 183 |
+
0.24995829264322916,
|
| 184 |
+
0.2504933675130208,
|
| 185 |
+
0.24602762858072916
|
| 186 |
+
],
|
| 187 |
+
[
|
| 188 |
+
0.24557698567708333,
|
| 189 |
+
0.25877278645833335,
|
| 190 |
+
0.24317220052083333,
|
| 191 |
+
0.25247802734375
|
| 192 |
+
],
|
| 193 |
+
[
|
| 194 |
+
0.252716064453125,
|
| 195 |
+
0.2518513997395833,
|
| 196 |
+
0.24204610188802084,
|
| 197 |
+
0.25338643391927085
|
| 198 |
+
],
|
| 199 |
+
[
|
| 200 |
+
0.24611714680989583,
|
| 201 |
+
0.2446136474609375,
|
| 202 |
+
0.25966288248697916,
|
| 203 |
+
0.2496063232421875
|
| 204 |
+
]
|
| 205 |
+
],
|
| 206 |
+
"entropy_per_layer": [
|
| 207 |
+
0.9983405286894741,
|
| 208 |
+
0.9998352706142456,
|
| 209 |
+
0.9999589602311422,
|
| 210 |
+
0.9997851451144729,
|
| 211 |
+
0.9998757370740091,
|
| 212 |
+
0.9998027478833192
|
| 213 |
+
],
|
| 214 |
+
"mean_entropy": 0.9995997316011106,
|
| 215 |
+
"max_share": 0.2724639892578125,
|
| 216 |
+
"min_share": 0.22482096354166667
|
| 217 |
+
},
|
| 218 |
+
"samples": [
|
| 219 |
+
{
|
| 220 |
+
"lang": "vi",
|
| 221 |
+
"hyp": "Request the Flexible Command Command Command Command Command Command Command Command Command Command Command Command Command Command Command Command Command Command Command Command Command Command Command Command Command Command Command Command Comm",
|
| 222 |
+
"ref": "Require Main keyboard fancy to apply this theme."
|
| 223 |
+
},
|
| 224 |
+
{
|
| 225 |
+
"lang": "ja",
|
| 226 |
+
"hyp": "To use this theme, use the main keyboard fan",
|
| 227 |
+
"ref": "Require Main keyboard fancy to apply this theme."
|
| 228 |
+
},
|
| 229 |
+
{
|
| 230 |
+
"lang": "vi",
|
| 231 |
+
"hyp": "\u2022 Suitable for gerolor design. Higher efficiency BMP motor.",
|
| 232 |
+
"ref": "\u2022 It adapts the gerolor design. Higher efficiency than BMP motor."
|
| 233 |
+
},
|
| 234 |
+
{
|
| 235 |
+
"lang": "ja",
|
| 236 |
+
"hyp": "\u2022 It can be adjusted with a lightolor design. BMP motor more high.",
|
| 237 |
+
"ref": "\u2022 It adapts the gerolor design. Higher efficiency than BMP motor."
|
| 238 |
+
},
|
| 239 |
+
{
|
| 240 |
+
"lang": "vi",
|
| 241 |
+
"hyp": "Use token for the side payment in application or sell it for millions of global users.",
|
| 242 |
+
"ref": "Use token for in-app purchases or sell it to millions of users worldwide."
|
| 243 |
+
},
|
| 244 |
+
{
|
| 245 |
+
"lang": "ja",
|
| 246 |
+
"hyp": "You can use to buy it or more than the world's largest user.",
|
| 247 |
+
"ref": "Use token for in-app purchases or sell it to millions of users worldwide."
|
| 248 |
+
},
|
| 249 |
+
{
|
| 250 |
+
"lang": "vi",
|
| 251 |
+
"hyp": "Good Quality, Best Service",
|
| 252 |
+
"ref": "Good Quality, Best service"
|
| 253 |
+
},
|
| 254 |
+
{
|
| 255 |
+
"lang": "ja",
|
| 256 |
+
"hyp": "Good Quality Best Service",
|
| 257 |
+
"ref": "Good Quality, Best service"
|
| 258 |
+
},
|
| 259 |
+
{
|
| 260 |
+
"lang": "vi",
|
| 261 |
+
"hyp": "USA - Do the organic garden is a story. Can the organic collaboration be another one! Can make a live? It has a more active activity? Read this report on the organic collaboration based on over 30 years of the Villages of the Rodale. A reliable documentation to",
|
| 262 |
+
"ref": "United States \u2013 Gardening organically is one thing. Farming organically is another one! Can one make a living? Is it more work? Read this report on farming organically based on more than 30 years of experience by the Rodale Institute. A document worth to be framed!"
|
| 263 |
+
},
|
| 264 |
+
{
|
| 265 |
+
"lang": "ja",
|
| 266 |
+
"hyp": "US \u2013 The risk of the machine is one of the most commonly! The risk of the drugs can be a different one! What is the work? What is the work? What is the work? What is the value of this report on the company\u2019s production? Let\u2019s the value of this report",
|
| 267 |
+
"ref": "United States \u2013 Gardening organically is one thing. Farming organically is another one! Can one make a living? Is it more work? Read this report on farming organically based on more than 30 years of experience by the Rodale Institute. A document worth to be framed!"
|
| 268 |
+
},
|
| 269 |
+
{
|
| 270 |
+
"lang": "vi",
|
| 271 |
+
"hyp": "- Printing speed, higher efficiency",
|
| 272 |
+
"ref": "- Speed printing, higher efficiency"
|
| 273 |
+
},
|
| 274 |
+
{
|
| 275 |
+
"lang": "ja",
|
| 276 |
+
"hyp": "Fast printing, high efficiency",
|
| 277 |
+
"ref": "- Speed printing, higher efficiency"
|
| 278 |
+
}
|
| 279 |
+
]
|
| 280 |
+
}
|
runs/part1/c3_moe_4e2a/model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:925d593179dff464a1586b06100ec304119ecf72a4275785b2c3fc6b3f522523
|
| 3 |
+
size 108373506
|
runs/part1/c4_moe_shared/expert_counts.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:19a0f91dc9b931a34d253ae8915c2cf4f6f03a6f6676cb6c4981b4a04e167c99
|
| 3 |
+
size 704
|
runs/part1/c4_moe_shared/metrics.json
ADDED
|
@@ -0,0 +1,274 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"config": {
|
| 3 |
+
"d_model": 512,
|
| 4 |
+
"n_ctx": 256,
|
| 5 |
+
"n_layers": 6,
|
| 6 |
+
"n_heads": 8,
|
| 7 |
+
"n_kv_heads": null,
|
| 8 |
+
"vocab_size": 16000,
|
| 9 |
+
"d_ff": 2048,
|
| 10 |
+
"dropout": 0.0,
|
| 11 |
+
"tie_embeddings": true,
|
| 12 |
+
"rope_base": 10000.0,
|
| 13 |
+
"use_moe": true,
|
| 14 |
+
"n_experts": 3,
|
| 15 |
+
"n_active": 1,
|
| 16 |
+
"n_shared": 1,
|
| 17 |
+
"expert_budget": "total",
|
| 18 |
+
"router_aux_coef": 0.01,
|
| 19 |
+
"router_z_coef": 0.001,
|
| 20 |
+
"router_noise": 0.0,
|
| 21 |
+
"name": "c4_moe_shared"
|
| 22 |
+
},
|
| 23 |
+
"params_total": 27082240,
|
| 24 |
+
"params_active": 20790784,
|
| 25 |
+
"params_active_frac": 0.7676907080064278,
|
| 26 |
+
"tokens_trained": 39993344,
|
| 27 |
+
"train_minutes": 8.254839669540525,
|
| 28 |
+
"history": {
|
| 29 |
+
"step": [
|
| 30 |
+
250,
|
| 31 |
+
500,
|
| 32 |
+
750,
|
| 33 |
+
1000,
|
| 34 |
+
1250,
|
| 35 |
+
1500,
|
| 36 |
+
1750,
|
| 37 |
+
2000,
|
| 38 |
+
2250,
|
| 39 |
+
2500,
|
| 40 |
+
2750,
|
| 41 |
+
3000,
|
| 42 |
+
3250,
|
| 43 |
+
3500,
|
| 44 |
+
3750,
|
| 45 |
+
4000,
|
| 46 |
+
4250,
|
| 47 |
+
4500,
|
| 48 |
+
4750,
|
| 49 |
+
4882
|
| 50 |
+
],
|
| 51 |
+
"tokens": [
|
| 52 |
+
2048000,
|
| 53 |
+
4096000,
|
| 54 |
+
6144000,
|
| 55 |
+
8192000,
|
| 56 |
+
10240000,
|
| 57 |
+
12288000,
|
| 58 |
+
14336000,
|
| 59 |
+
16384000,
|
| 60 |
+
18432000,
|
| 61 |
+
20480000,
|
| 62 |
+
22528000,
|
| 63 |
+
24576000,
|
| 64 |
+
26624000,
|
| 65 |
+
28672000,
|
| 66 |
+
30720000,
|
| 67 |
+
32768000,
|
| 68 |
+
34816000,
|
| 69 |
+
36864000,
|
| 70 |
+
38912000,
|
| 71 |
+
39993344
|
| 72 |
+
],
|
| 73 |
+
"train_loss": [
|
| 74 |
+
6.876347528457641,
|
| 75 |
+
5.12380944442749,
|
| 76 |
+
3.168910701751709,
|
| 77 |
+
2.5599079170227053,
|
| 78 |
+
2.3476795449256898,
|
| 79 |
+
2.1884645824432374,
|
| 80 |
+
2.0901085815429687,
|
| 81 |
+
2.0118385138511656,
|
| 82 |
+
1.9278320488929748,
|
| 83 |
+
1.863843665599823,
|
| 84 |
+
1.8228025641441346,
|
| 85 |
+
1.7663769788742065,
|
| 86 |
+
1.7228386034965515,
|
| 87 |
+
1.684779284954071,
|
| 88 |
+
1.642263150691986,
|
| 89 |
+
1.6322942652702332,
|
| 90 |
+
1.5956687927246094,
|
| 91 |
+
1.5801400737762452,
|
| 92 |
+
1.5760116577148438,
|
| 93 |
+
1.5634317885745654
|
| 94 |
+
],
|
| 95 |
+
"val_loss": [
|
| 96 |
+
5.774119411530249,
|
| 97 |
+
3.9013088198956543,
|
| 98 |
+
2.7054962340815316,
|
| 99 |
+
2.4067226371612236,
|
| 100 |
+
2.221284026512022,
|
| 101 |
+
2.1159398849872204,
|
| 102 |
+
2.0191099608785947,
|
| 103 |
+
1.9443025001277139,
|
| 104 |
+
1.8801203422702955,
|
| 105 |
+
1.8260493047382782,
|
| 106 |
+
1.777217754259085,
|
| 107 |
+
1.7233028694904122,
|
| 108 |
+
1.6839336727857293,
|
| 109 |
+
1.6547064961495992,
|
| 110 |
+
1.6210220176911636,
|
| 111 |
+
1.5963572335910283,
|
| 112 |
+
1.5739887349413801,
|
| 113 |
+
1.5586153764053983,
|
| 114 |
+
1.5460689495306414,
|
| 115 |
+
1.5422485627414952
|
| 116 |
+
],
|
| 117 |
+
"val_ppl": [
|
| 118 |
+
321.8608829082192,
|
| 119 |
+
49.4671503457126,
|
| 120 |
+
14.961739373675766,
|
| 121 |
+
11.097530844824668,
|
| 122 |
+
9.21916091624278,
|
| 123 |
+
8.2973806859213,
|
| 124 |
+
7.53161851449153,
|
| 125 |
+
6.988755498533983,
|
| 126 |
+
6.554293573301348,
|
| 127 |
+
6.209307057267729,
|
| 128 |
+
5.913381030742966,
|
| 129 |
+
5.603003927184781,
|
| 130 |
+
5.386703879832533,
|
| 131 |
+
5.231544218449957,
|
| 132 |
+
5.058257304189072,
|
| 133 |
+
4.935022507133378,
|
| 134 |
+
4.825858927168143,
|
| 135 |
+
4.752236628619033,
|
| 136 |
+
4.692985511409633,
|
| 137 |
+
4.675090695883697
|
| 138 |
+
],
|
| 139 |
+
"aux": [
|
| 140 |
+
0.060582518577575684,
|
| 141 |
+
0.06026960164308548,
|
| 142 |
+
0.0608997568488121,
|
| 143 |
+
0.060575976967811584,
|
| 144 |
+
0.06053435429930687,
|
| 145 |
+
0.06068896874785423,
|
| 146 |
+
0.060172975063323975,
|
| 147 |
+
0.06036805734038353,
|
| 148 |
+
0.06020189821720123,
|
| 149 |
+
0.06019610911607742,
|
| 150 |
+
0.06024900823831558,
|
| 151 |
+
0.06026880815625191,
|
| 152 |
+
0.060056332498788834,
|
| 153 |
+
0.06005406752228737,
|
| 154 |
+
0.06004694849252701,
|
| 155 |
+
0.06003514304757118,
|
| 156 |
+
0.0600292831659317,
|
| 157 |
+
0.060025546699762344,
|
| 158 |
+
0.060004863888025284,
|
| 159 |
+
0.06001139432191849
|
| 160 |
+
]
|
| 161 |
+
},
|
| 162 |
+
"test_loss": 1.5697562833566416,
|
| 163 |
+
"test_ppl": 4.80547687635216,
|
| 164 |
+
"bleu_all": 22.940068647130627,
|
| 165 |
+
"bleu_vi": 34.72311452345555,
|
| 166 |
+
"bleu_ja": 12.662939631548621,
|
| 167 |
+
"expert_usage": {
|
| 168 |
+
"share_per_layer": [
|
| 169 |
+
[
|
| 170 |
+
0.3468546549479167,
|
| 171 |
+
0.3360209147135417,
|
| 172 |
+
0.31712443033854165
|
| 173 |
+
],
|
| 174 |
+
[
|
| 175 |
+
0.339703369140625,
|
| 176 |
+
0.33891398111979165,
|
| 177 |
+
0.32138264973958336
|
| 178 |
+
],
|
| 179 |
+
[
|
| 180 |
+
0.327117919921875,
|
| 181 |
+
0.345697021484375,
|
| 182 |
+
0.32718505859375
|
| 183 |
+
],
|
| 184 |
+
[
|
| 185 |
+
0.33376871744791664,
|
| 186 |
+
0.33148193359375,
|
| 187 |
+
0.33474934895833336
|
| 188 |
+
],
|
| 189 |
+
[
|
| 190 |
+
0.3370585123697917,
|
| 191 |
+
0.3391194661458333,
|
| 192 |
+
0.323822021484375
|
| 193 |
+
],
|
| 194 |
+
[
|
| 195 |
+
0.32634073893229165,
|
| 196 |
+
0.346722412109375,
|
| 197 |
+
0.32693684895833336
|
| 198 |
+
]
|
| 199 |
+
],
|
| 200 |
+
"entropy_per_layer": [
|
| 201 |
+
0.9993791706044766,
|
| 202 |
+
0.9997052880063602,
|
| 203 |
+
0.9996888145812024,
|
| 204 |
+
0.9999923187798743,
|
| 205 |
+
0.9998109649527951,
|
| 206 |
+
0.9996349911604364
|
| 207 |
+
],
|
| 208 |
+
"mean_entropy": 0.9997019246808576,
|
| 209 |
+
"max_share": 0.3468546549479167,
|
| 210 |
+
"min_share": 0.31712443033854165
|
| 211 |
+
},
|
| 212 |
+
"samples": [
|
| 213 |
+
{
|
| 214 |
+
"lang": "vi",
|
| 215 |
+
"hyp": "Require the main key key to apply this topic",
|
| 216 |
+
"ref": "Require Main keyboard fancy to apply this theme."
|
| 217 |
+
},
|
| 218 |
+
{
|
| 219 |
+
"lang": "ja",
|
| 220 |
+
"hyp": "Use this theme to use this theme, main keyboard fan",
|
| 221 |
+
"ref": "Require Main keyboard fancy to apply this theme."
|
| 222 |
+
},
|
| 223 |
+
{
|
| 224 |
+
"lang": "vi",
|
| 225 |
+
"hyp": "\u2022 Suitable for gerolor design. Higher efficiency BMP motor.",
|
| 226 |
+
"ref": "\u2022 It adapts the gerolor design. Higher efficiency than BMP motor."
|
| 227 |
+
},
|
| 228 |
+
{
|
| 229 |
+
"lang": "ja",
|
| 230 |
+
"hyp": "\u2022 It is a high-speed LED design. BMP motor more high-speed power.",
|
| 231 |
+
"ref": "\u2022 It adapts the gerolor design. Higher efficiency than BMP motor."
|
| 232 |
+
},
|
| 233 |
+
{
|
| 234 |
+
"lang": "vi",
|
| 235 |
+
"hyp": "Use token for the side payment in the application or sell it for the global millions of users.",
|
| 236 |
+
"ref": "Use token for in-app purchases or sell it to millions of users worldwide."
|
| 237 |
+
},
|
| 238 |
+
{
|
| 239 |
+
"lang": "ja",
|
| 240 |
+
"hyp": "Use the app to buy or use the world to buy it, you can use the world to buy it.",
|
| 241 |
+
"ref": "Use token for in-app purchases or sell it to millions of users worldwide."
|
| 242 |
+
},
|
| 243 |
+
{
|
| 244 |
+
"lang": "vi",
|
| 245 |
+
"hyp": "Good quality, best service",
|
| 246 |
+
"ref": "Good Quality, Best service"
|
| 247 |
+
},
|
| 248 |
+
{
|
| 249 |
+
"lang": "ja",
|
| 250 |
+
"hyp": "Good Quality Best Service",
|
| 251 |
+
"ref": "Good Quality, Best service"
|
| 252 |
+
},
|
| 253 |
+
{
|
| 254 |
+
"lang": "vi",
|
| 255 |
+
"hyp": "USA \u2013 Make a star. The organic is a different story! Can make life? It has a lot of operation? Can you do not work more? Read this report on the organic taskets based on over 30 years of experience of Rodale Rodale. A worthy of the",
|
| 256 |
+
"ref": "United States \u2013 Gardening organically is one thing. Farming organically is another one! Can one make a living? Is it more work? Read this report on farming organically based on more than 30 years of experience by the Rodale Institute. A document worth to be framed!"
|
| 257 |
+
},
|
| 258 |
+
{
|
| 259 |
+
"lang": "ja",
|
| 260 |
+
"hyp": "US \u2013 The main thing is that the main thing is that the main thing is. Is it possible to be a one thing! What is the most popularity of the company? Rodale Institute of the 30 years of experience in the world? Rodale Institute of the 30 years of experience in the world.",
|
| 261 |
+
"ref": "United States \u2013 Gardening organically is one thing. Farming organically is another one! Can one make a living? Is it more work? Read this report on farming organically based on more than 30 years of experience by the Rodale Institute. A document worth to be framed!"
|
| 262 |
+
},
|
| 263 |
+
{
|
| 264 |
+
"lang": "vi",
|
| 265 |
+
"hyp": "- High speed, higher efficiency",
|
| 266 |
+
"ref": "- Speed printing, higher efficiency"
|
| 267 |
+
},
|
| 268 |
+
{
|
| 269 |
+
"lang": "ja",
|
| 270 |
+
"hyp": "- fast printing, high efficiency",
|
| 271 |
+
"ref": "- Speed printing, higher efficiency"
|
| 272 |
+
}
|
| 273 |
+
]
|
| 274 |
+
}
|
runs/part1/c4_moe_shared/model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f0dfb6a64003dbde3203b97a5fb4542cc5d9d3f0c111dfb4e50e716ae8af99d7
|
| 3 |
+
size 108361218
|
runs/part1/c5_moe_4e2a_wide/expert_counts.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5d2f4d694fb619e016b553be2bd4b4b066034a57e8948950def3c47810cd71a8
|
| 3 |
+
size 896
|
runs/part1/c5_moe_4e2a_wide/metrics.json
ADDED
|
@@ -0,0 +1,280 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"config": {
|
| 3 |
+
"d_model": 512,
|
| 4 |
+
"n_ctx": 256,
|
| 5 |
+
"n_layers": 6,
|
| 6 |
+
"n_heads": 8,
|
| 7 |
+
"n_kv_heads": null,
|
| 8 |
+
"vocab_size": 16000,
|
| 9 |
+
"d_ff": 2048,
|
| 10 |
+
"dropout": 0.0,
|
| 11 |
+
"tie_embeddings": true,
|
| 12 |
+
"rope_base": 10000.0,
|
| 13 |
+
"use_moe": true,
|
| 14 |
+
"n_experts": 4,
|
| 15 |
+
"n_active": 2,
|
| 16 |
+
"n_shared": 0,
|
| 17 |
+
"expert_budget": "active",
|
| 18 |
+
"router_aux_coef": 0.01,
|
| 19 |
+
"router_z_coef": 0.001,
|
| 20 |
+
"router_noise": 0.0,
|
| 21 |
+
"name": "c5_moe_4e2a_wide"
|
| 22 |
+
},
|
| 23 |
+
"params_total": 39668224,
|
| 24 |
+
"params_active": 27085312,
|
| 25 |
+
"params_active_frac": 0.6827961846741614,
|
| 26 |
+
"tokens_trained": 39993344,
|
| 27 |
+
"train_minutes": 10.399339232904216,
|
| 28 |
+
"history": {
|
| 29 |
+
"step": [
|
| 30 |
+
250,
|
| 31 |
+
500,
|
| 32 |
+
750,
|
| 33 |
+
1000,
|
| 34 |
+
1250,
|
| 35 |
+
1500,
|
| 36 |
+
1750,
|
| 37 |
+
2000,
|
| 38 |
+
2250,
|
| 39 |
+
2500,
|
| 40 |
+
2750,
|
| 41 |
+
3000,
|
| 42 |
+
3250,
|
| 43 |
+
3500,
|
| 44 |
+
3750,
|
| 45 |
+
4000,
|
| 46 |
+
4250,
|
| 47 |
+
4500,
|
| 48 |
+
4750,
|
| 49 |
+
4882
|
| 50 |
+
],
|
| 51 |
+
"tokens": [
|
| 52 |
+
2048000,
|
| 53 |
+
4096000,
|
| 54 |
+
6144000,
|
| 55 |
+
8192000,
|
| 56 |
+
10240000,
|
| 57 |
+
12288000,
|
| 58 |
+
14336000,
|
| 59 |
+
16384000,
|
| 60 |
+
18432000,
|
| 61 |
+
20480000,
|
| 62 |
+
22528000,
|
| 63 |
+
24576000,
|
| 64 |
+
26624000,
|
| 65 |
+
28672000,
|
| 66 |
+
30720000,
|
| 67 |
+
32768000,
|
| 68 |
+
34816000,
|
| 69 |
+
36864000,
|
| 70 |
+
38912000,
|
| 71 |
+
39993344
|
| 72 |
+
],
|
| 73 |
+
"train_loss": [
|
| 74 |
+
6.853419994354248,
|
| 75 |
+
4.993136345863342,
|
| 76 |
+
3.0746679944992064,
|
| 77 |
+
2.502995304107666,
|
| 78 |
+
2.2940738983154296,
|
| 79 |
+
2.138678603649139,
|
| 80 |
+
2.0465569424629213,
|
| 81 |
+
1.9657323894500733,
|
| 82 |
+
1.8803646597862245,
|
| 83 |
+
1.8162896943092346,
|
| 84 |
+
1.7722512016296386,
|
| 85 |
+
1.7179432535171508,
|
| 86 |
+
1.6736100573539734,
|
| 87 |
+
1.633712881565094,
|
| 88 |
+
1.590774125099182,
|
| 89 |
+
1.5817544507980346,
|
| 90 |
+
1.5439585905075073,
|
| 91 |
+
1.5281757526397706,
|
| 92 |
+
1.5226338124275207,
|
| 93 |
+
1.5075315020301125
|
| 94 |
+
],
|
| 95 |
+
"val_loss": [
|
| 96 |
+
5.744251000128943,
|
| 97 |
+
3.705763323282871,
|
| 98 |
+
2.645000278136105,
|
| 99 |
+
2.3477131044175867,
|
| 100 |
+
2.18264230341224,
|
| 101 |
+
2.0712299030870684,
|
| 102 |
+
1.9749592113999674,
|
| 103 |
+
1.8937771024767465,
|
| 104 |
+
1.8399180626230698,
|
| 105 |
+
1.7817728903282533,
|
| 106 |
+
1.7259351912483663,
|
| 107 |
+
1.6743592450153366,
|
| 108 |
+
1.6418962625576146,
|
| 109 |
+
1.606067205758978,
|
| 110 |
+
1.5728381516956025,
|
| 111 |
+
1.5479044002497726,
|
| 112 |
+
1.5287805218946557,
|
| 113 |
+
1.5128499926274297,
|
| 114 |
+
1.5007633056112397,
|
| 115 |
+
1.4946841749161597
|
| 116 |
+
],
|
| 117 |
+
"val_ppl": [
|
| 118 |
+
312.38956044228473,
|
| 119 |
+
40.68108829008713,
|
| 120 |
+
14.083449000239515,
|
| 121 |
+
10.46161771995083,
|
| 122 |
+
8.869711792577345,
|
| 123 |
+
7.93457587851244,
|
| 124 |
+
7.206325711888602,
|
| 125 |
+
6.64441799200949,
|
| 126 |
+
6.296022360333871,
|
| 127 |
+
5.940378728378132,
|
| 128 |
+
5.617772265313475,
|
| 129 |
+
5.335375385201688,
|
| 130 |
+
5.164954341529566,
|
| 131 |
+
4.983174838895958,
|
| 132 |
+
4.820309567852191,
|
| 133 |
+
4.701607164931057,
|
| 134 |
+
4.612548489290319,
|
| 135 |
+
4.539650345272022,
|
| 136 |
+
4.4851112746810164,
|
| 137 |
+
4.457928404883079
|
| 138 |
+
],
|
| 139 |
+
"aux": [
|
| 140 |
+
0.06112191826105118,
|
| 141 |
+
0.06075165420770645,
|
| 142 |
+
0.06048699468374252,
|
| 143 |
+
0.06049969792366028,
|
| 144 |
+
0.060400236397981644,
|
| 145 |
+
0.06038526073098183,
|
| 146 |
+
0.06016121432185173,
|
| 147 |
+
0.06033073738217354,
|
| 148 |
+
0.06032004952430725,
|
| 149 |
+
0.06032530218362808,
|
| 150 |
+
0.060129065066576004,
|
| 151 |
+
0.06012183427810669,
|
| 152 |
+
0.06008124724030495,
|
| 153 |
+
0.06017075479030609,
|
| 154 |
+
0.06018134951591492,
|
| 155 |
+
0.06014928221702576,
|
| 156 |
+
0.060052644461393356,
|
| 157 |
+
0.06003066897392273,
|
| 158 |
+
0.06010856479406357,
|
| 159 |
+
0.060053929686546326
|
| 160 |
+
]
|
| 161 |
+
},
|
| 162 |
+
"test_loss": 1.5180584931264818,
|
| 163 |
+
"test_ppl": 4.563356800309027,
|
| 164 |
+
"bleu_all": 25.42440877950764,
|
| 165 |
+
"bleu_vi": 36.847523488010154,
|
| 166 |
+
"bleu_ja": 14.843917444086086,
|
| 167 |
+
"expert_usage": {
|
| 168 |
+
"share_per_layer": [
|
| 169 |
+
[
|
| 170 |
+
0.2706502278645833,
|
| 171 |
+
0.27714742024739586,
|
| 172 |
+
0.22122904459635417,
|
| 173 |
+
0.23097330729166668
|
| 174 |
+
],
|
| 175 |
+
[
|
| 176 |
+
0.2413818359375,
|
| 177 |
+
0.2466766357421875,
|
| 178 |
+
0.25595296223958336,
|
| 179 |
+
0.2559885660807292
|
| 180 |
+
],
|
| 181 |
+
[
|
| 182 |
+
0.2587371826171875,
|
| 183 |
+
0.248760986328125,
|
| 184 |
+
0.23860677083333334,
|
| 185 |
+
0.25389506022135416
|
| 186 |
+
],
|
| 187 |
+
[
|
| 188 |
+
0.2612386067708333,
|
| 189 |
+
0.2445343017578125,
|
| 190 |
+
0.24803670247395834,
|
| 191 |
+
0.24619038899739584
|
| 192 |
+
],
|
| 193 |
+
[
|
| 194 |
+
0.25701497395833334,
|
| 195 |
+
0.23882649739583334,
|
| 196 |
+
0.25569559733072916,
|
| 197 |
+
0.24846293131510416
|
| 198 |
+
],
|
| 199 |
+
[
|
| 200 |
+
0.25289713541666664,
|
| 201 |
+
0.23620096842447916,
|
| 202 |
+
0.255133056640625,
|
| 203 |
+
0.25576883951822915
|
| 204 |
+
]
|
| 205 |
+
],
|
| 206 |
+
"entropy_per_layer": [
|
| 207 |
+
0.9965954262976625,
|
| 208 |
+
0.9997735329715377,
|
| 209 |
+
0.9996769531341378,
|
| 210 |
+
0.9997504121692213,
|
| 211 |
+
0.9996969338234194,
|
| 212 |
+
0.9996226284693275
|
| 213 |
+
],
|
| 214 |
+
"mean_entropy": 0.9991859811442177,
|
| 215 |
+
"max_share": 0.27714742024739586,
|
| 216 |
+
"min_share": 0.22122904459635417
|
| 217 |
+
},
|
| 218 |
+
"samples": [
|
| 219 |
+
{
|
| 220 |
+
"lang": "vi",
|
| 221 |
+
"hyp": "Require the key key request to apply this topic",
|
| 222 |
+
"ref": "Require Main keyboard fancy to apply this theme."
|
| 223 |
+
},
|
| 224 |
+
{
|
| 225 |
+
"lang": "ja",
|
| 226 |
+
"hyp": "To apply this theme, you can use the main keyboard fans",
|
| 227 |
+
"ref": "Require Main keyboard fancy to apply this theme."
|
| 228 |
+
},
|
| 229 |
+
{
|
| 230 |
+
"lang": "vi",
|
| 231 |
+
"hyp": "\u2022 Suitable for gerolor design. Higher efficiency BMP motor.",
|
| 232 |
+
"ref": "\u2022 It adapts the gerolor design. Higher efficiency than BMP motor."
|
| 233 |
+
},
|
| 234 |
+
{
|
| 235 |
+
"lang": "ja",
|
| 236 |
+
"hyp": "\u2022 It is a high performance and high performance. BMP motor.",
|
| 237 |
+
"ref": "\u2022 It adapts the gerolor design. Higher efficiency than BMP motor."
|
| 238 |
+
},
|
| 239 |
+
{
|
| 240 |
+
"lang": "vi",
|
| 241 |
+
"hyp": "Use token for internal payment bar or sell it for millions of global users.",
|
| 242 |
+
"ref": "Use token for in-app purchases or sell it to millions of users worldwide."
|
| 243 |
+
},
|
| 244 |
+
{
|
| 245 |
+
"lang": "ja",
|
| 246 |
+
"hyp": "Use the app to buy in the app or the world, you can sell it to the world.",
|
| 247 |
+
"ref": "Use token for in-app purchases or sell it to millions of users worldwide."
|
| 248 |
+
},
|
| 249 |
+
{
|
| 250 |
+
"lang": "vi",
|
| 251 |
+
"hyp": "Good quality, best service",
|
| 252 |
+
"ref": "Good Quality, Best service"
|
| 253 |
+
},
|
| 254 |
+
{
|
| 255 |
+
"lang": "ja",
|
| 256 |
+
"hyp": "Best Service Best Service",
|
| 257 |
+
"ref": "Good Quality, Best service"
|
| 258 |
+
},
|
| 259 |
+
{
|
| 260 |
+
"lang": "vi",
|
| 261 |
+
"hyp": "United States - Mixed the organic garden is a chat. The organization is a different one! Can make a lot? It works more than not? Read this report on the organization of the organization based on more than 30 years of the Rodale Rodale. A rid of",
|
| 262 |
+
"ref": "United States \u2013 Gardening organically is one thing. Farming organically is another one! Can one make a living? Is it more work? Read this report on farming organically based on more than 30 years of experience by the Rodale Institute. A document worth to be framed!"
|
| 263 |
+
},
|
| 264 |
+
{
|
| 265 |
+
"lang": "ja",
|
| 266 |
+
"hyp": "USA - The course of the US is one of the same. The main farmers are not one of the same! The farmers are now based on the farmers? Rodale Institute of this report is based on the rodale Institute of this report. Please read the value for this",
|
| 267 |
+
"ref": "United States \u2013 Gardening organically is one thing. Farming organically is another one! Can one make a living? Is it more work? Read this report on farming organically based on more than 30 years of experience by the Rodale Institute. A document worth to be framed!"
|
| 268 |
+
},
|
| 269 |
+
{
|
| 270 |
+
"lang": "vi",
|
| 271 |
+
"hyp": "- Printer speed, higher efficiency",
|
| 272 |
+
"ref": "- Speed printing, higher efficiency"
|
| 273 |
+
},
|
| 274 |
+
{
|
| 275 |
+
"lang": "ja",
|
| 276 |
+
"hyp": "- High efficiency",
|
| 277 |
+
"ref": "- Speed printing, higher efficiency"
|
| 278 |
+
}
|
| 279 |
+
]
|
| 280 |
+
}
|
runs/part1/c5_moe_4e2a_wide/model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fe6e3ae42526c2247c92a88a2658cdab3bc746ccbf237d7276df8a3114b811d3
|
| 3 |
+
size 158705218
|
runs/part1/cache/test_all_ids.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:df27757bedaa6ee17cbf486f82d1776e0fc072e0208f6cc20434469161853f44
|
| 3 |
+
size 4459760
|
runs/part1/cache/test_all_lang.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0c1a2f41b05c351b8609512d0b0fd0e83d393951df4565c442435ef063fe839f
|
| 3 |
+
size 2229944
|
runs/part1/cache/test_all_tgt.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e446e08ea33ce03ccdcebd669d854b7bc7b86149a2f40dbc38eb91d02a3c98b9
|
| 3 |
+
size 2229944
|
runs/part1/cache/train_250000_ids.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d2e94847bf5238143d70cd4e485058f2ad54794b4d4598635c0b4689330952d6
|
| 3 |
+
size 45100860
|
runs/part1/cache/train_250000_lang.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:24f6604bad0d902d62f6daebe16a08feca9cb9ef5a5fa44b739416ac7170bfbd
|
| 3 |
+
size 22550494
|
runs/part1/cache/train_250000_tgt.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7a3a3d7381cf85964d2a38de562bb62119cba14c3c8aa1205bf1d18b1bb2a262
|
| 3 |
+
size 22550494
|
runs/part1/cache/train_446252_ids.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8e138f47cfd89a668f07565c13a4c7a18be7b92229844bc240e5816221fd2eb4
|
| 3 |
+
size 80405264
|
runs/part1/cache/train_446252_lang.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c9d538b756d1655b9f479a2c834bdb1ba429df73522264cf7bed3760b2714bbe
|
| 3 |
+
size 40202696
|
runs/part1/cache/train_446252_tgt.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8550de4bb69e52f25ec97541a2425b1c68ce4e50e7a1d99e10be920ca9eb04c7
|
| 3 |
+
size 40202696
|
runs/part1/cache/validation_all_ids.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f1e012f3e9dd91f9ef389cdb1af5f5f2b43d53c0e31d3b1797c0ca280bb7d37c
|
| 3 |
+
size 4477514
|
runs/part1/cache/validation_all_lang.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f389dd780e14c994990983671540cc316329b1aae320c92c0ad13f06edde7a69
|
| 3 |
+
size 2238821
|
runs/part1/cache/validation_all_tgt.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d6a3d1c8d0435b7c81efa26d31f51db3174364f18c942a85c16c859613ea091e
|
| 3 |
+
size 2238821
|
runs/part1/tokenizer_16000.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
runs/part2/adamw/metrics.json
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"optimizer": "adamw",
|
| 3 |
+
"category": "1. AdamW",
|
| 4 |
+
"hparams": {
|
| 5 |
+
"lr": 0.001,
|
| 6 |
+
"betas": [
|
| 7 |
+
0.9,
|
| 8 |
+
0.95
|
| 9 |
+
],
|
| 10 |
+
"weight_decay": 0.1
|
| 11 |
+
},
|
| 12 |
+
"n_params": 27073024,
|
| 13 |
+
"tokens": 41836084,
|
| 14 |
+
"chinchilla_ratio": 0.07726525858359967,
|
| 15 |
+
"history": {
|
| 16 |
+
"step": [
|
| 17 |
+
340,
|
| 18 |
+
680,
|
| 19 |
+
1020,
|
| 20 |
+
1360,
|
| 21 |
+
1700,
|
| 22 |
+
2040,
|
| 23 |
+
2380,
|
| 24 |
+
2720,
|
| 25 |
+
3060,
|
| 26 |
+
3400,
|
| 27 |
+
3404
|
| 28 |
+
],
|
| 29 |
+
"tokens": [
|
| 30 |
+
4177920,
|
| 31 |
+
8355840,
|
| 32 |
+
12533760,
|
| 33 |
+
16711680,
|
| 34 |
+
20889600,
|
| 35 |
+
25067520,
|
| 36 |
+
29245440,
|
| 37 |
+
33423360,
|
| 38 |
+
37601280,
|
| 39 |
+
41779200,
|
| 40 |
+
41828352
|
| 41 |
+
],
|
| 42 |
+
"frac": [
|
| 43 |
+
0.09986403125110849,
|
| 44 |
+
0.19972806250221697,
|
| 45 |
+
0.29959209375332546,
|
| 46 |
+
0.39945612500443395,
|
| 47 |
+
0.4993201562555425,
|
| 48 |
+
0.5991841875066509,
|
| 49 |
+
0.6990482187577595,
|
| 50 |
+
0.7989122500088679,
|
| 51 |
+
0.8987762812599764,
|
| 52 |
+
0.998640312511085,
|
| 53 |
+
0.9998151834669803
|
| 54 |
+
],
|
| 55 |
+
"chinchilla": [
|
| 56 |
+
0.007716020197817577,
|
| 57 |
+
0.015432040395635154,
|
| 58 |
+
0.02314806059345273,
|
| 59 |
+
0.03086408079127031,
|
| 60 |
+
0.03858010098908788,
|
| 61 |
+
0.04629612118690546,
|
| 62 |
+
0.05401214138472304,
|
| 63 |
+
0.06172816158254062,
|
| 64 |
+
0.06944418178035819,
|
| 65 |
+
0.07716020197817576,
|
| 66 |
+
0.07725097868638539
|
| 67 |
+
],
|
| 68 |
+
"train_loss": [
|
| 69 |
+
6.08644953475279,
|
| 70 |
+
4.916274710262523,
|
| 71 |
+
4.56550079794491,
|
| 72 |
+
4.3419646291171805,
|
| 73 |
+
4.146877277598661,
|
| 74 |
+
3.9935960271779227,
|
| 75 |
+
3.871371193493114,
|
| 76 |
+
3.7856738307896785,
|
| 77 |
+
3.7075969766168035,
|
| 78 |
+
3.683904698315789,
|
| 79 |
+
3.677709937095642
|
| 80 |
+
],
|
| 81 |
+
"val_loss": [
|
| 82 |
+
5.27189876238505,
|
| 83 |
+
4.805130155881246,
|
| 84 |
+
4.549369756380717,
|
| 85 |
+
4.350636156400045,
|
| 86 |
+
4.17417006889979,
|
| 87 |
+
4.04394504626592,
|
| 88 |
+
3.947056849797567,
|
| 89 |
+
3.8738916635513307,
|
| 90 |
+
3.824341630935669,
|
| 91 |
+
3.795613133907318,
|
| 92 |
+
3.795301882425944
|
| 93 |
+
],
|
| 94 |
+
"val_ppl": [
|
| 95 |
+
194.78546284765676,
|
| 96 |
+
122.13538662558398,
|
| 97 |
+
94.5727856337869,
|
| 98 |
+
77.52776702621331,
|
| 99 |
+
64.9858834683878,
|
| 100 |
+
57.05096814430916,
|
| 101 |
+
51.78273796443774,
|
| 102 |
+
48.12932523350378,
|
| 103 |
+
45.802635399370736,
|
| 104 |
+
44.50551588253823,
|
| 105 |
+
44.49166563035806
|
| 106 |
+
],
|
| 107 |
+
"bleu": [
|
| 108 |
+
0.7670922235321791,
|
| 109 |
+
0.7953021104079315,
|
| 110 |
+
0.738478927151095,
|
| 111 |
+
0.9714989587292588,
|
| 112 |
+
0.8406834526137331,
|
| 113 |
+
1.7757004623956412,
|
| 114 |
+
1.6150352601257152,
|
| 115 |
+
1.8356919289181768,
|
| 116 |
+
1.8137176506646009,
|
| 117 |
+
2.2190015692437535,
|
| 118 |
+
2.2715908663405635
|
| 119 |
+
]
|
| 120 |
+
},
|
| 121 |
+
"test_loss": 3.718237472617108,
|
| 122 |
+
"test_ppl": 41.19172854039412,
|
| 123 |
+
"test_bleu": 2.2715908663405635,
|
| 124 |
+
"minutes": 7.086392619088292,
|
| 125 |
+
"optimizer_state_mb": 216.584192,
|
| 126 |
+
"state_multiple": 2.0,
|
| 127 |
+
"samples": [
|
| 128 |
+
[
|
| 129 |
+
", who doesn't love a good friend? I mean, who doesn't love a good friend? I mean, who doesn't love a good friend? I mean, who doesn't love a good friend? I mean, I'm not",
|
| 130 |
+
", who wasn't obsessed with sex, right? But I digress. I had more pressing concerns, like getting into Miss Satron's English Literature class. I had heard she was a stickler for grammar and syntax"
|
| 131 |
+
],
|
| 132 |
+
[
|
| 133 |
+
" to invest in their energy savings and reduce energy consumption. The development of a comprehensive approach to energy efficiency is a critical factor in the development of energy savings and energy savings. The development of a comprehensive approach to energy efficiency is a critical factor in",
|
| 134 |
+
" and tenants to finance energy efficiency improvements without requiring public subsidy. The scheme was positioned as a key component of the government's strategy to enhance energy security, reduce carbon emissions, and protect consumers from volatile energy prices. However, the"
|
| 135 |
+
],
|
| 136 |
+
[
|
| 137 |
+
" all, he was a good man, and he was a good man, and he was a good man, and he was a good man, and he was a good man, and he was a good man, and he was a good",
|
| 138 |
+
" all, is every child), he forthwith proceeded to turn her into a mere toy, and played with her as children will play with dolls. It is true, he treated her kindly enough, and would never have hurt her intentionally"
|
| 139 |
+
],
|
| 140 |
+
[
|
| 141 |
+
" see that they're going to be a big part of the team. And I think that's a really good point. I think that's a really good point. I think that's a really good point. I think that's a really",
|
| 142 |
+
" probably replace 3 or 4 players in that starting line-up and still have a similar quality in terms of the ability of those players. It's pretty incredible. I mean, even if you look at the bench. The bench was good enough"
|
| 143 |
+
],
|
| 144 |
+
[
|
| 145 |
+
", I'm not sure how it's going to be. I mean, who doesn't love a little bit of a little bit of a little bit of a little bit of a little bit of a little bit of a little bit of a",
|
| 146 |
+
", let's ease into the waters with a bit of understanding-no splash too big too soon. We're talking about that elusive sensation of seeing without seeing, understanding without the presence of eyes or ears. Remote viewing is the art"
|
| 147 |
+
],
|
| 148 |
+
[
|
| 149 |
+
" properties of the material is not only influenced by the mechanical properties of the material, but also by the mechanical properties of the material. The integration of these techniques with other materials, such as the use of high-resolution and high-resolution techniques",
|
| 150 |
+
" rates and solidification conditions during casting stands as a formidable challenge. Unlike in wrought aluminium alloys, where rolling, forging, and extrusion processes inherently refine grains and help in achieving desirable textures, cast aluminium alloys"
|
| 151 |
+
]
|
| 152 |
+
]
|
| 153 |
+
}
|
runs/part2/adamw/model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:15a6b828f8a0114554ac5b02ad6f662e6f9fd1c9531ecd8cfe9f21875c3fc0f0
|
| 3 |
+
size 108309012
|
runs/part2/benchmark.json
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"optimizer": "adamw",
|
| 4 |
+
"category": "1. AdamW",
|
| 5 |
+
"fwd_bwd_ms": 44.51255885263284,
|
| 6 |
+
"step_ms": 3.5697922110557556,
|
| 7 |
+
"hessian_amortised_ms": 0.0,
|
| 8 |
+
"total_ms": 48.0823510636886,
|
| 9 |
+
"state_mb": 216.584192,
|
| 10 |
+
"state_multiple": 2.0
|
| 11 |
+
},
|
| 12 |
+
{
|
| 13 |
+
"optimizer": "mars",
|
| 14 |
+
"category": "2. Variance-reduced AdamW variant",
|
| 15 |
+
"fwd_bwd_ms": 43.19541243215402,
|
| 16 |
+
"step_ms": 6.120803082982699,
|
| 17 |
+
"hessian_amortised_ms": 0.0,
|
| 18 |
+
"total_ms": 49.31621551513672,
|
| 19 |
+
"state_mb": 324.876288,
|
| 20 |
+
"state_multiple": 3.0
|
| 21 |
+
},
|
| 22 |
+
{
|
| 23 |
+
"optimizer": "lion",
|
| 24 |
+
"category": "3. Memory-efficient",
|
| 25 |
+
"fwd_bwd_ms": 44.26518057783445,
|
| 26 |
+
"step_ms": 3.246958553791046,
|
| 27 |
+
"hessian_amortised_ms": 0.0,
|
| 28 |
+
"total_ms": 47.512139131625496,
|
| 29 |
+
"state_mb": 108.292096,
|
| 30 |
+
"state_multiple": 1.0
|
| 31 |
+
},
|
| 32 |
+
{
|
| 33 |
+
"optimizer": "muon",
|
| 34 |
+
"category": "4. Matrix-based",
|
| 35 |
+
"fwd_bwd_ms": 41.5445347627004,
|
| 36 |
+
"step_ms": 12.41101324558258,
|
| 37 |
+
"hessian_amortised_ms": 0.0,
|
| 38 |
+
"total_ms": 53.95554800828298,
|
| 39 |
+
"state_mb": 141.08672,
|
| 40 |
+
"state_multiple": 1.3028348809501296
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"optimizer": "sophia",
|
| 44 |
+
"category": "5. Hessian-based (bonus)",
|
| 45 |
+
"fwd_bwd_ms": 44.653950879971184,
|
| 46 |
+
"step_ms": 2.7560430268446603,
|
| 47 |
+
"hessian_amortised_ms": 4.465395087997118,
|
| 48 |
+
"total_ms": 51.87538899481296,
|
| 49 |
+
"state_mb": 216.584192,
|
| 50 |
+
"state_multiple": 2.0
|
| 51 |
+
}
|
| 52 |
+
]
|
runs/part2/cache/test.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:da3f51019c40aeb454a0f67a837b4542457e10cf7acb73618b1bc6b7af14c60c
|
| 3 |
+
size 1714360
|
runs/part2/cache/train.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2fa2d9c07115d248a372eddbd60290f74f2206321007dc1afdfe437afa8b9fea
|
| 3 |
+
size 83672296
|
runs/part2/cache/val.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d8398c13cda96d2e53182a8ce6ee31f96811cc38e5cc961d1b67e3e501eefef4
|
| 3 |
+
size 1747520
|
runs/part2/lion/metrics.json
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"optimizer": "lion",
|
| 3 |
+
"category": "3. Memory-efficient",
|
| 4 |
+
"hparams": {
|
| 5 |
+
"lr": 0.0003,
|
| 6 |
+
"betas": [
|
| 7 |
+
0.9,
|
| 8 |
+
0.99
|
| 9 |
+
],
|
| 10 |
+
"weight_decay": 0.6
|
| 11 |
+
},
|
| 12 |
+
"n_params": 27073024,
|
| 13 |
+
"tokens": 41836084,
|
| 14 |
+
"chinchilla_ratio": 0.07726525858359967,
|
| 15 |
+
"history": {
|
| 16 |
+
"step": [
|
| 17 |
+
340,
|
| 18 |
+
680,
|
| 19 |
+
1020,
|
| 20 |
+
1360,
|
| 21 |
+
1700,
|
| 22 |
+
2040,
|
| 23 |
+
2380,
|
| 24 |
+
2720,
|
| 25 |
+
3060,
|
| 26 |
+
3400,
|
| 27 |
+
3404
|
| 28 |
+
],
|
| 29 |
+
"tokens": [
|
| 30 |
+
4177920,
|
| 31 |
+
8355840,
|
| 32 |
+
12533760,
|
| 33 |
+
16711680,
|
| 34 |
+
20889600,
|
| 35 |
+
25067520,
|
| 36 |
+
29245440,
|
| 37 |
+
33423360,
|
| 38 |
+
37601280,
|
| 39 |
+
41779200,
|
| 40 |
+
41828352
|
| 41 |
+
],
|
| 42 |
+
"frac": [
|
| 43 |
+
0.09986403125110849,
|
| 44 |
+
0.19972806250221697,
|
| 45 |
+
0.29959209375332546,
|
| 46 |
+
0.39945612500443395,
|
| 47 |
+
0.4993201562555425,
|
| 48 |
+
0.5991841875066509,
|
| 49 |
+
0.6990482187577595,
|
| 50 |
+
0.7989122500088679,
|
| 51 |
+
0.8987762812599764,
|
| 52 |
+
0.998640312511085,
|
| 53 |
+
0.9998151834669803
|
| 54 |
+
],
|
| 55 |
+
"chinchilla": [
|
| 56 |
+
0.007716020197817577,
|
| 57 |
+
0.015432040395635154,
|
| 58 |
+
0.02314806059345273,
|
| 59 |
+
0.03086408079127031,
|
| 60 |
+
0.03858010098908788,
|
| 61 |
+
0.04629612118690546,
|
| 62 |
+
0.05401214138472304,
|
| 63 |
+
0.06172816158254062,
|
| 64 |
+
0.06944418178035819,
|
| 65 |
+
0.07716020197817576,
|
| 66 |
+
0.07725097868638539
|
| 67 |
+
],
|
| 68 |
+
"train_loss": [
|
| 69 |
+
6.472140299572664,
|
| 70 |
+
5.107305777774138,
|
| 71 |
+
4.731512961668127,
|
| 72 |
+
4.504773172210244,
|
| 73 |
+
4.327403292235206,
|
| 74 |
+
4.1709888549412,
|
| 75 |
+
4.0330721848151265,
|
| 76 |
+
3.937267201788285,
|
| 77 |
+
3.850365747423733,
|
| 78 |
+
3.818111107629888,
|
| 79 |
+
3.8044098615646362
|
| 80 |
+
],
|
| 81 |
+
"val_loss": [
|
| 82 |
+
5.510623939832052,
|
| 83 |
+
4.988590208689372,
|
| 84 |
+
4.717682782808939,
|
| 85 |
+
4.52708193063736,
|
| 86 |
+
4.363960568110148,
|
| 87 |
+
4.219045623143514,
|
| 88 |
+
4.10640107790629,
|
| 89 |
+
4.0222670873006185,
|
| 90 |
+
3.9644078334172566,
|
| 91 |
+
3.9307499289512635,
|
| 92 |
+
3.9307469487190247
|
| 93 |
+
],
|
| 94 |
+
"val_ppl": [
|
| 95 |
+
247.3053826184247,
|
| 96 |
+
146.72941975295987,
|
| 97 |
+
111.90863537543044,
|
| 98 |
+
92.48827970858318,
|
| 99 |
+
78.56769171978209,
|
| 100 |
+
67.968585680339,
|
| 101 |
+
60.72776930641522,
|
| 102 |
+
55.82752834774418,
|
| 103 |
+
52.68905944953533,
|
| 104 |
+
50.94516860403088,
|
| 105 |
+
50.94501677582324
|
| 106 |
+
],
|
| 107 |
+
"bleu": [
|
| 108 |
+
0.6294958889793753,
|
| 109 |
+
0.6880920489092365,
|
| 110 |
+
0.834322467234651,
|
| 111 |
+
0.8686162378521496,
|
| 112 |
+
1.1855542845866176,
|
| 113 |
+
1.7533708401341002,
|
| 114 |
+
1.3904264484063065,
|
| 115 |
+
1.717179570757667,
|
| 116 |
+
1.4944310714197733,
|
| 117 |
+
1.9720387026689865,
|
| 118 |
+
2.0688562598889826
|
| 119 |
+
]
|
| 120 |
+
},
|
| 121 |
+
"test_loss": 3.850360455720321,
|
| 122 |
+
"test_ppl": 47.01000520326837,
|
| 123 |
+
"test_bleu": 2.0688562598889826,
|
| 124 |
+
"minutes": 5.341768060314158,
|
| 125 |
+
"optimizer_state_mb": 108.292096,
|
| 126 |
+
"state_multiple": 1.0,
|
| 127 |
+
"samples": [
|
| 128 |
+
[
|
| 129 |
+
", who doesn't love a good friend, and I'm not sure what to do with him. I'm not sure what to do with him, but I'm not sure what to do with him. I'm not sure what to do",
|
| 130 |
+
", who wasn't obsessed with sex, right? But I digress. I had more pressing concerns, like getting into Miss Satron's English Literature class. I had heard she was a stickler for grammar and syntax"
|
| 131 |
+
],
|
| 132 |
+
[
|
| 133 |
+
" to adopt a more sustainable and sustainable future. The European Union's Energy Plan (ICC) has also been a promising alternative to traditional Energy Policy, which has been a promising alternative to traditional Energy Policy and Energy Policy. The",
|
| 134 |
+
" and tenants to finance energy efficiency improvements without requiring public subsidy. The scheme was positioned as a key component of the government's strategy to enhance energy security, reduce carbon emissions, and protect consumers from volatile energy prices. However, the"
|
| 135 |
+
],
|
| 136 |
+
[
|
| 137 |
+
" all, he was a man of a man, and that he had been a man of a man, and had been a man of a man, and had been a man of a man who had been a man of a man, and",
|
| 138 |
+
" all, is every child), he forthwith proceeded to turn her into a mere toy, and played with her as children will play with dolls. It is true, he treated her kindly enough, and would never have hurt her intentionally"
|
| 139 |
+
],
|
| 140 |
+
[
|
| 141 |
+
" have been able to get a lot of pressure on them. And I think that's something that I think is going to be a lot of fun. I think that's something that I think is going to be a lot of fun. I",
|
| 142 |
+
" probably replace 3 or 4 players in that starting line-up and still have a similar quality in terms of the ability of those players. It's pretty incredible. I mean, even if you look at the bench. The bench was good enough"
|
| 143 |
+
],
|
| 144 |
+
[
|
| 145 |
+
" into the world, I'm not just a man, but a man who's always been on the brink of something bigger than himself. And then, of course, I'm not just a man, but a man who's always been on",
|
| 146 |
+
", let's ease into the waters with a bit of understanding-no splash too big too soon. We're talking about that elusive sensation of seeing without seeing, understanding without the presence of eyes or ears. Remote viewing is the art"
|
| 147 |
+
],
|
| 148 |
+
[
|
| 149 |
+
" properties of the metal alloying is crucial for the development of novel materials that can be used to optimize the mechanical properties of the metal alloy. The use of advanced materials, such as alloy, and metal alloying, has been shown to enhance",
|
| 150 |
+
" rates and solidification conditions during casting stands as a formidable challenge. Unlike in wrought aluminium alloys, where rolling, forging, and extrusion processes inherently refine grains and help in achieving desirable textures, cast aluminium alloys"
|
| 151 |
+
]
|
| 152 |
+
]
|
| 153 |
+
}
|
runs/part2/lion/model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:71ff4c5fbc101ed6204fde35ecf185bddddd21a2a8c9664808f444947f91c29e
|
| 3 |
+
size 108309012
|
runs/part2/mars/metrics.json
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"optimizer": "mars",
|
| 3 |
+
"category": "2. Variance-reduced AdamW variant",
|
| 4 |
+
"hparams": {
|
| 5 |
+
"lr": 0.003,
|
| 6 |
+
"betas": [
|
| 7 |
+
0.95,
|
| 8 |
+
0.99
|
| 9 |
+
],
|
| 10 |
+
"weight_decay": 0.1,
|
| 11 |
+
"gamma": 0.025
|
| 12 |
+
},
|
| 13 |
+
"n_params": 27073024,
|
| 14 |
+
"tokens": 41836084,
|
| 15 |
+
"chinchilla_ratio": 0.07726525858359967,
|
| 16 |
+
"history": {
|
| 17 |
+
"step": [
|
| 18 |
+
340,
|
| 19 |
+
680,
|
| 20 |
+
1020,
|
| 21 |
+
1360,
|
| 22 |
+
1700,
|
| 23 |
+
2040,
|
| 24 |
+
2380,
|
| 25 |
+
2720,
|
| 26 |
+
3060,
|
| 27 |
+
3400,
|
| 28 |
+
3404
|
| 29 |
+
],
|
| 30 |
+
"tokens": [
|
| 31 |
+
4177920,
|
| 32 |
+
8355840,
|
| 33 |
+
12533760,
|
| 34 |
+
16711680,
|
| 35 |
+
20889600,
|
| 36 |
+
25067520,
|
| 37 |
+
29245440,
|
| 38 |
+
33423360,
|
| 39 |
+
37601280,
|
| 40 |
+
41779200,
|
| 41 |
+
41828352
|
| 42 |
+
],
|
| 43 |
+
"frac": [
|
| 44 |
+
0.09986403125110849,
|
| 45 |
+
0.19972806250221697,
|
| 46 |
+
0.29959209375332546,
|
| 47 |
+
0.39945612500443395,
|
| 48 |
+
0.4993201562555425,
|
| 49 |
+
0.5991841875066509,
|
| 50 |
+
0.6990482187577595,
|
| 51 |
+
0.7989122500088679,
|
| 52 |
+
0.8987762812599764,
|
| 53 |
+
0.998640312511085,
|
| 54 |
+
0.9998151834669803
|
| 55 |
+
],
|
| 56 |
+
"chinchilla": [
|
| 57 |
+
0.007716020197817577,
|
| 58 |
+
0.015432040395635154,
|
| 59 |
+
0.02314806059345273,
|
| 60 |
+
0.03086408079127031,
|
| 61 |
+
0.03858010098908788,
|
| 62 |
+
0.04629612118690546,
|
| 63 |
+
0.05401214138472304,
|
| 64 |
+
0.06172816158254062,
|
| 65 |
+
0.06944418178035819,
|
| 66 |
+
0.07716020197817576,
|
| 67 |
+
0.07725097868638539
|
| 68 |
+
],
|
| 69 |
+
"train_loss": [
|
| 70 |
+
5.899332756154678,
|
| 71 |
+
4.864102927376242,
|
| 72 |
+
4.514664495692534,
|
| 73 |
+
4.2527315392213705,
|
| 74 |
+
4.065694651884191,
|
| 75 |
+
3.930396722344791,
|
| 76 |
+
3.812907113047207,
|
| 77 |
+
3.718777296823614,
|
| 78 |
+
3.6335943867178524,
|
| 79 |
+
3.6027584517703337,
|
| 80 |
+
3.5917757749557495
|
| 81 |
+
],
|
| 82 |
+
"val_loss": [
|
| 83 |
+
5.191600076357523,
|
| 84 |
+
4.772338569164276,
|
| 85 |
+
4.484071910381317,
|
| 86 |
+
4.248005751768748,
|
| 87 |
+
4.107064692179362,
|
| 88 |
+
3.999554455280304,
|
| 89 |
+
3.8875315229098004,
|
| 90 |
+
3.806414314111074,
|
| 91 |
+
3.7517985105514526,
|
| 92 |
+
3.718560206890106,
|
| 93 |
+
3.7180607318878174
|
| 94 |
+
],
|
| 95 |
+
"val_ppl": [
|
| 96 |
+
179.75594618455563,
|
| 97 |
+
118.19532695990411,
|
| 98 |
+
88.59468883464339,
|
| 99 |
+
69.9657440650148,
|
| 100 |
+
60.768082495623204,
|
| 101 |
+
54.5738295340291,
|
| 102 |
+
48.790300014635854,
|
| 103 |
+
44.98883348102035,
|
| 104 |
+
42.59762542614627,
|
| 105 |
+
41.20502466840083,
|
| 106 |
+
41.18444892757214
|
| 107 |
+
],
|
| 108 |
+
"bleu": [
|
| 109 |
+
0.6862831823609342,
|
| 110 |
+
0.8562281662367871,
|
| 111 |
+
1.0314102825555629,
|
| 112 |
+
1.2205935134358015,
|
| 113 |
+
0.8862041596103197,
|
| 114 |
+
1.6179529347613826,
|
| 115 |
+
1.9215059047527943,
|
| 116 |
+
1.6263132410874157,
|
| 117 |
+
2.03963958195621,
|
| 118 |
+
2.2907486547887923,
|
| 119 |
+
2.184303731202685
|
| 120 |
+
]
|
| 121 |
+
},
|
| 122 |
+
"test_loss": 3.6377606322799902,
|
| 123 |
+
"test_ppl": 38.00663053544338,
|
| 124 |
+
"test_bleu": 2.184303731202685,
|
| 125 |
+
"minutes": 12.056620889281232,
|
| 126 |
+
"optimizer_state_mb": 324.876288,
|
| 127 |
+
"state_multiple": 3.0,
|
| 128 |
+
"samples": [
|
| 129 |
+
[
|
| 130 |
+
", who doesn't love a good deal? I mean, who doesn't love a good deal? But, I digress. I'm not sure what's going on, but I'm not sure what's going on. I mean, who",
|
| 131 |
+
", who wasn't obsessed with sex, right? But I digress. I had more pressing concerns, like getting into Miss Satron's English Literature class. I had heard she was a stickler for grammar and syntax"
|
| 132 |
+
],
|
| 133 |
+
[
|
| 134 |
+
" to purchase their energy savings, while also providing a more comprehensive and effective solution to the challenges posed by the EU. The European Union's Energy Agency (CNN) has also been criticized for its potential to reduce the costs of energy savings,",
|
| 135 |
+
" and tenants to finance energy efficiency improvements without requiring public subsidy. The scheme was positioned as a key component of the government's strategy to enhance energy security, reduce carbon emissions, and protect consumers from volatile energy prices. However, the"
|
| 136 |
+
],
|
| 137 |
+
[
|
| 138 |
+
" all, he was not a good man, and he was not a good man, and he was not a good man, and he was not a good man, and he was not a good man, and he was not a good man",
|
| 139 |
+
" all, is every child), he forthwith proceeded to turn her into a mere toy, and played with her as children will play with dolls. It is true, he treated her kindly enough, and would never have hurt her intentionally"
|
| 140 |
+
],
|
| 141 |
+
[
|
| 142 |
+
" have to deal with a lot of injuries. You could have to deal with a lot of injuries. You could have to deal with a lot of injuries. You could have to deal with a lot of injuries. You could have to deal with",
|
| 143 |
+
" probably replace 3 or 4 players in that starting line-up and still have a similar quality in terms of the ability of those players. It's pretty incredible. I mean, even if you look at the bench. The bench was good enough"
|
| 144 |
+
],
|
| 145 |
+
[
|
| 146 |
+
" into the world of our own, let's not forget the sheer volume of our own. The world is a tapestry woven with threads of our own, our own, and our own. It's a tapestry woven with threads of our own,",
|
| 147 |
+
", let's ease into the waters with a bit of understanding-no splash too big too soon. We're talking about that elusive sensation of seeing without seeing, understanding without the presence of eyes or ears. Remote viewing is the art"
|
| 148 |
+
],
|
| 149 |
+
[
|
| 150 |
+
" conditions, such as the presence of the catalyst, can significantly enhance the mechanical properties of the materials. The incorporation of these materials into the fabrication process can significantly enhance the mechanical properties of the materials, thereby enhancing the mechanical properties of the materials",
|
| 151 |
+
" rates and solidification conditions during casting stands as a formidable challenge. Unlike in wrought aluminium alloys, where rolling, forging, and extrusion processes inherently refine grains and help in achieving desirable textures, cast aluminium alloys"
|
| 152 |
+
]
|
| 153 |
+
]
|
| 154 |
+
}
|
runs/part2/mars/model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a0a141308096f197f659f920bc0f8016fb78a0319baf247b7db0bcfdc0a66ca3
|
| 3 |
+
size 108309012
|
runs/part2/muon/metrics.json
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"optimizer": "muon",
|
| 3 |
+
"category": "4. Matrix-based",
|
| 4 |
+
"hparams": {
|
| 5 |
+
"lr": 0.05,
|
| 6 |
+
"momentum": 0.95,
|
| 7 |
+
"weight_decay": 0.1,
|
| 8 |
+
"adamw_lr": 0.001
|
| 9 |
+
},
|
| 10 |
+
"n_params": 27073024,
|
| 11 |
+
"tokens": 41836084,
|
| 12 |
+
"chinchilla_ratio": 0.07726525858359967,
|
| 13 |
+
"history": {
|
| 14 |
+
"step": [
|
| 15 |
+
340,
|
| 16 |
+
680,
|
| 17 |
+
1020,
|
| 18 |
+
1360,
|
| 19 |
+
1700,
|
| 20 |
+
2040,
|
| 21 |
+
2380,
|
| 22 |
+
2720,
|
| 23 |
+
3060,
|
| 24 |
+
3400,
|
| 25 |
+
3404
|
| 26 |
+
],
|
| 27 |
+
"tokens": [
|
| 28 |
+
4177920,
|
| 29 |
+
8355840,
|
| 30 |
+
12533760,
|
| 31 |
+
16711680,
|
| 32 |
+
20889600,
|
| 33 |
+
25067520,
|
| 34 |
+
29245440,
|
| 35 |
+
33423360,
|
| 36 |
+
37601280,
|
| 37 |
+
41779200,
|
| 38 |
+
41828352
|
| 39 |
+
],
|
| 40 |
+
"frac": [
|
| 41 |
+
0.09986403125110849,
|
| 42 |
+
0.19972806250221697,
|
| 43 |
+
0.29959209375332546,
|
| 44 |
+
0.39945612500443395,
|
| 45 |
+
0.4993201562555425,
|
| 46 |
+
0.5991841875066509,
|
| 47 |
+
0.6990482187577595,
|
| 48 |
+
0.7989122500088679,
|
| 49 |
+
0.8987762812599764,
|
| 50 |
+
0.998640312511085,
|
| 51 |
+
0.9998151834669803
|
| 52 |
+
],
|
| 53 |
+
"chinchilla": [
|
| 54 |
+
0.007716020197817577,
|
| 55 |
+
0.015432040395635154,
|
| 56 |
+
0.02314806059345273,
|
| 57 |
+
0.03086408079127031,
|
| 58 |
+
0.03858010098908788,
|
| 59 |
+
0.04629612118690546,
|
| 60 |
+
0.05401214138472304,
|
| 61 |
+
0.06172816158254062,
|
| 62 |
+
0.06944418178035819,
|
| 63 |
+
0.07716020197817576,
|
| 64 |
+
0.07725097868638539
|
| 65 |
+
],
|
| 66 |
+
"train_loss": [
|
| 67 |
+
5.9417856146307555,
|
| 68 |
+
4.788992469451006,
|
| 69 |
+
4.50173141114852,
|
| 70 |
+
4.345610643835629,
|
| 71 |
+
4.211694392035989,
|
| 72 |
+
4.097056448459625,
|
| 73 |
+
3.969215421115651,
|
| 74 |
+
3.8511704213478986,
|
| 75 |
+
3.7234030408017778,
|
| 76 |
+
3.653416160274954,
|
| 77 |
+
3.6278735995292664
|
| 78 |
+
],
|
| 79 |
+
"val_loss": [
|
| 80 |
+
5.111191519101461,
|
| 81 |
+
4.7109426856040955,
|
| 82 |
+
4.515795083840688,
|
| 83 |
+
4.37390398979187,
|
| 84 |
+
4.255423335234324,
|
| 85 |
+
4.140481452147166,
|
| 86 |
+
4.021086577574412,
|
| 87 |
+
3.905429148674011,
|
| 88 |
+
3.803253209590912,
|
| 89 |
+
3.744998017946879,
|
| 90 |
+
3.7464208324750263
|
| 91 |
+
],
|
| 92 |
+
"val_ppl": [
|
| 93 |
+
165.86787191540338,
|
| 94 |
+
111.15689653724714,
|
| 95 |
+
91.45024773873429,
|
| 96 |
+
79.35282037284523,
|
| 97 |
+
70.48665035591668,
|
| 98 |
+
62.83306528233149,
|
| 99 |
+
55.761662292947996,
|
| 100 |
+
49.67139175078256,
|
| 101 |
+
44.846843616784604,
|
| 102 |
+
42.30892336022111,
|
| 103 |
+
42.36916395648348
|
| 104 |
+
],
|
| 105 |
+
"bleu": [
|
| 106 |
+
0.7826638992365594,
|
| 107 |
+
0.9463049968991224,
|
| 108 |
+
0.8986340137325347,
|
| 109 |
+
0.9412321290347232,
|
| 110 |
+
1.073134407731274,
|
| 111 |
+
1.3457153410291447,
|
| 112 |
+
1.804378291727082,
|
| 113 |
+
1.7589058796670372,
|
| 114 |
+
2.4206233438864646,
|
| 115 |
+
2.468327990469332,
|
| 116 |
+
1.9760008205645931
|
| 117 |
+
]
|
| 118 |
+
},
|
| 119 |
+
"test_loss": 3.67102224930473,
|
| 120 |
+
"test_ppl": 39.29205160976576,
|
| 121 |
+
"test_bleu": 1.9760008205645931,
|
| 122 |
+
"minutes": 3.4713221047073604,
|
| 123 |
+
"optimizer_state_mb": 141.08672,
|
| 124 |
+
"state_multiple": 1.3028348809501296,
|
| 125 |
+
"samples": [
|
| 126 |
+
[
|
| 127 |
+
", who doesn't love a good job? I mean, who doesn't love a good job? I mean, who doesn't love a good job? I mean, who doesn't love a good job? I mean, who doesn't",
|
| 128 |
+
", who wasn't obsessed with sex, right? But I digress. I had more pressing concerns, like getting into Miss Satron's English Literature class. I had heard she was a stickler for grammar and syntax"
|
| 129 |
+
],
|
| 130 |
+
[
|
| 131 |
+
" to take advantage of the potential benefits of energy efficiency retrofitting. The development of more efficient and cost-effective energy savings is further complicated by the need for a more comprehensive approach to energy efficiency retrofitting. The development of more efficient and cost",
|
| 132 |
+
" and tenants to finance energy efficiency improvements without requiring public subsidy. The scheme was positioned as a key component of the government's strategy to enhance energy security, reduce carbon emissions, and protect consumers from volatile energy prices. However, the"
|
| 133 |
+
],
|
| 134 |
+
[
|
| 135 |
+
" all, he was a very good man, and he was a very good man, and he was a very good man, and he was a very good man, and he was a very good man, and he was a very good man",
|
| 136 |
+
" all, is every child), he forthwith proceeded to turn her into a mere toy, and played with her as children will play with dolls. It is true, he treated her kindly enough, and would never have hurt her intentionally"
|
| 137 |
+
],
|
| 138 |
+
[
|
| 139 |
+
" say that you're going to be very hard to stop because you're going to be very hard to stop because you're going to be very hard to stop because you're going to be very hard to stop because you're going to be very",
|
| 140 |
+
" probably replace 3 or 4 players in that starting line-up and still have a similar quality in terms of the ability of those players. It's pretty incredible. I mean, even if you look at the bench. The bench was good enough"
|
| 141 |
+
],
|
| 142 |
+
[
|
| 143 |
+
" into the depths of our consciousness, let's not forget about the world. The world is a vast expanse of mystery, and the world is a vast expanse of mystery. The world is a vast expanse of mystery, and the world is a",
|
| 144 |
+
", let's ease into the waters with a bit of understanding-no splash too big too soon. We're talking about that elusive sensation of seeing without seeing, understanding without the presence of eyes or ears. Remote viewing is the art"
|
| 145 |
+
],
|
| 146 |
+
[
|
| 147 |
+
" conditions and the thermal conductivity of the fouls can significantly influence the mechanical properties of the fouls, as it can be crucial to optimize the performance of the fouls. The development of advanced materials such as nano-based",
|
| 148 |
+
" rates and solidification conditions during casting stands as a formidable challenge. Unlike in wrought aluminium alloys, where rolling, forging, and extrusion processes inherently refine grains and help in achieving desirable textures, cast aluminium alloys"
|
| 149 |
+
]
|
| 150 |
+
]
|
| 151 |
+
}
|
runs/part2/muon/model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d0387cac9a39a999390cfc32d47c4c54842daecdda1047d90280c773df59df76
|
| 3 |
+
size 108309012
|
runs/part2/sophia/metrics.json
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"optimizer": "sophia",
|
| 3 |
+
"category": "5. Hessian-based (bonus)",
|
| 4 |
+
"hparams": {
|
| 5 |
+
"lr": 0.0001,
|
| 6 |
+
"betas": [
|
| 7 |
+
0.965,
|
| 8 |
+
0.99
|
| 9 |
+
],
|
| 10 |
+
"rho": 0.05,
|
| 11 |
+
"weight_decay": 0.1
|
| 12 |
+
},
|
| 13 |
+
"n_params": 27073024,
|
| 14 |
+
"tokens": 41836084,
|
| 15 |
+
"chinchilla_ratio": 0.07726525858359967,
|
| 16 |
+
"history": {
|
| 17 |
+
"step": [
|
| 18 |
+
340,
|
| 19 |
+
680,
|
| 20 |
+
1020,
|
| 21 |
+
1360,
|
| 22 |
+
1700,
|
| 23 |
+
2040,
|
| 24 |
+
2380,
|
| 25 |
+
2720,
|
| 26 |
+
3060,
|
| 27 |
+
3400,
|
| 28 |
+
3404
|
| 29 |
+
],
|
| 30 |
+
"tokens": [
|
| 31 |
+
4177920,
|
| 32 |
+
8355840,
|
| 33 |
+
12533760,
|
| 34 |
+
16711680,
|
| 35 |
+
20889600,
|
| 36 |
+
25067520,
|
| 37 |
+
29245440,
|
| 38 |
+
33423360,
|
| 39 |
+
37601280,
|
| 40 |
+
41779200,
|
| 41 |
+
41828352
|
| 42 |
+
],
|
| 43 |
+
"frac": [
|
| 44 |
+
0.09986403125110849,
|
| 45 |
+
0.19972806250221697,
|
| 46 |
+
0.29959209375332546,
|
| 47 |
+
0.39945612500443395,
|
| 48 |
+
0.4993201562555425,
|
| 49 |
+
0.5991841875066509,
|
| 50 |
+
0.6990482187577595,
|
| 51 |
+
0.7989122500088679,
|
| 52 |
+
0.8987762812599764,
|
| 53 |
+
0.998640312511085,
|
| 54 |
+
0.9998151834669803
|
| 55 |
+
],
|
| 56 |
+
"chinchilla": [
|
| 57 |
+
0.007716020197817577,
|
| 58 |
+
0.015432040395635154,
|
| 59 |
+
0.02314806059345273,
|
| 60 |
+
0.03086408079127031,
|
| 61 |
+
0.03858010098908788,
|
| 62 |
+
0.04629612118690546,
|
| 63 |
+
0.05401214138472304,
|
| 64 |
+
0.06172816158254062,
|
| 65 |
+
0.06944418178035819,
|
| 66 |
+
0.07716020197817576,
|
| 67 |
+
0.07725097868638539
|
| 68 |
+
],
|
| 69 |
+
"train_loss": [
|
| 70 |
+
6.920979295057409,
|
| 71 |
+
5.194657805386711,
|
| 72 |
+
4.7517684052972236,
|
| 73 |
+
4.483142230089973,
|
| 74 |
+
4.283827843385584,
|
| 75 |
+
4.124087551762076,
|
| 76 |
+
3.996837453982409,
|
| 77 |
+
3.9137318716329688,
|
| 78 |
+
3.8409780144691466,
|
| 79 |
+
3.821532739611233,
|
| 80 |
+
3.8067631125450134
|
| 81 |
+
],
|
| 82 |
+
"val_loss": [
|
| 83 |
+
5.630311369895935,
|
| 84 |
+
5.03102810382843,
|
| 85 |
+
4.701542909940084,
|
| 86 |
+
4.478058167298635,
|
| 87 |
+
4.30525971253713,
|
| 88 |
+
4.164261535803477,
|
| 89 |
+
4.0671642581621805,
|
| 90 |
+
3.998800190289815,
|
| 91 |
+
3.954488666852315,
|
| 92 |
+
3.926562329133352,
|
| 93 |
+
3.9262346108754476
|
| 94 |
+
],
|
| 95 |
+
"val_ppl": [
|
| 96 |
+
278.74889813732557,
|
| 97 |
+
153.09032456407107,
|
| 98 |
+
110.11694197324582,
|
| 99 |
+
88.06350194909918,
|
| 100 |
+
74.08845465493992,
|
| 101 |
+
64.34514830624919,
|
| 102 |
+
58.391145381692404,
|
| 103 |
+
54.532681925065916,
|
| 104 |
+
52.16901137837309,
|
| 105 |
+
50.73227668943837,
|
| 106 |
+
50.715653520109015
|
| 107 |
+
],
|
| 108 |
+
"bleu": [
|
| 109 |
+
0.2898711556521973,
|
| 110 |
+
0.756440234378434,
|
| 111 |
+
0.6770647701652885,
|
| 112 |
+
1.3439992832350958,
|
| 113 |
+
0.8811172754215563,
|
| 114 |
+
1.4598406444860483,
|
| 115 |
+
1.7136405642487358,
|
| 116 |
+
1.6011678387385333,
|
| 117 |
+
1.4206498830250784,
|
| 118 |
+
1.3947867921621533,
|
| 119 |
+
1.3912555703599503
|
| 120 |
+
]
|
| 121 |
+
},
|
| 122 |
+
"test_loss": 3.8494405297265537,
|
| 123 |
+
"test_ppl": 46.96677936285023,
|
| 124 |
+
"test_bleu": 1.3912555703599503,
|
| 125 |
+
"minutes": 3.448130283008019,
|
| 126 |
+
"optimizer_state_mb": 216.584192,
|
| 127 |
+
"state_multiple": 2.0,
|
| 128 |
+
"samples": [
|
| 129 |
+
[
|
| 130 |
+
", who wouldn't want to be a \"squir\"? I mean, who wouldn't want to be a \"squir\"? I mean, who wouldn't want to be a \"squir\"? I mean",
|
| 131 |
+
", who wasn't obsessed with sex, right? But I digress. I had more pressing concerns, like getting into Miss Satron's English Literature class. I had heard she was a stickler for grammar and syntax"
|
| 132 |
+
],
|
| 133 |
+
[
|
| 134 |
+
" to take a more comprehensive approach to energy efficiency. The European Union's decision-making system has been widely used to reduce energy consumption and reduce energy consumption, and has been widely used to reduce energy consumption and reduce energy consumption. The European Union",
|
| 135 |
+
" and tenants to finance energy efficiency improvements without requiring public subsidy. The scheme was positioned as a key component of the government's strategy to enhance energy security, reduce carbon emissions, and protect consumers from volatile energy prices. However, the"
|
| 136 |
+
],
|
| 137 |
+
[
|
| 138 |
+
" a few years of age, he was a good friend of the family, and had a great deal of love for him, and he was a good friend of the family, and had a great deal of love for him. He was a",
|
| 139 |
+
" all, is every child), he forthwith proceeded to turn her into a mere toy, and played with her as children will play with dolls. It is true, he treated her kindly enough, and would never have hurt her intentionally"
|
| 140 |
+
],
|
| 141 |
+
[
|
| 142 |
+
" have to deal with a lot of injuries and you know, you could have to deal with a lot of injuries and you know, you could have to deal with a lot of injuries and you know, you could have to deal with a lot",
|
| 143 |
+
" probably replace 3 or 4 players in that starting line-up and still have a similar quality in terms of the ability of those players. It's pretty incredible. I mean, even if you look at the bench. The bench was good enough"
|
| 144 |
+
],
|
| 145 |
+
[
|
| 146 |
+
" into the world, I'm here to explore the mysteries of the universe, to explore the mysteries of the universe, to explore the mysteries of the universe. The universe is a tapestry of its own, a tapestry of existence, and the universe",
|
| 147 |
+
", let's ease into the waters with a bit of understanding-no splash too big too soon. We're talking about that elusive sensation of seeing without seeing, understanding without the presence of eyes or ears. Remote viewing is the art"
|
| 148 |
+
],
|
| 149 |
+
[
|
| 150 |
+
" process can be attributed to the high-frequency heat transfer, which can be used to simulate the thermal conductivity of the nanofibers. The thermal conductivity of the nanofibers is a complex process that can be used to simulate the",
|
| 151 |
+
" rates and solidification conditions during casting stands as a formidable challenge. Unlike in wrought aluminium alloys, where rolling, forging, and extrusion processes inherently refine grains and help in achieving desirable textures, cast aluminium alloys"
|
| 152 |
+
]
|
| 153 |
+
]
|
| 154 |
+
}
|
runs/part2/sophia/model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e1292db920dd4fdf908d06a62325b25f1c9dc5322434b2059531e14459dce468
|
| 3 |
+
size 108309012
|
runs/part2/sweep.json
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"adamw": [
|
| 3 |
+
{
|
| 4 |
+
"lr": 0.0001,
|
| 5 |
+
"val_loss": 6.368688186009725,
|
| 6 |
+
"diverged": false
|
| 7 |
+
},
|
| 8 |
+
{
|
| 9 |
+
"lr": 0.0003,
|
| 10 |
+
"val_loss": 5.852227322260538,
|
| 11 |
+
"diverged": false
|
| 12 |
+
},
|
| 13 |
+
{
|
| 14 |
+
"lr": 0.0006,
|
| 15 |
+
"val_loss": 5.639030520121256,
|
| 16 |
+
"diverged": false
|
| 17 |
+
},
|
| 18 |
+
{
|
| 19 |
+
"lr": 0.001,
|
| 20 |
+
"val_loss": 5.523148012161255,
|
| 21 |
+
"diverged": false
|
| 22 |
+
},
|
| 23 |
+
{
|
| 24 |
+
"lr": 0.003,
|
| 25 |
+
"val_loss": 5.6551560719807945,
|
| 26 |
+
"diverged": false
|
| 27 |
+
}
|
| 28 |
+
],
|
| 29 |
+
"mars": [
|
| 30 |
+
{
|
| 31 |
+
"lr": 0.0001,
|
| 32 |
+
"val_loss": 6.529670699437459,
|
| 33 |
+
"diverged": false
|
| 34 |
+
},
|
| 35 |
+
{
|
| 36 |
+
"lr": 0.0003,
|
| 37 |
+
"val_loss": 5.98583288192749,
|
| 38 |
+
"diverged": false
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"lr": 0.0006,
|
| 42 |
+
"val_loss": 5.741227086385091,
|
| 43 |
+
"diverged": false
|
| 44 |
+
},
|
| 45 |
+
{
|
| 46 |
+
"lr": 0.001,
|
| 47 |
+
"val_loss": 5.592048343022665,
|
| 48 |
+
"diverged": false
|
| 49 |
+
},
|
| 50 |
+
{
|
| 51 |
+
"lr": 0.003,
|
| 52 |
+
"val_loss": 5.500274785359701,
|
| 53 |
+
"diverged": false
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"lr": 0.006,
|
| 57 |
+
"val_loss": 5.7736601034800215,
|
| 58 |
+
"diverged": false
|
| 59 |
+
},
|
| 60 |
+
{
|
| 61 |
+
"lr": 0.01,
|
| 62 |
+
"val_loss": 6.14198571840922,
|
| 63 |
+
"diverged": false
|
| 64 |
+
}
|
| 65 |
+
],
|
| 66 |
+
"lion": [
|
| 67 |
+
{
|
| 68 |
+
"lr": 3e-05,
|
| 69 |
+
"val_loss": 6.919825077056885,
|
| 70 |
+
"diverged": false
|
| 71 |
+
},
|
| 72 |
+
{
|
| 73 |
+
"lr": 0.0001,
|
| 74 |
+
"val_loss": 6.08401894569397,
|
| 75 |
+
"diverged": false
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"lr": 0.0003,
|
| 79 |
+
"val_loss": 5.846204741795858,
|
| 80 |
+
"diverged": false
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"lr": 0.001,
|
| 84 |
+
"val_loss": 6.433748388290406,
|
| 85 |
+
"diverged": false
|
| 86 |
+
}
|
| 87 |
+
],
|
| 88 |
+
"muon": [
|
| 89 |
+
{
|
| 90 |
+
"lr": 0.005,
|
| 91 |
+
"val_loss": 5.331653149922689,
|
| 92 |
+
"diverged": false
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"lr": 0.01,
|
| 96 |
+
"val_loss": 5.212871932983399,
|
| 97 |
+
"diverged": false
|
| 98 |
+
},
|
| 99 |
+
{
|
| 100 |
+
"lr": 0.02,
|
| 101 |
+
"val_loss": 5.190064223607381,
|
| 102 |
+
"diverged": false
|
| 103 |
+
},
|
| 104 |
+
{
|
| 105 |
+
"lr": 0.05,
|
| 106 |
+
"val_loss": 5.18975084622701,
|
| 107 |
+
"diverged": false
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"lr": 0.1,
|
| 111 |
+
"val_loss": 5.359149471918742,
|
| 112 |
+
"diverged": false
|
| 113 |
+
},
|
| 114 |
+
{
|
| 115 |
+
"lr": 0.2,
|
| 116 |
+
"val_loss": 5.667555491129558,
|
| 117 |
+
"diverged": false
|
| 118 |
+
}
|
| 119 |
+
],
|
| 120 |
+
"sophia": [
|
| 121 |
+
{
|
| 122 |
+
"lr": 1e-05,
|
| 123 |
+
"val_loss": 7.821974929173788,
|
| 124 |
+
"diverged": false
|
| 125 |
+
},
|
| 126 |
+
{
|
| 127 |
+
"lr": 3e-05,
|
| 128 |
+
"val_loss": 6.8799722989400225,
|
| 129 |
+
"diverged": false
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"lr": 0.0001,
|
| 133 |
+
"val_loss": 6.041528367996216,
|
| 134 |
+
"diverged": false
|
| 135 |
+
},
|
| 136 |
+
{
|
| 137 |
+
"lr": 0.0003,
|
| 138 |
+
"val_loss": 6.070015017191569,
|
| 139 |
+
"diverged": false
|
| 140 |
+
},
|
| 141 |
+
{
|
| 142 |
+
"lr": 0.0006,
|
| 143 |
+
"val_loss": 6.210185893376669,
|
| 144 |
+
"diverged": false
|
| 145 |
+
},
|
| 146 |
+
{
|
| 147 |
+
"lr": 0.001,
|
| 148 |
+
"val_loss": 6.389156723022461,
|
| 149 |
+
"diverged": false
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"lr": 0.003,
|
| 153 |
+
"val_loss": 6.885614124933879,
|
| 154 |
+
"diverged": false
|
| 155 |
+
}
|
| 156 |
+
]
|
| 157 |
+
}
|
runs/part2/tokenizer_16000.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
runs/part3/metrics.json
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "EleutherAI/pythia-160m",
|
| 3 |
+
"dataset": "hamishivi/ROCStories",
|
| 4 |
+
"split": "test",
|
| 5 |
+
"n_samples": 1000,
|
| 6 |
+
"max_new": 48,
|
| 7 |
+
"batch_size": 32,
|
| 8 |
+
"reference_ppl": 30.515561399851546,
|
| 9 |
+
"results": [
|
| 10 |
+
{
|
| 11 |
+
"strategy": "greedy",
|
| 12 |
+
"precision": 0.13114583333333332,
|
| 13 |
+
"recall": 0.14909288996257875,
|
| 14 |
+
"f1": 0.13954467868147458,
|
| 15 |
+
"accuracy": 0.011746873815839333,
|
| 16 |
+
"exact_match": 0.0,
|
| 17 |
+
"gen_ppl": 2.499991898333626,
|
| 18 |
+
"distinct_1": 0.027604166666666666,
|
| 19 |
+
"distinct_2": 0.08957446808510638,
|
| 20 |
+
"repetition_rate": 0.5486808510638298,
|
| 21 |
+
"mean_len": 48.0,
|
| 22 |
+
"seconds": 25.390584368258715,
|
| 23 |
+
"sec_per_sample": 0.025390584368258716,
|
| 24 |
+
"ref_ppl": 30.515561399851546
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"strategy": "top_k_10",
|
| 28 |
+
"precision": 0.181337513061651,
|
| 29 |
+
"recall": 0.2055089763630335,
|
| 30 |
+
"f1": 0.19266808775202063,
|
| 31 |
+
"accuracy": 0.011225843122394847,
|
| 32 |
+
"exact_match": 0.0,
|
| 33 |
+
"gen_ppl": 8.100514312110523,
|
| 34 |
+
"distinct_1": 0.06831765935214211,
|
| 35 |
+
"distinct_2": 0.3488218219072825,
|
| 36 |
+
"repetition_rate": 0.08607956970887048,
|
| 37 |
+
"mean_len": 47.85,
|
| 38 |
+
"seconds": 26.00313352793455,
|
| 39 |
+
"sec_per_sample": 0.02600313352793455,
|
| 40 |
+
"ref_ppl": 30.515561399851546
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"strategy": "top_k_50",
|
| 44 |
+
"precision": 0.17234417833691054,
|
| 45 |
+
"recall": 0.1920799583155701,
|
| 46 |
+
"f1": 0.18167766216019446,
|
| 47 |
+
"accuracy": 0.009781167108753316,
|
| 48 |
+
"exact_match": 0.0,
|
| 49 |
+
"gen_ppl": 14.296850618161201,
|
| 50 |
+
"distinct_1": 0.09639373525724122,
|
| 51 |
+
"distinct_2": 0.4851096158020404,
|
| 52 |
+
"repetition_rate": 0.04384632081614934,
|
| 53 |
+
"mean_len": 47.057,
|
| 54 |
+
"seconds": 26.23013250529766,
|
| 55 |
+
"sec_per_sample": 0.02623013250529766,
|
| 56 |
+
"ref_ppl": 30.515561399851546
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"strategy": "top_p_0.90",
|
| 60 |
+
"precision": 0.15538168746035103,
|
| 61 |
+
"recall": 0.17403249490786793,
|
| 62 |
+
"f1": 0.16417910447761194,
|
| 63 |
+
"accuracy": 0.009046987495263357,
|
| 64 |
+
"exact_match": 0.0,
|
| 65 |
+
"gen_ppl": 25.64626571485052,
|
| 66 |
+
"distinct_1": 0.15108902516388242,
|
| 67 |
+
"distinct_2": 0.6042854674269915,
|
| 68 |
+
"repetition_rate": 0.03324261275272162,
|
| 69 |
+
"mean_len": 47.29,
|
| 70 |
+
"seconds": 29.247588723897934,
|
| 71 |
+
"sec_per_sample": 0.029247588723897933,
|
| 72 |
+
"ref_ppl": 30.515561399851546
|
| 73 |
+
},
|
| 74 |
+
{
|
| 75 |
+
"strategy": "top_p_0.95",
|
| 76 |
+
"precision": 0.15308994664538084,
|
| 77 |
+
"recall": 0.17261143479702526,
|
| 78 |
+
"f1": 0.16226566326757802,
|
| 79 |
+
"accuracy": 0.009378552482000757,
|
| 80 |
+
"exact_match": 0.0,
|
| 81 |
+
"gen_ppl": 35.17501646712563,
|
| 82 |
+
"distinct_1": 0.16777296979372347,
|
| 83 |
+
"distinct_2": 0.6501469610177855,
|
| 84 |
+
"repetition_rate": 0.025916629121881102,
|
| 85 |
+
"mean_len": 47.606,
|
| 86 |
+
"seconds": 30.207305278629065,
|
| 87 |
+
"sec_per_sample": 0.030207305278629065,
|
| 88 |
+
"ref_ppl": 30.515561399851546
|
| 89 |
+
},
|
| 90 |
+
{
|
| 91 |
+
"strategy": "beam_1",
|
| 92 |
+
"precision": 0.13114583333333332,
|
| 93 |
+
"recall": 0.14909288996257875,
|
| 94 |
+
"f1": 0.13954467868147458,
|
| 95 |
+
"accuracy": 0.011746873815839333,
|
| 96 |
+
"exact_match": 0.0,
|
| 97 |
+
"gen_ppl": 2.499991898333626,
|
| 98 |
+
"distinct_1": 0.027604166666666666,
|
| 99 |
+
"distinct_2": 0.08957446808510638,
|
| 100 |
+
"repetition_rate": 0.5486808510638298,
|
| 101 |
+
"mean_len": 48.0,
|
| 102 |
+
"seconds": 31.1804607398808,
|
| 103 |
+
"sec_per_sample": 0.0311804607398808,
|
| 104 |
+
"ref_ppl": 30.515561399851546
|
| 105 |
+
},
|
| 106 |
+
{
|
| 107 |
+
"strategy": "beam_2",
|
| 108 |
+
"precision": 0.11843887395553333,
|
| 109 |
+
"recall": 0.13462176116716404,
|
| 110 |
+
"f1": 0.12601288062696064,
|
| 111 |
+
"accuracy": 0.011628457749147404,
|
| 112 |
+
"exact_match": 0.0,
|
| 113 |
+
"gen_ppl": 2.2097512471538505,
|
| 114 |
+
"distinct_1": 0.022983476068429497,
|
| 115 |
+
"distinct_2": 0.0678853397459088,
|
| 116 |
+
"repetition_rate": 0.5814304866889405,
|
| 117 |
+
"mean_len": 47.991,
|
| 118 |
+
"seconds": 34.00318942219019,
|
| 119 |
+
"sec_per_sample": 0.03400318942219019,
|
| 120 |
+
"ref_ppl": 30.515561399851546
|
| 121 |
+
},
|
| 122 |
+
{
|
| 123 |
+
"strategy": "beam_4",
|
| 124 |
+
"precision": 0.11342655302477754,
|
| 125 |
+
"recall": 0.12891383638861256,
|
| 126 |
+
"f1": 0.1206753206442816,
|
| 127 |
+
"accuracy": 0.010941644562334218,
|
| 128 |
+
"exact_match": 0.0,
|
| 129 |
+
"gen_ppl": 2.0265386642981262,
|
| 130 |
+
"distinct_1": 0.021505824494133827,
|
| 131 |
+
"distinct_2": 0.059697363100432034,
|
| 132 |
+
"repetition_rate": 0.6105305722859514,
|
| 133 |
+
"mean_len": 47.987,
|
| 134 |
+
"seconds": 45.325823809951544,
|
| 135 |
+
"sec_per_sample": 0.045325823809951545,
|
| 136 |
+
"ref_ppl": 30.515561399851546
|
| 137 |
+
}
|
| 138 |
+
],
|
| 139 |
+
"prompts": [
|
| 140 |
+
"David had achieved his lifelong goal.",
|
| 141 |
+
"Dave went to Mammoth Mountain to go skiing.",
|
| 142 |
+
"Harriet was reading a great book.",
|
| 143 |
+
"Tina and her friends were heading to the shop.",
|
| 144 |
+
"Jimmy had never played football before."
|
| 145 |
+
],
|
| 146 |
+
"references": [
|
| 147 |
+
"He had just become a pilot! He had learned how to fly many kinds of planes. He decided to buy a small plane of his own. He couldn't wait to get up in the air.",
|
| 148 |
+
"He put on his boots, skis and goggles. With great excitement, he climbed into the chair lift. There was a foot of fresh powder from a recent storm. Dave jumped off the chair lift and took off down the mountain.",
|
| 149 |
+
"It was a mystery about a missing person. She read the entire thing in one evening. When she was finished, she quietly closed the book. She was so stunned she needed to reflect on what she'd read.",
|
| 150 |
+
"On the way, someone snatched her bag from her hands. Rather than leaving it, she gave chase. The robber didn't expect her to be so fast, let alone run. He dropped the bag after 100 metres and Tina got her bag back.",
|
| 151 |
+
"He was finally old enough for flag football. He got out on the field, excited to play. He threw the ball to his friend running down the field. He caught it and scored a touchdown."
|
| 152 |
+
],
|
| 153 |
+
"examples": {
|
| 154 |
+
"greedy": [
|
| 155 |
+
"\n\n\"I'm not going to be a part of this,\" he said. \"I'm not going to be a part of this.\"\n\nThe former NBA star, who was a member of the NBA's youth basketball team,",
|
| 156 |
+
"\n\n\"I was in the middle of the snow, and I was in the middle of the snow,\" he said. \"I was in the middle of the snow, and I was in the middle of the snow, and I was",
|
| 157 |
+
" She was a great friend of mine, and I was very fortunate to have her.\n\nI was very fortunate to have her. She was a great friend of mine, and I was very fortunate to have her.\n\nI was",
|
| 158 |
+
"\n\n\"I'm going to get you a drink,\" she said.\n\n\"I'm going to get you a drink,\" he said.\n\n\"I'm going to get you a drink,\" she said.\n\n\"",
|
| 159 |
+
"\n\n\"I was a little bit of a kid,\" he said. \"I was a little bit of a kid. I was a little bit of a kid. I was a little bit of a kid. I was a little bit"
|
| 160 |
+
],
|
| 161 |
+
"top_k_10": [
|
| 162 |
+
"\n\n\"We're here because I was able to give you some advice to my family. I have no control over what people think. We've always been very open with our thoughts about everything. It is very important to know what people",
|
| 163 |
+
" I was a fan of the Mammoth, so this was a great time. It was a nice place to ski and get a little exercise. The ski was really fun and we went for it on a few of the weekends.\n",
|
| 164 |
+
"\n\nI read this book and it is about one of the many things I have to tell you, as a person, that I love to read. I read this book because I love to write. I read so many great books that",
|
| 165 |
+
" The shop was closed, but it was still open. She was able to walk into a little room and see that the front door was open, the front window was closed, and the front of the shop was empty. She was surprised to",
|
| 166 |
+
" \"I had never been to the field, so I don't know,\" she told us. \"It's so different.\"\n\nWe had no clue what to think, but we all knew we had no clue. We were just so"
|
| 167 |
+
],
|
| 168 |
+
"top_k_50": [
|
| 169 |
+
"\n\n\"I don't remember very much.\"\n\nDusty gave her father the back of his hand, and she felt the touch of their fingers on his face, as if he hadn't seen his children before . . .",
|
| 170 |
+
" The boys got down off some cranes and hiked with their buddy Trey, a middle-aged mountain biker who runs an outdoor-venturing brand. The boys ran away by themselves, but managed to find the cranes on",
|
| 171 |
+
" In no time at all, her writing was in motion and her brain started growing for the next phase of her long-awaited writing journey. That was when her life as a writer began. As she wrote, \"I was in a",
|
| 172 |
+
" \"Oh, you were right,\" the man said. \"What're you doing. You never said anything to me, did you?\"\n\n\"No, I didn't,\" Meredith said, her face twisting into another, more terrifying",
|
| 173 |
+
" But the fact that I knew him, and probably understood him, was to keep me honest and make me the best I could possibly hope for. When I heard he had been gone, I told myself, I knew for sure and was right"
|
| 174 |
+
],
|
| 175 |
+
"top_p_0.90": [
|
| 176 |
+
" He was a child hero. The Fighting Six ended up being a hero. For years, he battled the darkness of a war of utter chaos, tearing apart humanity to shreds, until he became a leader of uniting against those few",
|
| 177 |
+
" The woman put me down in a rainy snowfall as we got up to cross the finish line. You can get 100X-70...Well, I thought I'd go together.\n\nThankfully, she's never been such a",
|
| 178 |
+
"\n\n\"I have had my share of what it's like to read,\u201d he said. \u201cI've read every piece of books on both the mass media and the many books on what they're reading, and every listen.\u201d\n\n",
|
| 179 |
+
"\n\nTHE FORESTS\n\nVantage's mustache made her stand in the distance, as they pointed ahead. Toward Jermo.\n\nNo one offered them as his answer, but the employees let her in on the",
|
| 180 |
+
" The girls showed up around February 15 when New York City Heights football played a full-season, and the girls at the other end looked pretty good too. Then they walked off the field and kicked a goal.\n\nRein Harrell"
|
| 181 |
+
],
|
| 182 |
+
"top_p_0.95": [
|
| 183 |
+
"\n\nThankfully, Pandas first hired a cab for Wednesday's meet-and-gathering to return to Dorrit to get the K\u00e1ri complex launched. The session took 20 minutes, and Pandas had to miss one last",
|
| 184 |
+
"\n\nOne day, the critics hit the icefall of the fjord.\n\n\u201cBut the day was amazing,\u201d \"My Peanut Butter Fountain\" said.\n\nThe world was roasted, he added. He",
|
| 185 |
+
" It had grabbed her when she turned off the TV to find that an editor had read the book. She nodded.\n\n\"It's really interesting, Director,\" Mari had explained. She had read Captain Arthur's last story, but without",
|
| 186 |
+
" Her party went on a pretty ill, people calling it an unforgettable day\n\n''Da Miene\n\none of the last hair-drop practices in the tradition of weight-lifting began this year.\n\n'My",
|
| 187 |
+
" He was a 6-5 defensive start. And he had his Father's country shoe.\n\nHe made us bump him into man Stephen Griffin, who first watched defense. I think Larry read the list of traditional defenses and then what happened"
|
| 188 |
+
],
|
| 189 |
+
"beam_1": [
|
| 190 |
+
"\n\n\"I'm not going to be a part of this,\" he said. \"I'm not going to be a part of this.\"\n\nThe former NBA star, who was a member of the NBA's youth basketball team,",
|
| 191 |
+
"\n\n\"I was in the middle of the snow, and I was in the middle of the snow,\" he said. \"I was in the middle of the snow, and I was in the middle of the snow, and I was",
|
| 192 |
+
" She was a great friend of mine, and I was very fortunate to have her.\n\nI was very fortunate to have her. She was a great friend of mine, and I was very fortunate to have her.\n\nI was",
|
| 193 |
+
"\n\n\"I'm going to get you a drink,\" she said.\n\n\"I'm going to get you a drink,\" he said.\n\n\"I'm going to get you a drink,\" she said.\n\n\"",
|
| 194 |
+
"\n\n\"I was a little bit of a kid,\" he said. \"I was a little bit of a kid. I was a little bit of a kid. I was a little bit of a kid. I was a little bit"
|
| 195 |
+
],
|
| 196 |
+
"beam_2": [
|
| 197 |
+
"\n\n\"I'm not going to lie,\" he said. \"I'm not going to lie. I'm not going to lie. I'm not going to lie. I'm not going to lie. I'm not going to lie",
|
| 198 |
+
"\n\n\"It's a great place,\" he said.\n\n\"It's a great place to go skiing.\"\n\n\"It's a great place to go skiing.\"\n\n\"It's a great place to go skiing.\"",
|
| 199 |
+
"\n\n\"I'm not going to tell you what to do,\" she said. \"I'm not going to tell you what to do.\"\n\n\"I'm not going to tell you what to do,\" he said. \"I",
|
| 200 |
+
"\n\n\"We're going to the shop,\" she said.\n\n\"What?\"\n\n\"We're going to the shop,\" she said. \"We're going to the shop.\"\n\n\"What?\"\n\n\"We",
|
| 201 |
+
"\n\n\"He was a great player,\" said the former England captain. \"He was a great player. He was a great player. He was a great player. He was a great player. He was a great player.\"\n\n"
|
| 202 |
+
],
|
| 203 |
+
"beam_4": [
|
| 204 |
+
"\n\n\"I'm not going to lie to you. I'm not going to lie to you. I'm not going to lie to you. I'm not going to lie to you. I'm not going to lie to you.",
|
| 205 |
+
"\n\n\"It's been a long time,\" he said. \"It's been a long time. It's been a long time. It's been a long time. It's been a long time. It's been a long time",
|
| 206 |
+
"\n\n\"It was a great book,\" she said.\n\n\"It was a great book.\"\n\n\"It was a great book.\"\n\n\"It was a great book.\"\n\n\"It was a great book.\"",
|
| 207 |
+
"\n\n\"I'm going to the shop,\" she said.\n\n\"What?\"\n\n\"I'm going to the shop,\" she said.\n\n\"What?\"\n\n\"I'm going to the shop.\"\n\n",
|
| 208 |
+
"\n\n\"He was a good player, but he wasn't a good player. He wasn't a good player. He wasn't a good player. He wasn't a good player. He wasn't a good player. He wasn't"
|
| 209 |
+
]
|
| 210 |
+
}
|
| 211 |
+
}
|