RetentionLabs commited on
Commit
b31d677
·
verified ·
1 Parent(s): 751803b

shared-rotation: back up checkpoint-10000

Browse files
shared-rotation/checkpoint-10000/checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:935780dcca48ffbbdbdc12556af192ff92762c19c0c665256119145734c48032
3
+ size 13353035552
shared-rotation/checkpoint-10000/manifest.json ADDED
@@ -0,0 +1,328 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "step": 10000,
3
+ "files": [
4
+ "checkpoint.safetensors"
5
+ ],
6
+ "groups": {
7
+ "muon": [
8
+ {
9
+ "lr": 0.0,
10
+ "weight_decay": 0.0,
11
+ "momentum": 0.95,
12
+ "nesterov": true,
13
+ "ns_coefficients": [
14
+ 3.4445,
15
+ -4.775,
16
+ 2.0315
17
+ ],
18
+ "eps": 1e-07,
19
+ "ns_steps": 5,
20
+ "adjust_lr_fn": "match_rms_adamw",
21
+ "params": [
22
+ 0,
23
+ 1,
24
+ 2,
25
+ 3,
26
+ 4,
27
+ 5,
28
+ 6,
29
+ 7,
30
+ 8,
31
+ 9,
32
+ 10,
33
+ 11,
34
+ 12,
35
+ 13,
36
+ 14,
37
+ 15,
38
+ 16,
39
+ 17,
40
+ 18,
41
+ 19,
42
+ 20,
43
+ 21,
44
+ 22,
45
+ 23,
46
+ 24,
47
+ 25,
48
+ 26,
49
+ 27,
50
+ 28,
51
+ 29,
52
+ 30,
53
+ 31,
54
+ 32,
55
+ 33,
56
+ 34,
57
+ 35,
58
+ 36,
59
+ 37,
60
+ 38,
61
+ 39
62
+ ]
63
+ }
64
+ ],
65
+ "adamw": [
66
+ {
67
+ "weight_decay": 0.0,
68
+ "lr": 0.0,
69
+ "betas": [
70
+ 0.9,
71
+ 0.999
72
+ ],
73
+ "eps": 1e-08,
74
+ "amsgrad": false,
75
+ "maximize": false,
76
+ "foreach": null,
77
+ "capturable": false,
78
+ "differentiable": false,
79
+ "fused": null,
80
+ "decoupled_weight_decay": true,
81
+ "params": [
82
+ 0,
83
+ 1,
84
+ 2,
85
+ 3,
86
+ 4,
87
+ 5,
88
+ 6,
89
+ 7,
90
+ 8,
91
+ 9,
92
+ 10,
93
+ 11,
94
+ 12,
95
+ 13,
96
+ 14,
97
+ 15,
98
+ 16,
99
+ 17,
100
+ 18,
101
+ 19,
102
+ 20,
103
+ 21,
104
+ 22,
105
+ 23,
106
+ 24,
107
+ 25,
108
+ 26,
109
+ 27,
110
+ 28,
111
+ 29,
112
+ 30,
113
+ 31,
114
+ 32,
115
+ 33,
116
+ 34,
117
+ 35,
118
+ 36,
119
+ 37,
120
+ 38,
121
+ 39,
122
+ 40,
123
+ 41,
124
+ 42,
125
+ 43,
126
+ 44,
127
+ 45,
128
+ 46,
129
+ 47,
130
+ 48,
131
+ 49,
132
+ 50,
133
+ 51,
134
+ 52,
135
+ 53,
136
+ 54,
137
+ 55,
138
+ 56,
139
+ 57,
140
+ 58,
141
+ 59,
142
+ 60,
143
+ 61,
144
+ 62,
145
+ 63,
146
+ 64,
147
+ 65,
148
+ 66,
149
+ 67,
150
+ 68,
151
+ 69,
152
+ 70,
153
+ 71,
154
+ 72,
155
+ 73,
156
+ 74,
157
+ 75,
158
+ 76,
159
+ 77,
160
+ 78,
161
+ 79,
162
+ 80
163
+ ]
164
+ }
165
+ ],
166
+ "tables": [
167
+ {
168
+ "lr": 0.0,
169
+ "betas": [
170
+ 0.9,
171
+ 0.999
172
+ ],
173
+ "eps": 1e-08,
174
+ "params": [
175
+ 0,
176
+ 1
177
+ ]
178
+ }
179
+ ]
180
+ },
181
+ "extra": {
182
+ "documents_used": {
183
+ "fra_Latn": 7017,
184
+ "por_Latn": 5130,
185
+ "deu_Latn": 8532,
186
+ "ron_Latn": 2725,
187
+ "swe_Latn": 4067,
188
+ "dan_Latn": 4279,
189
+ "bul_Cyrl": 2586,
190
+ "ru": 4847,
191
+ "ces_Latn": 3025,
192
+ "ell_Grek": 2288,
193
+ "ukr_Cyrl": 1509,
194
+ "spa_Latn": 6935,
195
+ "nld_Latn": 5665,
196
+ "slk_Latn": 2419,
197
+ "hrv_Latn": 809,
198
+ "pol_Latn": 3284,
199
+ "lit_Latn": 2005,
200
+ "nob_Latn": 5062,
201
+ "nno_Latn": 1353,
202
+ "fas_Arab": 1427,
203
+ "slv_Latn": 2467,
204
+ "guj_Gujr": 643,
205
+ "lvs_Latn": 2368,
206
+ "ita_Latn": 5728,
207
+ "oci_Latn": 592,
208
+ "npi_Deva": 2365,
209
+ "mar_Deva": 1342,
210
+ "bel_Cyrl": 1212,
211
+ "srp_Cyrl": 1336,
212
+ "ltz_Latn": 1128,
213
+ "vec_Latn": 398,
214
+ "asm_Beng": 212,
215
+ "cym_Latn": 1943,
216
+ "szl_Latn": 292,
217
+ "ast_Latn": 490,
218
+ "hne_Deva": 345,
219
+ "awa_Deva": 240,
220
+ "mai_Deva": 111,
221
+ "bho_Deva": 511,
222
+ "snd_Arab": 253,
223
+ "gle_Latn": 962,
224
+ "fao_Latn": 1017,
225
+ "hi": 4016,
226
+ "pan_Guru": 1098,
227
+ "ben_Beng": 1160,
228
+ "ory_Orya": 796,
229
+ "tgk_Cyrl": 1403,
230
+ "ydd_Hebr": 595,
231
+ "lmo_Latn": 506,
232
+ "lij_Latn": 283,
233
+ "scn_Latn": 552,
234
+ "fur_Latn": 424,
235
+ "srd_Latn": 485,
236
+ "glg_Latn": 1793,
237
+ "cat_Latn": 2603,
238
+ "isl_Latn": 1474,
239
+ "als_Latn": 944,
240
+ "lim_Latn": 518,
241
+ "afr_Latn": 780,
242
+ "mkd_Cyrl": 1859,
243
+ "sin_Sinh": 1324,
244
+ "urd_Arab": 2350,
245
+ "mag_Deva": 24,
246
+ "bos_Latn": 1863,
247
+ "hye_Armn": 1030,
248
+ "zh": 7310,
249
+ "yue_Hani": 1443,
250
+ "mya_Mymr": 395,
251
+ "ar": 1451,
252
+ "ars_Arab": 136,
253
+ "apc_Arab": 364,
254
+ "arz_Arab": 988,
255
+ "ary_Arab": 2270,
256
+ "acm_Arab": 243,
257
+ "aeb_Arab": 206,
258
+ "heb_Hebr": 2158,
259
+ "mlt_Latn": 575,
260
+ "ind_Latn": 2725,
261
+ "zsm_Latn": 1880,
262
+ "fil_Latn": 928,
263
+ "ceb_Latn": 594,
264
+ "jav_Latn": 422,
265
+ "sun_Latn": 400,
266
+ "min_Latn": 123,
267
+ "ban_Latn": 180,
268
+ "bjn_Latn": 360,
269
+ "pag_Latn": 186,
270
+ "ilo_Latn": 437,
271
+ "war_Latn": 262,
272
+ "tam_Taml": 1842,
273
+ "tel_Telu": 393,
274
+ "kan_Knda": 608,
275
+ "mal_Mlym": 1149,
276
+ "tur_Latn": 2917,
277
+ "azj_Latn": 2652,
278
+ "uzn_Latn": 1112,
279
+ "kaz_Cyrl": 977,
280
+ "bak_Cyrl": 687,
281
+ "tat_Cyrl": 300,
282
+ "tha_Thai": 3482,
283
+ "lao_Laoo": 491,
284
+ "fin_Latn": 3618,
285
+ "ekk_Latn": 2409,
286
+ "hun_Latn": 1977,
287
+ "vie_Latn": 2811,
288
+ "khm_Khmr": 950,
289
+ "ja": 6875,
290
+ "ko": 5400,
291
+ "kat_Geor": 542,
292
+ "eus_Latn": 1397,
293
+ "hat_Latn": 694,
294
+ "pap_Latn": 191,
295
+ "kea_Latn": 101,
296
+ "tpi_Latn": 171,
297
+ "swh_Latn": 1020,
298
+ "en": 64202,
299
+ "fra_Latn@fineweb2-hq": 2048,
300
+ "por_Latn@fineweb2-hq": 1504,
301
+ "deu_Latn@fineweb2-hq": 2176,
302
+ "swe_Latn@fineweb2-hq": 1042,
303
+ "dan_Latn@fineweb2-hq": 1235,
304
+ "ru@fineweb2-hq": 1567,
305
+ "ces_Latn@fineweb2-hq": 1034,
306
+ "ell_Grek@fineweb2-hq": 896,
307
+ "spa_Latn@fineweb2-hq": 2006,
308
+ "nld_Latn@fineweb2-hq": 1888,
309
+ "pol_Latn@fineweb2-hq": 1259,
310
+ "fas_Arab@fineweb2-hq": 573,
311
+ "ita_Latn@fineweb2-hq": 1667,
312
+ "zh@fineweb2-hq": 2390,
313
+ "ar@fineweb2-hq": 651,
314
+ "ind_Latn@fineweb2-hq": 728,
315
+ "tur_Latn@fineweb2-hq": 883,
316
+ "hun_Latn@fineweb2-hq": 528,
317
+ "vie_Latn@fineweb2-hq": 786,
318
+ "ja@fineweb2-hq": 2722,
319
+ "ko@korean-webtext-edu": 146
320
+ },
321
+ "run_name": "shared-rotation",
322
+ "wandb": "https://wandb.ai/haan_modeling/polyplex/runs/unxz6ier",
323
+ "batch_size": 24,
324
+ "row_frames": 1024,
325
+ "lr_schedule": "cosine-restarts",
326
+ "language_stream_folded": true
327
+ }
328
+ }