Upload full checkpoint folder

best/config.json

@@ -0,0 +1,16 @@
+{
+  "vocab_size": 32000,
+  "dim": 1024,
+  "n_layers": 24,
+  "n_heads": 16,
+  "n_kv_heads": 4,
+  "max_seq_len": 1024,
+  "rope_theta": 1000000.0,
+  "norm_eps": 1e-06,
+  "ffn_multiple_of": 256,
+  "ffn_dim_multiplier": null,
+  "tie_embeddings": true,
+  "dropout": 0.0,
+  "use_grad_checkpoint": true,
+  "identity": "SOVYTHOS-V2"
+}

best/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7c4f7f74aed5ca6f7330305d9d01ab3bc02690176b686ac73c3e3e1cb8c89a5
+size 1344515752

best/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04f8301713938cf3f3b23066c15e832f936132e3b005796d625721c3410ea2eb
+size 2427058251

best/train_state.json

@@ -0,0 +1 @@
+{"step": 1035, "best_val_loss": 4.8648576736450195}

last/config.json

@@ -0,0 +1,16 @@
+{
+  "vocab_size": 32000,
+  "dim": 1024,
+  "n_layers": 24,
+  "n_heads": 16,
+  "n_kv_heads": 4,
+  "max_seq_len": 1024,
+  "rope_theta": 1000000.0,
+  "norm_eps": 1e-06,
+  "ffn_multiple_of": 256,
+  "ffn_dim_multiplier": null,
+  "tie_embeddings": true,
+  "dropout": 0.0,
+  "use_grad_checkpoint": true,
+  "identity": "SOVYTHOS-V2"
+}

last/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19972bf652246045e14e415fdc031697a4edb7839db81880742b301405c3597c
+size 1344515752

last/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5967f817c6139266839f70796a5366d52cda438b96dc5223f6b3818b83fa8955
+size 2427058251

last/train_state.json

@@ -0,0 +1 @@
+{"step": 1000, "best_val_loss": 4.8720598220825195}

train_log.csv

@@ -0,0 +1,212 @@
+step,train_loss,val_loss,lr,tok_per_sec
+0,10.9112,10.4971,1.234568e-07,3494
+0,10.9112,10.4971,1.234568e-07,3716
+0,10.9112,10.4971,1.234568e-07,3647
+5,10.4517,10.5088,7.407407e-07,3637
+10,10.6138,10.5406,1.358025e-06,3713
+15,10.6043,10.6834,1.975309e-06,3718
+20,10.6270,10.4677,2.592593e-06,3713
+25,10.8439,10.5793,3.209877e-06,3707
+30,10.0846,10.6479,3.827160e-06,3756
+35,10.1111,10.6505,4.444444e-06,3772
+40,10.7905,10.3880,5.061728e-06,3761
+45,11.1070,10.4632,5.679012e-06,3731
+50,10.7111,10.3795,6.296296e-06,3744
+55,10.7174,10.5085,6.913580e-06,3715
+60,10.6732,10.5885,7.530864e-06,3742
+65,10.6651,10.4095,8.148148e-06,3783
+70,10.4833,10.5147,8.765432e-06,3740
+75,10.8184,10.4139,9.382716e-06,3759
+80,10.3798,10.5609,1.000000e-05,3758
+85,10.3840,10.4332,1.061728e-05,3757
+90,10.4185,10.3512,1.123457e-05,3752
+95,10.2949,10.4681,1.185185e-05,3725
+100,10.3092,10.1948,1.246914e-05,3745
+105,10.5874,10.4473,1.308642e-05,3643
+110,9.7131,10.2977,1.370370e-05,3786
+115,10.1350,10.1289,1.432099e-05,3764
+120,10.2730,10.3133,1.493827e-05,3630
+125,10.8327,10.0885,1.555556e-05,3769
+130,10.2959,10.0770,1.617284e-05,3651
+135,10.4195,10.3138,1.679012e-05,3620
+140,10.1034,10.1827,1.740741e-05,3788
+145,10.4195,10.0783,1.802469e-05,3752
+150,9.4055,9.9608,1.864198e-05,3772
+155,10.4795,10.0450,1.925926e-05,3649
+160,10.0106,9.8834,1.987654e-05,3795
+165,9.4153,9.9071,1.999999e-05,3670
+170,9.8288,9.6883,1.999990e-05,3767
+175,10.0850,9.8642,1.999973e-05,3660
+180,10.2371,9.5905,1.999948e-05,3765
+185,9.8794,9.7468,1.999915e-05,3658
+190,9.9033,9.6532,1.999874e-05,3733
+195,9.7336,9.7696,1.999825e-05,3759
+200,9.3813,9.6925,1.999768e-05,3736
+205,9.6600,9.6392,1.999703e-05,3751
+210,9.4130,9.4400,1.999630e-05,3751
+215,9.2821,9.5107,1.999548e-05,3732
+220,9.6389,9.5400,1.999459e-05,3752
+225,9.0449,9.3598,1.999362e-05,3785
+230,9.4767,9.4461,1.999257e-05,3675
+235,9.6242,9.3021,1.999143e-05,3768
+240,9.5186,9.2188,1.999022e-05,3700
+245,9.4172,9.4415,1.998893e-05,3661
+250,9.5925,9.3873,1.998755e-05,3767
+255,8.6344,9.2743,1.998610e-05,3756
+260,9.1073,9.0064,1.998456e-05,3754
+265,8.7201,9.2447,1.998295e-05,3696
+270,8.8728,8.9627,1.998125e-05,3762
+275,8.7994,9.0302,1.997948e-05,3688
+280,9.1133,9.0228,1.997762e-05,3758
+285,8.7824,9.0435,1.997569e-05,3759
+290,9.1198,9.0367,1.997367e-05,3744
+295,8.8852,8.8261,1.997158e-05,3744
+300,8.9763,8.9376,1.996940e-05,3680
+305,8.7600,8.8829,1.996714e-05,3759
+310,8.2793,8.8488,1.996481e-05,3765
+315,8.9810,8.6538,1.996239e-05,3758
+320,8.6087,8.8295,1.995990e-05,3678
+325,8.8518,8.8161,1.995732e-05,3761
+330,8.8059,8.6472,1.995466e-05,3761
+335,8.8640,8.4923,1.995193e-05,3673
+340,8.6850,8.6009,1.994911e-05,3661
+345,8.5197,8.4518,1.994621e-05,3760
+350,8.2370,8.4011,1.994324e-05,3700
+355,7.9393,8.6889,1.994018e-05,3620
+360,8.2207,8.5162,1.993705e-05,3782
+365,8.6561,8.3625,1.993383e-05,3715
+370,8.1477,8.4674,1.993053e-05,3673
+375,8.3435,8.4003,1.992716e-05,3765
+380,8.3934,8.3060,1.992370e-05,3762
+385,8.4456,8.1257,1.992017e-05,3687
+390,8.1027,8.3619,1.991656e-05,3656
+395,8.5335,8.2656,1.991286e-05,3764
+400,7.6148,8.1133,1.990909e-05,3739
+405,8.2171,8.0826,1.990523e-05,3690
+410,8.3794,7.8770,1.990130e-05,3614
+415,8.0075,8.1164,1.989729e-05,3650
+420,7.7941,7.9268,1.989320e-05,3778
+425,8.1319,8.1677,1.988903e-05,3744
+430,7.6283,7.8489,1.988478e-05,3767
+435,7.9437,7.8033,1.988045e-05,3694
+440,8.1148,7.8978,1.987604e-05,3622
+445,7.9262,7.8263,1.987155e-05,3780
+450,7.8940,7.7561,1.986698e-05,3759
+455,7.7105,7.7842,1.986233e-05,3667
+460,7.6952,7.7598,1.985761e-05,3768
+465,7.5181,7.6652,1.985280e-05,3772
+470,7.4893,7.4035,1.984792e-05,3656
+475,7.2041,7.5229,1.984296e-05,3617
+480,7.9190,7.5440,1.983791e-05,3781
+485,7.3476,7.4843,1.983279e-05,3756
+490,7.3977,7.5204,1.982759e-05,3768
+495,6.9512,7.4113,1.982231e-05,3757
+500,7.6009,7.5011,1.981696e-05,3763
+505,7.4532,7.4641,1.981152e-05,3758
+510,7.4910,7.5049,1.980600e-05,3736
+515,7.7388,7.2478,1.980041e-05,3751
+520,7.0352,7.1976,1.979474e-05,3678
+525,6.9493,7.2728,1.978899e-05,3660
+530,7.1242,7.1452,1.978316e-05,3778
+535,7.2106,7.2657,1.977725e-05,3700
+540,7.9134,7.0492,1.977126e-05,3753
+545,6.9884,7.0495,1.976520e-05,3687
+550,6.9402,6.9147,1.975906e-05,3758
+555,7.1767,7.0462,1.975283e-05,3723
+560,7.3138,6.8653,1.974653e-05,3745
+565,7.0763,6.9463,1.974016e-05,3711
+570,7.0171,7.1615,1.973370e-05,3738
+575,6.7670,7.0529,1.972717e-05,3772
+580,6.7693,6.6696,1.972056e-05,3745
+585,7.1335,6.8630,1.971387e-05,3707
+590,6.9793,6.7214,1.970710e-05,3748
+595,6.8790,6.7984,1.970026e-05,3774
+600,6.6872,6.6837,1.969333e-05,3736
+605,6.9933,6.6740,1.968633e-05,3759
+610,6.4345,6.6842,1.967925e-05,3767
+615,6.6339,6.7207,1.967210e-05,3769
+620,7.1027,6.6582,1.966487e-05,3749
+625,6.4534,6.7176,1.965756e-05,3687
+630,6.5189,6.6587,1.965017e-05,3750
+635,6.3212,6.6642,1.964270e-05,3777
+640,6.6231,6.5201,1.963516e-05,3744
+645,6.4622,6.3905,1.962754e-05,3701
+650,6.6704,6.4112,1.961985e-05,3648
+655,6.8367,6.4772,1.961207e-05,3760
+660,6.2506,6.4934,1.960422e-05,3758
+665,6.5745,6.2712,1.959630e-05,3760
+670,6.3594,6.3216,1.958829e-05,3689
+675,6.8998,6.3016,1.958021e-05,3755
+680,6.3479,6.4399,1.957205e-05,3785
+685,6.2555,6.4131,1.956382e-05,3748
+690,6.0662,6.1448,1.955551e-05,3751
+695,6.1134,6.1140,1.954712e-05,3702
+700,6.3376,6.2484,1.953866e-05,3659
+705,6.2869,6.2330,1.953012e-05,3760
+710,6.3188,6.0334,1.952151e-05,3762
+715,5.9846,6.1682,1.951281e-05,3684
+720,6.3132,6.0579,1.950405e-05,3750
+725,5.9388,6.0106,1.949520e-05,3768
+730,6.3192,5.9095,1.948628e-05,3689
+735,6.1869,6.1003,1.947729e-05,3634
+740,5.7767,5.9216,1.946822e-05,3778
+745,6.0503,6.0561,1.945907e-05,3763
+750,5.7668,5.8983,1.944985e-05,3755
+755,5.8633,5.9404,1.944055e-05,3701
+760,5.6028,5.8223,1.943118e-05,3756
+765,5.5208,5.8048,1.942173e-05,3696
+770,5.8969,5.9655,1.941221e-05,3619
+775,6.0144,5.8877,1.940261e-05,3789
+780,5.7204,5.8346,1.939294e-05,3744
+785,6.1809,5.6561,1.938319e-05,3769
+790,5.9300,5.6917,1.937336e-05,3676
+795,5.8760,5.7121,1.936347e-05,3742
+800,5.5869,5.6907,1.935349e-05,3753
+805,5.6261,5.7272,1.934345e-05,3772
+810,5.7073,5.6937,1.933332e-05,3760
+815,5.3251,5.8421,1.932313e-05,3756
+820,5.3950,5.7228,1.931286e-05,3765
+825,5.5801,5.6757,1.930251e-05,3758
+830,5.7426,5.6937,1.929209e-05,3745
+835,5.5313,5.6269,1.928160e-05,3744
+840,5.2882,5.6539,1.927103e-05,3708
+845,5.4094,5.5266,1.926039e-05,3745
+850,5.4568,5.4187,1.924967e-05,3724
+855,5.1436,5.5411,1.923889e-05,3722
+860,5.3478,5.6897,1.922802e-05,3735
+865,5.3817,5.4134,1.921709e-05,3774
+870,5.6258,5.3834,1.920608e-05,3686
+875,5.1936,5.3110,1.919500e-05,3685
+880,5.3976,5.4505,1.918384e-05,3668
+885,5.4293,5.4300,1.917261e-05,3756
+890,5.4324,5.4582,1.916131e-05,3767
+895,5.1233,5.3866,1.914994e-05,3756
+900,5.2079,5.4437,1.913849e-05,3759
+905,5.6379,5.3787,1.912697e-05,3751
+910,4.8977,5.3001,1.911538e-05,3758
+915,5.2549,5.2675,1.910371e-05,3696
+920,5.2433,5.2976,1.909197e-05,3679
+925,5.2480,5.2367,1.908016e-05,3754
+930,5.4184,5.3054,1.906828e-05,3695
+935,5.0487,5.2159,1.905633e-05,3746
+940,5.1283,5.3724,1.904430e-05,3725
+945,5.2362,5.2342,1.903220e-05,3749
+950,5.0044,5.2391,1.902004e-05,3777
+955,5.1907,4.9603,1.900779e-05,3750
+960,5.4935,5.2582,1.899548e-05,3707
+965,5.0500,5.1159,1.898310e-05,3747
+970,5.3349,5.1027,1.897064e-05,3769
+975,5.0246,4.8721,1.895812e-05,3741
+980,5.0086,4.9964,1.894552e-05,3710
+985,4.9942,5.0104,1.893285e-05,3745
+990,5.1605,5.2444,1.892011e-05,3769
+995,4.8411,5.0955,1.890730e-05,3744
+1000,4.7949,5.0653,1.889442e-05,3750
+1005,4.9177,5.0215,1.888147e-05,3780
+1010,5.3035,4.9357,1.886845e-05,3726
+1015,4.9235,5.0090,1.885535e-05,3763
+1020,5.0670,4.8928,1.884219e-05,3758
+1025,4.8075,4.9390,1.882896e-05,3750
+1030,5.2121,4.9882,1.881566e-05,3752
+1035,4.8663,4.8649,1.880229e-05,3744
+1040,4.9257,4.9170,1.878884e-05,3702