{
"args": {
"vocab": 8192,
"dim": 320,
"layers": 8,
"heads": 5,
"ffn": 864,
"seq": 512,
"tokens": 1500000000.0,
"val_tokens": 4000000.0,
"epochs": 1.0,
"batch": 512,
"lr": 0.003,
"fim_rate": 0.5,
"mix": "fineweb=0.6,smoltalk=0.3,enron=0.1",
"fineweb_files": 2,
"smoltalk_files": 9,
"tokenizer_docs": 200000,
"eval_pauses": 1000,
"log_every": 50,
"repo": "cfahlgren1/tinynext",
"folder": "v2",
"name": "tinynext-v2",
"trackio_space": "cfahlgren1/tinynext-trackio",
"stage": "train",
"data_repo": "cfahlgren1/tinynext-data",
"smoke": false,
"no_push": false
},
"parameters": 12539200,
"data": {
"fineweb": {
"docs": 1062484,
"train_tokens": 900000000
},
"smoltalk": {
"docs": 671427,
"train_tokens": 450000000
},
"enron": {
"docs": 268120,
"train_tokens": 150000000
}
},
"data_minutes": 0.2,
"train_minutes": 28.1,
"history": [
{
"step": 0,
"train_loss": 9.0697,
"lr": 2.6e-05,
"tokens_per_second": 13248,
"minutes": 0.33,
"mfu_percent": 0.3,
"gpu_util_percent": 98,
"gpu_memory_gb": 70.5,
"val_loss": 9.0682
},
{
"step": 50,
"train_loss": 7.0154,
"lr": 0.001342,
"tokens_per_second": 932232,
"minutes": 0.61,
"mfu_percent": 22.5,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 100,
"train_loss": 6.0698,
"lr": 0.002656,
"tokens_per_second": 926160,
"minutes": 0.85,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 150,
"train_loss": 5.5539,
"lr": 0.002995,
"tokens_per_second": 923370,
"minutes": 1.09,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 200,
"train_loss": 5.0756,
"lr": 0.002992,
"tokens_per_second": 922463,
"minutes": 1.32,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 250,
"train_loss": 4.6591,
"lr": 0.002987,
"tokens_per_second": 923079,
"minutes": 1.56,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 300,
"train_loss": 4.3203,
"lr": 0.002982,
"tokens_per_second": 922918,
"minutes": 1.8,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 350,
"train_loss": 4.0751,
"lr": 0.002975,
"tokens_per_second": 922585,
"minutes": 2.03,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 400,
"train_loss": 3.8986,
"lr": 0.002967,
"tokens_per_second": 923709,
"minutes": 2.27,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 450,
"train_loss": 3.7565,
"lr": 0.002959,
"tokens_per_second": 923903,
"minutes": 2.51,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 500,
"train_loss": 3.6807,
"lr": 0.002949,
"tokens_per_second": 923528,
"minutes": 2.74,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 3.7221
},
{
"step": 550,
"train_loss": 3.6681,
"lr": 0.002938,
"tokens_per_second": 922910,
"minutes": 3.03,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 600,
"train_loss": 3.6275,
"lr": 0.002927,
"tokens_per_second": 921856,
"minutes": 3.26,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 650,
"train_loss": 3.5235,
"lr": 0.002914,
"tokens_per_second": 920847,
"minutes": 3.5,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 700,
"train_loss": 3.5007,
"lr": 0.002901,
"tokens_per_second": 920478,
"minutes": 3.74,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 750,
"train_loss": 3.5063,
"lr": 0.002886,
"tokens_per_second": 920413,
"minutes": 3.97,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 800,
"train_loss": 3.4786,
"lr": 0.002871,
"tokens_per_second": 919895,
"minutes": 4.21,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 850,
"train_loss": 3.4053,
"lr": 0.002855,
"tokens_per_second": 919668,
"minutes": 4.45,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 900,
"train_loss": 3.4465,
"lr": 0.002837,
"tokens_per_second": 919590,
"minutes": 4.69,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 950,
"train_loss": 3.3967,
"lr": 0.002819,
"tokens_per_second": 919159,
"minutes": 4.92,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1000,
"train_loss": 3.3874,
"lr": 0.0028,
"tokens_per_second": 918998,
"minutes": 5.16,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 3.4027
},
{
"step": 1050,
"train_loss": 3.3679,
"lr": 0.002781,
"tokens_per_second": 919250,
"minutes": 5.45,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1100,
"train_loss": 3.335,
"lr": 0.00276,
"tokens_per_second": 919050,
"minutes": 5.68,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1150,
"train_loss": 3.365,
"lr": 0.002738,
"tokens_per_second": 918695,
"minutes": 5.92,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1200,
"train_loss": 3.3143,
"lr": 0.002716,
"tokens_per_second": 918833,
"minutes": 6.16,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1250,
"train_loss": 3.2921,
"lr": 0.002693,
"tokens_per_second": 918924,
"minutes": 6.4,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1300,
"train_loss": 3.3627,
"lr": 0.002669,
"tokens_per_second": 919045,
"minutes": 6.64,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1350,
"train_loss": 3.3008,
"lr": 0.002644,
"tokens_per_second": 919010,
"minutes": 6.87,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1400,
"train_loss": 3.2552,
"lr": 0.002618,
"tokens_per_second": 919072,
"minutes": 7.11,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1450,
"train_loss": 3.2611,
"lr": 0.002592,
"tokens_per_second": 919238,
"minutes": 7.35,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1500,
"train_loss": 3.2515,
"lr": 0.002565,
"tokens_per_second": 919356,
"minutes": 7.59,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 3.2835
},
{
"step": 1550,
"train_loss": 3.2331,
"lr": 0.002538,
"tokens_per_second": 919186,
"minutes": 7.87,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1600,
"train_loss": 3.3018,
"lr": 0.002509,
"tokens_per_second": 919216,
"minutes": 8.11,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1650,
"train_loss": 3.2305,
"lr": 0.00248,
"tokens_per_second": 918761,
"minutes": 8.35,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1700,
"train_loss": 3.2899,
"lr": 0.002451,
"tokens_per_second": 918918,
"minutes": 8.58,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1750,
"train_loss": 3.2422,
"lr": 0.002421,
"tokens_per_second": 918821,
"minutes": 8.82,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1800,
"train_loss": 3.2045,
"lr": 0.00239,
"tokens_per_second": 918803,
"minutes": 9.06,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1850,
"train_loss": 3.2057,
"lr": 0.002358,
"tokens_per_second": 918327,
"minutes": 9.3,
"mfu_percent": 22.1,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1900,
"train_loss": 3.1794,
"lr": 0.002327,
"tokens_per_second": 918460,
"minutes": 9.54,
"mfu_percent": 22.1,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1950,
"train_loss": 3.1661,
"lr": 0.002294,
"tokens_per_second": 918253,
"minutes": 9.77,
"mfu_percent": 22.1,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2000,
"train_loss": 3.1771,
"lr": 0.002261,
"tokens_per_second": 918444,
"minutes": 10.01,
"mfu_percent": 22.1,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 3.2099
},
{
"step": 2050,
"train_loss": 3.1498,
"lr": 0.002228,
"tokens_per_second": 919237,
"minutes": 10.3,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2100,
"train_loss": 3.164,
"lr": 0.002194,
"tokens_per_second": 920523,
"minutes": 10.53,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2150,
"train_loss": 3.1412,
"lr": 0.00216,
"tokens_per_second": 921392,
"minutes": 10.77,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2200,
"train_loss": 3.1425,
"lr": 0.002125,
"tokens_per_second": 921489,
"minutes": 11.01,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2250,
"train_loss": 3.1489,
"lr": 0.00209,
"tokens_per_second": 921290,
"minutes": 11.24,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2300,
"train_loss": 3.1575,
"lr": 0.002055,
"tokens_per_second": 919970,
"minutes": 11.48,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2350,
"train_loss": 3.1278,
"lr": 0.00202,
"tokens_per_second": 919468,
"minutes": 11.72,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2400,
"train_loss": 3.1516,
"lr": 0.001984,
"tokens_per_second": 919200,
"minutes": 11.96,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2450,
"train_loss": 3.1655,
"lr": 0.001948,
"tokens_per_second": 918867,
"minutes": 12.2,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2500,
"train_loss": 3.1927,
"lr": 0.001911,
"tokens_per_second": 918862,
"minutes": 12.43,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 3.1622
},
{
"step": 2550,
"train_loss": 3.1609,
"lr": 0.001875,
"tokens_per_second": 918878,
"minutes": 12.72,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2600,
"train_loss": 3.1732,
"lr": 0.001838,
"tokens_per_second": 919151,
"minutes": 12.96,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2650,
"train_loss": 3.0875,
"lr": 0.001801,
"tokens_per_second": 919485,
"minutes": 13.19,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2700,
"train_loss": 3.1417,
"lr": 0.001764,
"tokens_per_second": 919430,
"minutes": 13.43,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2750,
"train_loss": 3.1589,
"lr": 0.001727,
"tokens_per_second": 919254,
"minutes": 13.67,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2800,
"train_loss": 3.052,
"lr": 0.00169,
"tokens_per_second": 919224,
"minutes": 13.91,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2850,
"train_loss": 3.0945,
"lr": 0.001653,
"tokens_per_second": 919252,
"minutes": 14.14,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2900,
"train_loss": 3.0436,
"lr": 0.001616,
"tokens_per_second": 919765,
"minutes": 14.38,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2950,
"train_loss": 3.0525,
"lr": 0.001579,
"tokens_per_second": 919937,
"minutes": 14.62,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3000,
"train_loss": 3.019,
"lr": 0.001542,
"tokens_per_second": 919762,
"minutes": 14.86,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 3.116
},
{
"step": 3050,
"train_loss": 3.13,
"lr": 0.001505,
"tokens_per_second": 920124,
"minutes": 15.14,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3100,
"train_loss": 3.0723,
"lr": 0.001468,
"tokens_per_second": 920009,
"minutes": 15.38,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3150,
"train_loss": 3.0721,
"lr": 0.001431,
"tokens_per_second": 919773,
"minutes": 15.62,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3200,
"train_loss": 3.082,
"lr": 0.001395,
"tokens_per_second": 919868,
"minutes": 15.85,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3250,
"train_loss": 3.1011,
"lr": 0.001358,
"tokens_per_second": 920018,
"minutes": 16.09,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3300,
"train_loss": 3.0397,
"lr": 0.001322,
"tokens_per_second": 919962,
"minutes": 16.33,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3350,
"train_loss": 3.0473,
"lr": 0.001286,
"tokens_per_second": 919990,
"minutes": 16.57,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3400,
"train_loss": 3.0615,
"lr": 0.001251,
"tokens_per_second": 923200,
"minutes": 16.8,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3450,
"train_loss": 3.0738,
"lr": 0.001215,
"tokens_per_second": 926033,
"minutes": 17.04,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3500,
"train_loss": 2.991,
"lr": 0.00118,
"tokens_per_second": 925099,
"minutes": 17.27,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 3.0811
},
{
"step": 3550,
"train_loss": 3.0385,
"lr": 0.001146,
"tokens_per_second": 924905,
"minutes": 17.56,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3600,
"train_loss": 3.0725,
"lr": 0.001111,
"tokens_per_second": 924717,
"minutes": 17.79,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3650,
"train_loss": 3.0169,
"lr": 0.001077,
"tokens_per_second": 925574,
"minutes": 18.03,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3700,
"train_loss": 3.0391,
"lr": 0.001044,
"tokens_per_second": 926820,
"minutes": 18.27,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3750,
"train_loss": 3.0948,
"lr": 0.001011,
"tokens_per_second": 926733,
"minutes": 18.5,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3800,
"train_loss": 3.0155,
"lr": 0.000979,
"tokens_per_second": 926219,
"minutes": 18.74,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3850,
"train_loss": 3.0926,
"lr": 0.000947,
"tokens_per_second": 924590,
"minutes": 18.97,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3900,
"train_loss": 3.0346,
"lr": 0.000915,
"tokens_per_second": 924607,
"minutes": 19.21,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3950,
"train_loss": 3.0385,
"lr": 0.000884,
"tokens_per_second": 923474,
"minutes": 19.45,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4000,
"train_loss": 2.9716,
"lr": 0.000854,
"tokens_per_second": 922129,
"minutes": 19.68,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 3.0478
},
{
"step": 4050,
"train_loss": 2.9443,
"lr": 0.000824,
"tokens_per_second": 920762,
"minutes": 19.97,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4100,
"train_loss": 2.9772,
"lr": 0.000795,
"tokens_per_second": 920133,
"minutes": 20.2,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4150,
"train_loss": 3.0015,
"lr": 0.000767,
"tokens_per_second": 919900,
"minutes": 20.44,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4200,
"train_loss": 3.0204,
"lr": 0.000739,
"tokens_per_second": 920172,
"minutes": 20.68,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4250,
"train_loss": 3.0105,
"lr": 0.000712,
"tokens_per_second": 923826,
"minutes": 20.92,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4300,
"train_loss": 3.0586,
"lr": 0.000686,
"tokens_per_second": 926224,
"minutes": 21.15,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4350,
"train_loss": 3.0093,
"lr": 0.00066,
"tokens_per_second": 925730,
"minutes": 21.39,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4400,
"train_loss": 3.0369,
"lr": 0.000635,
"tokens_per_second": 926216,
"minutes": 21.62,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4450,
"train_loss": 2.9631,
"lr": 0.000611,
"tokens_per_second": 926599,
"minutes": 21.86,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4500,
"train_loss": 3.0019,
"lr": 0.000588,
"tokens_per_second": 925142,
"minutes": 22.1,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 3.0207
},
{
"step": 4550,
"train_loss": 2.963,
"lr": 0.000565,
"tokens_per_second": 923970,
"minutes": 22.38,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4600,
"train_loss": 3.0014,
"lr": 0.000544,
"tokens_per_second": 924314,
"minutes": 22.62,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4650,
"train_loss": 3.0158,
"lr": 0.000523,
"tokens_per_second": 926032,
"minutes": 22.85,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4700,
"train_loss": 2.9781,
"lr": 0.000503,
"tokens_per_second": 926272,
"minutes": 23.09,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4750,
"train_loss": 3.0033,
"lr": 0.000484,
"tokens_per_second": 926281,
"minutes": 23.32,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4800,
"train_loss": 2.9712,
"lr": 0.000465,
"tokens_per_second": 925900,
"minutes": 23.56,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4850,
"train_loss": 2.9531,
"lr": 0.000448,
"tokens_per_second": 925757,
"minutes": 23.79,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4900,
"train_loss": 2.995,
"lr": 0.000432,
"tokens_per_second": 925796,
"minutes": 24.03,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4950,
"train_loss": 2.9912,
"lr": 0.000416,
"tokens_per_second": 925597,
"minutes": 24.27,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5000,
"train_loss": 2.9481,
"lr": 0.000401,
"tokens_per_second": 925036,
"minutes": 24.5,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 3.0002
},
{
"step": 5050,
"train_loss": 3.0146,
"lr": 0.000388,
"tokens_per_second": 924886,
"minutes": 24.79,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5100,
"train_loss": 2.9885,
"lr": 0.000375,
"tokens_per_second": 924816,
"minutes": 25.02,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5150,
"train_loss": 3.0184,
"lr": 0.000363,
"tokens_per_second": 924852,
"minutes": 25.26,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5200,
"train_loss": 2.9669,
"lr": 0.000353,
"tokens_per_second": 925481,
"minutes": 25.5,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5250,
"train_loss": 2.9598,
"lr": 0.000343,
"tokens_per_second": 926674,
"minutes": 25.73,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5300,
"train_loss": 2.9644,
"lr": 0.000334,
"tokens_per_second": 926065,
"minutes": 25.97,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5350,
"train_loss": 2.9615,
"lr": 0.000326,
"tokens_per_second": 926198,
"minutes": 26.2,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5400,
"train_loss": 2.9874,
"lr": 0.000319,
"tokens_per_second": 926366,
"minutes": 26.44,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5450,
"train_loss": 2.9591,
"lr": 0.000314,
"tokens_per_second": 926756,
"minutes": 26.67,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5500,
"train_loss": 2.9528,
"lr": 0.000309,
"tokens_per_second": 925140,
"minutes": 26.91,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 2.9883
},
{
"step": 5550,
"train_loss": 3.0098,
"lr": 0.000305,
"tokens_per_second": 925097,
"minutes": 27.19,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5600,
"train_loss": 3.0141,
"lr": 0.000302,
"tokens_per_second": 925982,
"minutes": 27.43,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5650,
"train_loss": 2.9269,
"lr": 0.000301,
"tokens_per_second": 927048,
"minutes": 27.67,
"mfu_percent": 22.4,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5700,
"train_loss": 2.9439,
"lr": 0.0003,
"tokens_per_second": 926122,
"minutes": 27.9,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5709,
"train_loss": 2.969,
"lr": 0.0003,
"tokens_per_second": 925339,
"minutes": 27.94,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 2.9852
}
],
"exports": {
"float": {
"megabytes": 60.81,
"same_top_token": true,
"cached_step_ms_p50_cpu": 2.882
},
"int8": {
"megabytes": 23.33,
"same_top_token": false,
"cached_step_ms_p50_cpu": 2.125
}
}
}
{
"args": {
"vocab": 8192,
"dim": 320,
"layers": 8,
"heads": 5,
"ffn": 864,
"seq": 512,
"tokens": 1500000000.0,
"val_tokens": 4000000.0,
"epochs": 1.0,
"batch": 512,
"lr": 0.003,
"fim_rate": 0.5,
"mix": "fineweb=0.6,smoltalk=0.3,enron=0.1",
"fineweb_files": 2,
"smoltalk_files": 9,
"tokenizer_docs": 200000,
"eval_pauses": 1000,
"log_every": 50,
"repo": "cfahlgren1/tinynext",
"folder": "v2",
"name": "tinynext-v2",
"trackio_space": "cfahlgren1/tinynext-trackio",
"stage": "train",
"data_repo": "cfahlgren1/tinynext-data",
"smoke": false,
"no_push": false
},
"parameters": 12539200,
"data": {
"fineweb": {
"docs": 1062484,
"train_tokens": 900000000
},
"smoltalk": {
"docs": 671427,
"train_tokens": 450000000
},
"enron": {
"docs": 268120,
"train_tokens": 150000000
}
},
"data_minutes": 0.2,
"train_minutes": 28.1,
"history": [
{
"step": 0,
"train_loss": 9.0697,
"lr": 2.6e-05,
"tokens_per_second": 13248,
"minutes": 0.33,
"mfu_percent": 0.3,
"gpu_util_percent": 98,
"gpu_memory_gb": 70.5,
"val_loss": 9.0682
},
{
"step": 50,
"train_loss": 7.0154,
"lr": 0.001342,
"tokens_per_second": 932232,
"minutes": 0.61,
"mfu_percent": 22.5,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 100,
"train_loss": 6.0698,
"lr": 0.002656,
"tokens_per_second": 926160,
"minutes": 0.85,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 150,
"train_loss": 5.5539,
"lr": 0.002995,
"tokens_per_second": 923370,
"minutes": 1.09,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 200,
"train_loss": 5.0756,
"lr": 0.002992,
"tokens_per_second": 922463,
"minutes": 1.32,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 250,