{
"args": {
"vocab": 8192,
"dim": 320,
"layers": 8,
"heads": 5,
"ffn": 864,
"seq": 512,
"tokens": 3000000000.0,
"val_tokens": 4000000.0,
"epochs": 1.0,
"batch": 512,
"lr": 0.003,
"fim_rate": 0.5,
"mix": "fineweb=0.6,smoltalk=0.3,enron=0.1",
"fineweb_files": 4,
"smoltalk_files": 9,
"tokenizer_docs": 200000,
"eval_pauses": 1000,
"log_every": 50,
"repo": "cfahlgren1/tinynext",
"folder": "v2/experiments/tinynext-v2-3b",
"eval_folder": "v2",
"name": "tinynext-v2-3b",
"trackio_space": "cfahlgren1/tinynext-trackio",
"stage": "train",
"data_repo": "cfahlgren1/tinynext-data",
"smoke": false,
"no_push": false
},
"parameters": 12539200,
"data": {
"fineweb": {
"docs": 2122073,
"train_tokens": 1800000000
},
"smoltalk": {
"docs": 1041801,
"train_tokens": 900000000
},
"enron": {
"docs": 268120,
"train_tokens": 300000000
}
},
"data_minutes": 0.2,
"train_minutes": 55.8,
"history": [
{
"step": 0,
"train_loss": 9.0702,
"lr": 1.3e-05,
"tokens_per_second": 12800,
"minutes": 0.34,
"mfu_percent": 0.3,
"gpu_util_percent": 98,
"gpu_memory_gb": 76.5,
"val_loss": 9.0663
},
{
"step": 50,
"train_loss": 7.215,
"lr": 0.000671,
"tokens_per_second": 930958,
"minutes": 0.62,
"mfu_percent": 22.4,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 100,
"train_loss": 6.1306,
"lr": 0.001329,
"tokens_per_second": 925320,
"minutes": 0.86,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 150,
"train_loss": 5.5618,
"lr": 0.001986,
"tokens_per_second": 924266,
"minutes": 1.1,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 200,
"train_loss": 5.1433,
"lr": 0.002643,
"tokens_per_second": 923447,
"minutes": 1.33,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 250,
"train_loss": 4.6259,
"lr": 0.002997,
"tokens_per_second": 923503,
"minutes": 1.57,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 300,
"train_loss": 4.2702,
"lr": 0.002995,
"tokens_per_second": 923163,
"minutes": 1.81,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 350,
"train_loss": 4.0271,
"lr": 0.002994,
"tokens_per_second": 922754,
"minutes": 2.04,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 400,
"train_loss": 3.8697,
"lr": 0.002992,
"tokens_per_second": 922255,
"minutes": 2.28,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 450,
"train_loss": 3.6918,
"lr": 0.00299,
"tokens_per_second": 921900,
"minutes": 2.52,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 500,
"train_loss": 3.6502,
"lr": 0.002987,
"tokens_per_second": 921603,
"minutes": 2.75,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 3.6813
},
{
"step": 550,
"train_loss": 3.6069,
"lr": 0.002985,
"tokens_per_second": 922139,
"minutes": 3.04,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 600,
"train_loss": 3.6009,
"lr": 0.002982,
"tokens_per_second": 921841,
"minutes": 3.28,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 650,
"train_loss": 3.5049,
"lr": 0.002978,
"tokens_per_second": 921515,
"minutes": 3.51,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 700,
"train_loss": 3.4728,
"lr": 0.002975,
"tokens_per_second": 921918,
"minutes": 3.75,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 750,
"train_loss": 3.4877,
"lr": 0.002971,
"tokens_per_second": 921386,
"minutes": 3.99,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 800,
"train_loss": 3.4264,
"lr": 0.002967,
"tokens_per_second": 921002,
"minutes": 4.22,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 850,
"train_loss": 3.3612,
"lr": 0.002963,
"tokens_per_second": 920889,
"minutes": 4.46,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 900,
"train_loss": 3.3516,
"lr": 0.002959,
"tokens_per_second": 920832,
"minutes": 4.7,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 950,
"train_loss": 3.3742,
"lr": 0.002954,
"tokens_per_second": 920862,
"minutes": 4.94,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 1000,
"train_loss": 3.3632,
"lr": 0.002949,
"tokens_per_second": 920554,
"minutes": 5.17,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 3.3796
},
{
"step": 1050,
"train_loss": 3.3061,
"lr": 0.002944,
"tokens_per_second": 920467,
"minutes": 5.46,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 1100,
"train_loss": 3.3351,
"lr": 0.002939,
"tokens_per_second": 921248,
"minutes": 5.69,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 1150,
"train_loss": 3.3238,
"lr": 0.002933,
"tokens_per_second": 921674,
"minutes": 5.93,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 1200,
"train_loss": 3.3208,
"lr": 0.002927,
"tokens_per_second": 921693,
"minutes": 6.17,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 1250,
"train_loss": 3.286,
"lr": 0.002921,
"tokens_per_second": 921893,
"minutes": 6.41,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 1300,
"train_loss": 3.2941,
"lr": 0.002914,
"tokens_per_second": 922496,
"minutes": 6.64,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 1350,
"train_loss": 3.2475,
"lr": 0.002908,
"tokens_per_second": 922535,
"minutes": 6.88,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 1400,
"train_loss": 3.21,
"lr": 0.002901,
"tokens_per_second": 921917,
"minutes": 7.12,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 1450,
"train_loss": 3.2507,
"lr": 0.002894,
"tokens_per_second": 921722,
"minutes": 7.35,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 1500,
"train_loss": 3.2548,
"lr": 0.002887,
"tokens_per_second": 920926,
"minutes": 7.59,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 3.2689
},
{
"step": 1550,
"train_loss": 3.2187,
"lr": 0.002879,
"tokens_per_second": 920076,
"minutes": 7.87,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 1600,
"train_loss": 3.2435,
"lr": 0.002871,
"tokens_per_second": 920390,
"minutes": 8.11,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 1650,
"train_loss": 3.1958,
"lr": 0.002863,
"tokens_per_second": 920901,
"minutes": 8.35,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 1700,
"train_loss": 3.1797,
"lr": 0.002855,
"tokens_per_second": 920930,
"minutes": 8.59,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 1750,
"train_loss": 3.2016,
"lr": 0.002846,
"tokens_per_second": 921134,
"minutes": 8.82,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 1800,
"train_loss": 3.1736,
"lr": 0.002838,
"tokens_per_second": 922208,
"minutes": 9.06,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 1850,
"train_loss": 3.1912,
"lr": 0.002829,
"tokens_per_second": 922453,
"minutes": 9.3,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 1900,
"train_loss": 3.1827,
"lr": 0.00282,
"tokens_per_second": 922660,
"minutes": 9.53,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 1950,
"train_loss": 3.202,
"lr": 0.00281,
"tokens_per_second": 922623,
"minutes": 9.77,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 2000,
"train_loss": 3.2288,
"lr": 0.002801,
"tokens_per_second": 922474,
"minutes": 10.01,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 3.208
},
{
"step": 2050,
"train_loss": 3.1591,
"lr": 0.002791,
"tokens_per_second": 922197,
"minutes": 10.29,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 2100,
"train_loss": 3.2074,
"lr": 0.002781,
"tokens_per_second": 922026,
"minutes": 10.53,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 2150,
"train_loss": 3.1784,
"lr": 0.002771,
"tokens_per_second": 922145,
"minutes": 10.77,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 2200,
"train_loss": 3.1416,
"lr": 0.00276,
"tokens_per_second": 921690,
"minutes": 11.0,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 2250,
"train_loss": 3.1531,
"lr": 0.002749,
"tokens_per_second": 921458,
"minutes": 11.24,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 2300,
"train_loss": 3.1148,
"lr": 0.002738,
"tokens_per_second": 920727,
"minutes": 11.48,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 2350,
"train_loss": 3.1005,
"lr": 0.002727,
"tokens_per_second": 921746,
"minutes": 11.71,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 2400,
"train_loss": 3.2112,
"lr": 0.002716,
"tokens_per_second": 922555,
"minutes": 11.95,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 2450,
"train_loss": 3.1437,
"lr": 0.002705,
"tokens_per_second": 922219,
"minutes": 12.19,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 2500,
"train_loss": 3.114,
"lr": 0.002693,
"tokens_per_second": 922404,
"minutes": 12.42,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 3.1664
},
{
"step": 2550,
"train_loss": 3.1538,
"lr": 0.002681,
"tokens_per_second": 922093,
"minutes": 12.71,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 2600,
"train_loss": 3.1874,
"lr": 0.002669,
"tokens_per_second": 921696,
"minutes": 12.95,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 2650,
"train_loss": 3.1451,
"lr": 0.002657,
"tokens_per_second": 920726,
"minutes": 13.18,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 2700,
"train_loss": 3.1138,
"lr": 0.002644,
"tokens_per_second": 920915,
"minutes": 13.42,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 2750,
"train_loss": 3.1194,
"lr": 0.002632,
"tokens_per_second": 920910,
"minutes": 13.66,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 2800,
"train_loss": 3.0835,
"lr": 0.002619,
"tokens_per_second": 920819,
"minutes": 13.89,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 2850,
"train_loss": 3.1095,
"lr": 0.002606,
"tokens_per_second": 920420,
"minutes": 14.13,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 2900,
"train_loss": 3.1067,
"lr": 0.002593,
"tokens_per_second": 920261,
"minutes": 14.37,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 2950,
"train_loss": 3.0488,
"lr": 0.002579,
"tokens_per_second": 921100,
"minutes": 14.61,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 3000,
"train_loss": 3.1089,
"lr": 0.002566,
"tokens_per_second": 921915,
"minutes": 14.84,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 3.1334
},
{
"step": 3050,
"train_loss": 3.0736,
"lr": 0.002552,
"tokens_per_second": 922085,
"minutes": 15.13,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 3100,
"train_loss": 3.0915,
"lr": 0.002538,
"tokens_per_second": 921538,
"minutes": 15.36,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 3150,
"train_loss": 3.0751,
"lr": 0.002524,
"tokens_per_second": 921498,
"minutes": 15.6,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 3200,
"train_loss": 3.1093,
"lr": 0.00251,
"tokens_per_second": 920747,
"minutes": 15.84,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 3250,
"train_loss": 3.1425,
"lr": 0.002495,
"tokens_per_second": 920376,
"minutes": 16.08,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 3300,
"train_loss": 3.0762,
"lr": 0.002481,
"tokens_per_second": 920083,
"minutes": 16.31,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 3350,
"train_loss": 3.1273,
"lr": 0.002466,
"tokens_per_second": 919889,
"minutes": 16.55,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 3400,
"train_loss": 3.1,
"lr": 0.002451,
"tokens_per_second": 921166,
"minutes": 16.79,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 3450,
"train_loss": 3.1428,
"lr": 0.002436,
"tokens_per_second": 921093,
"minutes": 17.02,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 3500,
"train_loss": 3.0703,
"lr": 0.002421,
"tokens_per_second": 920938,
"minutes": 17.26,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 3.1114
},
{
"step": 3550,
"train_loss": 3.1048,
"lr": 0.002406,
"tokens_per_second": 921446,
"minutes": 17.55,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 3600,
"train_loss": 3.0576,
"lr": 0.00239,
"tokens_per_second": 921917,
"minutes": 17.78,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 3650,
"train_loss": 3.0815,
"lr": 0.002375,
"tokens_per_second": 921369,
"minutes": 18.02,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 3700,
"train_loss": 3.102,
"lr": 0.002359,
"tokens_per_second": 921425,
"minutes": 18.26,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 3750,
"train_loss": 3.0362,
"lr": 0.002343,
"tokens_per_second": 921350,
"minutes": 18.49,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 3800,
"train_loss": 3.0936,
"lr": 0.002327,
"tokens_per_second": 920994,
"minutes": 18.73,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 3850,
"train_loss": 3.0723,
"lr": 0.002311,
"tokens_per_second": 920745,
"minutes": 18.97,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 3900,
"train_loss": 3.0621,
"lr": 0.002295,
"tokens_per_second": 920434,
"minutes": 19.21,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 3950,
"train_loss": 3.1088,
"lr": 0.002278,
"tokens_per_second": 919905,
"minutes": 19.44,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 4000,
"train_loss": 3.019,
"lr": 0.002262,
"tokens_per_second": 919709,
"minutes": 19.68,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 3.0871
},
{
"step": 4050,
"train_loss": 3.0537,
"lr": 0.002245,
"tokens_per_second": 919817,
"minutes": 19.97,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 4100,
"train_loss": 3.0237,
"lr": 0.002228,
"tokens_per_second": 919826,
"minutes": 20.2,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 4150,
"train_loss": 3.0252,
"lr": 0.002212,
"tokens_per_second": 920702,
"minutes": 20.44,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 4200,
"train_loss": 2.9956,
"lr": 0.002195,
"tokens_per_second": 920999,
"minutes": 20.68,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 4250,
"train_loss": 3.0576,
"lr": 0.002178,
"tokens_per_second": 920918,
"minutes": 20.92,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 4300,
"train_loss": 3.0944,
"lr": 0.00216,
"tokens_per_second": 921070,
"minutes": 21.15,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 4350,
"train_loss": 3.0655,
"lr": 0.002143,
"tokens_per_second": 921341,
"minutes": 21.39,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 4400,
"train_loss": 3.0756,
"lr": 0.002126,
"tokens_per_second": 920880,
"minutes": 21.63,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 4450,
"train_loss": 3.0513,
"lr": 0.002108,
"tokens_per_second": 921004,
"minutes": 21.86,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 4500,
"train_loss": 3.0553,
"lr": 0.002091,
"tokens_per_second": 921256,
"minutes": 22.1,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 3.0698
},
{
"step": 4550,
"train_loss": 2.967,
"lr": 0.002073,
"tokens_per_second": 920961,
"minutes": 22.39,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 4600,
"train_loss": 3.0567,
"lr": 0.002056,
"tokens_per_second": 920043,
"minutes": 22.62,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 4650,
"train_loss": 3.0559,
"lr": 0.002038,
"tokens_per_second": 919645,
"minutes": 22.86,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 4700,
"train_loss": 3.0584,
"lr": 0.00202,
"tokens_per_second": 920036,
"minutes": 23.1,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 4750,
"train_loss": 3.0321,
"lr": 0.002002,
"tokens_per_second": 920584,
"minutes": 23.34,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 4800,
"train_loss": 2.9678,
"lr": 0.001984,
"tokens_per_second": 920212,
"minutes": 23.57,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 4850,
"train_loss": 2.9969,
"lr": 0.001966,
"tokens_per_second": 921129,
"minutes": 23.81,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 4900,
"train_loss": 3.0317,
"lr": 0.001948,
"tokens_per_second": 921336,
"minutes": 24.05,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 4950,
"train_loss": 3.0314,
"lr": 0.00193,
"tokens_per_second": 921674,
"minutes": 24.28,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 5000,
"train_loss": 3.0467,
"lr": 0.001912,
"tokens_per_second": 922491,
"minutes": 24.52,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 3.0507
},
{
"step": 5050,
"train_loss": 3.0208,
"lr": 0.001894,
"tokens_per_second": 922889,
"minutes": 24.8,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 5100,
"train_loss": 2.9957,
"lr": 0.001875,
"tokens_per_second": 922887,
"minutes": 25.04,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 5150,
"train_loss": 3.0355,
"lr": 0.001857,
"tokens_per_second": 922331,
"minutes": 25.28,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 5200,
"train_loss": 3.0702,
"lr": 0.001839,
"tokens_per_second": 922142,
"minutes": 25.51,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 5250,
"train_loss": 3.0055,
"lr": 0.00182,
"tokens_per_second": 922504,
"minutes": 25.75,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 5300,
"train_loss": 2.9704,
"lr": 0.001802,
"tokens_per_second": 922274,
"minutes": 25.99,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 5350,
"train_loss": 2.9492,
"lr": 0.001783,
"tokens_per_second": 921682,
"minutes": 26.23,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 5400,
"train_loss": 3.0025,
"lr": 0.001765,
"tokens_per_second": 921085,
"minutes": 26.46,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 5450,
"train_loss": 2.9997,
"lr": 0.001746,
"tokens_per_second": 921161,
"minutes": 26.7,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 5500,
"train_loss": 3.012,
"lr": 0.001728,
"tokens_per_second": 921331,
"minutes": 26.94,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 3.0312
},
{
"step": 5550,
"train_loss": 3.0025,
"lr": 0.001709,
"tokens_per_second": 921305,
"minutes": 27.22,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 5600,
"train_loss": 2.9965,
"lr": 0.001691,
"tokens_per_second": 921367,
"minutes": 27.46,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 5650,
"train_loss": 3.0261,
"lr": 0.001672,
"tokens_per_second": 921494,
"minutes": 27.7,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 5700,
"train_loss": 2.9857,
"lr": 0.001654,
"tokens_per_second": 921931,
"minutes": 27.93,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 5750,
"train_loss": 2.9916,
"lr": 0.001635,
"tokens_per_second": 921797,
"minutes": 28.17,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 5800,
"train_loss": 2.9868,
"lr": 0.001616,
"tokens_per_second": 921701,
"minutes": 28.41,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 5850,
"train_loss": 3.0135,
"lr": 0.001598,
"tokens_per_second": 921157,
"minutes": 28.64,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 5900,
"train_loss": 3.0171,
"lr": 0.001579,
"tokens_per_second": 921347,
"minutes": 28.88,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 5950,
"train_loss": 3.0381,
"lr": 0.001561,
"tokens_per_second": 921107,
"minutes": 29.12,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 6000,
"train_loss": 3.0234,
"lr": 0.001542,
"tokens_per_second": 920618,
"minutes": 29.35,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 3.014
},
{
"step": 6050,
"train_loss": 2.9671,
"lr": 0.001524,
"tokens_per_second": 920459,
"minutes": 29.64,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 6100,
"train_loss": 3.0432,
"lr": 0.001505,
"tokens_per_second": 919920,
"minutes": 29.88,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 6150,
"train_loss": 3.0124,
"lr": 0.001487,
"tokens_per_second": 920638,
"minutes": 30.11,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 6200,
"train_loss": 3.0045,
"lr": 0.001468,
"tokens_per_second": 919978,
"minutes": 30.35,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 6250,
"train_loss": 3.0096,
"lr": 0.00145,
"tokens_per_second": 919568,
"minutes": 30.59,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 6300,
"train_loss": 2.9723,
"lr": 0.001432,
"tokens_per_second": 919292,
"minutes": 30.83,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 6350,
"train_loss": 2.9286,
"lr": 0.001413,
"tokens_per_second": 919775,
"minutes": 31.06,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 6400,
"train_loss": 3.0349,
"lr": 0.001395,
"tokens_per_second": 920505,
"minutes": 31.3,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 6450,
"train_loss": 2.9374,
"lr": 0.001377,
"tokens_per_second": 920417,
"minutes": 31.54,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 6500,
"train_loss": 2.9625,
"lr": 0.001359,
"tokens_per_second": 919773,
"minutes": 31.78,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 2.9999
},
{
"step": 6550,
"train_loss": 2.9994,
"lr": 0.001341,
"tokens_per_second": 919974,
"minutes": 32.06,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 6600,
"train_loss": 2.9899,
"lr": 0.001323,
"tokens_per_second": 920286,
"minutes": 32.3,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 6650,
"train_loss": 2.9663,
"lr": 0.001305,
"tokens_per_second": 920398,
"minutes": 32.54,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 6700,
"train_loss": 2.9463,
"lr": 0.001287,
"tokens_per_second": 921337,
"minutes": 32.77,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 6750,
"train_loss": 2.9973,
"lr": 0.001269,
"tokens_per_second": 921370,
"minutes": 33.01,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 6800,
"train_loss": 2.9206,
"lr": 0.001251,
"tokens_per_second": 920742,
"minutes": 33.25,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 6850,
"train_loss": 2.9574,
"lr": 0.001233,
"tokens_per_second": 920284,
"minutes": 33.48,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 6900,
"train_loss": 2.9734,
"lr": 0.001216,
"tokens_per_second": 919909,
"minutes": 33.72,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 6950,
"train_loss": 2.992,
"lr": 0.001198,
"tokens_per_second": 919380,
"minutes": 33.96,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 7000,
"train_loss": 2.9482,
"lr": 0.001181,
"tokens_per_second": 919305,
"minutes": 34.2,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 2.9834
},
{
"step": 7050,
"train_loss": 2.9944,
"lr": 0.001163,
"tokens_per_second": 919402,
"minutes": 34.48,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 7100,
"train_loss": 2.9649,
"lr": 0.001146,
"tokens_per_second": 919490,
"minutes": 34.72,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 7150,
"train_loss": 2.9533,
"lr": 0.001129,
"tokens_per_second": 919450,
"minutes": 34.96,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 7200,
"train_loss": 2.9467,
"lr": 0.001112,
"tokens_per_second": 919809,
"minutes": 35.19,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 7250,
"train_loss": 2.9482,
"lr": 0.001095,
"tokens_per_second": 920118,
"minutes": 35.43,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 7300,
"train_loss": 2.9288,
"lr": 0.001078,
"tokens_per_second": 920161,
"minutes": 35.67,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 7350,
"train_loss": 2.9711,
"lr": 0.001061,
"tokens_per_second": 920166,
"minutes": 35.91,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 7400,
"train_loss": 2.9711,
"lr": 0.001045,
"tokens_per_second": 920031,
"minutes": 36.14,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 7450,
"train_loss": 2.9619,
"lr": 0.001028,
"tokens_per_second": 920234,
"minutes": 36.38,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 7500,
"train_loss": 2.9199,
"lr": 0.001012,
"tokens_per_second": 920421,
"minutes": 36.62,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 2.9683
},
{
"step": 7550,
"train_loss": 2.9091,
"lr": 0.000995,
"tokens_per_second": 921260,
"minutes": 36.9,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 7600,
"train_loss": 2.9747,
"lr": 0.000979,
"tokens_per_second": 920906,
"minutes": 37.14,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 7650,
"train_loss": 2.9445,
"lr": 0.000963,
"tokens_per_second": 920371,
"minutes": 37.38,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 7700,
"train_loss": 2.9593,
"lr": 0.000947,
"tokens_per_second": 920018,
"minutes": 37.62,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 7750,
"train_loss": 2.9826,
"lr": 0.000931,
"tokens_per_second": 920997,
"minutes": 37.85,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 7800,
"train_loss": 2.933,
"lr": 0.000916,
"tokens_per_second": 921462,
"minutes": 38.09,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 7850,
"train_loss": 2.9202,
"lr": 0.0009,
"tokens_per_second": 921036,
"minutes": 38.33,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 7900,
"train_loss": 2.9152,
"lr": 0.000885,
"tokens_per_second": 920538,
"minutes": 38.56,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 7950,
"train_loss": 2.926,
"lr": 0.00087,
"tokens_per_second": 919615,
"minutes": 38.8,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 8000,
"train_loss": 2.8966,
"lr": 0.000855,
"tokens_per_second": 919288,
"minutes": 39.04,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 2.9544
},
{
"step": 8050,
"train_loss": 2.9517,
"lr": 0.00084,
"tokens_per_second": 919884,
"minutes": 39.32,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 8100,
"train_loss": 2.9257,
"lr": 0.000825,
"tokens_per_second": 921250,
"minutes": 39.56,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 8150,
"train_loss": 2.9299,
"lr": 0.00081,
"tokens_per_second": 921425,
"minutes": 39.8,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 8200,
"train_loss": 2.934,
"lr": 0.000796,
"tokens_per_second": 921296,
"minutes": 40.04,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 8250,
"train_loss": 2.8999,
"lr": 0.000782,
"tokens_per_second": 920909,
"minutes": 40.27,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 8300,
"train_loss": 2.9048,
"lr": 0.000767,
"tokens_per_second": 920173,
"minutes": 40.51,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 8350,
"train_loss": 2.8714,
"lr": 0.000753,
"tokens_per_second": 919853,
"minutes": 40.75,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 8400,
"train_loss": 2.8722,
"lr": 0.00074,
"tokens_per_second": 919551,
"minutes": 40.98,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 8450,
"train_loss": 2.9693,
"lr": 0.000726,
"tokens_per_second": 920679,
"minutes": 41.22,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 8500,
"train_loss": 2.9393,
"lr": 0.000713,
"tokens_per_second": 921776,
"minutes": 41.46,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 2.9392
},
{
"step": 8550,
"train_loss": 2.8964,
"lr": 0.000699,
"tokens_per_second": 920913,
"minutes": 41.74,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 8600,
"train_loss": 2.9192,
"lr": 0.000686,
"tokens_per_second": 919992,
"minutes": 41.98,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 8650,
"train_loss": 2.9288,
"lr": 0.000673,
"tokens_per_second": 919914,
"minutes": 42.22,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 8700,
"train_loss": 2.8246,
"lr": 0.000661,
"tokens_per_second": 919626,
"minutes": 42.46,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 8750,
"train_loss": 2.9473,
"lr": 0.000648,
"tokens_per_second": 919635,
"minutes": 42.69,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 8800,
"train_loss": 2.8781,
"lr": 0.000636,
"tokens_per_second": 921027,
"minutes": 42.93,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 8850,
"train_loss": 2.8393,
"lr": 0.000624,
"tokens_per_second": 921693,
"minutes": 43.17,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 8900,
"train_loss": 2.9105,
"lr": 0.000612,
"tokens_per_second": 922213,
"minutes": 43.4,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 8950,
"train_loss": 2.9286,
"lr": 0.0006,
"tokens_per_second": 923184,
"minutes": 43.64,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 9000,
"train_loss": 2.8668,
"lr": 0.000588,
"tokens_per_second": 923153,
"minutes": 43.88,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 2.9286
},
{
"step": 9050,
"train_loss": 2.9063,
"lr": 0.000577,
"tokens_per_second": 922088,
"minutes": 44.16,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 9100,
"train_loss": 2.9428,
"lr": 0.000566,
"tokens_per_second": 921296,
"minutes": 44.4,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 9150,
"train_loss": 2.9146,
"lr": 0.000555,
"tokens_per_second": 920478,
"minutes": 44.64,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 9200,
"train_loss": 2.9337,
"lr": 0.000544,
"tokens_per_second": 919857,
"minutes": 44.87,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 9250,
"train_loss": 2.8898,
"lr": 0.000533,
"tokens_per_second": 919677,
"minutes": 45.11,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 9300,
"train_loss": 2.8907,
"lr": 0.000523,
"tokens_per_second": 919018,
"minutes": 45.35,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 9350,
"train_loss": 2.9104,
"lr": 0.000513,
"tokens_per_second": 919151,
"minutes": 45.59,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 9400,
"train_loss": 2.9346,
"lr": 0.000503,
"tokens_per_second": 919820,
"minutes": 45.82,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 9450,
"train_loss": 2.864,
"lr": 0.000493,
"tokens_per_second": 920435,
"minutes": 46.06,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 9500,
"train_loss": 2.8496,
"lr": 0.000484,
"tokens_per_second": 920689,
"minutes": 46.3,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 2.9166
},
{
"step": 9550,
"train_loss": 2.9794,
"lr": 0.000475,
"tokens_per_second": 920761,
"minutes": 46.58,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 9600,
"train_loss": 2.8557,
"lr": 0.000466,
"tokens_per_second": 921094,
"minutes": 46.82,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 9650,
"train_loss": 2.8992,
"lr": 0.000457,
"tokens_per_second": 921528,
"minutes": 47.06,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 9700,
"train_loss": 2.835,
"lr": 0.000448,
"tokens_per_second": 920716,
"minutes": 47.29,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 9750,
"train_loss": 2.9188,
"lr": 0.00044,
"tokens_per_second": 920417,
"minutes": 47.53,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 9800,
"train_loss": 2.925,
"lr": 0.000432,
"tokens_per_second": 920052,
"minutes": 47.77,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 9850,
"train_loss": 2.9457,
"lr": 0.000424,
"tokens_per_second": 919561,
"minutes": 48.01,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 9900,
"train_loss": 2.9236,
"lr": 0.000416,
"tokens_per_second": 919301,
"minutes": 48.24,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 9950,
"train_loss": 2.8542,
"lr": 0.000409,
"tokens_per_second": 919176,
"minutes": 48.48,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 10000,
"train_loss": 2.8907,
"lr": 0.000402,
"tokens_per_second": 919788,
"minutes": 48.72,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 2.9086
},
{
"step": 10050,
"train_loss": 2.7981,
"lr": 0.000395,
"tokens_per_second": 920621,
"minutes": 49.0,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 10100,
"train_loss": 2.8218,
"lr": 0.000388,
"tokens_per_second": 920907,
"minutes": 49.24,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 10150,
"train_loss": 2.8762,
"lr": 0.000382,
"tokens_per_second": 921101,
"minutes": 49.48,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 10200,
"train_loss": 2.8884,
"lr": 0.000375,
"tokens_per_second": 920843,
"minutes": 49.72,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 10250,
"train_loss": 2.8617,
"lr": 0.000369,
"tokens_per_second": 921002,
"minutes": 49.95,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 10300,
"train_loss": 2.8414,
"lr": 0.000364,
"tokens_per_second": 921151,
"minutes": 50.19,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 10350,
"train_loss": 2.8439,
"lr": 0.000358,
"tokens_per_second": 920309,
"minutes": 50.43,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 10400,
"train_loss": 2.8992,
"lr": 0.000353,
"tokens_per_second": 919693,
"minutes": 50.66,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 10450,
"train_loss": 2.8711,
"lr": 0.000348,
"tokens_per_second": 919270,
"minutes": 50.9,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 10500,
"train_loss": 2.8533,
"lr": 0.000343,
"tokens_per_second": 920712,
"minutes": 51.14,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 2.9005
},
{
"step": 10550,
"train_loss": 2.8638,
"lr": 0.000338,
"tokens_per_second": 921492,
"minutes": 51.42,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 10600,
"train_loss": 2.8673,
"lr": 0.000334,
"tokens_per_second": 921466,
"minutes": 51.66,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 10650,
"train_loss": 2.8849,
"lr": 0.00033,
"tokens_per_second": 920910,
"minutes": 51.9,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 10700,
"train_loss": 2.8652,
"lr": 0.000326,
"tokens_per_second": 920277,
"minutes": 52.14,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 10750,
"train_loss": 2.8523,
"lr": 0.000323,
"tokens_per_second": 919903,
"minutes": 52.37,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 10800,
"train_loss": 2.8797,
"lr": 0.00032,
"tokens_per_second": 919671,
"minutes": 52.61,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 10850,
"train_loss": 2.858,
"lr": 0.000317,
"tokens_per_second": 919122,
"minutes": 52.85,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 10900,
"train_loss": 2.8607,
"lr": 0.000314,
"tokens_per_second": 919161,
"minutes": 53.09,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 10950,
"train_loss": 2.8733,
"lr": 0.000311,
"tokens_per_second": 920086,
"minutes": 53.32,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 11000,
"train_loss": 2.8539,
"lr": 0.000309,
"tokens_per_second": 920688,
"minutes": 53.56,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 2.8962
},
{
"step": 11050,
"train_loss": 2.8992,
"lr": 0.000307,
"tokens_per_second": 920928,
"minutes": 53.84,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 11100,
"train_loss": 2.8486,
"lr": 0.000305,
"tokens_per_second": 920127,
"minutes": 54.08,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 11150,
"train_loss": 2.8515,
"lr": 0.000304,
"tokens_per_second": 919904,
"minutes": 54.32,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 11200,
"train_loss": 2.8647,
"lr": 0.000302,
"tokens_per_second": 919524,
"minutes": 54.56,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 11250,
"train_loss": 2.8809,
"lr": 0.000301,
"tokens_per_second": 920127,
"minutes": 54.79,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 11300,
"train_loss": 2.8752,
"lr": 0.000301,
"tokens_per_second": 921273,
"minutes": 55.03,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 11350,
"train_loss": 2.9045,
"lr": 0.0003,
"tokens_per_second": 920772,
"minutes": 55.27,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 11400,
"train_loss": 2.7934,
"lr": 0.0003,
"tokens_per_second": 920858,
"minutes": 55.51,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 11420,
"train_loss": 2.8882,
"lr": 0.0003,
"tokens_per_second": 921250,
"minutes": 55.6,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1,
"val_loss": 2.8934
}
],
"exports": {
"parity_prompts": 300,
"parity_steps_per_prompt": 5,
"float": {
"megabytes": 60.81,
"same_first_token": 1.0,
"same_five_greedy_tokens": 1.0,
"cached_step_ms_p50_cpu": 5.544
},
"int8": {
"megabytes": 15.64,
"same_first_token": 0.0,
"same_five_greedy_tokens": 0.0,
"cached_step_ms_p50_cpu": 4.808
}
}
}
{
"args": {
"vocab": 8192,
"dim": 320,
"layers": 8,
"heads": 5,
"ffn": 864,
"seq": 512,
"tokens": 3000000000.0,
"val_tokens": 4000000.0,
"epochs": 1.0,
"batch": 512,
"lr": 0.003,
"fim_rate": 0.5,
"mix": "fineweb=0.6,smoltalk=0.3,enron=0.1",
"fineweb_files": 4,
"smoltalk_files": 9,
"tokenizer_docs": 200000,
"eval_pauses": 1000,
"log_every": 50,
"repo": "cfahlgren1/tinynext",
"folder": "v2/experiments/tinynext-v2-3b",
"eval_folder": "v2",
"name": "tinynext-v2-3b",
"trackio_space": "cfahlgren1/tinynext-trackio",
"stage": "train",
"data_repo": "cfahlgren1/tinynext-data",
"smoke": false,
"no_push": false
},
"parameters": 12539200,
"data": {
"fineweb": {
"docs": 2122073,
"train_tokens": 1800000000
},
"smoltalk": {
"docs": 1041801,
"train_tokens": 900000000
},
"enron": {
"docs": 268120,
"train_tokens": 300000000
}
},
"data_minutes": 0.2,
"train_minutes": 55.8,
"history": [
{
"step": 0,
"train_loss": 9.0702,
"lr": 1.3e-05,
"tokens_per_second": 12800,
"minutes": 0.34,
"mfu_percent": 0.3,
"gpu_util_percent": 98,
"gpu_memory_gb": 76.5,
"val_loss": 9.0663
},
{
"step": 50,
"train_loss": 7.215,
"lr": 0.000671,
"tokens_per_second": 930958,
"minutes": 0.62,
"mfu_percent": 22.4,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 100,
"train_loss": 6.1306,
"lr": 0.001329,
"tokens_per_second": 925320,
"minutes": 0.86,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 150,
"train_loss": 5.5618,
"lr": 0.001986,
"tokens_per_second": 924266,
"minutes": 1.1,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{
"step": 200,
"train_loss": 5.1433,
"lr": 0.002643,
"tokens_per_second": 923447,
"minutes": 1.33,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 85.1
},
{