Download v2/experiments/tinynext-v2-nofim/report.json from paperweight/tinynext: direct link, hf CLI and curl.
- Browser
- Download file 24.7 kB
-
https://huggingface.co/paperweight/tinynext/resolve/main/v2/experiments/tinynext-v2-nofim/report.json
- Command line
-
hf download hf://paperweight/tinynext/v2/experiments/tinynext-v2-nofim/report.json
-
curl -L -o report.json https://huggingface.co/paperweight/tinynext/resolve/main/v2/experiments/tinynext-v2-nofim/report.json
24.7 kB
{
"args": {
"vocab": 8192,
"dim": 320,
"layers": 8,
"heads": 5,
"ffn": 864,
"seq": 512,
"tokens": 1500000000.0,
"val_tokens": 4000000.0,
"epochs": 1.0,
"batch": 512,
"lr": 0.003,
"fim_rate": 0.0,
"mix": "fineweb=0.6,smoltalk=0.3,enron=0.1",
"fineweb_files": 2,
"smoltalk_files": 9,
"tokenizer_docs": 200000,
"eval_pauses": 1000,
"log_every": 50,
"repo": "cfahlgren1/tinynext",
"folder": "v2/experiments/tinynext-v2-nofim",
"eval_folder": "v2",
"name": "tinynext-v2-nofim",
"trackio_space": "cfahlgren1/tinynext-trackio",
"stage": "train",
"data_repo": "cfahlgren1/tinynext-data",
"smoke": false,
"no_push": false
},
"parameters": 12539200,
"data": {
"fineweb": {
"docs": 1062484,
"train_tokens": 900000000
},
"smoltalk": {
"docs": 671427,
"train_tokens": 450000000
},
"enron": {
"docs": 268120,
"train_tokens": 150000000
}
},
"data_minutes": 0.2,
"train_minutes": 28.1,
"history": [
{
"step": 0,
"train_loss": 9.066,
"lr": 2.6e-05,
"tokens_per_second": 13050,
"minutes": 0.33,
"mfu_percent": 0.3,
"gpu_util_percent": 97,
"gpu_memory_gb": 70.5,
"val_loss": 9.0674
},
{
"step": 50,
"train_loss": 7.0033,
"lr": 0.001342,
"tokens_per_second": 933112,
"minutes": 0.62,
"mfu_percent": 22.5,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 100,
"train_loss": 6.1098,
"lr": 0.002656,
"tokens_per_second": 926865,
"minutes": 0.85,
"mfu_percent": 22.4,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 150,
"train_loss": 5.5684,
"lr": 0.002995,
"tokens_per_second": 926389,
"minutes": 1.09,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 200,
"train_loss": 5.0866,
"lr": 0.002992,
"tokens_per_second": 925033,
"minutes": 1.33,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 250,
"train_loss": 4.6613,
"lr": 0.002987,
"tokens_per_second": 925563,
"minutes": 1.56,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 300,
"train_loss": 4.2932,
"lr": 0.002982,
"tokens_per_second": 923984,
"minutes": 1.8,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 350,
"train_loss": 4.0354,
"lr": 0.002975,
"tokens_per_second": 923469,
"minutes": 2.03,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 400,
"train_loss": 3.901,
"lr": 0.002967,
"tokens_per_second": 922950,
"minutes": 2.27,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 450,
"train_loss": 3.7236,
"lr": 0.002959,
"tokens_per_second": 922140,
"minutes": 2.51,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 500,
"train_loss": 3.6749,
"lr": 0.002949,
"tokens_per_second": 921720,
"minutes": 2.74,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 3.6931
},
{
"step": 550,
"train_loss": 3.6001,
"lr": 0.002938,
"tokens_per_second": 921925,
"minutes": 3.03,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 600,
"train_loss": 3.5615,
"lr": 0.002927,
"tokens_per_second": 922017,
"minutes": 3.27,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 650,
"train_loss": 3.5618,
"lr": 0.002914,
"tokens_per_second": 922210,
"minutes": 3.5,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 700,
"train_loss": 3.453,
"lr": 0.002901,
"tokens_per_second": 922179,
"minutes": 3.74,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 750,
"train_loss": 3.4649,
"lr": 0.002886,
"tokens_per_second": 922152,
"minutes": 3.98,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 800,
"train_loss": 3.3789,
"lr": 0.002871,
"tokens_per_second": 921856,
"minutes": 4.21,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 850,
"train_loss": 3.4502,
"lr": 0.002855,
"tokens_per_second": 922127,
"minutes": 4.45,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 900,
"train_loss": 3.361,
"lr": 0.002837,
"tokens_per_second": 921952,
"minutes": 4.69,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 950,
"train_loss": 3.398,
"lr": 0.002819,
"tokens_per_second": 922148,
"minutes": 4.92,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1000,
"train_loss": 3.3632,
"lr": 0.0028,
"tokens_per_second": 922256,
"minutes": 5.16,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 3.3573
},
{
"step": 1050,
"train_loss": 3.276,
"lr": 0.002781,
"tokens_per_second": 923002,
"minutes": 5.44,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1100,
"train_loss": 3.2999,
"lr": 0.00276,
"tokens_per_second": 923065,
"minutes": 5.68,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1150,
"train_loss": 3.346,
"lr": 0.002738,
"tokens_per_second": 923089,
"minutes": 5.92,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1200,
"train_loss": 3.2729,
"lr": 0.002716,
"tokens_per_second": 922983,
"minutes": 6.15,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1250,
"train_loss": 3.2381,
"lr": 0.002693,
"tokens_per_second": 922842,
"minutes": 6.39,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1300,
"train_loss": 3.2686,
"lr": 0.002669,
"tokens_per_second": 922738,
"minutes": 6.63,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1350,
"train_loss": 3.2762,
"lr": 0.002644,
"tokens_per_second": 922862,
"minutes": 6.86,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1400,
"train_loss": 3.2013,
"lr": 0.002618,
"tokens_per_second": 922796,
"minutes": 7.1,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1450,
"train_loss": 3.1987,
"lr": 0.002592,
"tokens_per_second": 923059,
"minutes": 7.34,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1500,
"train_loss": 3.2144,
"lr": 0.002565,
"tokens_per_second": 922750,
"minutes": 7.57,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 3.2337
},
{
"step": 1550,
"train_loss": 3.1748,
"lr": 0.002538,
"tokens_per_second": 922943,
"minutes": 7.86,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1600,
"train_loss": 3.2102,
"lr": 0.002509,
"tokens_per_second": 922876,
"minutes": 8.1,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1650,
"train_loss": 3.1717,
"lr": 0.00248,
"tokens_per_second": 923021,
"minutes": 8.33,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1700,
"train_loss": 3.215,
"lr": 0.002451,
"tokens_per_second": 922818,
"minutes": 8.57,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1750,
"train_loss": 3.2434,
"lr": 0.002421,
"tokens_per_second": 923028,
"minutes": 8.81,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1800,
"train_loss": 3.1329,
"lr": 0.00239,
"tokens_per_second": 922698,
"minutes": 9.04,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1850,
"train_loss": 3.1663,
"lr": 0.002358,
"tokens_per_second": 922517,
"minutes": 9.28,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1900,
"train_loss": 3.1417,
"lr": 0.002327,
"tokens_per_second": 922580,
"minutes": 9.52,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 1950,
"train_loss": 3.1055,
"lr": 0.002294,
"tokens_per_second": 922863,
"minutes": 9.75,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2000,
"train_loss": 3.1488,
"lr": 0.002261,
"tokens_per_second": 922511,
"minutes": 9.99,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 3.1599
},
{
"step": 2050,
"train_loss": 3.0741,
"lr": 0.002228,
"tokens_per_second": 922742,
"minutes": 10.27,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2100,
"train_loss": 3.1161,
"lr": 0.002194,
"tokens_per_second": 922930,
"minutes": 10.51,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2150,
"train_loss": 3.1425,
"lr": 0.00216,
"tokens_per_second": 922993,
"minutes": 10.75,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2200,
"train_loss": 3.1001,
"lr": 0.002125,
"tokens_per_second": 922882,
"minutes": 10.98,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2250,
"train_loss": 3.1289,
"lr": 0.00209,
"tokens_per_second": 922732,
"minutes": 11.22,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2300,
"train_loss": 3.1166,
"lr": 0.002055,
"tokens_per_second": 922818,
"minutes": 11.46,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2350,
"train_loss": 3.0936,
"lr": 0.00202,
"tokens_per_second": 923024,
"minutes": 11.69,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2400,
"train_loss": 3.1052,
"lr": 0.001984,
"tokens_per_second": 922709,
"minutes": 11.93,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2450,
"train_loss": 3.0901,
"lr": 0.001948,
"tokens_per_second": 922648,
"minutes": 12.17,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2500,
"train_loss": 3.094,
"lr": 0.001911,
"tokens_per_second": 922464,
"minutes": 12.4,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 3.1067
},
{
"step": 2550,
"train_loss": 3.1594,
"lr": 0.001875,
"tokens_per_second": 922698,
"minutes": 12.69,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2600,
"train_loss": 3.1005,
"lr": 0.001838,
"tokens_per_second": 922773,
"minutes": 12.92,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2650,
"train_loss": 3.0486,
"lr": 0.001801,
"tokens_per_second": 922597,
"minutes": 13.16,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2700,
"train_loss": 3.045,
"lr": 0.001764,
"tokens_per_second": 922925,
"minutes": 13.4,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2750,
"train_loss": 3.0978,
"lr": 0.001727,
"tokens_per_second": 922957,
"minutes": 13.63,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2800,
"train_loss": 3.0068,
"lr": 0.00169,
"tokens_per_second": 922829,
"minutes": 13.87,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2850,
"train_loss": 2.999,
"lr": 0.001653,
"tokens_per_second": 922756,
"minutes": 14.11,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2900,
"train_loss": 3.0705,
"lr": 0.001616,
"tokens_per_second": 922667,
"minutes": 14.34,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 2950,
"train_loss": 3.0176,
"lr": 0.001579,
"tokens_per_second": 922610,
"minutes": 14.58,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3000,
"train_loss": 3.015,
"lr": 0.001542,
"tokens_per_second": 922714,
"minutes": 14.82,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 3.0619
},
{
"step": 3050,
"train_loss": 3.0686,
"lr": 0.001505,
"tokens_per_second": 922937,
"minutes": 15.1,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3100,
"train_loss": 3.0109,
"lr": 0.001468,
"tokens_per_second": 922847,
"minutes": 15.34,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3150,
"train_loss": 2.9785,
"lr": 0.001431,
"tokens_per_second": 922659,
"minutes": 15.58,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3200,
"train_loss": 3.0064,
"lr": 0.001395,
"tokens_per_second": 922762,
"minutes": 15.81,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3250,
"train_loss": 3.0779,
"lr": 0.001358,
"tokens_per_second": 923029,
"minutes": 16.05,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3300,
"train_loss": 3.0103,
"lr": 0.001322,
"tokens_per_second": 922850,
"minutes": 16.29,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3350,
"train_loss": 3.0329,
"lr": 0.001286,
"tokens_per_second": 922649,
"minutes": 16.52,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3400,
"train_loss": 2.9922,
"lr": 0.001251,
"tokens_per_second": 922830,
"minutes": 16.76,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3450,
"train_loss": 2.9825,
"lr": 0.001215,
"tokens_per_second": 922466,
"minutes": 17.0,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3500,
"train_loss": 2.9975,
"lr": 0.00118,
"tokens_per_second": 922482,
"minutes": 17.23,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 3.0243
},
{
"step": 3550,
"train_loss": 3.0041,
"lr": 0.001146,
"tokens_per_second": 922239,
"minutes": 17.52,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3600,
"train_loss": 3.0208,
"lr": 0.001111,
"tokens_per_second": 921706,
"minutes": 17.75,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3650,
"train_loss": 2.9875,
"lr": 0.001077,
"tokens_per_second": 921485,
"minutes": 17.99,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3700,
"train_loss": 3.0218,
"lr": 0.001044,
"tokens_per_second": 921567,
"minutes": 18.23,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3750,
"train_loss": 2.9973,
"lr": 0.001011,
"tokens_per_second": 922249,
"minutes": 18.46,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3800,
"train_loss": 2.9779,
"lr": 0.000979,
"tokens_per_second": 923901,
"minutes": 18.7,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3850,
"train_loss": 3.0253,
"lr": 0.000947,
"tokens_per_second": 924971,
"minutes": 18.94,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3900,
"train_loss": 2.9768,
"lr": 0.000915,
"tokens_per_second": 924295,
"minutes": 19.17,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 3950,
"train_loss": 2.9955,
"lr": 0.000884,
"tokens_per_second": 924314,
"minutes": 19.41,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4000,
"train_loss": 2.9672,
"lr": 0.000854,
"tokens_per_second": 923365,
"minutes": 19.65,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 2.9912
},
{
"step": 4050,
"train_loss": 2.982,
"lr": 0.000824,
"tokens_per_second": 923158,
"minutes": 19.93,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4100,
"train_loss": 2.9381,
"lr": 0.000795,
"tokens_per_second": 922493,
"minutes": 20.17,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4150,
"train_loss": 2.96,
"lr": 0.000767,
"tokens_per_second": 922454,
"minutes": 20.4,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4200,
"train_loss": 2.9844,
"lr": 0.000739,
"tokens_per_second": 924196,
"minutes": 20.64,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4250,
"train_loss": 2.9594,
"lr": 0.000712,
"tokens_per_second": 925585,
"minutes": 20.88,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4300,
"train_loss": 2.9844,
"lr": 0.000686,
"tokens_per_second": 925297,
"minutes": 21.11,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4350,
"train_loss": 2.9955,
"lr": 0.00066,
"tokens_per_second": 924922,
"minutes": 21.35,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4400,
"train_loss": 2.9842,
"lr": 0.000635,
"tokens_per_second": 925657,
"minutes": 21.58,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4450,
"train_loss": 2.9217,
"lr": 0.000611,
"tokens_per_second": 926080,
"minutes": 21.82,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4500,
"train_loss": 2.9781,
"lr": 0.000588,
"tokens_per_second": 925039,
"minutes": 22.06,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 2.9623
},
{
"step": 4550,
"train_loss": 2.9359,
"lr": 0.000565,
"tokens_per_second": 924164,
"minutes": 22.34,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4600,
"train_loss": 2.9604,
"lr": 0.000544,
"tokens_per_second": 924462,
"minutes": 22.58,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4650,
"train_loss": 2.937,
"lr": 0.000523,
"tokens_per_second": 926111,
"minutes": 22.81,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4700,
"train_loss": 2.9574,
"lr": 0.000503,
"tokens_per_second": 925254,
"minutes": 23.05,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4750,
"train_loss": 2.9611,
"lr": 0.000484,
"tokens_per_second": 926355,
"minutes": 23.28,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4800,
"train_loss": 2.9299,
"lr": 0.000465,
"tokens_per_second": 925442,
"minutes": 23.52,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4850,
"train_loss": 2.8933,
"lr": 0.000448,
"tokens_per_second": 924963,
"minutes": 23.76,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4900,
"train_loss": 2.9737,
"lr": 0.000432,
"tokens_per_second": 924506,
"minutes": 23.99,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 4950,
"train_loss": 2.9918,
"lr": 0.000416,
"tokens_per_second": 925264,
"minutes": 24.23,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5000,
"train_loss": 2.8923,
"lr": 0.000401,
"tokens_per_second": 923776,
"minutes": 24.47,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 2.9413
},
{
"step": 5050,
"train_loss": 2.9833,
"lr": 0.000388,
"tokens_per_second": 924281,
"minutes": 24.75,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5100,
"train_loss": 2.916,
"lr": 0.000375,
"tokens_per_second": 924275,
"minutes": 24.98,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5150,
"train_loss": 2.9783,
"lr": 0.000363,
"tokens_per_second": 925466,
"minutes": 25.22,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5200,
"train_loss": 2.9437,
"lr": 0.000353,
"tokens_per_second": 924296,
"minutes": 25.46,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5250,
"train_loss": 2.8472,
"lr": 0.000343,
"tokens_per_second": 924268,
"minutes": 25.69,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5300,
"train_loss": 2.9167,
"lr": 0.000334,
"tokens_per_second": 923137,
"minutes": 25.93,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5350,
"train_loss": 2.9399,
"lr": 0.000326,
"tokens_per_second": 923097,
"minutes": 26.17,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5400,
"train_loss": 2.9526,
"lr": 0.000319,
"tokens_per_second": 925239,
"minutes": 26.4,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5450,
"train_loss": 2.9353,
"lr": 0.000314,
"tokens_per_second": 926014,
"minutes": 26.64,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5500,
"train_loss": 2.8749,
"lr": 0.000309,
"tokens_per_second": 925013,
"minutes": 26.88,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 2.9287
},
{
"step": 5550,
"train_loss": 2.9738,
"lr": 0.000305,
"tokens_per_second": 924722,
"minutes": 27.16,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5600,
"train_loss": 2.9329,
"lr": 0.000302,
"tokens_per_second": 924135,
"minutes": 27.39,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5650,
"train_loss": 2.9227,
"lr": 0.000301,
"tokens_per_second": 922406,
"minutes": 27.63,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5700,
"train_loss": 2.9227,
"lr": 0.0003,
"tokens_per_second": 921979,
"minutes": 27.87,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 5709,
"train_loss": 2.9206,
"lr": 0.0003,
"tokens_per_second": 921700,
"minutes": 27.91,
"mfu_percent": 22.2,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1,
"val_loss": 2.9258
}
],
"exports": {
"parity_prompts": 300,
"parity_steps_per_prompt": 5,
"float": {
"megabytes": 60.81,
"same_first_token": 1.0,
"same_five_greedy_tokens": 1.0,
"cached_step_ms_p50_cpu": 5.492
},
"int8": {
"megabytes": 15.64,
"same_first_token": 0.0033,
"same_five_greedy_tokens": 0.0,
"cached_step_ms_p50_cpu": 6.308
}
}
}
{
"args": {
"vocab": 8192,
"dim": 320,
"layers": 8,
"heads": 5,
"ffn": 864,
"seq": 512,
"tokens": 1500000000.0,
"val_tokens": 4000000.0,
"epochs": 1.0,
"batch": 512,
"lr": 0.003,
"fim_rate": 0.0,
"mix": "fineweb=0.6,smoltalk=0.3,enron=0.1",
"fineweb_files": 2,
"smoltalk_files": 9,
"tokenizer_docs": 200000,
"eval_pauses": 1000,
"log_every": 50,
"repo": "cfahlgren1/tinynext",
"folder": "v2/experiments/tinynext-v2-nofim",
"eval_folder": "v2",
"name": "tinynext-v2-nofim",
"trackio_space": "cfahlgren1/tinynext-trackio",
"stage": "train",
"data_repo": "cfahlgren1/tinynext-data",
"smoke": false,
"no_push": false
},
"parameters": 12539200,
"data": {
"fineweb": {
"docs": 1062484,
"train_tokens": 900000000
},
"smoltalk": {
"docs": 671427,
"train_tokens": 450000000
},
"enron": {
"docs": 268120,
"train_tokens": 150000000
}
},
"data_minutes": 0.2,
"train_minutes": 28.1,
"history": [
{
"step": 0,
"train_loss": 9.066,
"lr": 2.6e-05,
"tokens_per_second": 13050,
"minutes": 0.33,
"mfu_percent": 0.3,
"gpu_util_percent": 97,
"gpu_memory_gb": 70.5,
"val_loss": 9.0674
},
{
"step": 50,
"train_loss": 7.0033,
"lr": 0.001342,
"tokens_per_second": 933112,
"minutes": 0.62,
"mfu_percent": 22.5,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 100,
"train_loss": 6.1098,
"lr": 0.002656,
"tokens_per_second": 926865,
"minutes": 0.85,
"mfu_percent": 22.4,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 150,
"train_loss": 5.5684,
"lr": 0.002995,
"tokens_per_second": 926389,
"minutes": 1.09,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 200,
"train_loss": 5.0866,
"lr": 0.002992,
"tokens_per_second": 925033,
"minutes": 1.33,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{