cfahlgren1's picture
cfahlgren1 HF Staff
tinynext-v2-nofim: 12.5M params, 28 min on GPU
e5dde63 verified
Raw History Blame Contribute Delete
24.7 kB
{ "args": { "vocab": 8192, "dim": 320, "layers": 8, "heads": 5, "ffn": 864, "seq": 512, "tokens": 1500000000.0, "val_tokens": 4000000.0, "epochs": 1.0, "batch": 512, "lr": 0.003, "fim_rate": 0.0, "mix": "fineweb=0.6,smoltalk=0.3,enron=0.1", "fineweb_files": 2, "smoltalk_files": 9, "tokenizer_docs": 200000, "eval_pauses": 1000, "log_every": 50, "repo": "cfahlgren1/tinynext", "folder": "v2/experiments/tinynext-v2-nofim", "eval_folder": "v2", "name": "tinynext-v2-nofim", "trackio_space": "cfahlgren1/tinynext-trackio", "stage": "train", "data_repo": "cfahlgren1/tinynext-data", "smoke": false, "no_push": false }, "parameters": 12539200, "data": { "fineweb": { "docs": 1062484, "train_tokens": 900000000 }, "smoltalk": { "docs": 671427, "train_tokens": 450000000 }, "enron": { "docs": 268120, "train_tokens": 150000000 } }, "data_minutes": 0.2, "train_minutes": 28.1, "history": [ { "step": 0, "train_loss": 9.066, "lr": 2.6e-05, "tokens_per_second": 13050, "minutes": 0.33, "mfu_percent": 0.3, "gpu_util_percent": 97, "gpu_memory_gb": 70.5, "val_loss": 9.0674 }, { "step": 50, "train_loss": 7.0033, "lr": 0.001342, "tokens_per_second": 933112, "minutes": 0.62, "mfu_percent": 22.5, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 100, "train_loss": 6.1098, "lr": 0.002656, "tokens_per_second": 926865, "minutes": 0.85, "mfu_percent": 22.4, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 150, "train_loss": 5.5684, "lr": 0.002995, "tokens_per_second": 926389, "minutes": 1.09, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 200, "train_loss": 5.0866, "lr": 0.002992, "tokens_per_second": 925033, "minutes": 1.33, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 250, "train_loss": 4.6613, "lr": 0.002987, "tokens_per_second": 925563, "minutes": 1.56, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 300, "train_loss": 4.2932, "lr": 0.002982, "tokens_per_second": 923984, "minutes": 1.8, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 350, "train_loss": 4.0354, "lr": 0.002975, "tokens_per_second": 923469, "minutes": 2.03, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 400, "train_loss": 3.901, "lr": 0.002967, "tokens_per_second": 922950, "minutes": 2.27, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 450, "train_loss": 3.7236, "lr": 0.002959, "tokens_per_second": 922140, "minutes": 2.51, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 500, "train_loss": 3.6749, "lr": 0.002949, "tokens_per_second": 921720, "minutes": 2.74, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1, "val_loss": 3.6931 }, { "step": 550, "train_loss": 3.6001, "lr": 0.002938, "tokens_per_second": 921925, "minutes": 3.03, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 600, "train_loss": 3.5615, "lr": 0.002927, "tokens_per_second": 922017, "minutes": 3.27, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 650, "train_loss": 3.5618, "lr": 0.002914, "tokens_per_second": 922210, "minutes": 3.5, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 700, "train_loss": 3.453, "lr": 0.002901, "tokens_per_second": 922179, "minutes": 3.74, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 750, "train_loss": 3.4649, "lr": 0.002886, "tokens_per_second": 922152, "minutes": 3.98, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 800, "train_loss": 3.3789, "lr": 0.002871, "tokens_per_second": 921856, "minutes": 4.21, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 850, "train_loss": 3.4502, "lr": 0.002855, "tokens_per_second": 922127, "minutes": 4.45, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 900, "train_loss": 3.361, "lr": 0.002837, "tokens_per_second": 921952, "minutes": 4.69, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 950, "train_loss": 3.398, "lr": 0.002819, "tokens_per_second": 922148, "minutes": 4.92, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 1000, "train_loss": 3.3632, "lr": 0.0028, "tokens_per_second": 922256, "minutes": 5.16, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1, "val_loss": 3.3573 }, { "step": 1050, "train_loss": 3.276, "lr": 0.002781, "tokens_per_second": 923002, "minutes": 5.44, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 1100, "train_loss": 3.2999, "lr": 0.00276, "tokens_per_second": 923065, "minutes": 5.68, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 1150, "train_loss": 3.346, "lr": 0.002738, "tokens_per_second": 923089, "minutes": 5.92, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 1200, "train_loss": 3.2729, "lr": 0.002716, "tokens_per_second": 922983, "minutes": 6.15, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 1250, "train_loss": 3.2381, "lr": 0.002693, "tokens_per_second": 922842, "minutes": 6.39, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 1300, "train_loss": 3.2686, "lr": 0.002669, "tokens_per_second": 922738, "minutes": 6.63, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 1350, "train_loss": 3.2762, "lr": 0.002644, "tokens_per_second": 922862, "minutes": 6.86, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 1400, "train_loss": 3.2013, "lr": 0.002618, "tokens_per_second": 922796, "minutes": 7.1, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 1450, "train_loss": 3.1987, "lr": 0.002592, "tokens_per_second": 923059, "minutes": 7.34, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 1500, "train_loss": 3.2144, "lr": 0.002565, "tokens_per_second": 922750, "minutes": 7.57, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1, "val_loss": 3.2337 }, { "step": 1550, "train_loss": 3.1748, "lr": 0.002538, "tokens_per_second": 922943, "minutes": 7.86, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 1600, "train_loss": 3.2102, "lr": 0.002509, "tokens_per_second": 922876, "minutes": 8.1, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 1650, "train_loss": 3.1717, "lr": 0.00248, "tokens_per_second": 923021, "minutes": 8.33, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 1700, "train_loss": 3.215, "lr": 0.002451, "tokens_per_second": 922818, "minutes": 8.57, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 1750, "train_loss": 3.2434, "lr": 0.002421, "tokens_per_second": 923028, "minutes": 8.81, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 1800, "train_loss": 3.1329, "lr": 0.00239, "tokens_per_second": 922698, "minutes": 9.04, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 1850, "train_loss": 3.1663, "lr": 0.002358, "tokens_per_second": 922517, "minutes": 9.28, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 1900, "train_loss": 3.1417, "lr": 0.002327, "tokens_per_second": 922580, "minutes": 9.52, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 1950, "train_loss": 3.1055, "lr": 0.002294, "tokens_per_second": 922863, "minutes": 9.75, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 2000, "train_loss": 3.1488, "lr": 0.002261, "tokens_per_second": 922511, "minutes": 9.99, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1, "val_loss": 3.1599 }, { "step": 2050, "train_loss": 3.0741, "lr": 0.002228, "tokens_per_second": 922742, "minutes": 10.27, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 2100, "train_loss": 3.1161, "lr": 0.002194, "tokens_per_second": 922930, "minutes": 10.51, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 2150, "train_loss": 3.1425, "lr": 0.00216, "tokens_per_second": 922993, "minutes": 10.75, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 2200, "train_loss": 3.1001, "lr": 0.002125, "tokens_per_second": 922882, "minutes": 10.98, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 2250, "train_loss": 3.1289, "lr": 0.00209, "tokens_per_second": 922732, "minutes": 11.22, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 2300, "train_loss": 3.1166, "lr": 0.002055, "tokens_per_second": 922818, "minutes": 11.46, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 2350, "train_loss": 3.0936, "lr": 0.00202, "tokens_per_second": 923024, "minutes": 11.69, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 2400, "train_loss": 3.1052, "lr": 0.001984, "tokens_per_second": 922709, "minutes": 11.93, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 2450, "train_loss": 3.0901, "lr": 0.001948, "tokens_per_second": 922648, "minutes": 12.17, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 2500, "train_loss": 3.094, "lr": 0.001911, "tokens_per_second": 922464, "minutes": 12.4, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1, "val_loss": 3.1067 }, { "step": 2550, "train_loss": 3.1594, "lr": 0.001875, "tokens_per_second": 922698, "minutes": 12.69, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 2600, "train_loss": 3.1005, "lr": 0.001838, "tokens_per_second": 922773, "minutes": 12.92, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 2650, "train_loss": 3.0486, "lr": 0.001801, "tokens_per_second": 922597, "minutes": 13.16, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 2700, "train_loss": 3.045, "lr": 0.001764, "tokens_per_second": 922925, "minutes": 13.4, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 2750, "train_loss": 3.0978, "lr": 0.001727, "tokens_per_second": 922957, "minutes": 13.63, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 2800, "train_loss": 3.0068, "lr": 0.00169, "tokens_per_second": 922829, "minutes": 13.87, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 2850, "train_loss": 2.999, "lr": 0.001653, "tokens_per_second": 922756, "minutes": 14.11, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 2900, "train_loss": 3.0705, "lr": 0.001616, "tokens_per_second": 922667, "minutes": 14.34, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 2950, "train_loss": 3.0176, "lr": 0.001579, "tokens_per_second": 922610, "minutes": 14.58, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 3000, "train_loss": 3.015, "lr": 0.001542, "tokens_per_second": 922714, "minutes": 14.82, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1, "val_loss": 3.0619 }, { "step": 3050, "train_loss": 3.0686, "lr": 0.001505, "tokens_per_second": 922937, "minutes": 15.1, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 3100, "train_loss": 3.0109, "lr": 0.001468, "tokens_per_second": 922847, "minutes": 15.34, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 3150, "train_loss": 2.9785, "lr": 0.001431, "tokens_per_second": 922659, "minutes": 15.58, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 3200, "train_loss": 3.0064, "lr": 0.001395, "tokens_per_second": 922762, "minutes": 15.81, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 3250, "train_loss": 3.0779, "lr": 0.001358, "tokens_per_second": 923029, "minutes": 16.05, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 3300, "train_loss": 3.0103, "lr": 0.001322, "tokens_per_second": 922850, "minutes": 16.29, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 3350, "train_loss": 3.0329, "lr": 0.001286, "tokens_per_second": 922649, "minutes": 16.52, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 3400, "train_loss": 2.9922, "lr": 0.001251, "tokens_per_second": 922830, "minutes": 16.76, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 3450, "train_loss": 2.9825, "lr": 0.001215, "tokens_per_second": 922466, "minutes": 17.0, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 3500, "train_loss": 2.9975, "lr": 0.00118, "tokens_per_second": 922482, "minutes": 17.23, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1, "val_loss": 3.0243 }, { "step": 3550, "train_loss": 3.0041, "lr": 0.001146, "tokens_per_second": 922239, "minutes": 17.52, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 3600, "train_loss": 3.0208, "lr": 0.001111, "tokens_per_second": 921706, "minutes": 17.75, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 3650, "train_loss": 2.9875, "lr": 0.001077, "tokens_per_second": 921485, "minutes": 17.99, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 3700, "train_loss": 3.0218, "lr": 0.001044, "tokens_per_second": 921567, "minutes": 18.23, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 3750, "train_loss": 2.9973, "lr": 0.001011, "tokens_per_second": 922249, "minutes": 18.46, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 3800, "train_loss": 2.9779, "lr": 0.000979, "tokens_per_second": 923901, "minutes": 18.7, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 3850, "train_loss": 3.0253, "lr": 0.000947, "tokens_per_second": 924971, "minutes": 18.94, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 3900, "train_loss": 2.9768, "lr": 0.000915, "tokens_per_second": 924295, "minutes": 19.17, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 3950, "train_loss": 2.9955, "lr": 0.000884, "tokens_per_second": 924314, "minutes": 19.41, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 4000, "train_loss": 2.9672, "lr": 0.000854, "tokens_per_second": 923365, "minutes": 19.65, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1, "val_loss": 2.9912 }, { "step": 4050, "train_loss": 2.982, "lr": 0.000824, "tokens_per_second": 923158, "minutes": 19.93, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 4100, "train_loss": 2.9381, "lr": 0.000795, "tokens_per_second": 922493, "minutes": 20.17, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 4150, "train_loss": 2.96, "lr": 0.000767, "tokens_per_second": 922454, "minutes": 20.4, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 4200, "train_loss": 2.9844, "lr": 0.000739, "tokens_per_second": 924196, "minutes": 20.64, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 4250, "train_loss": 2.9594, "lr": 0.000712, "tokens_per_second": 925585, "minutes": 20.88, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 4300, "train_loss": 2.9844, "lr": 0.000686, "tokens_per_second": 925297, "minutes": 21.11, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 4350, "train_loss": 2.9955, "lr": 0.00066, "tokens_per_second": 924922, "minutes": 21.35, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 4400, "train_loss": 2.9842, "lr": 0.000635, "tokens_per_second": 925657, "minutes": 21.58, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 4450, "train_loss": 2.9217, "lr": 0.000611, "tokens_per_second": 926080, "minutes": 21.82, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 4500, "train_loss": 2.9781, "lr": 0.000588, "tokens_per_second": 925039, "minutes": 22.06, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1, "val_loss": 2.9623 }, { "step": 4550, "train_loss": 2.9359, "lr": 0.000565, "tokens_per_second": 924164, "minutes": 22.34, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 4600, "train_loss": 2.9604, "lr": 0.000544, "tokens_per_second": 924462, "minutes": 22.58, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 4650, "train_loss": 2.937, "lr": 0.000523, "tokens_per_second": 926111, "minutes": 22.81, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 4700, "train_loss": 2.9574, "lr": 0.000503, "tokens_per_second": 925254, "minutes": 23.05, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 4750, "train_loss": 2.9611, "lr": 0.000484, "tokens_per_second": 926355, "minutes": 23.28, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 4800, "train_loss": 2.9299, "lr": 0.000465, "tokens_per_second": 925442, "minutes": 23.52, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 4850, "train_loss": 2.8933, "lr": 0.000448, "tokens_per_second": 924963, "minutes": 23.76, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 4900, "train_loss": 2.9737, "lr": 0.000432, "tokens_per_second": 924506, "minutes": 23.99, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 4950, "train_loss": 2.9918, "lr": 0.000416, "tokens_per_second": 925264, "minutes": 24.23, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 5000, "train_loss": 2.8923, "lr": 0.000401, "tokens_per_second": 923776, "minutes": 24.47, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1, "val_loss": 2.9413 }, { "step": 5050, "train_loss": 2.9833, "lr": 0.000388, "tokens_per_second": 924281, "minutes": 24.75, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 5100, "train_loss": 2.916, "lr": 0.000375, "tokens_per_second": 924275, "minutes": 24.98, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 5150, "train_loss": 2.9783, "lr": 0.000363, "tokens_per_second": 925466, "minutes": 25.22, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 5200, "train_loss": 2.9437, "lr": 0.000353, "tokens_per_second": 924296, "minutes": 25.46, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 5250, "train_loss": 2.8472, "lr": 0.000343, "tokens_per_second": 924268, "minutes": 25.69, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 5300, "train_loss": 2.9167, "lr": 0.000334, "tokens_per_second": 923137, "minutes": 25.93, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 5350, "train_loss": 2.9399, "lr": 0.000326, "tokens_per_second": 923097, "minutes": 26.17, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 5400, "train_loss": 2.9526, "lr": 0.000319, "tokens_per_second": 925239, "minutes": 26.4, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 5450, "train_loss": 2.9353, "lr": 0.000314, "tokens_per_second": 926014, "minutes": 26.64, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 5500, "train_loss": 2.8749, "lr": 0.000309, "tokens_per_second": 925013, "minutes": 26.88, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1, "val_loss": 2.9287 }, { "step": 5550, "train_loss": 2.9738, "lr": 0.000305, "tokens_per_second": 924722, "minutes": 27.16, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 5600, "train_loss": 2.9329, "lr": 0.000302, "tokens_per_second": 924135, "minutes": 27.39, "mfu_percent": 22.3, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 5650, "train_loss": 2.9227, "lr": 0.000301, "tokens_per_second": 922406, "minutes": 27.63, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 5700, "train_loss": 2.9227, "lr": 0.0003, "tokens_per_second": 921979, "minutes": 27.87, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1 }, { "step": 5709, "train_loss": 2.9206, "lr": 0.0003, "tokens_per_second": 921700, "minutes": 27.91, "mfu_percent": 22.2, "gpu_util_percent": 100, "gpu_memory_gb": 79.1, "val_loss": 2.9258 } ], "exports": { "parity_prompts": 300, "parity_steps_per_prompt": 5, "float": { "megabytes": 60.81, "same_first_token": 1.0, "same_five_greedy_tokens": 1.0, "cached_step_ms_p50_cpu": 5.492 }, "int8": { "megabytes": 15.64, "same_first_token": 0.0033, "same_five_greedy_tokens": 0.0, "cached_step_ms_p50_cpu": 6.308 } } }
{
"args": {
"vocab": 8192,
"dim": 320,
"layers": 8,
"heads": 5,
"ffn": 864,
"seq": 512,
"tokens": 1500000000.0,
"val_tokens": 4000000.0,
"epochs": 1.0,
"batch": 512,
"lr": 0.003,
"fim_rate": 0.0,
"mix": "fineweb=0.6,smoltalk=0.3,enron=0.1",
"fineweb_files": 2,
"smoltalk_files": 9,
"tokenizer_docs": 200000,
"eval_pauses": 1000,
"log_every": 50,
"repo": "cfahlgren1/tinynext",
"folder": "v2/experiments/tinynext-v2-nofim",
"eval_folder": "v2",
"name": "tinynext-v2-nofim",
"trackio_space": "cfahlgren1/tinynext-trackio",
"stage": "train",
"data_repo": "cfahlgren1/tinynext-data",
"smoke": false,
"no_push": false
},
"parameters": 12539200,
"data": {
"fineweb": {
"docs": 1062484,
"train_tokens": 900000000
},
"smoltalk": {
"docs": 671427,
"train_tokens": 450000000
},
"enron": {
"docs": 268120,
"train_tokens": 150000000
}
},
"data_minutes": 0.2,
"train_minutes": 28.1,
"history": [
{
"step": 0,
"train_loss": 9.066,
"lr": 2.6e-05,
"tokens_per_second": 13050,
"minutes": 0.33,
"mfu_percent": 0.3,
"gpu_util_percent": 97,
"gpu_memory_gb": 70.5,
"val_loss": 9.0674
},
{
"step": 50,
"train_loss": 7.0033,
"lr": 0.001342,
"tokens_per_second": 933112,
"minutes": 0.62,
"mfu_percent": 22.5,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 100,
"train_loss": 6.1098,
"lr": 0.002656,
"tokens_per_second": 926865,
"minutes": 0.85,
"mfu_percent": 22.4,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 150,
"train_loss": 5.5684,
"lr": 0.002995,
"tokens_per_second": 926389,
"minutes": 1.09,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{
"step": 200,
"train_loss": 5.0866,
"lr": 0.002992,
"tokens_per_second": 925033,
"minutes": 1.33,
"mfu_percent": 22.3,
"gpu_util_percent": 100,
"gpu_memory_gb": 79.1
},
{