| { |
| "config": { |
| "vocab_size": 32009, |
| "d_model": 1280, |
| "n_layers": 23, |
| "n_heads": 20, |
| "d_ff": 3456, |
| "max_seq_len": 768, |
| "rope_theta": 10000.0, |
| "rope_pct": 0.25, |
| "dropout": 0.0, |
| "tie_embeddings": true |
| }, |
| "train_args": { |
| "steps": 500, |
| "batch_size": 4, |
| "grad_accum": 8, |
| "seq_len": 768, |
| "lr": 5e-05, |
| "warmup": 20, |
| "save_every": 100, |
| "log_every": 10, |
| "use_8bit_adam": true |
| }, |
| "total_params": 496998400, |
| "steps": [ |
| { |
| "step": 0, |
| "loss": 6.937812924385071, |
| "ema_loss": 0.3468906462192536, |
| "lr": 2.5e-06, |
| "elapsed_s": 4.15720272064209, |
| "steps_per_sec": 0.2405463642738951, |
| "vram_gb": 9.164018176 |
| }, |
| { |
| "step": 10, |
| "loss": 2.739780366420746, |
| "ema_loss": 1.9638136853068686, |
| "lr": 2.75e-05, |
| "elapsed_s": 40.6308798789978, |
| "steps_per_sec": 0.2707300465251781, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 20, |
| "loss": 1.2084012106060982, |
| "ema_loss": 1.882433597453757, |
| "lr": 5e-05, |
| "elapsed_s": 77.03947591781616, |
| "steps_per_sec": 0.27258752412078047, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 30, |
| "loss": 1.0680485740303993, |
| "ema_loss": 1.5722938780650435, |
| "lr": 4.995182577286858e-05, |
| "elapsed_s": 113.55863189697266, |
| "steps_per_sec": 0.27298673365601217, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 40, |
| "loss": 0.7858129814267159, |
| "ema_loss": 1.2813095330935025, |
| "lr": 4.980750938091074e-05, |
| "elapsed_s": 149.9709255695343, |
| "steps_per_sec": 0.27338632367772026, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 50, |
| "loss": 0.6811780035495758, |
| "ema_loss": 1.0847747264114023, |
| "lr": 4.956766880907269e-05, |
| "elapsed_s": 186.4907727241516, |
| "steps_per_sec": 0.2734719753423769, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 60, |
| "loss": 0.6801792122423649, |
| "ema_loss": 0.9220971225814172, |
| "lr": 4.9233331091504034e-05, |
| "elapsed_s": 222.90015697479248, |
| "steps_per_sec": 0.27366512804608933, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 70, |
| "loss": 0.4824926480650902, |
| "ema_loss": 0.8074420473552227, |
| "lr": 4.8805927913639885e-05, |
| "elapsed_s": 259.41771626472473, |
| "steps_per_sec": 0.2736898659902916, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 80, |
| "loss": 0.4920233264565468, |
| "ema_loss": 0.7012675245586312, |
| "lr": 4.8287289481503954e-05, |
| "elapsed_s": 295.8301751613617, |
| "steps_per_sec": 0.2738057399175667, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 90, |
| "loss": 0.6347984820604324, |
| "ema_loss": 0.6271864858820199, |
| "lr": 4.767963668448549e-05, |
| "elapsed_s": 332.34874868392944, |
| "steps_per_sec": 0.2738087637168837, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 100, |
| "loss": 0.6809600777924061, |
| "ema_loss": 0.5743832670414681, |
| "lr": 4.6985571585149876e-05, |
| "elapsed_s": 414.7752454280853, |
| "steps_per_sec": 0.24350537095278896, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 110, |
| "loss": 0.4763348139822483, |
| "ema_loss": 0.521244228570747, |
| "lr": 4.620806627680727e-05, |
| "elapsed_s": 451.26919865608215, |
| "steps_per_sec": 0.24597291446118502, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 120, |
| "loss": 0.5240486040711403, |
| "ema_loss": 0.494888873638692, |
| "lr": 4.535045015655279e-05, |
| "elapsed_s": 487.7379126548767, |
| "steps_per_sec": 0.24808405674549147, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 130, |
| "loss": 0.286451730877161, |
| "ema_loss": 0.46101876181189894, |
| "lr": 4.4416395668276995e-05, |
| "elapsed_s": 524.1727938652039, |
| "steps_per_sec": 0.24991758735514977, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 140, |
| "loss": 0.35978504456579685, |
| "ema_loss": 0.4399799286456892, |
| "lr": 4.340990257669732e-05, |
| "elapsed_s": 560.6376507282257, |
| "steps_per_sec": 0.25149934153878484, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 150, |
| "loss": 0.3526405543088913, |
| "ema_loss": 0.40866163158804786, |
| "lr": 4.2335280839751554e-05, |
| "elapsed_s": 597.072151184082, |
| "steps_per_sec": 0.25290075864457046, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 160, |
| "loss": 0.3449416421353817, |
| "ema_loss": 0.36948892724765275, |
| "lr": 4.1197132152696215e-05, |
| "elapsed_s": 633.5287528038025, |
| "steps_per_sec": 0.2541321120587878, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 170, |
| "loss": 0.32148140482604504, |
| "ema_loss": 0.3509791943321394, |
| "lr": 4.000033024294105e-05, |
| "elapsed_s": 669.9639327526093, |
| "steps_per_sec": 0.2552376204752255, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 180, |
| "loss": 0.40944845974445343, |
| "ema_loss": 0.3403033324976888, |
| "lr": 3.875e-05, |
| "elapsed_s": 706.4267771244049, |
| "steps_per_sec": 0.2562190532142372, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 190, |
| "loss": 0.2508099125698209, |
| "ema_loss": 0.31061072625747166, |
| "lr": 3.7451495529927534e-05, |
| "elapsed_s": 742.8610553741455, |
| "steps_per_sec": 0.25711403043440195, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 200, |
| "loss": 0.2846630644053221, |
| "ema_loss": 0.29097493865487023, |
| "lr": 3.6110377228214525e-05, |
| "elapsed_s": 823.1199431419373, |
| "steps_per_sec": 0.2441928441592138, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 210, |
| "loss": 0.1874328227713704, |
| "ema_loss": 0.2691413417018003, |
| "lr": 3.473238796932114e-05, |
| "elapsed_s": 859.5507295131683, |
| "steps_per_sec": 0.24547707628554513, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 220, |
| "loss": 0.18770729657262564, |
| "ema_loss": 0.2469535500748924, |
| "lr": 3.332342851480672e-05, |
| "elapsed_s": 896.0201919078827, |
| "steps_per_sec": 0.24664622739073316, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 230, |
| "loss": 0.20278130657970905, |
| "ema_loss": 0.2411670786782403, |
| "lr": 3.1889532245362895e-05, |
| "elapsed_s": 932.4535627365112, |
| "steps_per_sec": 0.2477335164252839, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 240, |
| "loss": 0.24111063033342361, |
| "ema_loss": 0.2326045524210834, |
| "lr": 3.0436839324951166e-05, |
| "elapsed_s": 968.919579744339, |
| "steps_per_sec": 0.24873065323294502, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 250, |
| "loss": 0.2577468641102314, |
| "ema_loss": 0.22216086817509004, |
| "lr": 2.897157040767822e-05, |
| "elapsed_s": 1005.3552136421204, |
| "steps_per_sec": 0.24966300128956143, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 260, |
| "loss": 0.21209231484681368, |
| "ema_loss": 0.2089893973874345, |
| "lr": 2.75e-05, |
| "elapsed_s": 1041.8542084693909, |
| "steps_per_sec": 0.25051489726517534, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 270, |
| "loss": 0.16555726807564497, |
| "ema_loss": 0.20300413108044887, |
| "lr": 2.602842959232178e-05, |
| "elapsed_s": 1078.29061460495, |
| "steps_per_sec": 0.25132371211381216, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 280, |
| "loss": 0.20196319418027997, |
| "ema_loss": 0.1959523809991305, |
| "lr": 2.4563160675048846e-05, |
| "elapsed_s": 1114.7595565319061, |
| "steps_per_sec": 0.25207229519001423, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 290, |
| "loss": 0.1890458231791854, |
| "ema_loss": 0.19173463181336906, |
| "lr": 2.3110467754637118e-05, |
| "elapsed_s": 1151.1741054058075, |
| "steps_per_sec": 0.2527853941758165, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 300, |
| "loss": 0.14224259229376912, |
| "ema_loss": 0.170971237515532, |
| "lr": 2.1676571485193282e-05, |
| "elapsed_s": 1231.5103888511658, |
| "steps_per_sec": 0.2444153153111381, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 310, |
| "loss": 0.1485865106806159, |
| "ema_loss": 0.17176215697934846, |
| "lr": 2.026761203067887e-05, |
| "elapsed_s": 1267.9178903102875, |
| "steps_per_sec": 0.24528402223577067, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 320, |
| "loss": 0.17956801364198327, |
| "ema_loss": 0.1700756775799304, |
| "lr": 1.888962277178548e-05, |
| "elapsed_s": 1304.4403862953186, |
| "steps_per_sec": 0.2460825372876237, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 330, |
| "loss": 0.12682513892650604, |
| "ema_loss": 0.15713850400483031, |
| "lr": 1.7548504470072478e-05, |
| "elapsed_s": 1340.8541731834412, |
| "steps_per_sec": 0.24685756782495105, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 340, |
| "loss": 0.12957867234945297, |
| "ema_loss": 0.1510191265222257, |
| "lr": 1.6250000000000005e-05, |
| "elapsed_s": 1377.3766162395477, |
| "steps_per_sec": 0.24757208448258908, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 350, |
| "loss": 0.12622385285794735, |
| "ema_loss": 0.14607439549355963, |
| "lr": 1.4999669757058956e-05, |
| "elapsed_s": 1413.787192106247, |
| "steps_per_sec": 0.24826933074495, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 360, |
| "loss": 0.1389128677546978, |
| "ema_loss": 0.1373236412001944, |
| "lr": 1.3802867847303787e-05, |
| "elapsed_s": 1450.3096079826355, |
| "steps_per_sec": 0.24891236878872158, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 370, |
| "loss": 0.11274528503417969, |
| "ema_loss": 0.1358663910950191, |
| "lr": 1.2664719160248452e-05, |
| "elapsed_s": 1486.721604347229, |
| "steps_per_sec": 0.24954234801941552, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 380, |
| "loss": 0.12164316419512033, |
| "ema_loss": 0.12515423124366815, |
| "lr": 1.1590097423302684e-05, |
| "elapsed_s": 1523.1735746860504, |
| "steps_per_sec": 0.2501356420121259, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 390, |
| "loss": 0.10157549497671425, |
| "ema_loss": 0.1216169996836354, |
| "lr": 1.0583604331723012e-05, |
| "elapsed_s": 1559.6568088531494, |
| "steps_per_sec": 0.25069617737732386, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 400, |
| "loss": 0.10004387237131596, |
| "ema_loss": 0.12196290672243944, |
| "lr": 9.649549843447213e-06, |
| "elapsed_s": 1641.5576264858246, |
| "steps_per_sec": 0.24428018458203227, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 410, |
| "loss": 0.15670533804222941, |
| "ema_loss": 0.11768572540664866, |
| "lr": 8.79193372319273e-06, |
| "elapsed_s": 1678.030652999878, |
| "steps_per_sec": 0.24492997149083062, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 420, |
| "loss": 0.0967125273309648, |
| "ema_loss": 0.12051557885116983, |
| "lr": 8.01442841485013e-06, |
| "elapsed_s": 1714.4823727607727, |
| "steps_per_sec": 0.24555516387262588, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 430, |
| "loss": 0.08109346427954733, |
| "ema_loss": 0.11255813170642848, |
| "lr": 7.320363315514515e-06, |
| "elapsed_s": 1750.9651441574097, |
| "steps_per_sec": 0.24614995988820987, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 440, |
| "loss": 0.06976099917665124, |
| "ema_loss": 0.10574265881842762, |
| "lr": 6.712710518496049e-06, |
| "elapsed_s": 1787.420247077942, |
| "steps_per_sec": 0.24672429481591848, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 450, |
| "loss": 0.08222721493802965, |
| "ema_loss": 0.09805770252970188, |
| "lr": 6.194072086360125e-06, |
| "elapsed_s": 1823.9047615528107, |
| "steps_per_sec": 0.24727168298855356, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 460, |
| "loss": 0.09003248857334256, |
| "ema_loss": 0.09465487723876762, |
| "lr": 5.766668908495966e-06, |
| "elapsed_s": 1860.354943037033, |
| "steps_per_sec": 0.24780217437830257, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 470, |
| "loss": 0.050968108931556344, |
| "ema_loss": 0.0877422503218975, |
| "lr": 5.432331190927316e-06, |
| "elapsed_s": 1896.8366348743439, |
| "steps_per_sec": 0.24830815228914083, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 480, |
| "loss": 0.09658047696575522, |
| "ema_loss": 0.08559591419795948, |
| "lr": 5.192490619089267e-06, |
| "elapsed_s": 1933.280499458313, |
| "steps_per_sec": 0.24879990261877244, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 490, |
| "loss": 0.057105707470327616, |
| "ema_loss": 0.08398783932396543, |
| "lr": 5.048174227131422e-06, |
| "elapsed_s": 1969.7602517604828, |
| "steps_per_sec": 0.24926891461088543, |
| "vram_gb": 10.17390848 |
| }, |
| { |
| "step": 499, |
| "loss": 0.0780253498815, |
| "ema_loss": 0.08407452203461949, |
| "lr": 5.000481912557093e-06, |
| "elapsed_s": 2002.5768718719482, |
| "steps_per_sec": 0.24967830549876227, |
| "vram_gb": 10.17390848 |
| } |
| ] |
| } |
