{
  "config": {
    "vocab_size": 32009,
    "d_model": 1280,
    "n_layers": 23,
    "n_heads": 20,
    "d_ff": 3456,
    "max_seq_len": 768,
    "rope_theta": 10000.0,
    "rope_pct": 0.25,
    "dropout": 0.0,
    "tie_embeddings": true
  },
  "train_args": {
    "steps": 500,
    "batch_size": 4,
    "grad_accum": 8,
    "seq_len": 768,
    "lr": 5e-05,
    "warmup": 20,
    "save_every": 100,
    "log_every": 10,
    "use_8bit_adam": true
  },
  "total_params": 496998400,
  "steps": [
    {
      "step": 0,
      "loss": 6.937812924385071,
      "ema_loss": 0.3468906462192536,
      "lr": 2.5e-06,
      "elapsed_s": 4.15720272064209,
      "steps_per_sec": 0.2405463642738951,
      "vram_gb": 9.164018176
    },
    {
      "step": 10,
      "loss": 2.739780366420746,
      "ema_loss": 1.9638136853068686,
      "lr": 2.75e-05,
      "elapsed_s": 40.6308798789978,
      "steps_per_sec": 0.2707300465251781,
      "vram_gb": 10.17390848
    },
    {
      "step": 20,
      "loss": 1.2084012106060982,
      "ema_loss": 1.882433597453757,
      "lr": 5e-05,
      "elapsed_s": 77.03947591781616,
      "steps_per_sec": 0.27258752412078047,
      "vram_gb": 10.17390848
    },
    {
      "step": 30,
      "loss": 1.0680485740303993,
      "ema_loss": 1.5722938780650435,
      "lr": 4.995182577286858e-05,
      "elapsed_s": 113.55863189697266,
      "steps_per_sec": 0.27298673365601217,
      "vram_gb": 10.17390848
    },
    {
      "step": 40,
      "loss": 0.7858129814267159,
      "ema_loss": 1.2813095330935025,
      "lr": 4.980750938091074e-05,
      "elapsed_s": 149.9709255695343,
      "steps_per_sec": 0.27338632367772026,
      "vram_gb": 10.17390848
    },
    {
      "step": 50,
      "loss": 0.6811780035495758,
      "ema_loss": 1.0847747264114023,
      "lr": 4.956766880907269e-05,
      "elapsed_s": 186.4907727241516,
      "steps_per_sec": 0.2734719753423769,
      "vram_gb": 10.17390848
    },
    {
      "step": 60,
      "loss": 0.6801792122423649,
      "ema_loss": 0.9220971225814172,
      "lr": 4.9233331091504034e-05,
      "elapsed_s": 222.90015697479248,
      "steps_per_sec": 0.27366512804608933,
      "vram_gb": 10.17390848
    },
    {
      "step": 70,
      "loss": 0.4824926480650902,
      "ema_loss": 0.8074420473552227,
      "lr": 4.8805927913639885e-05,
      "elapsed_s": 259.41771626472473,
      "steps_per_sec": 0.2736898659902916,
      "vram_gb": 10.17390848
    },
    {
      "step": 80,
      "loss": 0.4920233264565468,
      "ema_loss": 0.7012675245586312,
      "lr": 4.8287289481503954e-05,
      "elapsed_s": 295.8301751613617,
      "steps_per_sec": 0.2738057399175667,
      "vram_gb": 10.17390848
    },
    {
      "step": 90,
      "loss": 0.6347984820604324,
      "ema_loss": 0.6271864858820199,
      "lr": 4.767963668448549e-05,
      "elapsed_s": 332.34874868392944,
      "steps_per_sec": 0.2738087637168837,
      "vram_gb": 10.17390848
    },
    {
      "step": 100,
      "loss": 0.6809600777924061,
      "ema_loss": 0.5743832670414681,
      "lr": 4.6985571585149876e-05,
      "elapsed_s": 414.7752454280853,
      "steps_per_sec": 0.24350537095278896,
      "vram_gb": 10.17390848
    },
    {
      "step": 110,
      "loss": 0.4763348139822483,
      "ema_loss": 0.521244228570747,
      "lr": 4.620806627680727e-05,
      "elapsed_s": 451.26919865608215,
      "steps_per_sec": 0.24597291446118502,
      "vram_gb": 10.17390848
    },
    {
      "step": 120,
      "loss": 0.5240486040711403,
      "ema_loss": 0.494888873638692,
      "lr": 4.535045015655279e-05,
      "elapsed_s": 487.7379126548767,
      "steps_per_sec": 0.24808405674549147,
      "vram_gb": 10.17390848
    },
    {
      "step": 130,
      "loss": 0.286451730877161,
      "ema_loss": 0.46101876181189894,
      "lr": 4.4416395668276995e-05,
      "elapsed_s": 524.1727938652039,
      "steps_per_sec": 0.24991758735514977,
      "vram_gb": 10.17390848
    },
    {
      "step": 140,
      "loss": 0.35978504456579685,
      "ema_loss": 0.4399799286456892,
      "lr": 4.340990257669732e-05,
      "elapsed_s": 560.6376507282257,
      "steps_per_sec": 0.25149934153878484,
      "vram_gb": 10.17390848
    },
    {
      "step": 150,
      "loss": 0.3526405543088913,
      "ema_loss": 0.40866163158804786,
      "lr": 4.2335280839751554e-05,
      "elapsed_s": 597.072151184082,
      "steps_per_sec": 0.25290075864457046,
      "vram_gb": 10.17390848
    },
    {
      "step": 160,
      "loss": 0.3449416421353817,
      "ema_loss": 0.36948892724765275,
      "lr": 4.1197132152696215e-05,
      "elapsed_s": 633.5287528038025,
      "steps_per_sec": 0.2541321120587878,
      "vram_gb": 10.17390848
    },
    {
      "step": 170,
      "loss": 0.32148140482604504,
      "ema_loss": 0.3509791943321394,
      "lr": 4.000033024294105e-05,
      "elapsed_s": 669.9639327526093,
      "steps_per_sec": 0.2552376204752255,
      "vram_gb": 10.17390848
    },
    {
      "step": 180,
      "loss": 0.40944845974445343,
      "ema_loss": 0.3403033324976888,
      "lr": 3.875e-05,
      "elapsed_s": 706.4267771244049,
      "steps_per_sec": 0.2562190532142372,
      "vram_gb": 10.17390848
    },
    {
      "step": 190,
      "loss": 0.2508099125698209,
      "ema_loss": 0.31061072625747166,
      "lr": 3.7451495529927534e-05,
      "elapsed_s": 742.8610553741455,
      "steps_per_sec": 0.25711403043440195,
      "vram_gb": 10.17390848
    },
    {
      "step": 200,
      "loss": 0.2846630644053221,
      "ema_loss": 0.29097493865487023,
      "lr": 3.6110377228214525e-05,
      "elapsed_s": 823.1199431419373,
      "steps_per_sec": 0.2441928441592138,
      "vram_gb": 10.17390848
    },
    {
      "step": 210,
      "loss": 0.1874328227713704,
      "ema_loss": 0.2691413417018003,
      "lr": 3.473238796932114e-05,
      "elapsed_s": 859.5507295131683,
      "steps_per_sec": 0.24547707628554513,
      "vram_gb": 10.17390848
    },
    {
      "step": 220,
      "loss": 0.18770729657262564,
      "ema_loss": 0.2469535500748924,
      "lr": 3.332342851480672e-05,
      "elapsed_s": 896.0201919078827,
      "steps_per_sec": 0.24664622739073316,
      "vram_gb": 10.17390848
    },
    {
      "step": 230,
      "loss": 0.20278130657970905,
      "ema_loss": 0.2411670786782403,
      "lr": 3.1889532245362895e-05,
      "elapsed_s": 932.4535627365112,
      "steps_per_sec": 0.2477335164252839,
      "vram_gb": 10.17390848
    },
    {
      "step": 240,
      "loss": 0.24111063033342361,
      "ema_loss": 0.2326045524210834,
      "lr": 3.0436839324951166e-05,
      "elapsed_s": 968.919579744339,
      "steps_per_sec": 0.24873065323294502,
      "vram_gb": 10.17390848
    },
    {
      "step": 250,
      "loss": 0.2577468641102314,
      "ema_loss": 0.22216086817509004,
      "lr": 2.897157040767822e-05,
      "elapsed_s": 1005.3552136421204,
      "steps_per_sec": 0.24966300128956143,
      "vram_gb": 10.17390848
    },
    {
      "step": 260,
      "loss": 0.21209231484681368,
      "ema_loss": 0.2089893973874345,
      "lr": 2.75e-05,
      "elapsed_s": 1041.8542084693909,
      "steps_per_sec": 0.25051489726517534,
      "vram_gb": 10.17390848
    },
    {
      "step": 270,
      "loss": 0.16555726807564497,
      "ema_loss": 0.20300413108044887,
      "lr": 2.602842959232178e-05,
      "elapsed_s": 1078.29061460495,
      "steps_per_sec": 0.25132371211381216,
      "vram_gb": 10.17390848
    },
    {
      "step": 280,
      "loss": 0.20196319418027997,
      "ema_loss": 0.1959523809991305,
      "lr": 2.4563160675048846e-05,
      "elapsed_s": 1114.7595565319061,
      "steps_per_sec": 0.25207229519001423,
      "vram_gb": 10.17390848
    },
    {
      "step": 290,
      "loss": 0.1890458231791854,
      "ema_loss": 0.19173463181336906,
      "lr": 2.3110467754637118e-05,
      "elapsed_s": 1151.1741054058075,
      "steps_per_sec": 0.2527853941758165,
      "vram_gb": 10.17390848
    },
    {
      "step": 300,
      "loss": 0.14224259229376912,
      "ema_loss": 0.170971237515532,
      "lr": 2.1676571485193282e-05,
      "elapsed_s": 1231.5103888511658,
      "steps_per_sec": 0.2444153153111381,
      "vram_gb": 10.17390848
    },
    {
      "step": 310,
      "loss": 0.1485865106806159,
      "ema_loss": 0.17176215697934846,
      "lr": 2.026761203067887e-05,
      "elapsed_s": 1267.9178903102875,
      "steps_per_sec": 0.24528402223577067,
      "vram_gb": 10.17390848
    },
    {
      "step": 320,
      "loss": 0.17956801364198327,
      "ema_loss": 0.1700756775799304,
      "lr": 1.888962277178548e-05,
      "elapsed_s": 1304.4403862953186,
      "steps_per_sec": 0.2460825372876237,
      "vram_gb": 10.17390848
    },
    {
      "step": 330,
      "loss": 0.12682513892650604,
      "ema_loss": 0.15713850400483031,
      "lr": 1.7548504470072478e-05,
      "elapsed_s": 1340.8541731834412,
      "steps_per_sec": 0.24685756782495105,
      "vram_gb": 10.17390848
    },
    {
      "step": 340,
      "loss": 0.12957867234945297,
      "ema_loss": 0.1510191265222257,
      "lr": 1.6250000000000005e-05,
      "elapsed_s": 1377.3766162395477,
      "steps_per_sec": 0.24757208448258908,
      "vram_gb": 10.17390848
    },
    {
      "step": 350,
      "loss": 0.12622385285794735,
      "ema_loss": 0.14607439549355963,
      "lr": 1.4999669757058956e-05,
      "elapsed_s": 1413.787192106247,
      "steps_per_sec": 0.24826933074495,
      "vram_gb": 10.17390848
    },
    {
      "step": 360,
      "loss": 0.1389128677546978,
      "ema_loss": 0.1373236412001944,
      "lr": 1.3802867847303787e-05,
      "elapsed_s": 1450.3096079826355,
      "steps_per_sec": 0.24891236878872158,
      "vram_gb": 10.17390848
    },
    {
      "step": 370,
      "loss": 0.11274528503417969,
      "ema_loss": 0.1358663910950191,
      "lr": 1.2664719160248452e-05,
      "elapsed_s": 1486.721604347229,
      "steps_per_sec": 0.24954234801941552,
      "vram_gb": 10.17390848
    },
    {
      "step": 380,
      "loss": 0.12164316419512033,
      "ema_loss": 0.12515423124366815,
      "lr": 1.1590097423302684e-05,
      "elapsed_s": 1523.1735746860504,
      "steps_per_sec": 0.2501356420121259,
      "vram_gb": 10.17390848
    },
    {
      "step": 390,
      "loss": 0.10157549497671425,
      "ema_loss": 0.1216169996836354,
      "lr": 1.0583604331723012e-05,
      "elapsed_s": 1559.6568088531494,
      "steps_per_sec": 0.25069617737732386,
      "vram_gb": 10.17390848
    },
    {
      "step": 400,
      "loss": 0.10004387237131596,
      "ema_loss": 0.12196290672243944,
      "lr": 9.649549843447213e-06,
      "elapsed_s": 1641.5576264858246,
      "steps_per_sec": 0.24428018458203227,
      "vram_gb": 10.17390848
    },
    {
      "step": 410,
      "loss": 0.15670533804222941,
      "ema_loss": 0.11768572540664866,
      "lr": 8.79193372319273e-06,
      "elapsed_s": 1678.030652999878,
      "steps_per_sec": 0.24492997149083062,
      "vram_gb": 10.17390848
    },
    {
      "step": 420,
      "loss": 0.0967125273309648,
      "ema_loss": 0.12051557885116983,
      "lr": 8.01442841485013e-06,
      "elapsed_s": 1714.4823727607727,
      "steps_per_sec": 0.24555516387262588,
      "vram_gb": 10.17390848
    },
    {
      "step": 430,
      "loss": 0.08109346427954733,
      "ema_loss": 0.11255813170642848,
      "lr": 7.320363315514515e-06,
      "elapsed_s": 1750.9651441574097,
      "steps_per_sec": 0.24614995988820987,
      "vram_gb": 10.17390848
    },
    {
      "step": 440,
      "loss": 0.06976099917665124,
      "ema_loss": 0.10574265881842762,
      "lr": 6.712710518496049e-06,
      "elapsed_s": 1787.420247077942,
      "steps_per_sec": 0.24672429481591848,
      "vram_gb": 10.17390848
    },
    {
      "step": 450,
      "loss": 0.08222721493802965,
      "ema_loss": 0.09805770252970188,
      "lr": 6.194072086360125e-06,
      "elapsed_s": 1823.9047615528107,
      "steps_per_sec": 0.24727168298855356,
      "vram_gb": 10.17390848
    },
    {
      "step": 460,
      "loss": 0.09003248857334256,
      "ema_loss": 0.09465487723876762,
      "lr": 5.766668908495966e-06,
      "elapsed_s": 1860.354943037033,
      "steps_per_sec": 0.24780217437830257,
      "vram_gb": 10.17390848
    },
    {
      "step": 470,
      "loss": 0.050968108931556344,
      "ema_loss": 0.0877422503218975,
      "lr": 5.432331190927316e-06,
      "elapsed_s": 1896.8366348743439,
      "steps_per_sec": 0.24830815228914083,
      "vram_gb": 10.17390848
    },
    {
      "step": 480,
      "loss": 0.09658047696575522,
      "ema_loss": 0.08559591419795948,
      "lr": 5.192490619089267e-06,
      "elapsed_s": 1933.280499458313,
      "steps_per_sec": 0.24879990261877244,
      "vram_gb": 10.17390848
    },
    {
      "step": 490,
      "loss": 0.057105707470327616,
      "ema_loss": 0.08398783932396543,
      "lr": 5.048174227131422e-06,
      "elapsed_s": 1969.7602517604828,
      "steps_per_sec": 0.24926891461088543,
      "vram_gb": 10.17390848
    },
    {
      "step": 499,
      "loss": 0.0780253498815,
      "ema_loss": 0.08407452203461949,
      "lr": 5.000481912557093e-06,
      "elapsed_s": 2002.5768718719482,
      "steps_per_sec": 0.24967830549876227,
      "vram_gb": 10.17390848
    }
  ]
}