Upload folder using huggingface_hub

390M_700kbs_10k/blumodel_muon_4M_step00008001.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3405f8d0554ccf6e5f902442e6e90ee381e3c61497b5e0cf1fa8de33276ec5bb
+size 4205803653

390M_700kbs_10k/blumodel_muon_4M_step00009000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28c48b00e453fbd2ccd627f003ab188e86d286946f3162e7163684c2f75e761b
+size 4205803653

390M_700kbs_10k/blumodel_muon_4M_step00009967.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34bfc300fcb6cdd1528d50480b9ddf755d1811513ab821c394bf5a198ae245b9
+size 4205803653

train_390M.py

@@ -448,9 +448,9 @@ class ModelConfig:
     qk_eps: float = 1e-6                  # QK-norm (fused) epsilon
 
     # ---- optimization / data ----
-    B: int = 8                           # micro-batch
+    B: int = 4                           # micro-batch
     T: int = 4096                         # sequence length (<= context_length)
-    global_batch: int = 262144         # tokens per optimizer step
+    global_batch: int = 786432        # tokens per optimizer step
     grad_clip: float = 1.0
     weight_decay: float = 0.1
 
@@ -463,7 +463,7 @@ class ModelConfig:
     muon_lr: float = 0.001
     muon_min_lr: float = 0.0001
     
-    max_steps: int = 53406
+    max_steps: int = 17802
 
     warmup_steps: int = 500
     stable_steps: int = int(max_steps * 0.99) - warmup_steps           # steps held at max_lr after warmup