Hugging Face's logo

jayasriii
/
40M_121M

Model card Files
xet
 Community
jayasriii commited on
Commit
5ccfc8e
·
verified ·
1 Parent(s): 4ddf731

Upload folder using huggingface_hub

Browse files
Files changed (39) hide show
  1. 40M_121M_12e3_10B.csv +0 -0
  2. 40M_121M_12e3_10B/blumodel_muon_step00000953.pt +3 -0
  3. 40M_121M_12e3_10B/blumodel_muon_step00001906.pt +3 -0
  4. 40M_121M_12e3_10B/blumodel_muon_step00002859.pt +3 -0
  5. 40M_121M_12e3_10B/blumodel_muon_step00003812.pt +3 -0
  6. 40M_121M_12e3_10B/blumodel_muon_step00004765.pt +3 -0
  7. 40M_121M_12e3_10B/blumodel_muon_step00005718.pt +3 -0
  8. 40M_121M_12e3_10B/blumodel_muon_step00006671.pt +3 -0
  9. 40M_121M_12e3_10B/blumodel_muon_step00007624.pt +3 -0
  10. 40M_121M_12e3_10B/blumodel_muon_step00008577.pt +3 -0
  11. 40M_121M_12e3_10B/blumodel_muon_step00009530.pt +3 -0
  12. 40M_121M_12e3_10B/blumodel_muon_step00010483.pt +3 -0
  13. 40M_121M_12e3_10B/blumodel_muon_step00011436.pt +3 -0
  14. 40M_121M_12e3_10B/blumodel_muon_step00012389.pt +3 -0
  15. 40M_121M_12e3_10B/blumodel_muon_step00013342.pt +3 -0
  16. 40M_121M_12e3_10B/blumodel_muon_step00014295.pt +3 -0
  17. 40M_121M_12e3_10B/blumodel_muon_step00015248.pt +3 -0
  18. 40M_121M_12e3_10B/blumodel_muon_step00016201.pt +3 -0
  19. 40M_121M_12e3_10B/blumodel_muon_step00017154.pt +3 -0
  20. 40M_121M_12e3_10B/blumodel_muon_step00018107.pt +3 -0
  21. 40M_121M_12e3_10B/blumodel_muon_step00019060.pt +3 -0
  22. 40M_121M_12e3_10B/blumodel_muon_step00019074.pt +3 -0
  23. 40M_121M_15e4_10B_half.csv +0 -0
  24. 40M_121M_15e4_10B_half/blumodel_muon_step00000953.pt +3 -0
  25. 40M_121M_15e4_10B_half/blumodel_muon_step00001906.pt +3 -0
  26. 40M_121M_15e4_10B_half/blumodel_muon_step00002859.pt +3 -0
  27. 40M_121M_15e4_10B_half/blumodel_muon_step00003812.pt +3 -0
  28. 40M_121M_15e4_10B_half/blumodel_muon_step00004765.pt +3 -0
  29. 40M_121M_15e4_10B_half/blumodel_muon_step00005718.pt +3 -0
  30. 40M_121M_15e4_10B_half/blumodel_muon_step00006671.pt +3 -0
  31. 40M_121M_15e4_10B_half/blumodel_muon_step00007624.pt +3 -0
  32. 40M_121M_15e4_10B_half/blumodel_muon_step00008577.pt +3 -0
  33. 40M_121M_15e4_10B_half/blumodel_muon_step00009472.pt +3 -0
  34. 40M_121M_15e4_10B_half/blumodel_muon_step00009530.pt +3 -0
  35. 40M_121M_15e4_10B_half/blumodel_muon_step00010483.pt +3 -0
  36. 40M_121M_15e4_10B_half/blumodel_muon_step00011436.pt +3 -0
  37. 40M_121M_15e4_10B_half/blumodel_muon_step00012389.pt +3 -0
  38. 40M_121M_15e4_10B_half/blumodel_muon_step00012561.pt +3 -0
  39. train_muon_40M_121M.py +6 -4
40M_121M_12e3_10B.csv ADDED
The diff for this file is too large to render. See raw diff
 
40M_121M_12e3_10B/blumodel_muon_step00000953.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7023e87ad346da92f43f809029cb5f8f7a34018e4f21c0fc1f3b9a5761e974de
3
+ size 1390854167
40M_121M_12e3_10B/blumodel_muon_step00001906.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:024b0cdb822abc37af81af46503653d0a9e92df76b441a70cf2a40d190566a60
3
+ size 1390854167
40M_121M_12e3_10B/blumodel_muon_step00002859.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ebdf793279d2e667b1031c84e1b9a946f1a69bc15544e2aadf87f3ebf2381ef9
3
+ size 1390854167
40M_121M_12e3_10B/blumodel_muon_step00003812.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c0c59e1cc35793516af3d385924f8a9c359c644e30c10dc696d4848fac8b778
3
+ size 1390854167
40M_121M_12e3_10B/blumodel_muon_step00004765.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:02f1680d344746846118ac959e95e18ea7b38a58c6321021bb1738ab0702620c
3
+ size 1390854167
40M_121M_12e3_10B/blumodel_muon_step00005718.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:30b024d80fbbb6af3e708a89c4d4f2974aee4fd515564e439ddd77adb5762789
3
+ size 1390854167
40M_121M_12e3_10B/blumodel_muon_step00006671.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:975d1bc6a707a76270c296b06ab6243e8220ebf647e4cb04cf9fe3328bbad15e
3
+ size 1390854167
40M_121M_12e3_10B/blumodel_muon_step00007624.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c07f4a1ad205088d63d00550703f247f5b19d47ac1047dda0755966ffa44bcfa
3
+ size 1390854167
40M_121M_12e3_10B/blumodel_muon_step00008577.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a4f57e1e23b4b499c622759ede3eb7b70174b0d2456a4895972b75aa0facca92
3
+ size 1390854167
40M_121M_12e3_10B/blumodel_muon_step00009530.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c28d33e1583ca639cfb544e385909a5d39e789a7f56bf3e130036d94c4008737
3
+ size 1390854167
40M_121M_12e3_10B/blumodel_muon_step00010483.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:38739953a5a7a5b1f8bf1296854109c7068abdd821a677e1e822d8adf9b583e5
3
+ size 1390854167
40M_121M_12e3_10B/blumodel_muon_step00011436.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:00095f5458f6ec025a13846fc1cff11d76c2442ca72d52cdf6f9dfb0aa9f5c31
3
+ size 1390854167
40M_121M_12e3_10B/blumodel_muon_step00012389.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:70037094f573ed2fb371f801fa5f34294dacc9029c06668141712fcf5012034c
3
+ size 1390854167
40M_121M_12e3_10B/blumodel_muon_step00013342.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f04325d91731dda25082e0442461e72b53cd25fdef48714a547e715a25a82653
3
+ size 1390854359
40M_121M_12e3_10B/blumodel_muon_step00014295.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9f62fd8b98c3d49acce8adaa7f8b2db87c5b62016a601f470175e3f253faae9c
3
+ size 1390854359
40M_121M_12e3_10B/blumodel_muon_step00015248.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eb917d9e89c50faa1b3c7be90a0d59c8bdd4581a1d8e9ce57fe149f50146fa02
3
+ size 1390854359
40M_121M_12e3_10B/blumodel_muon_step00016201.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d5509afe86bbdf8ff3dec0c7c0cd45f91724ff1ec57be4b3401c6a1c8cde3af4
3
+ size 1390854359
40M_121M_12e3_10B/blumodel_muon_step00017154.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c0e3e51d5f375ed4935d38ec28998fdeb604880b5061b78fdc8962d777243cc
3
+ size 1390854359
40M_121M_12e3_10B/blumodel_muon_step00018107.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6eb836f3b012e79ec06281deb8b0d5bb4d7ef379109843eb14413d2a0fcd8e31
3
+ size 1390854359
40M_121M_12e3_10B/blumodel_muon_step00019060.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1511e1c34beda4c21b9df1c811aa99bfcdbfdd330fa073782f68bc7e04b4425b
3
+ size 1390854359
40M_121M_12e3_10B/blumodel_muon_step00019074.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d2df289c8a615240fbc63a112524ceed945c30ec744aeed8fbb42c6b0ae8eb7e
3
+ size 1390854359
40M_121M_15e4_10B_half.csv ADDED
The diff for this file is too large to render. See raw diff
 
40M_121M_15e4_10B_half/blumodel_muon_step00000953.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd28f86f2d54d49465b6e2ce54a94d0e10ed892bbbdba45099024d6abcfff279
3
+ size 1390854167
40M_121M_15e4_10B_half/blumodel_muon_step00001906.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7547afd16f9da6f8088fda6f7a77713bcd7d78a036ba8ab652036669ac7500ed
3
+ size 1390854167
40M_121M_15e4_10B_half/blumodel_muon_step00002859.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ddd72a910f5193b6188b0325a128ff2bcb6e2c28d83b8ce136eaccd90fc6ee08
3
+ size 1390854167
40M_121M_15e4_10B_half/blumodel_muon_step00003812.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2359d3f89c2f6e33a8d40957e5716dd4fbd7dd6cb22234699c7b3c8a5e94ee60
3
+ size 1390854167
40M_121M_15e4_10B_half/blumodel_muon_step00004765.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3eb51edba26156e6006acd25d4a7bc40db5d00bc5a26d50aedb3d6296d0110c4
3
+ size 1390854167
40M_121M_15e4_10B_half/blumodel_muon_step00005718.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dfc63632d803657fc389e2b43f04944ffd177ae69881bde27189da65bf258c7c
3
+ size 1390854167
40M_121M_15e4_10B_half/blumodel_muon_step00006671.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:045666e7b9cf91c8e6b47d6fccf422fdfc5db2cdf633ebb51a155540bc06ade0
3
+ size 1390854167
40M_121M_15e4_10B_half/blumodel_muon_step00007624.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7f2af442bbf058277aa227d32857674fdb6db8932517aa5041f8b387d90f177f
3
+ size 1390854167
40M_121M_15e4_10B_half/blumodel_muon_step00008577.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:446152e130a27a9dbf9b38474b73c15ad15f75401dae2da51c5eb9124986e3b7
3
+ size 1390854167
40M_121M_15e4_10B_half/blumodel_muon_step00009472.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dd45086b3f03269986861c1165b308afe6ecec165c9cfb44f637d50adc7b86ef
3
+ size 1390854167
40M_121M_15e4_10B_half/blumodel_muon_step00009530.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7669df868493732e06beaab855ea534243991afb90cdd743f80d2ed29e1fa583
3
+ size 1390854359
40M_121M_15e4_10B_half/blumodel_muon_step00010483.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:63d820065ac285914a8de9d81e68b8c0f37961c0694b101f8e18284855f8912b
3
+ size 1390854359
40M_121M_15e4_10B_half/blumodel_muon_step00011436.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f00489f260da72c811ba5427b02d2535b22d8338723303791e382bfde1546130
3
+ size 1390854359
40M_121M_15e4_10B_half/blumodel_muon_step00012389.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6ea2e91c1068284ae4172fd1b2ee665ac5b2a05982d5927ce425e87e179a061c
3
+ size 1390854359
40M_121M_15e4_10B_half/blumodel_muon_step00012561.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0feccda8e9f75c1c046e11a37b25e1f03c4f095bdf81056cf0b167408c6e3e36
3
+ size 1390854359
train_muon_40M_121M.py CHANGED
@@ -1,3 +1,4 @@
 
1
  #!/usr/bin/env python3
2
  # =============================================================================
3
  # train_muon.py — the MUON arm of train.py. Same model, same data order, same
@@ -457,11 +458,11 @@ class ModelConfig:
457
  # WSD: linear warmup -> constant at max_lr -> cosine decay to min_lr.
458
  # Requires warmup_steps + stable_steps < max_steps - 1 (the remainder is
459
  # the decay tail).
460
- max_lr: float = 0.003
461
- min_lr: float = 0.0003 # 10% of max_lr
462
 
463
- muon_lr: float = 0.003
464
- muon_min_lr: float = 0.0003
465
 
466
  max_steps: int = 19074
467
  warmup_steps: int = 500
@@ -1806,3 +1807,4 @@ if __name__ == "__main__":
1806
 
1807
 
1808
 
 
 
1
+
2
  #!/usr/bin/env python3
3
  # =============================================================================
4
  # train_muon.py — the MUON arm of train.py. Same model, same data order, same
 
458
  # WSD: linear warmup -> constant at max_lr -> cosine decay to min_lr.
459
  # Requires warmup_steps + stable_steps < max_steps - 1 (the remainder is
460
  # the decay tail).
461
+ max_lr: float = 0.0015
462
+ min_lr: float = 0.00015 # 10% of max_lr
463
 
464
+ muon_lr: float = 0.0015
465
+ muon_min_lr: float = 0.00015
466
 
467
  max_steps: int = 19074
468
  warmup_steps: int = 500
 
1807
 
1808
 
1809
 
1810
+