python-124m-muon-5b-20261021-v1: 1250043430 training targets

README.md

@@ -11,7 +11,7 @@ tags:
 # Simple LLM Python 124M / 5B
 
 Run: `python-124m-muon-5b-20261021-v1`. This is an **intermediate** base-model checkpoint, freshly initialized.
-Processed **1,000,070,440 nonpadding training targets** in **16,446 optimizer updates**.
+Processed **1,250,043,430 nonpadding training targets** in **20,819 optimizer updates**.
 The exact target is 5,000,000,000. This model is not instruction-tuned.
 
 124,389,632 parameters; 14 layers, width 768, 12 attention heads, RMSNorm,

metrics.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7bc3dacc96821bbd8a93dd183962e33f815b6c42a3dc5a27e2a79872dbf1a51b
-size 26543001
+oid sha256:fe3adad5156fb7fd04a929740f261a6c2b6c0e5d5010615326cee2934b83880f
+size 33610823

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:128ac2d57ca86eef82fadbcdda3857a541036a6b17e75c063d9f93bddc5d4eec
+oid sha256:084abd8bfcc152474b883737a749be51945bdd1e9d0a43f1d0a75d02b57ec8a2
 size 598238584

provenance.json

@@ -49,20 +49,20 @@
   "torch_version": "2.6.0+cu124",
   "gpu": "NVIDIA H100 80GB HBM3",
   "parameters": 124389632,
-  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s005-retry-001/step-000012072.pt",
-  "step": 16446,
-  "tokens": 1000070440,
+  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s006/step-000016446.pt",
+  "step": 20819,
+  "tokens": 1250043430,
   "run_id": "python-124m-muon-5b-20261021-v1",
-  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s006/step-000016446.pt",
-  "weights_sha256": "128ac2d57ca86eef82fadbcdda3857a541036a6b17e75c063d9f93bddc5d4eec",
+  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s007/step-000020819.pt",
+  "weights_sha256": "084abd8bfcc152474b883737a749be51945bdd1e9d0a43f1d0a75d02b57ec8a2",
   "weight_roundtrip_exact": true,
   "cpu_logits_roundtrip_exact": true,
   "loss_token_exposures": {
-    "python_implementation": 650035375,
-    "python_tests_examples": 150015147,
-    "python_documentation": 100005889,
-    "cs_math": 50005950,
-    "general_education": 30001554,
-    "books": 20006525
+    "python_implementation": 812527466,
+    "python_tests_examples": 187505735,
+    "python_documentation": 125004796,
+    "cs_math": 62501707,
+    "general_education": 37501065,
+    "books": 25002661
   }
 }

runtime-cost-estimate.json

@@ -1,11 +1,11 @@
 {
   "kind": "GPU compute estimate, not measured invoice",
-  "training_worker_reported_seconds": 6999.435524978,
+  "training_worker_reported_seconds": 8451.06975036,
   "h100_usd_per_second": 0.001097,
   "excludes": [
     "GPU work outside recorded training segments",
     "CPU preparation and evaluation",
     "storage and transfer"
   ],
-  "training_gpu_usd_estimate": 61.42704616720693
+  "training_gpu_usd_estimate": 74.16658812915936
 }

validation.jsonl

@@ -44,3 +44,14 @@
 {"step": 15500, "tokens": 947077614, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8594222856914986, "python_tests_examples": 1.6748200916824856, "python_documentation": 2.7956311417561435, "cs_math": 2.985422776822322, "general_education": 3.950244911336941, "books": 4.0800848224163495}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8594222856914986, "python_tests_examples": 1.6748200916824856, "python_documentation": 2.7956311417561435, "cs_math": 2.985422776822322, "general_education": 3.950244911336941, "books": 4.0800848224163495}, "weighted_loss": 2.0887907962570127}
 {"step": 16000, "tokens": 974882707, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.854393777780219, "python_tests_examples": 1.6702474014735955, "python_documentation": 2.8046355190348904, "cs_math": 2.890126670573612, "general_education": 3.955185241842711, "books": 4.10421407147964}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.854393777780219, "python_tests_examples": 1.6702474014735955, "python_documentation": 2.8046355190348904, "cs_math": 2.890126670573612, "general_education": 3.955185241842711, "books": 4.10421407147964}, "weighted_loss": 2.0816027898952254}
 {"step": 16446, "tokens": 1000070440, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.825571772312484, "python_tests_examples": 1.6630119510485815, "python_documentation": 2.7780784043286477, "cs_math": 2.880713280455128, "general_education": 3.928751808829358, "books": 4.058927748896857}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.825571772312484, "python_tests_examples": 1.6630119510485815, "python_documentation": 2.7780784043286477, "cs_math": 2.880713280455128, "general_education": 3.928751808829358, "books": 4.058927748896857}, "weighted_loss": 2.0569580583588407}
+{"step": 16446, "tokens": 1000070440, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.825571772312484, "python_tests_examples": 1.6630119510485815, "python_documentation": 2.7780784043286477, "cs_math": 2.880713280455128, "general_education": 3.928751808829358, "books": 4.058927748896857}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.825571772312484, "python_tests_examples": 1.6630119510485815, "python_documentation": 2.7780784043286477, "cs_math": 2.880713280455128, "general_education": 3.928751808829358, "books": 4.058927748896857}, "weighted_loss": 2.0569580583588407}
+{"step": 16500, "tokens": 1002960689, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8472072849429315, "python_tests_examples": 1.6819528796235308, "python_documentation": 2.7942138899547326, "cs_math": 2.8885196824929826, "general_education": 3.955763078098368, "books": 4.113734100432582}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8472072849429315, "python_tests_examples": 1.6819528796235308, "python_documentation": 2.7942138899547326, "cs_math": 2.8885196824929826, "general_education": 3.955763078098368, "books": 4.113734100432582}, "weighted_loss": 2.0777726146281603}
+{"step": 17000, "tokens": 1031521309, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8363403994697347, "python_tests_examples": 1.6787470956535708, "python_documentation": 2.792027160499074, "cs_math": 2.881142310302406, "general_education": 3.942886925710231, "books": 4.097817113640329}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8363403994697347, "python_tests_examples": 1.6787470956535708, "python_documentation": 2.792027160499074, "cs_math": 2.881142310302406, "general_education": 3.942886925710231, "books": 4.097817113640329}, "weighted_loss": 2.0689361056125044}
+{"step": 17500, "tokens": 1059432052, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8285059506276997, "python_tests_examples": 1.6525428593991052, "python_documentation": 2.7946218366026403, "cs_math": 2.8562418263644602, "general_education": 3.956512814685161, "books": 4.107554022373072}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8285059506276997, "python_tests_examples": 1.6525428593991052, "python_documentation": 2.7946218366026403, "cs_math": 2.8562418263644602, "general_education": 3.956512814685161, "books": 4.107554022373072}, "weighted_loss": 2.059531036684374}
+{"step": 18000, "tokens": 1088311058, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8295966278064526, "python_tests_examples": 1.6523509331871298, "python_documentation": 2.786419180017372, "cs_math": 2.9621296860068385, "general_education": 3.973182637965154, "books": 4.067668265598987}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8295966278064526, "python_tests_examples": 1.6523509331871298, "python_documentation": 2.786419180017372, "cs_math": 2.9621296860068385, "general_education": 3.973182637965154, "books": 4.067668265598987}, "weighted_loss": 2.0643876948052773}
+{"step": 18500, "tokens": 1117012399, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8465943370542282, "python_tests_examples": 1.6645568777466322, "python_documentation": 2.8064508116299205, "cs_math": 2.879292727516501, "general_education": 3.953716436116762, "books": 4.105160729768152}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8465943370542282, "python_tests_examples": 1.6645568777466322, "python_documentation": 2.8064508116299205, "cs_math": 2.879292727516501, "general_education": 3.953716436116762, "books": 4.105160729768152}, "weighted_loss": 2.075294275964926}
+{"step": 19000, "tokens": 1145629560, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8270632919762684, "python_tests_examples": 1.6460989169814384, "python_documentation": 2.769483320759075, "cs_math": 2.9001806056785266, "general_education": 3.9234352739594285, "books": 4.075284648190621}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8270632919762684, "python_tests_examples": 1.6460989169814384, "python_documentation": 2.769483320759075, "cs_math": 2.9001806056785266, "general_education": 3.9234352739594285, "books": 4.075284648190621}, "weighted_loss": 2.055672090874219}
+{"step": 19500, "tokens": 1174140311, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.812806008779639, "python_tests_examples": 1.637071411461201, "python_documentation": 2.7598965977220895, "cs_math": 2.8532476080342066, "general_education": 3.9392088998741044, "books": 4.042620210208864}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.812806008779639, "python_tests_examples": 1.637071411461201, "python_documentation": 2.7598965977220895, "cs_math": 2.8532476080342066, "general_education": 3.9392088998741044, "books": 4.042620210208864}, "weighted_loss": 2.041565328800265}
+{"step": 20000, "tokens": 1202829235, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8195488412400105, "python_tests_examples": 1.6549985057441632, "python_documentation": 2.7682803908320976, "cs_math": 2.855537448354571, "general_education": 3.951401117957689, "books": 4.087104636168625}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8195488412400105, "python_tests_examples": 1.6549985057441632, "python_documentation": 2.7682803908320976, "cs_math": 2.855537448354571, "general_education": 3.951401117957689, "books": 4.087104636168625}, "weighted_loss": 2.050845560430673}
+{"step": 20500, "tokens": 1231597289, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8230518750977858, "python_tests_examples": 1.641852645034251, "python_documentation": 2.757211575758287, "cs_math": 2.9303158158619897, "general_education": 3.940093853837251, "books": 4.0772245839636545}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8230518750977858, "python_tests_examples": 1.641852645034251, "python_documentation": 2.757211575758287, "cs_math": 2.9303158158619897, "general_education": 3.940093853837251, "books": 4.0772245839636545}, "weighted_loss": 2.053245871232017}
+{"step": 20819, "tokens": 1250043430, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.8134269705343335, "python_tests_examples": 1.6431167731638394, "python_documentation": 2.7631289270946664, "cs_math": 2.931464609876069, "general_education": 3.9224856050651864, "books": 4.08663177344478}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.8134269705343335, "python_tests_examples": 1.6431167731638394, "python_documentation": 2.7631289270946664, "cs_math": 2.931464609876069, "general_education": 3.9224856050651864, "books": 4.08663177344478}, "weighted_loss": 2.047488373646014}