python-124m-muon-5b-20261021-v1: 4250017085 training targets

README.md

@@ -11,7 +11,7 @@ tags:
 # Simple LLM Python 124M / 5B
 
 Run: `python-124m-muon-5b-20261021-v1`. This is an **intermediate** base-model checkpoint, freshly initialized.
-Processed **4,000,050,828 nonpadding training targets** in **68,695 optimizer updates**.
+Processed **4,250,017,085 nonpadding training targets** in **72,999 optimizer updates**.
 The exact target is 5,000,000,000. This model is not instruction-tuned.
 
 124,389,632 parameters; 14 layers, width 768, 12 attention heads, RMSNorm,

metrics.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae5382bf286a446fa472e7c4cf6b9d914623e4797f2c42e203a3bc84cffd3326
-size 111005287
+oid sha256:466c6594214d3697d6c8efe34f77fea99052529fca9df71f9e03a7bd03bf1bba
+size 117963862

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:598fac790da185d17ed86ca1e0dd669cbafadc452e2cf6259b23a7e1837d9057
+oid sha256:e1076b1277d50b8ef937721c5286fc6653264b3caff9158ccd27bf3333e57c86
 size 598238584

provenance.json

@@ -49,20 +49,20 @@
   "torch_version": "2.6.0+cu124",
   "gpu": "NVIDIA H100 80GB HBM3",
   "parameters": 124389632,
-  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s017/step-000064314.pt",
-  "step": 68695,
-  "tokens": 4000050828,
+  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s018/step-000068695.pt",
+  "step": 72999,
+  "tokens": 4250017085,
   "run_id": "python-124m-muon-5b-20261021-v1",
-  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s018/step-000068695.pt",
-  "weights_sha256": "598fac790da185d17ed86ca1e0dd669cbafadc452e2cf6259b23a7e1837d9057",
+  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s019/step-000072999.pt",
+  "weights_sha256": "e1076b1277d50b8ef937721c5286fc6653264b3caff9158ccd27bf3333e57c86",
   "weight_roundtrip_exact": true,
   "cpu_logits_roundtrip_exact": true,
   "loss_token_exposures": {
-    "python_implementation": 2600025355,
-    "python_tests_examples": 600009749,
-    "python_documentation": 400003840,
-    "cs_math": 200002118,
-    "general_education": 120003954,
-    "books": 80005812
+    "python_implementation": 2762506345,
+    "python_tests_examples": 637506476,
+    "python_documentation": 425001012,
+    "cs_math": 212500292,
+    "general_education": 127501121,
+    "books": 85001839
   }
 }

runtime-cost-estimate.json

@@ -1,11 +1,11 @@
 {
   "kind": "GPU compute estimate, not measured invoice",
-  "training_worker_reported_seconds": 24398.398226481,
+  "training_worker_reported_seconds": 25860.754632223,
   "h100_usd_per_second": 0.001097,
   "excludes": [
     "GPU work outside recorded training segments",
     "CPU preparation and evaluation",
     "storage and transfer"
   ],
-  "training_gpu_usd_estimate": 214.12034283559728
+  "training_gpu_usd_estimate": 226.95398265238907
 }

validation.jsonl

@@ -174,3 +174,13 @@
 {"step": 68000, "tokens": 3960163873, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.491658087374827, "python_tests_examples": 1.3296636027108473, "python_documentation": 2.3974830933903113, "cs_math": 2.427966778466422, "general_education": 3.476407078937841, "books": 3.64512588462247}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.491658087374827, "python_tests_examples": 1.3296636027108473, "python_documentation": 2.3974830933903113, "cs_math": 2.427966778466422, "general_education": 3.476407078937841, "books": 3.64512588462247}, "weighted_loss": 1.7073686755232016}
 {"step": 68500, "tokens": 3989329448, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.4878267151674192, "python_tests_examples": 1.3381985103004923, "python_documentation": 2.3813420782779957, "cs_math": 2.4293124916263187, "general_education": 3.4613178178039776, "books": 3.5946787210921385}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.4878267151674192, "python_tests_examples": 1.3381985103004923, "python_documentation": 2.3813420782779957, "cs_math": 2.4293124916263187, "general_education": 3.4613178178039776, "books": 3.5946787210921385}, "weighted_loss": 1.7031500827689738}
 {"step": 68695, "tokens": 4000050828, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.4843846831903258, "python_tests_examples": 1.3291989281511627, "python_documentation": 2.372109309818256, "cs_math": 2.408725010838461, "general_education": 3.469720559747707, "books": 3.5792315186490185}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.4843846831903258, "python_tests_examples": 1.3291989281511627, "python_documentation": 2.372109309818256, "cs_math": 2.408725010838461, "general_education": 3.469720559747707, "books": 3.5792315186490185}, "weighted_loss": 1.6975533119855464}
+{"step": 68695, "tokens": 4000050828, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.484394063018389, "python_tests_examples": 1.329200125140119, "python_documentation": 2.372151901407966, "cs_math": 2.4086142992028257, "general_education": 3.4696517568864977, "books": 3.5792427470065022}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.484394063018389, "python_tests_examples": 1.329200125140119, "python_documentation": 2.372151901407966, "cs_math": 2.4086142992028257, "general_education": 3.4696517568864977, "books": 3.5792427470065022}, "weighted_loss": 1.6975564724806336}
+{"step": 69000, "tokens": 4018039242, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.4770220838454517, "python_tests_examples": 1.3326947832804743, "python_documentation": 2.3719101091808543, "cs_math": 2.426386419924502, "general_education": 3.471886515399392, "books": 3.640629854579787}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.4770220838454517, "python_tests_examples": 1.3326947832804743, "python_documentation": 2.3719101091808543, "cs_math": 2.426386419924502, "general_education": 3.471886515399392, "books": 3.640629854579787}, "weighted_loss": 1.695448096459503}
+{"step": 69500, "tokens": 4047369029, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.4873870438378811, "python_tests_examples": 1.3224833883126028, "python_documentation": 2.3784988830732945, "cs_math": 2.413154070848377, "general_education": 3.459723370406927, "books": 3.61773668623688}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.4873870438378811, "python_tests_examples": 1.3224833883126028, "python_documentation": 2.3784988830732945, "cs_math": 2.413154070848377, "general_education": 3.459723370406927, "books": 3.61773668623688}, "weighted_loss": 1.699828113428207}
+{"step": 70000, "tokens": 4075301815, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.476828547317982, "python_tests_examples": 1.3228050856749296, "python_documentation": 2.3721368729263266, "cs_math": 2.4216621556475584, "general_education": 3.4578033107667685, "books": 3.6075947437305786}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.476828547317982, "python_tests_examples": 1.3228050856749296, "python_documentation": 2.3721368729263266, "cs_math": 2.4216621556475584, "general_education": 3.4578033107667685, "books": 3.6075947437305786}, "weighted_loss": 1.692542107880553}
+{"step": 70500, "tokens": 4104897532, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.4710989894600393, "python_tests_examples": 1.3189767643562067, "python_documentation": 2.3594717417661544, "cs_math": 2.414913587663, "general_education": 3.4509738622977997, "books": 3.6023476492746354}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.4710989894600393, "python_tests_examples": 1.3189767643562067, "python_documentation": 2.3594717417661544, "cs_math": 2.414913587663, "general_education": 3.4509738622977997, "books": 3.6023476492746354}, "weighted_loss": 1.6863298802166486}
+{"step": 71000, "tokens": 4133340706, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.4631781358913152, "python_tests_examples": 1.3159947743924238, "python_documentation": 2.360290443020666, "cs_math": 2.431155230955795, "general_education": 3.450355027164264, "books": 3.6033699152636336}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.4631781358913152, "python_tests_examples": 1.3159947743924238, "python_documentation": 2.360290443020666, "cs_math": 2.431155230955795, "general_education": 3.450355027164264, "books": 3.6033699152636336}, "weighted_loss": 1.6816298594582755}
+{"step": 71500, "tokens": 4161999523, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.4610051099925678, "python_tests_examples": 1.3142394186025852, "python_documentation": 2.3605923344177957, "cs_math": 2.390226444268832, "general_education": 3.4389445444028035, "books": 3.5917996011797078}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.4610051099925678, "python_tests_examples": 1.3142394186025852, "python_documentation": 2.3605923344177957, "cs_math": 2.390226444268832, "general_education": 3.4389445444028035, "books": 3.5917996011797078}, "weighted_loss": 1.6773641182964563}
+{"step": 72000, "tokens": 4191378329, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.4621123048860443, "python_tests_examples": 1.3141731589968928, "python_documentation": 2.353899680810144, "cs_math": 2.4273978093821316, "general_education": 3.4530181637640363, "books": 3.6280979757917677}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.4621123048860443, "python_tests_examples": 1.3141731589968928, "python_documentation": 2.353899680810144, "cs_math": 2.4273978093821316, "general_education": 3.4530181637640363, "books": 3.6280979757917677}, "weighted_loss": 1.6804113350043401}
+{"step": 72500, "tokens": 4220576095, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.4524975155407107, "python_tests_examples": 1.2874576690182498, "python_documentation": 2.375524896441134, "cs_math": 2.409801992323874, "general_education": 3.41783198769218, "books": 3.566619371763918}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.4524975155407107, "python_tests_examples": 1.2874576690182498, "python_documentation": 2.375524896441134, "cs_math": 2.409801992323874, "general_education": 3.41783198769218, "books": 3.566619371763918}, "weighted_loss": 1.6691519717805503}
+{"step": 72999, "tokens": 4250017085, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.4449019714130305, "python_tests_examples": 1.302770082559299, "python_documentation": 2.348897355095165, "cs_math": 2.419067476980199, "general_education": 3.4275809564496287, "books": 3.5823515418480465}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.4449019714130305, "python_tests_examples": 1.302770082559299, "python_documentation": 2.348897355095165, "cs_math": 2.419067476980199, "general_education": 3.4275809564496287, "books": 3.5823515418480465}, "weighted_loss": 1.664919362691341}