python-124m-muon-5b-20261021-v1: 3500002820 training targets

README.md

@@ -11,7 +11,7 @@ tags:
 # Simple LLM Python 124M / 5B
 
 Run: `python-124m-muon-5b-20261021-v1`. This is an **intermediate** base-model checkpoint, freshly initialized.
-Processed **3,250,020,404 nonpadding training targets** in **55,578 optimizer updates**.
+Processed **3,500,002,820 nonpadding training targets** in **59,962 optimizer updates**.
 The exact target is 5,000,000,000. This model is not instruction-tuned.
 
 124,389,632 parameters; 14 layers, width 768, 12 attention heads, RMSNorm,

metrics.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4571519e038fe0a4f7747a5ae25c6ff8ae24d298ad5145ac9a7ca819450c101f
-size 89809575
+oid sha256:011a6c210020adeff56053a51cb5e69c0aaab398479e9b875440cc57c21fe6ef
+size 96894059

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4a1dd2c5c5daf33829c336c57c09f67a1b016a6ba8ef445815764dbb305e1d20
+oid sha256:ec6a6d4863359857391cdd3fcecb1128126c18000f640af17487fc4a73f2bd55
 size 598238584

provenance.json

@@ -49,20 +49,20 @@
   "torch_version": "2.6.0+cu124",
   "gpu": "NVIDIA H100 80GB HBM3",
   "parameters": 124389632,
-  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s015/step-000052000.pt",
-  "step": 55578,
-  "tokens": 3250020404,
+  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s015-retry-001/step-000055578.pt",
+  "step": 59962,
+  "tokens": 3500002820,
   "run_id": "python-124m-muon-5b-20261021-v1",
-  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s015-retry-001/step-000055578.pt",
-  "weights_sha256": "4a1dd2c5c5daf33829c336c57c09f67a1b016a6ba8ef445815764dbb305e1d20",
+  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s016/step-000059962.pt",
+  "weights_sha256": "ec6a6d4863359857391cdd3fcecb1128126c18000f640af17487fc4a73f2bd55",
   "weight_roundtrip_exact": true,
   "cpu_logits_roundtrip_exact": true,
   "loss_token_exposures": {
-    "python_implementation": 2112506782,
-    "python_tests_examples": 487502611,
-    "python_documentation": 325001565,
-    "cs_math": 162501588,
-    "general_education": 97501257,
-    "books": 65006601
+    "python_implementation": 2274996049,
+    "python_tests_examples": 525000419,
+    "python_documentation": 349999178,
+    "cs_math": 174999629,
+    "general_education": 105001083,
+    "books": 70006462
   }
 }

runtime-cost-estimate.json

@@ -1,11 +1,11 @@
 {
   "kind": "GPU compute estimate, not measured invoice",
-  "training_worker_reported_seconds": 20103.543753661,
+  "training_worker_reported_seconds": 21568.491258613,
   "h100_usd_per_second": 0.001097,
   "excludes": [
     "GPU work outside recorded training segments",
     "CPU preparation and evaluation",
     "storage and transfer"
   ],
-  "training_gpu_usd_estimate": 176.42869998212896
+  "training_gpu_usd_estimate": 189.28507928558773
 }

validation.jsonl

@@ -142,3 +142,13 @@
 {"step": 55000, "tokens": 3217131091, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5625315793460448, "python_tests_examples": 1.427025033704201, "python_documentation": 2.4945047995350222, "cs_math": 2.551059848155503, "general_education": 3.605662266616817, "books": 3.7320141816782204}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5625315793460448, "python_tests_examples": 1.427025033704201, "python_documentation": 2.4945047995350222, "cs_math": 2.551059848155503, "general_education": 3.605662266616817, "books": 3.7320141816782204}, "weighted_loss": 1.7895129056239056}
 {"step": 55500, "tokens": 3245727964, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5580064021150448, "python_tests_examples": 1.4257301893339225, "python_documentation": 2.4808555694992207, "cs_math": 2.5460227793064174, "general_education": 3.5927341956030046, "books": 3.7304924479643873}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5580064021150448, "python_tests_examples": 1.4257301893339225, "python_documentation": 2.4808555694992207, "cs_math": 2.5460227793064174, "general_education": 3.5927341956030046, "books": 3.7304924479643873}, "weighted_loss": 1.7843422605174883}
 {"step": 55578, "tokens": 3250020404, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5688506460597964, "python_tests_examples": 1.4291687882825896, "python_documentation": 2.4936180137144413, "cs_math": 2.7126572116453707, "general_education": 3.6291224016963137, "books": 3.7664474305436424}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5688506460597964, "python_tests_examples": 1.4291687882825896, "python_documentation": 2.4936180137144413, "cs_math": 2.7126572116453707, "general_education": 3.6291224016963137, "books": 3.7664474305436424}, "weighted_loss": 1.8033255207967311}
+{"step": 55578, "tokens": 3250020404, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5688506460597964, "python_tests_examples": 1.4291687882825896, "python_documentation": 2.4936180137144413, "cs_math": 2.7126572116453707, "general_education": 3.6291224016963137, "books": 3.7664474305436424}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5688506460597964, "python_tests_examples": 1.4291687882825896, "python_documentation": 2.4936180137144413, "cs_math": 2.7126572116453707, "general_education": 3.6291224016963137, "books": 3.7664474305436424}, "weighted_loss": 1.8033255207967311}
+{"step": 56000, "tokens": 3274110251, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.575399134782875, "python_tests_examples": 1.4183193045099596, "python_documentation": 2.489968896418603, "cs_math": 2.5354743985890567, "general_education": 3.590567323196321, "books": 3.7508612633744245}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.575399134782875, "python_tests_examples": 1.4183193045099596, "python_documentation": 2.489968896418603, "cs_math": 2.5354743985890567, "general_education": 3.590567323196321, "books": 3.7508612633744245}, "weighted_loss": 1.7952621878200539}
+{"step": 56500, "tokens": 3302139050, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5774145731368858, "python_tests_examples": 1.4271482530634294, "python_documentation": 2.4913191261248024, "cs_math": 2.5308012058014486, "general_education": 3.603904378544318, "books": 3.783387538111688}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5774145731368858, "python_tests_examples": 1.4271482530634294, "python_documentation": 2.4913191261248024, "cs_math": 2.5308012058014486, "general_education": 3.603904378544318, "books": 3.783387538111688}, "weighted_loss": 1.7988485655196063}
+{"step": 57000, "tokens": 3330798005, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5551849406274243, "python_tests_examples": 1.4090709742681917, "python_documentation": 2.4705898074236154, "cs_math": 2.5953004283046353, "general_education": 3.587718163455369, "books": 3.7161494339719883}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5551849406274243, "python_tests_examples": 1.4090709742681917, "python_documentation": 2.4705898074236154, "cs_math": 2.5953004283046353, "general_education": 3.587718163455369, "books": 3.7161494339719883}, "weighted_loss": 1.7810093932887487}
+{"step": 57500, "tokens": 3358931999, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.564921253734051, "python_tests_examples": 1.4082292644813454, "python_documentation": 2.485529326101381, "cs_math": 2.5657149836349094, "general_education": 3.5860243796754188, "books": 3.7549223116456547}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.564921253734051, "python_tests_examples": 1.4082292644813454, "python_documentation": 2.485529326101381, "cs_math": 2.5657149836349094, "general_education": 3.5860243796754188, "books": 3.7549223116456547}, "weighted_loss": 1.7879510640143945}
+{"step": 58000, "tokens": 3387592601, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.552792273518009, "python_tests_examples": 1.3881421403303627, "python_documentation": 2.4606719666085985, "cs_math": 2.534990676625147, "general_education": 3.5772983867305865, "books": 3.7231046684868003}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.552792273518009, "python_tests_examples": 1.3881421403303627, "python_documentation": 2.4606719666085985, "cs_math": 2.534990676625147, "general_education": 3.5772983867305865, "books": 3.7231046684868003}, "weighted_loss": 1.7721340743000311}
+{"step": 58500, "tokens": 3415869898, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5677214259291266, "python_tests_examples": 1.3990249693325163, "python_documentation": 2.472103556454237, "cs_math": 2.5278719444706503, "general_education": 3.579709983296756, "books": 3.7410704174046137}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5677214259291266, "python_tests_examples": 1.3990249693325163, "python_documentation": 2.472103556454237, "cs_math": 2.5278719444706503, "general_education": 3.579709983296756, "books": 3.7410704174046137}, "weighted_loss": 1.7846893329697608}
+{"step": 59000, "tokens": 3444340752, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5613102577114222, "python_tests_examples": 1.3965378647389544, "python_documentation": 2.469179269847765, "cs_math": 2.5288061007205345, "general_education": 3.5913385373597784, "books": 3.7487922490842474}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5613102577114222, "python_tests_examples": 1.3965378647389544, "python_documentation": 2.469179269847765, "cs_math": 2.5288061007205345, "general_education": 3.5913385373597784, "books": 3.7487922490842474}, "weighted_loss": 1.780406580346549}
+{"step": 59500, "tokens": 3472633266, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.565028378602422, "python_tests_examples": 1.3982575599716787, "python_documentation": 2.475228831347491, "cs_math": 2.6625630188350464, "general_education": 3.5657097601378624, "books": 3.6792614658854097}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.565028378602422, "python_tests_examples": 1.3982575599716787, "python_documentation": 2.475228831347491, "cs_math": 2.6625630188350464, "general_education": 3.5657097601378624, "books": 3.6792614658854097}, "weighted_loss": 1.7882146362856715}
+{"step": 59962, "tokens": 3500002820, "target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "per_stratum": {"python_implementation": 1.5447765558799367, "python_tests_examples": 1.3814372709276486, "python_documentation": 2.453733376294258, "cs_math": 2.527201465021461, "general_education": 3.5492423770881225, "books": 3.7050187810936785}, "stratum_target_counts": {"python_implementation": 55027, "python_tests_examples": 97392, "python_documentation": 28953, "cs_math": 50556, "general_education": 38282, "books": 256483}, "windows": 192, "per_source": {"python_implementation": 1.5447765558799367, "python_tests_examples": 1.3814372709276486, "python_documentation": 2.453733376294258, "cs_math": 2.527201465021461, "general_education": 3.5492423770881225, "books": 3.7050187810936785}, "weighted_loss": 1.7636314097761223}