Qwen90M step 90000; complete resumable checkpoint

complete.json

@@ -1,18 +1,18 @@
 {
-  "step": 85000,
+  "step": 90000,
   "files": {
     "config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7",
     "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4",
-    "model.safetensors": "2683c5255a46fb5734cdf83842090f23a32fa503fc084bf57f67a828bfeab926",
+    "model.safetensors": "d1b9321795a955b99feefcdb828552461d21c50d7c0ed64202118da5f4a4eb80",
     "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03",
     "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300",
     "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263",
     "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6",
-    "trainer.pt": "b22b50c2a3f3cc11176d153ff6d2d48d52ac57f92ea3cf82eef967444d82426f",
+    "trainer.pt": "a7a66ded6461cef76b89f27ed3606c872345dc1f83d547f29c24c93e5c270c58",
     "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783",
-    "performance.json": "b9785e5ac65e7e9df73669f94ffdf686e237067bf54649bd2ebf893d5327bdc9",
-    "shard_scheduler.json": "3bcf783fa48a15540228bb1e4fd020f0a0cbc02a12b62595b447f4d19b8722cb",
-    "coverage.json": "05c62b88d1a99622c29ec20f235ec8f3ba77c21ab09cd74eeddfb98f8759e10a",
+    "performance.json": "7e827680f2cb27108bc66b58bb4d6ede7b8c07fab9033e664cac4bed5acb1a98",
+    "shard_scheduler.json": "1a342da44c6786263afa65805d94c6d3098e1c2f4ddb32204c37ab8faa15f028",
+    "coverage.json": "6ba08ee821b99cb552c6773108af7236f9e3e692b1eecc0b7f1943b6dafcefc2",
     "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"
   },
   "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063"

coverage.json

@@ -1,107 +1,107 @@
 {
-  "step": 85000,
+  "step": 90000,
   "stats": {
-    "climbmix/blocks": 3060000,
-    "climbmix/targets": 5113419097,
-    "climbmix/input_tokens": 5113742843,
-    "ultra_qa/blocks": 1147500,
-    "ultra_qa/targets": 1871965816,
-    "ultra_qa/input_tokens": 1871969468,
-    "cortex/blocks": 1147500,
-    "cortex/targets": 43652665,
-    "cortex/input_tokens": 1823889389,
-    "ultra_style/blocks": 1147500,
-    "ultra_style/targets": 1814807863,
-    "ultra_style/input_tokens": 1814902860,
-    "openmath/blocks": 765000,
-    "openmath/targets": 1182206521,
-    "openmath/input_tokens": 1182600938,
-    "rewrite6/blocks": 382500,
-    "rewrite6/targets": 627023097,
-    "rewrite6/input_tokens": 627710727
+    "climbmix/blocks": 3240000,
+    "climbmix/targets": 5413943761,
+    "climbmix/input_tokens": 5414286992,
+    "ultra_qa/blocks": 1215000,
+    "ultra_qa/targets": 1982074966,
+    "ultra_qa/input_tokens": 1982078809,
+    "cortex/blocks": 1215000,
+    "cortex/targets": 46211808,
+    "cortex/input_tokens": 1931141281,
+    "ultra_style/blocks": 1215000,
+    "ultra_style/targets": 1921585691,
+    "ultra_style/input_tokens": 1921686239,
+    "openmath/blocks": 810000,
+    "openmath/targets": 1251693248,
+    "openmath/input_tokens": 1252110755,
+    "rewrite6/blocks": 405000,
+    "rewrite6/targets": 663921893,
+    "rewrite6/input_tokens": 664649732
   },
   "sources": {
     "climbmix": {
-      "page": 419,
-      "row": 587,
+      "page": 444,
+      "row": 130,
       "start": {
-        "file": 20,
+        "file": 21,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 21,
+          "row_group": 37,
           "row": 1024
         }
       },
       "epoch": 0,
       "offset": 2048,
-      "documents": 1716811
+      "documents": 1818754
     },
     "rewrite6": {
-      "page": 38,
-      "row": 2375,
+      "page": 2,
+      "row": 1720,
       "start": {
         "file": 0,
         "reader": {
           "format": "jsonl-row-v1",
-          "row": 155648
+          "row": 8192
         }
       },
-      "epoch": 4,
+      "epoch": 5,
       "offset": 2048,
-      "documents": 979639
+      "documents": 1036932
     },
     "openmath": {
-      "page": 262,
-      "row": 252,
+      "page": 277,
+      "row": 1836,
       "start": {
         "file": 2,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 199,
-          "row": 852
+          "row_group": 261,
+          "row": 292
         }
       },
       "epoch": 0,
-      "offset": 2048,
-      "documents": 1073404
+      "offset": 0,
+      "documents": 1136428
     },
     "ultra_qa": {
-      "page": 113,
-      "row": 1205,
+      "page": 119,
+      "row": 3845,
       "start": {
         "file": 0,
         "reader": {
           "format": "parquet-row-v1",
           "row_group": 7,
-          "row": 10817
+          "row": 35393
         }
       },
       "epoch": 0,
       "offset": 2048,
-      "documents": 464053
+      "documents": 491269
     },
     "ultra_style": {
-      "page": 172,
-      "row": 3360,
+      "page": 182,
+      "row": 4014,
       "start": {
         "file": 1,
         "reader": {
           "format": "parquet-row-v1",
           "row_group": 0,
-          "row": 19573
+          "row": 60533
         }
       },
       "epoch": 0,
       "offset": 2048,
-      "documents": 707872
+      "documents": 749486
     },
     "cortex": {
-      "page": 22250,
-      "row": 49,
+      "page": 23557,
+      "row": 72,
       "start": null,
       "epoch": 0,
       "offset": 0,
-      "documents": 2848049
+      "documents": 3015368
     }
   }
 }

latest.json

@@ -1 +1 @@
-{"step": 85000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "2683c5255a46fb5734cdf83842090f23a32fa503fc084bf57f67a828bfeab926", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "b22b50c2a3f3cc11176d153ff6d2d48d52ac57f92ea3cf82eef967444d82426f", "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783", "performance.json": "b9785e5ac65e7e9df73669f94ffdf686e237067bf54649bd2ebf893d5327bdc9", "shard_scheduler.json": "3bcf783fa48a15540228bb1e4fd020f0a0cbc02a12b62595b447f4d19b8722cb", "coverage.json": "05c62b88d1a99622c29ec20f235ec8f3ba77c21ab09cd74eeddfb98f8759e10a", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000085000"}
+{"step": 90000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "d1b9321795a955b99feefcdb828552461d21c50d7c0ed64202118da5f4a4eb80", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "a7a66ded6461cef76b89f27ed3606c872345dc1f83d547f29c24c93e5c270c58", "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783", "performance.json": "7e827680f2cb27108bc66b58bb4d6ede7b8c07fab9033e664cac4bed5acb1a98", "shard_scheduler.json": "1a342da44c6786263afa65805d94c6d3098e1c2f4ddb32204c37ab8faa15f028", "coverage.json": "6ba08ee821b99cb552c6773108af7236f9e3e692b1eecc0b7f1943b6dafcefc2", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000090000"}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2683c5255a46fb5734cdf83842090f23a32fa503fc084bf57f67a828bfeab926
+oid sha256:d1b9321795a955b99feefcdb828552461d21c50d7c0ed64202118da5f4a4eb80
 size 360622744

performance.json

@@ -49,11 +49,11 @@
   "selected": {
     "microbatch": 86,
     "mode": "compiled",
-    "microbatch_seconds": 0.8982481310000026,
-    "peak_gb": 85.39913082122803,
+    "microbatch_seconds": 0.899068480999631,
+    "peak_gb": 85.39889430999756,
     "fits_budget": true,
-    "estimated_update_seconds": 0.9539577840000106,
-    "input_tokens_per_second": 196079.45056776243
+    "estimated_update_seconds": 0.9546496269995259,
+    "input_tokens_per_second": 195900.5389713715
   },
   "measurements": [
     {

shard_scheduler.json

@@ -22,8 +22,8 @@
       "tokens_per_second": 1062967.6168812835,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 75341.36127254891,
-      "required_pages_per_second_upper_estimate": 0.006530398244170153
+      "required_token_positions_per_second": 75156.2396899193,
+      "required_pages_per_second_upper_estimate": 0.006514352374574179
     },
     {
       "source": "rewrite6",
@@ -36,8 +36,8 @@
       "tokens_per_second": 1206672.263377472,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 9417.670159068613,
-      "required_pages_per_second_upper_estimate": 0.003756849433169225
+      "required_token_positions_per_second": 9394.529961239912,
+      "required_pages_per_second_upper_estimate": 0.0037476184622785673
     },
     {
       "source": "openmath",
@@ -50,8 +50,8 @@
       "tokens_per_second": 410042.7581134458,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 18835.340318137227,
-      "required_pages_per_second_upper_estimate": 0.0043572244919332604
+      "required_token_positions_per_second": 18789.059922479824,
+      "required_pages_per_second_upper_estimate": 0.004346518336905065
     },
     {
       "source": "ultra_qa",
@@ -64,8 +64,8 @@
       "tokens_per_second": 2163486.5624068812,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28253.010477205844,
-      "required_pages_per_second_upper_estimate": 0.0017204601059718416
+      "required_token_positions_per_second": 28183.589883719735,
+      "required_pages_per_second_upper_estimate": 0.0017162327560502429
     },
     {
       "source": "ultra_style",
@@ -78,8 +78,8 @@
       "tokens_per_second": 1546694.2737633225,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28253.010477205844,
-      "required_pages_per_second_upper_estimate": 0.0021749949558739273
+      "required_token_positions_per_second": 28183.589883719735,
+      "required_pages_per_second_upper_estimate": 0.0021696507664188645
     },
     {
       "source": "cortex",
@@ -92,8 +92,8 @@
       "tokens_per_second": 1015177.8395898147,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 22,
-      "required_token_positions_per_second": 28253.010477205844,
-      "required_pages_per_second_upper_estimate": 0.3508078333834864
+      "required_token_positions_per_second": 28183.589883719735,
+      "required_pages_per_second_upper_estimate": 0.34994586194816957
     }
   ],
   "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
@@ -158,188 +158,32 @@
     "prefetch_batches": 62,
     "prefetch_capacity_limit": 512,
     "batch_memory_estimate_bytes": 688946,
-    "producer_batch_seconds": 0.04785197319529999,
-    "producer_p95_seconds": 0.1573215110001911,
-    "gpu_update_seconds": 0.9785859819188487,
-    "producer_batches_per_second": 20.89777982443198,
-    "gpu_batches_per_second": 1.0218826127461604,
-    "producer_headroom": 20.450274389415675,
+    "producer_batch_seconds": 0.08091448022655356,
+    "producer_p95_seconds": 0.18719405599949823,
+    "gpu_update_seconds": 0.9809963923712529,
+    "producer_batches_per_second": 12.35872735263313,
+    "gpu_batches_per_second": 1.019371740585928,
+    "producer_headroom": 12.123866947233024,
     "sustainable": true,
-    "buffer_seconds": 60.67233087896862,
+    "buffer_seconds": 60.82177632701768,
     "buffer_drain_seconds": null,
-    "shard_target_mib": 32,
-    "upload_pages_per_commit": 164,
+    "shard_target_mib": 16,
+    "upload_pages_per_commit": 87,
     "max_pending_pages": 4096,
-    "generation_bytes_per_second": 611305.9510963782,
-    "upload_bytes_per_second": 5634180.224215531,
-    "upload_sustainable": true,
+    "generation_bytes_per_second": 397083.56006381963,
+    "upload_bytes_per_second": 43958.880398765425,
+    "upload_sustainable": false,
     "logical_page_rows": 4096,
     "logical_cortex_page_rows": 128,
-    "estimated_full_shards_per_hour": 65.58601331552748,
+    "estimated_full_shards_per_hour": 85.20488835750524,
     "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
   },
   "upload_measurements": [
     {
-      "seconds": 1.104392732000008,
-      "bytes": 36786,
+      "seconds": 0.9955667570011428,
+      "bytes": 43764,
       "pages": 1,
       "shards": 1
-    },
-    {
-      "seconds": 0.71818233099998,
-      "bytes": 253911,
-      "pages": 7,
-      "shards": 1
-    },
-    {
-      "seconds": 1.0387228129999357,
-      "bytes": 25849,
-      "pages": 1,
-      "shards": 1
-    },
-    {
-      "seconds": 1.3780422429999817,
-      "bytes": 2340308,
-      "pages": 30,
-      "shards": 2
-    },
-    {
-      "seconds": 1.0020929300000034,
-      "bytes": 10862030,
-      "pages": 33,
-      "shards": 3
-    },
-    {
-      "seconds": 0.8503723880000962,
-      "bytes": 6155947,
-      "pages": 35,
-      "shards": 2
-    },
-    {
-      "seconds": 0.8228095520000807,
-      "bytes": 2489087,
-      "pages": 34,
-      "shards": 2
-    },
-    {
-      "seconds": 1.2741382629999407,
-      "bytes": 11572130,
-      "pages": 33,
-      "shards": 3
-    },
-    {
-      "seconds": 1.0936447959998077,
-      "bytes": 6527222,
-      "pages": 37,
-      "shards": 3
-    },
-    {
-      "seconds": 0.8425330919999396,
-      "bytes": 6207711,
-      "pages": 34,
-      "shards": 2
-    },
-    {
-      "seconds": 1.0241649789998064,
-      "bytes": 5879797,
-      "pages": 26,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9143005570001606,
-      "bytes": 2623386,
-      "pages": 36,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1859510890001275,
-      "bytes": 10202108,
-      "pages": 38,
-      "shards": 3
-    },
-    {
-      "seconds": 1.2170790539998961,
-      "bytes": 6598891,
-      "pages": 27,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9556911770000625,
-      "bytes": 7849875,
-      "pages": 36,
-      "shards": 3
-    },
-    {
-      "seconds": 0.9569393540000419,
-      "bytes": 10947040,
-      "pages": 39,
-      "shards": 3
-    },
-    {
-      "seconds": 0.8917274570003428,
-      "bytes": 2243299,
-      "pages": 26,
-      "shards": 2
-    },
-    {
-      "seconds": 1.2289755599999808,
-      "bytes": 6161182,
-      "pages": 34,
-      "shards": 2
-    },
-    {
-      "seconds": 0.8229349630000797,
-      "bytes": 1172160,
-      "pages": 33,
-      "shards": 1
-    },
-    {
-      "seconds": 1.011977818000105,
-      "bytes": 13444268,
-      "pages": 40,
-      "shards": 5
-    },
-    {
-      "seconds": 1.0623180309999043,
-      "bytes": 9531595,
-      "pages": 31,
-      "shards": 3
-    },
-    {
-      "seconds": 0.9543692199999896,
-      "bytes": 1054926,
-      "pages": 32,
-      "shards": 1
-    },
-    {
-      "seconds": 1.207006858999648,
-      "bytes": 7769728,
-      "pages": 37,
-      "shards": 3
-    },
-    {
-      "seconds": 0.9489059839997935,
-      "bytes": 5950611,
-      "pages": 30,
-      "shards": 2
-    },
-    {
-      "seconds": 1.4311316419998548,
-      "bytes": 7407679,
-      "pages": 37,
-      "shards": 3
-    },
-    {
-      "seconds": 1.307497912999679,
-      "bytes": 11505681,
-      "pages": 31,
-      "shards": 3
-    },
-    {
-      "seconds": 0.7801257850001093,
-      "bytes": 1090489,
-      "pages": 33,
-      "shards": 1
     }
   ],
   "cache_key": {
@@ -384,7 +228,7 @@
   },
   "cache_reused": true,
   "resume_probe_seconds": [
-    2.5095109330000014,
-    0.10111166000001504
+    0.5039531120000902,
+    0.014973258001191425
   ]
 }

trainer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b22b50c2a3f3cc11176d153ff6d2d48d52ac57f92ea3cf82eef967444d82426f
-size 721760447
+oid sha256:a7a66ded6461cef76b89f27ed3606c872345dc1f83d547f29c24c93e5c270c58
+size 721778879