Qwen90M step 237163; complete resumable checkpoint

complete.json

@@ -1,18 +1,18 @@
 {
-  "step": 235000,
+  "step": 237163,
   "files": {
     "config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7",
     "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4",
-    "model.safetensors": "2e35e5395cb287d1981044a5822eddab4ba9c62bc411de4f902b706cfb265591",
+    "model.safetensors": "2b20b64846b5ec5f1a1266ad70e66bdcf4161c507eb849f41d4457b347e1fef8",
     "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03",
     "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300",
     "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263",
     "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6",
-    "trainer.pt": "bdfb4bcc1f514f4993f7557ffa73c6c78e0fbe0f7eae146e44f89fbac2919cdd",
-    "training_config.json": "2122ad6f8105987f9b87dd55d13b632084853908566b081c829ffe801259e2bf",
+    "trainer.pt": "3817012a1ce6a6ac292529de34664fef83f2285d26044dba167d3aaa20957a39",
+    "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783",
     "performance.json": "7f72f0a0e868a4baec06f238f922d0c5d7e397a51386c11ac32acc776d4d3522",
-    "shard_scheduler.json": "b38e01cf77934ef444db8086750e69ecf4d69bd7e1c45ee0fec8d3051a6abc4d",
-    "coverage.json": "d2f27535dad5eb059b3164dfa9e026883a87ba1676e6c7dc163b3f5eb36c7ad5",
+    "shard_scheduler.json": "7642e374dd789aa75440578f87a3db0e21df171d6322760efedf97e6eb520c03",
+    "coverage.json": "70e3c6c517ccfd837f7cb477fd294e1034ff341fbf2983c3135f90eb9c011c05",
     "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"
   },
   "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063"

coverage.json

@@ -1,107 +1,107 @@
 {
-  "step": 235000,
+  "step": 237163,
   "stats": {
-    "climbmix/blocks": 8460000,
-    "climbmix/targets": 14142088161,
-    "climbmix/input_tokens": 14142984336,
-    "ultra_qa/blocks": 3172500,
-    "ultra_qa/targets": 5175699264,
-    "ultra_qa/input_tokens": 5175709288,
-    "cortex/blocks": 3172500,
-    "cortex/targets": 120690717,
-    "cortex/input_tokens": 5042698249,
-    "ultra_style/blocks": 3172500,
-    "ultra_style/targets": 5018546024,
-    "ultra_style/input_tokens": 5018809138,
-    "openmath/blocks": 2115000,
-    "openmath/targets": 3267999434,
-    "openmath/input_tokens": 3269088966,
-    "rewrite6/blocks": 1057500,
-    "rewrite6/targets": 1733635352,
-    "rewrite6/input_tokens": 1735535955
+    "climbmix/blocks": 8537867,
+    "climbmix/targets": 14271950160,
+    "climbmix/input_tokens": 14272854649,
+    "ultra_qa/blocks": 3201701,
+    "ultra_qa/targets": 5223297877,
+    "ultra_qa/input_tokens": 5223307978,
+    "cortex/blocks": 3201700,
+    "cortex/targets": 121802303,
+    "cortex/input_tokens": 5089120155,
+    "ultra_style/blocks": 3201701,
+    "ultra_style/targets": 5064464924,
+    "ultra_style/input_tokens": 5064730515,
+    "openmath/blocks": 2134467,
+    "openmath/targets": 3298047515,
+    "openmath/input_tokens": 3299146987,
+    "rewrite6/blocks": 1067234,
+    "rewrite6/targets": 1749558797,
+    "rewrite6/input_tokens": 1751477016
   },
   "sources": {
     "climbmix": {
-      "page": 1158,
-      "row": 2810,
+      "page": 1169,
+      "row": 1585,
       "start": {
         "file": 56,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 4,
+          "row_group": 48,
           "row": 1024
         }
       },
       "epoch": 0,
       "offset": 2048,
-      "documents": 4745978
+      "documents": 4789809
     },
     "rewrite6": {
-      "page": 9,
-      "row": 555,
+      "page": 15,
+      "row": 1043,
       "start": {
         "file": 0,
         "reader": {
           "format": "jsonl-row-v1",
-          "row": 36864
+          "row": 61440
         }
       },
       "epoch": 13,
       "offset": 0,
-      "documents": 2707671
+      "documents": 2732735
     },
     "openmath": {
-      "page": 724,
-      "row": 1423,
+      "page": 730,
+      "row": 4073,
       "start": {
         "file": 6,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 345,
-          "row": 604
+          "row_group": 370,
+          "row": 180
         }
       },
       "epoch": 0,
       "offset": 0,
-      "documents": 2966927
+      "documents": 2994153
     },
     "ultra_qa": {
-      "page": 313,
-      "row": 492,
+      "page": 315,
+      "row": 4076,
       "start": {
         "file": 2,
         "reader": {
           "format": "parquet-row-v1",
           "row_group": 3,
-          "row": 49063
+          "row": 57255
         }
       },
       "epoch": 0,
       "offset": 2048,
-      "documents": 1282540
+      "documents": 1294316
     },
     "ultra_style": {
-      "page": 477,
-      "row": 3572,
+      "page": 482,
+      "row": 1439,
       "start": {
         "file": 2,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 6,
-          "row": 73569
+          "row_group": 7,
+          "row": 8799
         }
       },
       "epoch": 0,
       "offset": 2048,
-      "documents": 1957364
+      "documents": 1975711
     },
     "cortex": {
-      "page": 61524,
-      "row": 82,
+      "page": 62090,
+      "row": 107,
       "start": null,
       "epoch": 0,
       "offset": 0,
-      "documents": 7875154
+      "documents": 7947627
     }
   }
 }

latest.json

@@ -1 +1 @@
-{"step": 235000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "2e35e5395cb287d1981044a5822eddab4ba9c62bc411de4f902b706cfb265591", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "bdfb4bcc1f514f4993f7557ffa73c6c78e0fbe0f7eae146e44f89fbac2919cdd", "training_config.json": "2122ad6f8105987f9b87dd55d13b632084853908566b081c829ffe801259e2bf", "performance.json": "7f72f0a0e868a4baec06f238f922d0c5d7e397a51386c11ac32acc776d4d3522", "shard_scheduler.json": "b38e01cf77934ef444db8086750e69ecf4d69bd7e1c45ee0fec8d3051a6abc4d", "coverage.json": "d2f27535dad5eb059b3164dfa9e026883a87ba1676e6c7dc163b3f5eb36c7ad5", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000235000"}
+{"step": 237163, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "2b20b64846b5ec5f1a1266ad70e66bdcf4161c507eb849f41d4457b347e1fef8", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "3817012a1ce6a6ac292529de34664fef83f2285d26044dba167d3aaa20957a39", "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783", "performance.json": "7f72f0a0e868a4baec06f238f922d0c5d7e397a51386c11ac32acc776d4d3522", "shard_scheduler.json": "7642e374dd789aa75440578f87a3db0e21df171d6322760efedf97e6eb520c03", "coverage.json": "70e3c6c517ccfd837f7cb477fd294e1034ff341fbf2983c3135f90eb9c011c05", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000237163"}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e35e5395cb287d1981044a5822eddab4ba9c62bc411de4f902b706cfb265591
+oid sha256:2b20b64846b5ec5f1a1266ad70e66bdcf4161c507eb849f41d4457b347e1fef8
 size 360622744

shard_scheduler.json

@@ -22,8 +22,8 @@
       "tokens_per_second": 1062967.6168812835,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 73759.62675692099,
-      "required_pages_per_second_upper_estimate": 0.006393297505225003
+      "required_token_positions_per_second": 75472.3444404108,
+      "required_pages_per_second_upper_estimate": 0.006541751533186617
     },
     {
       "source": "rewrite6",
@@ -36,8 +36,8 @@
       "tokens_per_second": 1206672.263377472,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 9219.953344615124,
-      "required_pages_per_second_upper_estimate": 0.003677977239753919
+      "required_token_positions_per_second": 9434.04305505135,
+      "required_pages_per_second_upper_estimate": 0.0037633808261733487
     },
     {
       "source": "openmath",
@@ -50,8 +50,8 @@
       "tokens_per_second": 410042.7581134458,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 18439.906689230247,
-      "required_pages_per_second_upper_estimate": 0.004265747881279806
+      "required_token_positions_per_second": 18868.0861101027,
+      "required_pages_per_second_upper_estimate": 0.004364799654598218
     },
     {
       "source": "ultra_qa",
@@ -64,8 +64,8 @@
       "tokens_per_second": 2163486.5624068812,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 27659.86003384537,
-      "required_pages_per_second_upper_estimate": 0.0016843403559910553
+      "required_token_positions_per_second": 28302.129165154052,
+      "required_pages_per_second_upper_estimate": 0.0017234511763620484
     },
     {
       "source": "ultra_style",
@@ -78,8 +78,8 @@
       "tokens_per_second": 1546694.2737633225,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 27659.86003384537,
-      "required_pages_per_second_upper_estimate": 0.0021293325927985215
+      "required_token_positions_per_second": 28302.129165154052,
+      "required_pages_per_second_upper_estimate": 0.002178776248441411
     },
     {
       "source": "cortex",
@@ -91,9 +91,9 @@
       "read_seconds": 0.042780299000014566,
       "tokens_per_second": 1015177.8395898147,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
-      "estimated_pages_for_buffer": 21,
-      "required_token_positions_per_second": 27659.86003384537,
-      "required_pages_per_second_upper_estimate": 0.3434428900237825
+      "estimated_pages_for_buffer": 22,
+      "required_token_positions_per_second": 28302.129165154052,
+      "required_pages_per_second_upper_estimate": 0.35141772309812946
     }
   ],
   "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
@@ -155,138 +155,30 @@
   "shard_benchmark_scope": "Local packing/read measurements; upload capacity measured only on real data commits. Targets larger than available pages are sample-limited.",
   "estimated_fresh_generation_batch_seconds": 0.1434574083060457,
   "selected": {
-    "prefetch_batches": 61,
+    "prefetch_batches": 62,
     "prefetch_capacity_limit": 512,
     "batch_memory_estimate_bytes": 688946,
-    "producer_batch_seconds": 0.48818316624266345,
-    "producer_p95_seconds": 0.43546763699851,
-    "gpu_update_seconds": 0.9995712185878433,
-    "producer_batches_per_second": 2.0484114757511436,
-    "gpu_batches_per_second": 1.0004289653445229,
-    "producer_headroom": 2.0475331549858926,
+    "producer_batch_seconds": 0.12409712750809376,
+    "producer_p95_seconds": 0.4301016780009377,
+    "gpu_update_seconds": 0.9768876340950552,
+    "producer_batches_per_second": 8.05820424759452,
+    "gpu_batches_per_second": 1.0236591856609538,
+    "producer_headroom": 7.871960082487336,
     "sustainable": true,
-    "buffer_seconds": 60.97384433385844,
+    "buffer_seconds": 60.56703331389342,
     "buffer_drain_seconds": null,
     "shard_target_mib": 16,
-    "upload_pages_per_commit": 81,
+    "upload_pages_per_commit": 84,
     "max_pending_pages": 4096,
-    "generation_bytes_per_second": 293726.8938308259,
-    "upload_bytes_per_second": 5635627.239740325,
+    "generation_bytes_per_second": 379406.6242124819,
+    "upload_bytes_per_second": 5575493.562052467,
     "upload_sustainable": true,
     "logical_page_rows": 4096,
     "logical_cortex_page_rows": 128,
-    "estimated_full_shards_per_hour": 63.02695380395492,
+    "estimated_full_shards_per_hour": 81.41182942181437,
     "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
   },
   "upload_measurements": [
-    {
-      "seconds": 0.9491155960022297,
-      "bytes": 6968517,
-      "pages": 33,
-      "shards": 3
-    },
-    {
-      "seconds": 1.2095077359990682,
-      "bytes": 6033276,
-      "pages": 36,
-      "shards": 2
-    },
-    {
-      "seconds": 2.290356701996643,
-      "bytes": 13205046,
-      "pages": 34,
-      "shards": 4
-    },
-    {
-      "seconds": 1.0317470599984517,
-      "bytes": 1158559,
-      "pages": 34,
-      "shards": 1
-    },
-    {
-      "seconds": 1.1566925789957168,
-      "bytes": 9706808,
-      "pages": 33,
-      "shards": 3
-    },
-    {
-      "seconds": 0.9952422389978892,
-      "bytes": 7289939,
-      "pages": 30,
-      "shards": 3
-    },
-    {
-      "seconds": 0.823028221995628,
-      "bytes": 1028230,
-      "pages": 34,
-      "shards": 1
-    },
-    {
-      "seconds": 1.2517391190049239,
-      "bytes": 5921573,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 1.3183760180036188,
-      "bytes": 12970748,
-      "pages": 31,
-      "shards": 4
-    },
-    {
-      "seconds": 1.064266913002939,
-      "bytes": 6180437,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 1.110571836004965,
-      "bytes": 7330551,
-      "pages": 29,
-      "shards": 3
-    },
-    {
-      "seconds": 1.0676300510021974,
-      "bytes": 1203686,
-      "pages": 35,
-      "shards": 1
-    },
-    {
-      "seconds": 1.0235948650006321,
-      "bytes": 9738901,
-      "pages": 35,
-      "shards": 3
-    },
-    {
-      "seconds": 0.8542503440039582,
-      "bytes": 2305951,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9167553060033242,
-      "bytes": 5839408,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 1.0627733270011959,
-      "bytes": 12238667,
-      "pages": 44,
-      "shards": 3
-    },
-    {
-      "seconds": 1.0165252450024127,
-      "bytes": 7299775,
-      "pages": 36,
-      "shards": 4
-    },
-    {
-      "seconds": 1.2853216939984122,
-      "bytes": 5894461,
-      "pages": 29,
-      "shards": 2
-    },
     {
       "seconds": 1.3212683820020175,
       "bytes": 7578218,
@@ -562,6 +454,114 @@
       "bytes": 7202195,
       "pages": 30,
       "shards": 3
+    },
+    {
+      "seconds": 1.457639926004049,
+      "bytes": 7115441,
+      "pages": 39,
+      "shards": 2
+    },
+    {
+      "seconds": 1.29948361699644,
+      "bytes": 9783282,
+      "pages": 32,
+      "shards": 3
+    },
+    {
+      "seconds": 1.0601626389980083,
+      "bytes": 7547567,
+      "pages": 29,
+      "shards": 3
+    },
+    {
+      "seconds": 1.118395095996675,
+      "bytes": 1248062,
+      "pages": 35,
+      "shards": 1
+    },
+    {
+      "seconds": 0.9650491309948848,
+      "bytes": 5725695,
+      "pages": 29,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1484884869933012,
+      "bytes": 6640918,
+      "pages": 32,
+      "shards": 3
+    },
+    {
+      "seconds": 1.1328601960049127,
+      "bytes": 11867525,
+      "pages": 34,
+      "shards": 3
+    },
+    {
+      "seconds": 1.0216524100032984,
+      "bytes": 7553152,
+      "pages": 40,
+      "shards": 3
+    },
+    {
+      "seconds": 0.8235062559979269,
+      "bytes": 854296,
+      "pages": 26,
+      "shards": 1
+    },
+    {
+      "seconds": 1.2072945199979586,
+      "bytes": 10797475,
+      "pages": 40,
+      "shards": 3
+    },
+    {
+      "seconds": 1.3132626469960087,
+      "bytes": 2377167,
+      "pages": 28,
+      "shards": 2
+    },
+    {
+      "seconds": 0.8975952609980595,
+      "bytes": 5947609,
+      "pages": 37,
+      "shards": 2
+    },
+    {
+      "seconds": 1.3013317000004463,
+      "bytes": 11518633,
+      "pages": 35,
+      "shards": 3
+    },
+    {
+      "seconds": 1.1227065580023918,
+      "bytes": 6254929,
+      "pages": 31,
+      "shards": 3
+    },
+    {
+      "seconds": 1.5046920809982112,
+      "bytes": 5774998,
+      "pages": 29,
+      "shards": 2
+    },
+    {
+      "seconds": 0.8562071050037048,
+      "bytes": 2686484,
+      "pages": 37,
+      "shards": 3
+    },
+    {
+      "seconds": 1.2383149749948643,
+      "bytes": 6204112,
+      "pages": 30,
+      "shards": 2
+    },
+    {
+      "seconds": 1.4738188229966909,
+      "bytes": 10769021,
+      "pages": 39,
+      "shards": 3
     }
   ],
   "cache_key": {

trainer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bdfb4bcc1f514f4993f7557ffa73c6c78e0fbe0f7eae146e44f89fbac2919cdd
-size 721759615
+oid sha256:3817012a1ce6a6ac292529de34664fef83f2285d26044dba167d3aaa20957a39
+size 721779903

training_config.json

@@ -46,8 +46,8 @@
   "cpu_threads": 4,
   "page_rows": 4096,
   "cortex_page_rows": 128,
-  "prefetch_batches": 61,
-  "prefetch_low_water": 30,
+  "prefetch_batches": 62,
+  "prefetch_low_water": 31,
   "memory_pages": 12,
   "cache_gb": 20,
   "force_rebenchmark": false,