Qwen90M step 400000; complete resumable checkpoint

complete.json

@@ -1,18 +1,18 @@
 {
-  "step": 397106,
+  "step": 400000,
   "files": {
     "config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7",
     "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4",
-    "model.safetensors": "a47c91999dd0ebcdc282336b24f357db9c84c594fb3b2642a3894a2b6a3275cc",
+    "model.safetensors": "a131696398b13dbcafb16c35950eb04cbd669d17fb0d4873cd34a9793ddb041d",
     "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03",
     "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300",
     "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263",
     "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6",
-    "trainer.pt": "f7cd41c0ad064fdb32d106a04f2baeeeeed5930763f4ca1400aa877b835cb4d4",
-    "training_config.json": "2122ad6f8105987f9b87dd55d13b632084853908566b081c829ffe801259e2bf",
-    "performance.json": "e3923fa86352f6d878fd8d37b6b9fc374577f52be522203b0b865badf630da15",
-    "shard_scheduler.json": "73efbc129590a5a3737fdb30c517ab49d4777d335d99a458b160010d41de4d70",
-    "coverage.json": "861a5c400ef29ce2ebe8dbeb84b40a4ce490ac63f7b453e5e8d6bfed66907dad",
+    "trainer.pt": "0883783165f69503eba3b62bf8c9c7a6e664527b2bcaeee310413d02bf169eb3",
+    "training_config.json": "c8bdf052510c2d9092e5a6824b119dd6584dc7697f8c4a8e9b7ecf3057c5bbb6",
+    "performance.json": "a6ae17d5ca4d76a29c6e069dddb352125dfbf6f18e59e9553ab94c741a962b18",
+    "shard_scheduler.json": "c40659570cd21160457c25f9debd82b3f0d1ed40efab78c7456bc3d67e11e8d3",
+    "coverage.json": "ccc6e483b973caea3627baa83892ef110132ad12b30133bfa601b18a54ee08eb",
     "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"
   },
   "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063"

coverage.json

@@ -1,107 +1,107 @@
 {
-  "step": 397106,
+  "step": 400000,
   "stats": {
-    "climbmix/blocks": 14295816,
-    "climbmix/targets": 23897781997,
-    "climbmix/input_tokens": 23899295733,
-    "ultra_qa/blocks": 5360931,
-    "ultra_qa/targets": 8745343329,
-    "ultra_qa/input_tokens": 8745360186,
-    "cortex/blocks": 5360931,
-    "cortex/targets": 203917069,
-    "cortex/input_tokens": 8521943593,
-    "ultra_style/blocks": 5360931,
-    "ultra_style/targets": 8479672034,
-    "ultra_style/input_tokens": 8480116529,
-    "openmath/blocks": 3573954,
-    "openmath/targets": 5522855410,
-    "openmath/input_tokens": 5524695746,
-    "rewrite6/blocks": 1786977,
-    "rewrite6/targets": 2929490049,
-    "rewrite6/input_tokens": 2932701855
+    "climbmix/blocks": 14400000,
+    "climbmix/targets": 24072071047,
+    "climbmix/input_tokens": 24073595873,
+    "ultra_qa/blocks": 5400000,
+    "ultra_qa/targets": 8809069980,
+    "ultra_qa/input_tokens": 8809086942,
+    "cortex/blocks": 5400000,
+    "cortex/targets": 205392043,
+    "cortex/input_tokens": 8584044366,
+    "ultra_style/blocks": 5400000,
+    "ultra_style/targets": 8541509026,
+    "ultra_style/input_tokens": 8541956871,
+    "openmath/blocks": 3600000,
+    "openmath/targets": 5563081132,
+    "openmath/input_tokens": 5564934909,
+    "rewrite6/blocks": 1800000,
+    "rewrite6/targets": 2950834067,
+    "rewrite6/input_tokens": 2954069608
   },
   "sources": {
     "climbmix": {
-      "page": 1957,
-      "row": 3523,
+      "page": 1972,
+      "row": 642,
       "start": {
-        "file": 94,
+        "file": 95,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 60,
+          "row_group": 38,
           "row": 1024
         }
       },
       "epoch": 0,
       "offset": 0,
-      "documents": 8019395
+      "documents": 8077954
     },
     "rewrite6": {
-      "page": 13,
-      "row": 3512,
+      "page": 22,
+      "row": 377,
       "start": {
         "file": 0,
         "reader": {
           "format": "jsonl-row-v1",
-          "row": 53248
+          "row": 90112
         }
       },
       "epoch": 22,
       "offset": 0,
-      "documents": 4575648
+      "documents": 4609377
     },
     "openmath": {
-      "page": 1223,
-      "row": 2487,
+      "page": 1232,
+      "row": 2237,
       "start": {
         "file": 11,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 206,
-          "row": 258
+          "row_group": 243,
+          "row": 122
         }
       },
       "epoch": 0,
       "offset": 0,
-      "documents": 5011895
+      "documents": 5048509
     },
     "ultra_qa": {
-      "page": 529,
-      "row": 178,
+      "page": 532,
+      "row": 3708,
       "start": {
         "file": 4,
         "reader": {
           "format": "parquet-row-v1",
           "row_group": 1,
-          "row": 23796
+          "row": 36084
         }
       },
       "epoch": 0,
-      "offset": 4096,
-      "documents": 2166962
+      "offset": 0,
+      "documents": 2182780
     },
     "ultra_style": {
-      "page": 807,
-      "row": 1879,
+      "page": 813,
+      "row": 1709,
       "start": {
         "file": 4,
         "reader": {
           "format": "parquet-row-v1",
           "row_group": 6,
-          "row": 55903
+          "row": 80479
         }
       },
       "epoch": 0,
-      "offset": 0,
-      "documents": 3307351
+      "offset": 4096,
+      "documents": 3331757
     },
     "cortex": {
-      "page": 103976,
-      "row": 115,
+      "page": 104734,
+      "row": 30,
       "start": null,
       "epoch": 0,
       "offset": 0,
-      "documents": 13309043
+      "documents": 13405982
     }
   }
 }

latest.json

@@ -1 +1 @@
-{"step": 397106, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "a47c91999dd0ebcdc282336b24f357db9c84c594fb3b2642a3894a2b6a3275cc", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "f7cd41c0ad064fdb32d106a04f2baeeeeed5930763f4ca1400aa877b835cb4d4", "training_config.json": "2122ad6f8105987f9b87dd55d13b632084853908566b081c829ffe801259e2bf", "performance.json": "e3923fa86352f6d878fd8d37b6b9fc374577f52be522203b0b865badf630da15", "shard_scheduler.json": "73efbc129590a5a3737fdb30c517ab49d4777d335d99a458b160010d41de4d70", "coverage.json": "861a5c400ef29ce2ebe8dbeb84b40a4ce490ac63f7b453e5e8d6bfed66907dad", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000397106"}
+{"step": 400000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "a131696398b13dbcafb16c35950eb04cbd669d17fb0d4873cd34a9793ddb041d", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "0883783165f69503eba3b62bf8c9c7a6e664527b2bcaeee310413d02bf169eb3", "training_config.json": "c8bdf052510c2d9092e5a6824b119dd6584dc7697f8c4a8e9b7ecf3057c5bbb6", "performance.json": "a6ae17d5ca4d76a29c6e069dddb352125dfbf6f18e59e9553ab94c741a962b18", "shard_scheduler.json": "c40659570cd21160457c25f9debd82b3f0d1ed40efab78c7456bc3d67e11e8d3", "coverage.json": "ccc6e483b973caea3627baa83892ef110132ad12b30133bfa601b18a54ee08eb", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000400000"}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a47c91999dd0ebcdc282336b24f357db9c84c594fb3b2642a3894a2b6a3275cc
+oid sha256:a131696398b13dbcafb16c35950eb04cbd669d17fb0d4873cd34a9793ddb041d
 size 360622744

performance.json

@@ -39,148 +39,148 @@
       "compile_model": true,
       "autotune_microbatch": true,
       "attention_backend": "flash",
-      "vram_fraction": 0.95
+      "vram_fraction": 0.98
     },
     "source": "e6d135a4487a9fd1907e12903cda79e75fd61305fad149fda7ec5281279dc18b",
     "requested_microbatch_ceiling": 90
   },
-  "cache_reused": true,
+  "cache_reused": false,
   "attention": "PyTorch FlashAttention (forced; forward/backward verified)",
   "selected": {
-    "microbatch": 86,
+    "microbatch": 90,
     "mode": "compiled",
-    "microbatch_seconds": 0.9009280120000085,
-    "peak_gb": 85.39913082122803,
+    "microbatch_seconds": 0.9302335780000135,
+    "peak_gb": 89.3772840499878,
     "fits_budget": true,
-    "estimated_update_seconds": 0.9565700770000376,
-    "input_tokens_per_second": 195496.19687038695
+    "estimated_update_seconds": 0.9302335780000135,
+    "input_tokens_per_second": 198143.7827650609
   },
   "measurements": [
     {
       "microbatch": 8,
       "mode": "eager",
-      "microbatch_seconds": 0.15782159200000478,
-      "peak_gb": 14.079077243804932,
+      "microbatch_seconds": 0.15844926500000156,
+      "peak_gb": 14.764304637908936,
       "fits_budget": true,
-      "estimated_update_seconds": 1.793421506500053,
-      "input_tokens_per_second": 103813.42497165725
+      "estimated_update_seconds": 1.8007989265000077,
+      "input_tokens_per_second": 103402.18365796673
     },
     {
       "microbatch": 16,
       "mode": "eager",
-      "microbatch_seconds": 0.3450626039999918,
-      "peak_gb": 27.397329807281494,
+      "microbatch_seconds": 0.34631374599999276,
+      "peak_gb": 28.084960460662842,
       "fits_budget": true,
-      "estimated_update_seconds": 1.9234990809999601,
-      "input_tokens_per_second": 94962.47817106481
+      "estimated_update_seconds": 1.929804849999968,
+      "input_tokens_per_second": 94619.40329680326
     },
     {
       "microbatch": 32,
       "mode": "eager",
-      "microbatch_seconds": 0.7818640030000097,
-      "peak_gb": 54.14767265319824,
+      "microbatch_seconds": 0.7831460159999892,
+      "peak_gb": 54.827155113220215,
       "fits_budget": true,
-      "estimated_update_seconds": 2.185234111500023,
-      "input_tokens_per_second": 83820.20370363462
+      "estimated_update_seconds": 2.188403235499976,
+      "input_tokens_per_second": 83682.98971210101
     },
     {
       "microbatch": 48,
       "mode": "eager",
-      "microbatch_seconds": 1.1685763079999845,
-      "peak_gb": 80.94732475280762,
+      "microbatch_seconds": 1.1706683780000162,
+      "peak_gb": 81.63183498382568,
       "fits_budget": true,
-      "estimated_update_seconds": 2.190914332499986,
-      "input_tokens_per_second": 84122.87612457851
+      "estimated_update_seconds": 2.1949380675000043,
+      "input_tokens_per_second": 83972.54239321277
     },
     {
       "microbatch": 8,
       "mode": "compiled",
-      "microbatch_seconds": 0.08803204299999834,
-      "peak_gb": 8.428553104400635,
+      "microbatch_seconds": 0.08827481200000875,
+      "peak_gb": 9.113307476043701,
       "fits_budget": true,
-      "estimated_update_seconds": 1.0060347614999898,
-      "input_tokens_per_second": 186114.0494035826
+      "estimated_update_seconds": 1.008851873500106,
+      "input_tokens_per_second": 185602.20779624404
     },
     {
       "microbatch": 16,
       "mode": "compiled",
-      "microbatch_seconds": 0.1704058820000114,
-      "peak_gb": 16.191233158111572,
+      "microbatch_seconds": 0.17047579999996287,
+      "peak_gb": 16.87287473678589,
       "fits_budget": true,
-      "estimated_update_seconds": 0.9576335650000658,
-      "input_tokens_per_second": 192293.8317352086
+      "estimated_update_seconds": 0.9571819019997747,
+      "input_tokens_per_second": 192214.96540862185
     },
     {
       "microbatch": 32,
       "mode": "compiled",
-      "microbatch_seconds": 0.34082671699997036,
-      "peak_gb": 31.810625553131104,
+      "microbatch_seconds": 0.34263174000000163,
+      "peak_gb": 32.49641752243042,
       "fits_budget": true,
-      "estimated_update_seconds": 0.9493586249999737,
-      "input_tokens_per_second": 192285.39527904938
+      "estimated_update_seconds": 0.9525396829999977,
+      "input_tokens_per_second": 191272.41393339593
     },
     {
       "microbatch": 48,
       "mode": "compiled",
-      "microbatch_seconds": 0.5061537779999981,
-      "peak_gb": 47.43971490859985,
+      "microbatch_seconds": 0.5075005140000144,
+      "peak_gb": 48.123157024383545,
       "fits_budget": true,
-      "estimated_update_seconds": 0.9468006980000041,
-      "input_tokens_per_second": 194217.6553308279
+      "estimated_update_seconds": 0.9488392155000156,
+      "input_tokens_per_second": 193702.2668709991
     },
     {
       "microbatch": 64,
       "mode": "compiled",
-      "microbatch_seconds": 0.6658512800000267,
-      "peak_gb": 63.073320388793945,
+      "microbatch_seconds": 0.6643874849999634,
+      "peak_gb": 63.757601737976074,
       "fits_budget": true,
-      "estimated_update_seconds": 0.9295456565000109,
-      "input_tokens_per_second": 196848.76178355434
+      "estimated_update_seconds": 0.926719353999971,
+      "input_tokens_per_second": 197282.46386219515
     },
     {
       "microbatch": 72,
       "mode": "compiled",
-      "microbatch_seconds": 0.7430151979999664,
-      "peak_gb": 70.97307014465332,
+      "microbatch_seconds": 0.7446585950000326,
+      "peak_gb": 71.65071392059326,
       "fits_budget": true,
-      "estimated_update_seconds": 0.9250642054999503,
-      "input_tokens_per_second": 198456.23669195347
+      "estimated_update_seconds": 0.9283865335000314,
+      "input_tokens_per_second": 198018.26097232322
     },
     {
       "microbatch": 76,
       "mode": "compiled",
-      "microbatch_seconds": 0.78068104700003,
-      "peak_gb": 74.84185981750488,
+      "microbatch_seconds": 0.7858128550000174,
+      "peak_gb": 75.5242338180542,
       "fits_budget": true,
-      "estimated_update_seconds": 0.9241739715000392,
-      "input_tokens_per_second": 199374.63654089966
+      "estimated_update_seconds": 0.9300540555000225,
+      "input_tokens_per_second": 198072.60597689834
     },
     {
       "microbatch": 80,
       "mode": "compiled",
-      "microbatch_seconds": 0.8210239410000213,
-      "peak_gb": 78.72572612762451,
+      "microbatch_seconds": 0.8269688110000288,
+      "peak_gb": 79.40062236785889,
       "fits_budget": true,
-      "estimated_update_seconds": 0.9229792625000073,
-      "input_tokens_per_second": 199555.69115370748
+      "estimated_update_seconds": 0.9293605420000404,
+      "input_tokens_per_second": 198121.13567121493
     },
     {
       "microbatch": 86,
       "mode": "compiled",
-      "microbatch_seconds": 0.8818629400000191,
-      "peak_gb": 84.78368663787842,
+      "microbatch_seconds": 0.8882591199999865,
+      "peak_gb": 85.4687147140503,
       "fits_budget": true,
-      "estimated_update_seconds": 0.9349941874999956,
-      "input_tokens_per_second": 199722.64624250587
+      "estimated_update_seconds": 0.9413692904999778,
+      "input_tokens_per_second": 198284.4825730612
     },
     {
       "microbatch": 90,
       "mode": "compiled",
-      "microbatch_seconds": 0.9259720020000373,
-      "peak_gb": 88.6924238204956,
-      "fits_budget": false,
-      "estimated_update_seconds": 0.9259720020000373,
-      "input_tokens_per_second": 199055.69455867045
+      "microbatch_seconds": 0.9302335780000135,
+      "peak_gb": 89.3772840499878,
+      "fits_budget": true,
+      "estimated_update_seconds": 0.9302335780000135,
+      "input_tokens_per_second": 198143.7827650609
     }
   ],
   "failures": [
@@ -216,8 +216,7 @@
     }
   ],
   "selection_policy": "max_batch",
-  "budget_gb": 89.6964394569397,
+  "budget_gb": 92.52170804977416,
   "reserve_gb": 2.1791439056396484,
-  "note": "Forward/backward throughput; excludes optimizer, data, logging and checkpoint time.",
-  "legacy_seed_verified": false
+  "note": "Forward/backward throughput; excludes optimizer, data, logging and checkpoint time."
 }

shard_scheduler.json

@@ -1,99 +1,99 @@
 {
   "version": 1,
   "startup_batch_seconds": [
-    17.862166522000052,
-    0.01599195200003578,
-    0.012736249999989013,
-    0.10922379700002693,
-    0.015912060000005113,
-    0.014246264000007614,
-    0.01397435499995936,
-    0.10559937200002878
+    2.3260779750000324,
+    0.017838069999982054,
+    0.0158013189999906,
+    0.015721255000016754,
+    0.09197544800002788,
+    0.014327330000014626,
+    0.013824727999974584,
+    0.10075297500003444
   ],
   "source_benchmarks": [
     {
       "source": "climbmix",
       "rows": 256,
-      "tokens": 721064,
-      "bytes": 302776,
-      "generation_seconds": 0.6739276440000594,
-      "write_seconds": 0.00442227799999273,
-      "read_seconds": 0.04565144100001817,
-      "tokens_per_second": 1062967.6168812835,
+      "tokens": 903410,
+      "bytes": 372598,
+      "generation_seconds": 1.5044533369999726,
+      "write_seconds": 0.005470204000005197,
+      "read_seconds": 0.004841493000014907,
+      "tokens_per_second": 598315.063954628,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 74809.39388603426,
-      "required_pages_per_second_upper_estimate": 0.006484288659366078
+      "required_token_positions_per_second": 76764.09997329117,
+      "required_pages_per_second_upper_estimate": 0.00531071855340399
     },
     {
       "source": "rewrite6",
       "rows": 256,
-      "tokens": 156675,
-      "bytes": 68694,
-      "generation_seconds": 0.12810849499999222,
-      "write_seconds": 0.001732062999963091,
-      "read_seconds": 0.002349199000036606,
-      "tokens_per_second": 1206672.263377472,
+      "tokens": 149279,
+      "bytes": 66870,
+      "generation_seconds": 17.489590872999997,
+      "write_seconds": 0.0027630569999814725,
+      "read_seconds": 0.003109784999992371,
+      "tokens_per_second": 8533.957213384612,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 9351.174235754283,
-      "required_pages_per_second_upper_estimate": 0.0037303232151564875
+      "required_token_positions_per_second": 9595.512496661397,
+      "required_pages_per_second_upper_estimate": 0.004017440705265558
     },
     {
       "source": "openmath",
       "rows": 256,
-      "tokens": 270174,
-      "bytes": 90063,
-      "generation_seconds": 0.6564187429999038,
-      "write_seconds": 0.002473517999987962,
-      "read_seconds": 0.0032515120000198294,
-      "tokens_per_second": 410042.7581134458,
+      "tokens": 300086,
+      "bytes": 97823,
+      "generation_seconds": 0.7871533299999669,
+      "write_seconds": 0.0022467860000006112,
+      "read_seconds": 0.0028803160000165917,
+      "tokens_per_second": 380144.35761751566,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 18702.348471508565,
-      "required_pages_per_second_upper_estimate": 0.004326459168792279
+      "required_token_positions_per_second": 19191.024993322793,
+      "required_pages_per_second_upper_estimate": 0.003996984404746221
     },
     {
       "source": "ultra_qa",
       "rows": 256,
-      "tokens": 1026361,
-      "bytes": 365167,
-      "generation_seconds": 0.46967248600003586,
-      "write_seconds": 0.004728889000034542,
-      "read_seconds": 0.0050225659999796335,
-      "tokens_per_second": 2163486.5624068812,
+      "tokens": 1042562,
+      "bytes": 369139,
+      "generation_seconds": 23.827993708999998,
+      "write_seconds": 0.0062629279999555365,
+      "read_seconds": 0.006445001999964006,
+      "tokens_per_second": 43742.16556775435,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28053.522707262844,
-      "required_pages_per_second_upper_estimate": 0.0017083123474137538
+      "required_token_positions_per_second": 28786.537489984188,
+      "required_pages_per_second_upper_estimate": 0.0017257089680268527
     },
     {
       "source": "ultra_style",
       "rows": 256,
-      "tokens": 811870,
-      "bytes": 338806,
-      "generation_seconds": 0.5208254859999215,
-      "write_seconds": 0.004081093000081637,
-      "read_seconds": 0.004787273000033565,
-      "tokens_per_second": 1546694.2737633225,
+      "tokens": 803904,
+      "bytes": 337820,
+      "generation_seconds": 36.50458967899999,
+      "write_seconds": 0.00519779599994763,
+      "read_seconds": 0.005391688000031536,
+      "tokens_per_second": 22018.86276523721,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28053.522707262844,
-      "required_pages_per_second_upper_estimate": 0.0021596378351262245
+      "required_token_positions_per_second": 28786.537489984188,
+      "required_pages_per_second_upper_estimate": 0.002238026671249318
     },
     {
       "source": "cortex",
       "rows": 128,
-      "tokens": 80537,
-      "bytes": 22745,
-      "generation_seconds": 0.07537080299994159,
-      "write_seconds": 0.003962095000019872,
-      "read_seconds": 0.042780299000014566,
-      "tokens_per_second": 1015177.8395898147,
+      "tokens": 77961,
+      "bytes": 43256,
+      "generation_seconds": 0.08065045199998622,
+      "write_seconds": 0.004991313999994418,
+      "read_seconds": 0.04061180899998362,
+      "tokens_per_second": 910315.184299418,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
-      "estimated_pages_for_buffer": 21,
-      "required_token_positions_per_second": 28053.522707262844,
-      "required_pages_per_second_upper_estimate": 0.34833086292341214
+      "estimated_pages_for_buffer": 23,
+      "required_token_positions_per_second": 28786.537489984188,
+      "required_pages_per_second_upper_estimate": 0.3692427943456881
     }
   ],
   "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
@@ -108,460 +108,220 @@
   "local_shard_size_benchmark": [
     {
       "target_mib": 16,
-      "actual_bytes": 16107520,
-      "pages": 6,
-      "packing_seconds": 0.03508346299997811,
-      "read_seconds": 0.0015898530000413302,
-      "io_bytes_per_second": 439216350.1111124,
+      "actual_bytes": 15554560,
+      "pages": 7,
+      "packing_seconds": 0.03195600199990167,
+      "read_seconds": 0.0013301719999390116,
+      "io_bytes_per_second": 467297923.7588089,
       "target_reached": true
     },
     {
       "target_mib": 32,
-      "actual_bytes": 18657280,
-      "pages": 8,
-      "packing_seconds": 0.04401178199998412,
-      "read_seconds": 0.0016850460000341627,
-      "io_bytes_per_second": 408283918.5247723,
+      "actual_bytes": 18083840,
+      "pages": 9,
+      "packing_seconds": 0.0402841879999869,
+      "read_seconds": 0.0016377959999545055,
+      "io_bytes_per_second": 431368897.0451703,
       "target_reached": false
     },
     {
       "target_mib": 64,
-      "actual_bytes": 18657280,
-      "pages": 8,
-      "packing_seconds": 0.04403130000002875,
-      "read_seconds": 0.0015284270000393008,
-      "io_bytes_per_second": 409512550.4148024,
+      "actual_bytes": 18083840,
+      "pages": 9,
+      "packing_seconds": 0.0399535819999528,
+      "read_seconds": 0.0013251859999172666,
+      "io_bytes_per_second": 438090594.17802685,
       "target_reached": false
     },
     {
       "target_mib": 128,
-      "actual_bytes": 18657280,
-      "pages": 8,
-      "packing_seconds": 0.04190044699998907,
-      "read_seconds": 0.001583749000019452,
-      "io_bytes_per_second": 429058870.0316856,
+      "actual_bytes": 18083840,
+      "pages": 9,
+      "packing_seconds": 0.04405787199993938,
+      "read_seconds": 0.0014645830000290516,
+      "io_bytes_per_second": 397250983.05907583,
       "target_reached": false
     },
     {
       "target_mib": 256,
-      "actual_bytes": 18657280,
-      "pages": 8,
-      "packing_seconds": 0.042332877000035296,
-      "read_seconds": 0.0015621619999137693,
-      "io_bytes_per_second": 425043021.37700915,
+      "actual_bytes": 18083840,
+      "pages": 9,
+      "packing_seconds": 0.038781767000045875,
+      "read_seconds": 0.0013218610000649278,
+      "io_bytes_per_second": 450927781.39548963,
       "target_reached": false
     }
   ],
   "shard_benchmark_scope": "Local packing/read measurements; upload capacity measured only on real data commits. Targets larger than available pages are sample-limited.",
-  "estimated_fresh_generation_batch_seconds": 0.1434574083060457,
+  "estimated_fresh_generation_batch_seconds": 2.558590814491007,
   "selected": {
-    "prefetch_batches": 61,
+    "prefetch_batches": 63,
     "prefetch_capacity_limit": 512,
-    "batch_memory_estimate_bytes": 717798,
-    "producer_batch_seconds": 0.32073739904717513,
-    "producer_p95_seconds": 0.39812204299960285,
-    "gpu_update_seconds": 0.9855446778825442,
-    "producer_batches_per_second": 3.1178153934362878,
-    "gpu_batches_per_second": 1.01466734328931,
-    "producer_headroom": 3.072746367621404,
+    "batch_memory_estimate_bytes": 704566,
+    "producer_batch_seconds": 0.044507669203127875,
+    "producer_p95_seconds": 0.14738937200036162,
+    "gpu_update_seconds": 0.9604489601995264,
+    "producer_batches_per_second": 22.468037933779797,
+    "gpu_batches_per_second": 1.0411797413912105,
+    "producer_headroom": 21.579403671222323,
     "sustainable": true,
-    "buffer_seconds": 60.118225350835196,
+    "buffer_seconds": 60.50828449257016,
     "buffer_drain_seconds": null,
     "shard_target_mib": 16,
-    "upload_pages_per_commit": 78,
+    "upload_pages_per_commit": 83,
     "max_pending_pages": 4096,
-    "generation_bytes_per_second": 370873.33853366604,
-    "upload_bytes_per_second": 5789859.621793583,
+    "generation_bytes_per_second": 289702.9369927744,
+    "upload_bytes_per_second": 5452255.476764406,
     "upload_sustainable": true,
     "logical_page_rows": 4096,
     "logical_cortex_page_rows": 128,
-    "estimated_full_shards_per_hour": 79.58078495986449,
+    "estimated_full_shards_per_hour": 62.16350633943009,
     "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
   },
   "upload_measurements": [
     {
-      "seconds": 1.0140557359991362,
-      "bytes": 5030935,
-      "pages": 25,
-      "shards": 3
-    },
-    {
-      "seconds": 1.2526760300024762,
-      "bytes": 11691054,
-      "pages": 30,
-      "shards": 3
-    },
-    {
-      "seconds": 0.9398190409992822,
-      "bytes": 2819009,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1499773260002257,
-      "bytes": 6017503,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 1.6606815510021988,
-      "bytes": 10233181,
-      "pages": 23,
-      "shards": 3
-    },
-    {
-      "seconds": 1.0018192189963884,
-      "bytes": 366522,
-      "pages": 10,
+      "seconds": 0.9708695179999722,
+      "bytes": 43075,
+      "pages": 1,
       "shards": 1
     },
     {
-      "seconds": 0.8059965610009385,
-      "bytes": 2452391,
-      "pages": 29,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9446101400026237,
-      "bytes": 6258704,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1053826549978112,
-      "bytes": 8176480,
-      "pages": 32,
-      "shards": 3
+      "seconds": 0.8249086619999844,
+      "bytes": 105910,
+      "pages": 3,
+      "shards": 1
     },
     {
-      "seconds": 1.249701370994444,
-      "bytes": 10043413,
-      "pages": 32,
+      "seconds": 1.08548185799998,
+      "bytes": 2013983,
+      "pages": 18,
       "shards": 3
     },
     {
-      "seconds": 1.0213671140008955,
-      "bytes": 6189327,
-      "pages": 36,
-      "shards": 2
-    },
-    {
-      "seconds": 1.0084248750063125,
-      "bytes": 2332449,
-      "pages": 29,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9006169920030516,
-      "bytes": 6118351,
-      "pages": 36,
-      "shards": 2
-    },
-    {
-      "seconds": 1.270539192002616,
-      "bytes": 5502511,
-      "pages": 29,
-      "shards": 2
-    },
-    {
-      "seconds": 1.512666299000557,
-      "bytes": 13201899,
-      "pages": 33,
-      "shards": 4
-    },
-    {
-      "seconds": 1.000070736001362,
-      "bytes": 6089030,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 1.0753602199984016,
-      "bytes": 871659,
+      "seconds": 1.0607301400000324,
+      "bytes": 873319,
       "pages": 26,
       "shards": 1
     },
     {
-      "seconds": 1.484286307997536,
-      "bytes": 11527011,
-      "pages": 36,
-      "shards": 4
-    },
-    {
-      "seconds": 0.9214101749967085,
-      "bytes": 1181839,
-      "pages": 33,
-      "shards": 1
-    },
-    {
-      "seconds": 1.1708070859967847,
-      "bytes": 7634853,
-      "pages": 39,
-      "shards": 3
-    },
-    {
-      "seconds": 1.219036098998913,
-      "bytes": 6506587,
-      "pages": 23,
-      "shards": 2
-    },
-    {
-      "seconds": 1.234236194002733,
-      "bytes": 11430289,
-      "pages": 42,
+      "seconds": 1.4327509930000133,
+      "bytes": 9634876,
+      "pages": 38,
       "shards": 3
     },
     {
-      "seconds": 1.0073759259976214,
-      "bytes": 7448462,
-      "pages": 30,
+      "seconds": 1.0578138549999494,
+      "bytes": 7567895,
+      "pages": 37,
       "shards": 3
     },
     {
-      "seconds": 0.7627367220047745,
-      "bytes": 1172696,
-      "pages": 36,
+      "seconds": 0.8248641469999711,
+      "bytes": 1159359,
+      "pages": 34,
       "shards": 1
     },
     {
-      "seconds": 1.1723295699994196,
-      "bytes": 5714080,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1845541919974494,
-      "bytes": 7669056,
-      "pages": 38,
+      "seconds": 0.9606663819999994,
+      "bytes": 10228939,
+      "pages": 23,
       "shards": 3
     },
     {
-      "seconds": 0.9817348009964917,
-      "bytes": 4526983,
-      "pages": 22,
-      "shards": 2
-    },
-    {
-      "seconds": 1.2601451889931923,
-      "bytes": 13743089,
-      "pages": 49,
+      "seconds": 1.4356544900001609,
+      "bytes": 13537180,
+      "pages": 46,
       "shards": 4
     },
     {
-      "seconds": 0.8212157870002557,
-      "bytes": 890133,
-      "pages": 26,
+      "seconds": 0.9362600690001273,
+      "bytes": 1025985,
+      "pages": 34,
       "shards": 1
     },
     {
-      "seconds": 1.1462097159965197,
-      "bytes": 10417894,
-      "pages": 36,
-      "shards": 3
-    },
-    {
-      "seconds": 1.1751894160042866,
-      "bytes": 7506454,
-      "pages": 38,
+      "seconds": 1.7697618709999006,
+      "bytes": 7336838,
+      "pages": 30,
       "shards": 3
     },
     {
-      "seconds": 1.0263809830066748,
-      "bytes": 924946,
-      "pages": 27,
-      "shards": 1
-    },
-    {
-      "seconds": 1.1823451020027278,
-      "bytes": 13453015,
-      "pages": 42,
-      "shards": 4
-    },
-    {
-      "seconds": 1.2355715770027018,
-      "bytes": 9750066,
-      "pages": 35,
-      "shards": 4
-    },
-    {
-      "seconds": 1.382024664999335,
-      "bytes": 840789,
-      "pages": 26,
-      "shards": 1
-    },
-    {
-      "seconds": 1.7087262210043264,
-      "bytes": 7732105,
-      "pages": 39,
+      "seconds": 1.264844782000182,
+      "bytes": 9996183,
+      "pages": 40,
       "shards": 3
     },
     {
-      "seconds": 1.1756209270024556,
-      "bytes": 879254,
-      "pages": 27,
+      "seconds": 0.9787746179999885,
+      "bytes": 1132423,
+      "pages": 33,
       "shards": 1
     },
     {
-      "seconds": 1.4754508929981967,
-      "bytes": 10625968,
-      "pages": 36,
-      "shards": 3
-    },
-    {
-      "seconds": 0.9398709419983788,
-      "bytes": 2450871,
-      "pages": 30,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1338124570029322,
-      "bytes": 11954791,
-      "pages": 38,
-      "shards": 3
-    },
-    {
-      "seconds": 1.1746257199993124,
-      "bytes": 7528355,
-      "pages": 38,
+      "seconds": 0.8866051049999442,
+      "bytes": 7112636,
+      "pages": 23,
       "shards": 3
     },
     {
-      "seconds": 1.2720379320016946,
-      "bytes": 4790673,
-      "pages": 27,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9963719520019367,
-      "bytes": 6449131,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9008199800009606,
-      "bytes": 2360873,
-      "pages": 29,
+      "seconds": 1.117540433999693,
+      "bytes": 7055686,
+      "pages": 40,
       "shards": 2
     },
     {
-      "seconds": 1.193771635000303,
-      "bytes": 12283347,
-      "pages": 43,
+      "seconds": 1.0181373959999291,
+      "bytes": 10411775,
+      "pages": 36,
       "shards": 3
     },
     {
-      "seconds": 0.9776089399965713,
-      "bytes": 9862558,
-      "pages": 28,
+      "seconds": 1.183017940999889,
+      "bytes": 7545162,
+      "pages": 32,
       "shards": 3
     },
     {
-      "seconds": 0.8356411559943808,
-      "bytes": 2333118,
-      "pages": 29,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1188001900009112,
-      "bytes": 6081727,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9660166879984899,
-      "bytes": 2720285,
-      "pages": 38,
-      "shards": 2
+      "seconds": 0.9269013600001017,
+      "bytes": 1114116,
+      "pages": 34,
+      "shards": 1
     },
     {
-      "seconds": 1.24898188199586,
-      "bytes": 10600974,
+      "seconds": 1.037127201000203,
+      "bytes": 7612192,
       "pages": 37,
       "shards": 3
     },
     {
-      "seconds": 1.208974320004927,
-      "bytes": 5767450,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1749305560006178,
-      "bytes": 8197083,
-      "pages": 33,
-      "shards": 3
-    },
-    {
-      "seconds": 0.9447351959970547,
-      "bytes": 6335429,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 1.0603171299953829,
-      "bytes": 4753623,
-      "pages": 29,
+      "seconds": 0.9605875500001275,
+      "bytes": 5330342,
+      "pages": 25,
       "shards": 2
     },
     {
-      "seconds": 1.2698575760005042,
-      "bytes": 7643809,
-      "pages": 39,
+      "seconds": 1.2280645959999674,
+      "bytes": 12018597,
+      "pages": 45,
       "shards": 3
     },
     {
-      "seconds": 1.0007858850003686,
-      "bytes": 5847294,
+      "seconds": 1.1235430140000062,
+      "bytes": 7362856,
       "pages": 29,
-      "shards": 2
-    },
-    {
-      "seconds": 0.8758766960017965,
-      "bytes": 2703346,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 1.3512026129974402,
-      "bytes": 16969598,
-      "pages": 37,
-      "shards": 4
-    },
-    {
-      "seconds": 1.1678855200007092,
-      "bytes": 919043,
-      "pages": 27,
-      "shards": 1
-    },
-    {
-      "seconds": 1.127943243001937,
-      "bytes": 7825410,
-      "pages": 38,
       "shards": 3
     },
     {
-      "seconds": 1.1976684740002383,
-      "bytes": 5884805,
-      "pages": 31,
-      "shards": 2
-    },
-    {
-      "seconds": 0.8530800510052359,
-      "bytes": 740338,
-      "pages": 24,
+      "seconds": 0.8067839350001123,
+      "bytes": 1162221,
+      "pages": 35,
       "shards": 1
     },
     {
-      "seconds": 1.4493056469946168,
-      "bytes": 12401988,
-      "pages": 45,
+      "seconds": 1.3463661700002376,
+      "bytes": 11675037,
+      "pages": 32,
       "shards": 4
-    },
-    {
-      "seconds": 0.9416848679975374,
-      "bytes": 6631553,
-      "pages": 29,
-      "shards": 2
     }
   ],
   "cache_key": {
@@ -602,11 +362,7 @@
     "source": "f8722b66ff2f92bca174f36564336149e85c65c48b964d05ff3a4d5d85da308f",
     "cpu": "unknown",
     "cpu_count": 20,
-    "microbatch": 86
+    "microbatch": 90
   },
-  "cache_reused": true,
-  "resume_probe_seconds": [
-    3.301654667999969,
-    0.11550889399995867
-  ]
+  "cache_reused": false
 }

trainer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f7cd41c0ad064fdb32d106a04f2baeeeeed5930763f4ca1400aa877b835cb4d4
-size 721771839
+oid sha256:0883783165f69503eba3b62bf8c9c7a6e664527b2bcaeee310413d02bf169eb3
+size 721779455

training_config.json

@@ -4,7 +4,7 @@
   "run_name": "void-byte-v1",
   "context": 2048,
   "global_batch": 90,
-  "micro_batch": 86,
+  "micro_batch": 90,
   "microbatch_candidates": [
     8,
     16,
@@ -46,8 +46,8 @@
   "cpu_threads": 4,
   "page_rows": 4096,
   "cortex_page_rows": 128,
-  "prefetch_batches": 61,
-  "prefetch_low_water": 30,
+  "prefetch_batches": 63,
+  "prefetch_low_water": 31,
   "memory_pages": 12,
   "cache_gb": 20,
   "force_rebenchmark": false,
@@ -66,7 +66,7 @@
   "benchmark_compile": true,
   "autotune_microbatch": true,
   "attention_backend": "flash",
-  "vram_fraction": 0.95,
+  "vram_fraction": 0.98,
   "upload": true,
   "hub_resume": true,
   "work_dir": "/marimo/qwen90m_workspace/runs/void-byte-v1",