Qwen90M step 535000; complete resumable checkpoint

complete.json

@@ -1,18 +1,18 @@
 {
-  "step": 530000,
+  "step": 535000,
   "files": {
     "config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7",
     "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4",
-    "model.safetensors": "3345a1d3d37a56145174811805daadbe6545c69564f12badd37ca9980383c189",
+    "model.safetensors": "746a63165b4986046476ae84c2713e36ed3fa23737687659143ada4b831f1654",
     "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03",
     "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300",
     "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263",
     "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6",
-    "trainer.pt": "4273939d74c5226bc57df9a209306193dff7da001a42417dfdf439575063da35",
+    "trainer.pt": "2b45969c25feb44bef2bbc28e1a234c7ba36a717f6375ea0dba3053855528f94",
     "training_config.json": "c8bdf052510c2d9092e5a6824b119dd6584dc7697f8c4a8e9b7ecf3057c5bbb6",
     "performance.json": "3070ebbdf9b2b0fd6a0a237819043aa2c2a35c74a97edfa223e5c324471af4af",
-    "shard_scheduler.json": "3e8b88313d070ffdb6572b9af938d708afe66786c34590df9f577e00e5a7ad71",
-    "coverage.json": "1f166ed6ce6875dfc663aa1359c72063b2c84099952c88e471aa37b08f50927f",
+    "shard_scheduler.json": "dc3b4c9c9c5634b43786ba061a7ead1b9ed759c5b1eacff95382e91a9ef4f364",
+    "coverage.json": "7998ce90c7aacd5ffa1b3b45381b7a23cce87145599cd5217ab5d192996cc5db",
     "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"
   },
   "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063"

coverage.json

@@ -1,107 +1,101 @@
 {
-  "step": 530000,
+  "step": 535000,
   "stats": {
-    "climbmix/blocks": 19080000,
-    "climbmix/targets": 31893841018,
-    "climbmix/input_tokens": 31895861954,
-    "ultra_qa/blocks": 7155000,
-    "ultra_qa/targets": 11672418639,
-    "ultra_qa/input_tokens": 11672441161,
-    "cortex/blocks": 7155000,
-    "cortex/targets": 272094364,
-    "cortex/input_tokens": 11373535505,
-    "ultra_style/blocks": 7155000,
-    "ultra_style/targets": 11316260253,
-    "ultra_style/input_tokens": 11316853356,
-    "openmath/blocks": 4770000,
-    "openmath/targets": 7370634197,
-    "openmath/input_tokens": 7373089745,
-    "rewrite6/blocks": 2385000,
-    "rewrite6/targets": 3909838802,
-    "rewrite6/input_tokens": 3914125422
+    "climbmix/blocks": 19260000,
+    "climbmix/targets": 32195111082,
+    "climbmix/input_tokens": 32197151068,
+    "ultra_qa/blocks": 7222500,
+    "ultra_qa/targets": 11782590681,
+    "ultra_qa/input_tokens": 11782613431,
+    "cortex/blocks": 7222500,
+    "cortex/targets": 274657059,
+    "cortex/input_tokens": 11480802169,
+    "ultra_style/blocks": 7222500,
+    "ultra_style/targets": 11423290047,
+    "ultra_style/input_tokens": 11423888845,
+    "openmath/blocks": 4815000,
+    "openmath/targets": 7439991699,
+    "openmath/input_tokens": 7442470149,
+    "rewrite6/blocks": 2407500,
+    "rewrite6/targets": 3946780306,
+    "rewrite6/input_tokens": 3951107292
   },
   "sources": {
     "climbmix": {
-      "page": 2613,
-      "row": 1722,
+      "page": 2637,
+      "row": 4073,
       "start": {
-        "file": 126,
+        "file": 127,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 37,
+          "row_group": 51,
           "row": 1024
         }
       },
       "epoch": 0,
       "offset": 0,
-      "documents": 10704570
+      "documents": 10805225
     },
     "rewrite6": {
-      "page": 36,
-      "row": 2819,
-      "start": {
-        "file": 0,
-        "reader": {
-          "format": "jsonl-row-v1",
-          "row": 147456
-        }
-      },
-      "epoch": 29,
-      "offset": 0,
-      "documents": 6106991
+      "page": 0,
+      "row": 2348,
+      "start": null,
+      "epoch": 30,
+      "offset": 2048,
+      "documents": 6164468
     },
     "openmath": {
-      "page": 1633,
-      "row": 76,
+      "page": 1648,
+      "row": 1424,
       "start": {
         "file": 15,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 139,
-          "row": 18
+          "row_group": 200,
+          "row": 458
         }
       },
       "epoch": 0,
       "offset": 0,
-      "documents": 6688844
+      "documents": 6751632
     },
     "ultra_qa": {
-      "page": 706,
-      "row": 539,
+      "page": 712,
+      "row": 3263,
       "start": {
         "file": 5,
         "reader": {
           "format": "parquet-row-v1",
           "row_group": 4,
-          "row": 35556
+          "row": 60132
         }
       },
       "epoch": 0,
-      "offset": 2048,
-      "documents": 2892315
+      "offset": 6144,
+      "documents": 2919615
     },
     "ultra_style": {
-      "page": 1077,
-      "row": 2689,
+      "page": 1087,
+      "row": 3342,
       "start": {
         "file": 6,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 3,
-          "row": 45678
+          "row_group": 4,
+          "row": 2030
         }
       },
       "epoch": 0,
-      "offset": 2048,
-      "documents": 4414081
+      "offset": 4096,
+      "documents": 4455694
     },
     "cortex": {
-      "page": 138765,
-      "row": 110,
+      "page": 140074,
+      "row": 15,
       "start": null,
       "epoch": 0,
       "offset": 0,
-      "documents": 17762030
+      "documents": 17929487
     }
   }
 }

latest.json

@@ -1 +1 @@
-{"step": 530000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "3345a1d3d37a56145174811805daadbe6545c69564f12badd37ca9980383c189", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "4273939d74c5226bc57df9a209306193dff7da001a42417dfdf439575063da35", "training_config.json": "c8bdf052510c2d9092e5a6824b119dd6584dc7697f8c4a8e9b7ecf3057c5bbb6", "performance.json": "3070ebbdf9b2b0fd6a0a237819043aa2c2a35c74a97edfa223e5c324471af4af", "shard_scheduler.json": "3e8b88313d070ffdb6572b9af938d708afe66786c34590df9f577e00e5a7ad71", "coverage.json": "1f166ed6ce6875dfc663aa1359c72063b2c84099952c88e471aa37b08f50927f", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000530000"}
+{"step": 535000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "746a63165b4986046476ae84c2713e36ed3fa23737687659143ada4b831f1654", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "2b45969c25feb44bef2bbc28e1a234c7ba36a717f6375ea0dba3053855528f94", "training_config.json": "c8bdf052510c2d9092e5a6824b119dd6584dc7697f8c4a8e9b7ecf3057c5bbb6", "performance.json": "3070ebbdf9b2b0fd6a0a237819043aa2c2a35c74a97edfa223e5c324471af4af", "shard_scheduler.json": "dc3b4c9c9c5634b43786ba061a7ead1b9ed759c5b1eacff95382e91a9ef4f364", "coverage.json": "7998ce90c7aacd5ffa1b3b45381b7a23cce87145599cd5217ab5d192996cc5db", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000535000"}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3345a1d3d37a56145174811805daadbe6545c69564f12badd37ca9980383c189
+oid sha256:746a63165b4986046476ae84c2713e36ed3fa23737687659143ada4b831f1654
 size 360622744

shard_scheduler.json

@@ -22,8 +22,8 @@
       "tokens_per_second": 598315.063954628,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 77015.98464267456,
-      "required_pages_per_second_upper_estimate": 0.005328144519284888
+      "required_token_positions_per_second": 76499.76095720792,
+      "required_pages_per_second_upper_estimate": 0.00529243096692033
     },
     {
       "source": "rewrite6",
@@ -36,8 +36,8 @@
       "tokens_per_second": 8533.957213384612,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 9626.99808033432,
-      "required_pages_per_second_upper_estimate": 0.0040306230616556585
+      "required_token_positions_per_second": 9562.47011965099,
+      "required_pages_per_second_upper_estimate": 0.004003606552014596
     },
     {
       "source": "openmath",
@@ -50,8 +50,8 @@
       "tokens_per_second": 380144.35761751566,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 19253.99616066864,
-      "required_pages_per_second_upper_estimate": 0.004010099638243004
+      "required_token_positions_per_second": 19124.94023930198,
+      "required_pages_per_second_upper_estimate": 0.003983220693255846
     },
     {
       "source": "ultra_qa",
@@ -64,8 +64,8 @@
       "tokens_per_second": 43742.16556775435,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28880.99424100296,
-      "required_pages_per_second_upper_estimate": 0.0017313715060233205
+      "required_token_positions_per_second": 28687.41035895297,
+      "required_pages_per_second_upper_estimate": 0.0017197664478798965
     },
     {
       "source": "ultra_style",
@@ -78,8 +78,8 @@
       "tokens_per_second": 22018.86276523721,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28880.99424100296,
-      "required_pages_per_second_upper_estimate": 0.0022453702681696883
+      "required_token_positions_per_second": 28687.41035895297,
+      "required_pages_per_second_upper_estimate": 0.002230319972825811
     },
     {
       "source": "cortex",
@@ -92,8 +92,8 @@
       "tokens_per_second": 910315.184299418,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 23,
-      "required_token_positions_per_second": 28880.99424100296,
-      "required_pages_per_second_upper_estimate": 0.3704543841279994
+      "required_token_positions_per_second": 28687.41035895297,
+      "required_pages_per_second_upper_estimate": 0.36797129794324046
     }
   ],
   "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
@@ -158,45 +158,27 @@
     "prefetch_batches": 63,
     "prefetch_capacity_limit": 512,
     "batch_memory_estimate_bytes": 704566,
-    "producer_batch_seconds": 0.320096299234379,
-    "producer_p95_seconds": 0.15855196899974544,
-    "gpu_update_seconds": 0.95730776334381,
-    "producer_batches_per_second": 3.1240598607101857,
-    "gpu_batches_per_second": 1.0445961458696094,
-    "producer_headroom": 2.9906867578086427,
+    "producer_batch_seconds": 0.08245735881254745,
+    "producer_p95_seconds": 0.19908109500102,
+    "gpu_update_seconds": 0.9637677174081841,
+    "producer_batches_per_second": 12.127480365619363,
+    "gpu_batches_per_second": 1.03759441402463,
+    "producer_headroom": 11.688074069885543,
     "sustainable": true,
-    "buffer_seconds": 60.310389090660024,
+    "buffer_seconds": 60.7173661967156,
     "buffer_drain_seconds": null,
     "shard_target_mib": 16,
-    "upload_pages_per_commit": 79,
+    "upload_pages_per_commit": 87,
     "max_pending_pages": 4096,
-    "generation_bytes_per_second": 386229.8712076355,
-    "upload_bytes_per_second": 3476176.657096584,
+    "generation_bytes_per_second": 322888.0851025507,
+    "upload_bytes_per_second": 4628498.063736584,
     "upload_sustainable": true,
     "logical_page_rows": 4096,
     "logical_cortex_page_rows": 128,
-    "estimated_full_shards_per_hour": 82.87593938991354,
+    "estimated_full_shards_per_hour": 69.28426661307708,
     "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
   },
   "upload_measurements": [
-    {
-      "seconds": 1.0629590909998115,
-      "bytes": 25106,
-      "pages": 1,
-      "shards": 1
-    },
-    {
-      "seconds": 1.218460234000304,
-      "bytes": 6273746,
-      "pages": 34,
-      "shards": 3
-    },
-    {
-      "seconds": 1.1453777450001326,
-      "bytes": 12048299,
-      "pages": 34,
-      "shards": 3
-    },
     {
       "seconds": 1.011093388000063,
       "bytes": 1199201,
@@ -334,6 +316,252 @@
       "bytes": 6117097,
       "pages": 37,
       "shards": 2
+    },
+    {
+      "seconds": 1.1878640209997684,
+      "bytes": 11776096,
+      "pages": 38,
+      "shards": 3
+    },
+    {
+      "seconds": 1.224845865000134,
+      "bytes": 2065038,
+      "pages": 19,
+      "shards": 2
+    },
+    {
+      "seconds": 0.9610936170001878,
+      "bytes": 5798019,
+      "pages": 27,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1097549460000664,
+      "bytes": 5080154,
+      "pages": 36,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1122178149998945,
+      "bytes": 7607293,
+      "pages": 38,
+      "shards": 3
+    },
+    {
+      "seconds": 1.0907007900004828,
+      "bytes": 6092133,
+      "pages": 36,
+      "shards": 2
+    },
+    {
+      "seconds": 0.8937167260000933,
+      "bytes": 2214439,
+      "pages": 24,
+      "shards": 2
+    },
+    {
+      "seconds": 1.3153861170003438,
+      "bytes": 17177113,
+      "pages": 40,
+      "shards": 4
+    },
+    {
+      "seconds": 0.9797320220004622,
+      "bytes": 1103154,
+      "pages": 35,
+      "shards": 2
+    },
+    {
+      "seconds": 1.195222697000645,
+      "bytes": 7323521,
+      "pages": 29,
+      "shards": 3
+    },
+    {
+      "seconds": 1.158046825999918,
+      "bytes": 6457008,
+      "pages": 36,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1688710530006574,
+      "bytes": 6589194,
+      "pages": 40,
+      "shards": 3
+    },
+    {
+      "seconds": 1.0017207049995704,
+      "bytes": 5856566,
+      "pages": 27,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1244407199992565,
+      "bytes": 12346633,
+      "pages": 40,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9011206420000235,
+      "bytes": 2301339,
+      "pages": 25,
+      "shards": 2
+    },
+    {
+      "seconds": 1.4037659219993657,
+      "bytes": 10492690,
+      "pages": 42,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9956328799999028,
+      "bytes": 1276481,
+      "pages": 36,
+      "shards": 1
+    },
+    {
+      "seconds": 1.4530193070004316,
+      "bytes": 7690650,
+      "pages": 38,
+      "shards": 3
+    },
+    {
+      "seconds": 1.0416680760008603,
+      "bytes": 5628205,
+      "pages": 24,
+      "shards": 2
+    },
+    {
+      "seconds": 1.0915457699993567,
+      "bytes": 13214903,
+      "pages": 39,
+      "shards": 4
+    },
+    {
+      "seconds": 1.4333472569996957,
+      "bytes": 6210705,
+      "pages": 36,
+      "shards": 2
+    },
+    {
+      "seconds": 1.232096203000765,
+      "bytes": 1201273,
+      "pages": 35,
+      "shards": 1
+    },
+    {
+      "seconds": 1.2199994860002334,
+      "bytes": 7355179,
+      "pages": 26,
+      "shards": 3
+    },
+    {
+      "seconds": 1.1187948159986263,
+      "bytes": 10435269,
+      "pages": 45,
+      "shards": 3
+    },
+    {
+      "seconds": 0.8605066399995849,
+      "bytes": 757336,
+      "pages": 24,
+      "shards": 1
+    },
+    {
+      "seconds": 1.4440002779992938,
+      "bytes": 13477323,
+      "pages": 45,
+      "shards": 4
+    },
+    {
+      "seconds": 1.0930595740010176,
+      "bytes": 6005049,
+      "pages": 30,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1409170160004578,
+      "bytes": 7751245,
+      "pages": 38,
+      "shards": 3
+    },
+    {
+      "seconds": 1.385997141000189,
+      "bytes": 5854774,
+      "pages": 28,
+      "shards": 2
+    },
+    {
+      "seconds": 0.8962630369987892,
+      "bytes": 1124581,
+      "pages": 35,
+      "shards": 1
+    },
+    {
+      "seconds": 1.1323460120001982,
+      "bytes": 7094094,
+      "pages": 25,
+      "shards": 3
+    },
+    {
+      "seconds": 1.3368760009998368,
+      "bytes": 10726853,
+      "pages": 39,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9546346269999049,
+      "bytes": 5999756,
+      "pages": 29,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1835868019988993,
+      "bytes": 7840439,
+      "pages": 38,
+      "shards": 3
+    },
+    {
+      "seconds": 1.00542208799925,
+      "bytes": 5360697,
+      "pages": 33,
+      "shards": 2
+    },
+    {
+      "seconds": 1.2967056479992607,
+      "bytes": 7612762,
+      "pages": 38,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9550784620005288,
+      "bytes": 883731,
+      "pages": 26,
+      "shards": 1
+    },
+    {
+      "seconds": 1.1562814480002999,
+      "bytes": 11922123,
+      "pages": 40,
+      "shards": 3
+    },
+    {
+      "seconds": 1.2267546230013977,
+      "bytes": 11778233,
+      "pages": 33,
+      "shards": 4
+    },
+    {
+      "seconds": 0.9305929850015673,
+      "bytes": 1085853,
+      "pages": 35,
+      "shards": 2
+    },
+    {
+      "seconds": 1.4772485220000817,
+      "bytes": 7565247,
+      "pages": 36,
+      "shards": 3
     }
   ],
   "cache_key": {

trainer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4273939d74c5226bc57df9a209306193dff7da001a42417dfdf439575063da35
-size 721771007
+oid sha256:2b45969c25feb44bef2bbc28e1a234c7ba36a717f6375ea0dba3053855528f94
+size 721759935