Qwen90M step 495000; complete resumable checkpoint

complete.json

@@ -1,18 +1,18 @@
 {
-  "step": 490000,
+  "step": 495000,
   "files": {
     "config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7",
     "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4",
-    "model.safetensors": "f2108bb7a48fce23b36f33461a1e756b5caa7d1607df7ec77a36189ce14cd91a",
+    "model.safetensors": "04a9c9ab7806b229ff0c1f81bf062a0cb0f4ff40115f9dc54fcd35653143dba5",
     "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03",
     "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300",
     "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263",
     "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6",
-    "trainer.pt": "057bc6789cc9af9e9681c23890813cd87d445ef5153b156a985027190827e3c4",
+    "trainer.pt": "b09148f8bdb03fa1cdc8767b6fe281fcc58f0012b4c4869a74f3162a6d4c4a19",
     "training_config.json": "c8bdf052510c2d9092e5a6824b119dd6584dc7697f8c4a8e9b7ecf3057c5bbb6",
     "performance.json": "234744063e094dfd7b525da4e181ac493fac400c830f871c7e8c59127ed10dc9",
-    "shard_scheduler.json": "b69378129bd07cf55de152eb8802dc876a8d2b68eceb701d39f8a9e31bcbbbb5",
-    "coverage.json": "719ac7183a62a1284cd10c856bf1014ea4d9dbf7c92da34175071b83066b7d81",
+    "shard_scheduler.json": "5090f346773104837959c2e40fec42df5511a51ba2bc53922efca54c043ec515",
+    "coverage.json": "79dfd44233cf538b7f2dfa46bc0fdc879f8607366262089a9f9c1e5fcd1d4480",
     "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"
   },
   "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063"

coverage.json

@@ -1,107 +1,107 @@
 {
-  "step": 490000,
+  "step": 495000,
   "stats": {
-    "climbmix/blocks": 17640000,
-    "climbmix/targets": 29485571146,
-    "climbmix/input_tokens": 29487439993,
-    "ultra_qa/blocks": 6615000,
-    "ultra_qa/targets": 10791309156,
-    "ultra_qa/input_tokens": 10791329945,
-    "cortex/blocks": 6615000,
-    "cortex/targets": 251600465,
-    "cortex/input_tokens": 10515435320,
-    "ultra_style/blocks": 6615000,
-    "ultra_style/targets": 10462953915,
-    "ultra_style/input_tokens": 10463502402,
-    "openmath/blocks": 4410000,
-    "openmath/targets": 6814294301,
-    "openmath/input_tokens": 6816564814,
-    "rewrite6/blocks": 2205000,
-    "rewrite6/targets": 3614791751,
-    "rewrite6/input_tokens": 3618755339
+    "climbmix/blocks": 17820000,
+    "climbmix/targets": 29786461296,
+    "climbmix/input_tokens": 29788349282,
+    "ultra_qa/blocks": 6682500,
+    "ultra_qa/targets": 10901546468,
+    "ultra_qa/input_tokens": 10901567474,
+    "cortex/blocks": 6682500,
+    "cortex/targets": 254162427,
+    "cortex/input_tokens": 10622791425,
+    "ultra_style/blocks": 6682500,
+    "ultra_style/targets": 10570465550,
+    "ultra_style/input_tokens": 10571019587,
+    "openmath/blocks": 4455000,
+    "openmath/targets": 6883812928,
+    "openmath/input_tokens": 6886106549,
+    "rewrite6/blocks": 2227500,
+    "rewrite6/targets": 3651663877,
+    "rewrite6/input_tokens": 3655667526
   },
   "sources": {
     "climbmix": {
-      "page": 2416,
-      "row": 2276,
+      "page": 2441,
+      "row": 677,
       "start": {
-        "file": 116,
+        "file": 118,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 75,
+          "row_group": 9,
           "row": 1024
         }
       },
       "epoch": 0,
-      "offset": 0,
-      "documents": 9898212
+      "offset": 2048,
+      "documents": 9999013
     },
     "rewrite6": {
-      "page": 24,
-      "row": 2371,
+      "page": 38,
+      "row": 2269,
       "start": {
         "file": 0,
         "reader": {
           "format": "jsonl-row-v1",
-          "row": 98304
+          "row": 155648
         }
       },
       "epoch": 27,
       "offset": 0,
-      "documents": 5646583
+      "documents": 5703825
     },
     "openmath": {
-      "page": 1509,
-      "row": 3634,
+      "page": 1525,
+      "row": 1171,
       "start": {
         "file": 14,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 67,
-          "row": 764
+          "row_group": 133,
+          "row": 300
         }
       },
       "epoch": 0,
-      "offset": 0,
-      "documents": 6184498
+      "offset": 2048,
+      "documents": 6247571
     },
     "ultra_qa": {
-      "page": 652,
-      "row": 3429,
+      "page": 659,
+      "row": 2052,
       "start": {
         "file": 5,
         "reader": {
           "format": "parquet-row-v1",
           "row_group": 1,
-          "row": 8004
+          "row": 36676
         }
       },
       "epoch": 0,
-      "offset": 2048,
-      "documents": 2674021
+      "offset": 4096,
+      "documents": 2701316
     },
     "ultra_style": {
-      "page": 996,
-      "row": 1168,
+      "page": 1006,
+      "row": 1193,
       "start": {
-        "file": 5,
+        "file": 6,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 7,
-          "row": 59261
+          "row_group": 0,
+          "row": 10924
         }
       },
       "epoch": 0,
-      "offset": 2048,
-      "documents": 4080784
+      "offset": 0,
+      "documents": 4121769
     },
     "cortex": {
-      "page": 128297,
-      "row": 17,
+      "page": 129606,
+      "row": 60,
       "start": null,
       "epoch": 0,
       "offset": 0,
-      "documents": 16422033
+      "documents": 16589628
     }
   }
 }

latest.json

@@ -1 +1 @@
-{"step": 490000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "f2108bb7a48fce23b36f33461a1e756b5caa7d1607df7ec77a36189ce14cd91a", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "057bc6789cc9af9e9681c23890813cd87d445ef5153b156a985027190827e3c4", "training_config.json": "c8bdf052510c2d9092e5a6824b119dd6584dc7697f8c4a8e9b7ecf3057c5bbb6", "performance.json": "234744063e094dfd7b525da4e181ac493fac400c830f871c7e8c59127ed10dc9", "shard_scheduler.json": "b69378129bd07cf55de152eb8802dc876a8d2b68eceb701d39f8a9e31bcbbbb5", "coverage.json": "719ac7183a62a1284cd10c856bf1014ea4d9dbf7c92da34175071b83066b7d81", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000490000"}
+{"step": 495000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "04a9c9ab7806b229ff0c1f81bf062a0cb0f4ff40115f9dc54fcd35653143dba5", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "b09148f8bdb03fa1cdc8767b6fe281fcc58f0012b4c4869a74f3162a6d4c4a19", "training_config.json": "c8bdf052510c2d9092e5a6824b119dd6584dc7697f8c4a8e9b7ecf3057c5bbb6", "performance.json": "234744063e094dfd7b525da4e181ac493fac400c830f871c7e8c59127ed10dc9", "shard_scheduler.json": "5090f346773104837959c2e40fec42df5511a51ba2bc53922efca54c043ec515", "coverage.json": "79dfd44233cf538b7f2dfa46bc0fdc879f8607366262089a9f9c1e5fcd1d4480", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000495000"}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f2108bb7a48fce23b36f33461a1e756b5caa7d1607df7ec77a36189ce14cd91a
+oid sha256:04a9c9ab7806b229ff0c1f81bf062a0cb0f4ff40115f9dc54fcd35653143dba5
 size 360622744

shard_scheduler.json

@@ -22,8 +22,8 @@
       "tokens_per_second": 598315.063954628,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 76295.78030360074,
-      "required_pages_per_second_upper_estimate": 0.00527831911200346
+      "required_token_positions_per_second": 77246.17314450147,
+      "required_pages_per_second_upper_estimate": 0.005344069493952184
     },
     {
       "source": "rewrite6",
@@ -36,8 +36,8 @@
       "tokens_per_second": 8533.957213384612,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 9536.972537950092,
-      "required_pages_per_second_upper_estimate": 0.003992931247006483
+      "required_token_positions_per_second": 9655.771643062684,
+      "required_pages_per_second_upper_estimate": 0.004042669951509708
     },
     {
       "source": "openmath",
@@ -50,8 +50,8 @@
       "tokens_per_second": 380144.35761751566,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 19073.945075900185,
-      "required_pages_per_second_upper_estimate": 0.003972599745552147
+      "required_token_positions_per_second": 19311.54328612537,
+      "required_pages_per_second_upper_estimate": 0.0040220851868558866
     },
     {
       "source": "ultra_qa",
@@ -64,8 +64,8 @@
       "tokens_per_second": 43742.16556775435,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28610.917613850273,
-      "required_pages_per_second_upper_estimate": 0.0017151808246086487
+      "required_token_positions_per_second": 28967.31492918805,
+      "required_pages_per_second_upper_estimate": 0.0017365462994759575
     },
     {
       "source": "ultra_style",
@@ -78,8 +78,8 @@
       "tokens_per_second": 22018.86276523721,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28610.917613850273,
-      "required_pages_per_second_upper_estimate": 0.002224372998350104
+      "required_token_positions_per_second": 28967.31492918805,
+      "required_pages_per_second_upper_estimate": 0.002252081321991498
     },
     {
       "source": "cortex",
@@ -92,8 +92,8 @@
       "tokens_per_second": 910315.184299418,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 23,
-      "required_token_positions_per_second": 28610.917613850273,
-      "required_pages_per_second_upper_estimate": 0.36699013114057377
+      "required_token_positions_per_second": 28967.31492918805,
+      "required_pages_per_second_upper_estimate": 0.3715616132321039
     }
   ],
   "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
@@ -158,111 +158,27 @@
     "prefetch_batches": 63,
     "prefetch_capacity_limit": 512,
     "batch_memory_estimate_bytes": 704566,
-    "producer_batch_seconds": 0.38075650120313753,
-    "producer_p95_seconds": 0.17078465200029314,
-    "gpu_update_seconds": 0.9663443994755297,
-    "producer_batches_per_second": 2.6263504282662,
-    "gpu_batches_per_second": 1.034827749343543,
-    "producer_headroom": 2.537959027415201,
+    "producer_batch_seconds": 0.18283066840626816,
+    "producer_p95_seconds": 0.22040481899966835,
+    "gpu_update_seconds": 0.9544550493405005,
+    "producer_batches_per_second": 5.469541892052264,
+    "gpu_batches_per_second": 1.0477182772420448,
+    "producer_headroom": 5.220431876448678,
     "sustainable": true,
-    "buffer_seconds": 60.87969716695837,
+    "buffer_seconds": 60.13066810845153,
     "buffer_drain_seconds": null,
     "shard_target_mib": 16,
-    "upload_pages_per_commit": 84,
+    "upload_pages_per_commit": 78,
     "max_pending_pages": 4096,
-    "generation_bytes_per_second": 391819.6533964042,
-    "upload_bytes_per_second": 5833319.346352694,
+    "generation_bytes_per_second": 476173.2566712523,
+    "upload_bytes_per_second": 5837253.21659938,
     "upload_sustainable": true,
     "logical_page_rows": 4096,
     "logical_cortex_page_rows": 128,
-    "estimated_full_shards_per_hour": 84.07537652415365,
+    "estimated_full_shards_per_hour": 102.17569613555123,
     "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
   },
   "upload_measurements": [
-    {
-      "seconds": 1.0425156310002421,
-      "bytes": 44934,
-      "pages": 1,
-      "shards": 1
-    },
-    {
-      "seconds": 1.7656013979999443,
-      "bytes": 11283965,
-      "pages": 34,
-      "shards": 4
-    },
-    {
-      "seconds": 1.4391079690003608,
-      "bytes": 12352688,
-      "pages": 39,
-      "shards": 3
-    },
-    {
-      "seconds": 0.9268819079998138,
-      "bytes": 1009442,
-      "pages": 28,
-      "shards": 1
-    },
-    {
-      "seconds": 1.2650270520002778,
-      "bytes": 7586230,
-      "pages": 36,
-      "shards": 3
-    },
-    {
-      "seconds": 1.2436492840001847,
-      "bytes": 10846371,
-      "pages": 34,
-      "shards": 3
-    },
-    {
-      "seconds": 0.9011205990000235,
-      "bytes": 2637170,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9096160310000414,
-      "bytes": 6003921,
-      "pages": 36,
-      "shards": 2
-    },
-    {
-      "seconds": 1.0415576699997473,
-      "bytes": 6858425,
-      "pages": 32,
-      "shards": 2
-    },
-    {
-      "seconds": 1.2630921189993387,
-      "bytes": 11470878,
-      "pages": 37,
-      "shards": 4
-    },
-    {
-      "seconds": 1.1407325299996955,
-      "bytes": 6174556,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9178262779996658,
-      "bytes": 2279800,
-      "pages": 27,
-      "shards": 2
-    },
-    {
-      "seconds": 1.0028891179999846,
-      "bytes": 5661960,
-      "pages": 32,
-      "shards": 2
-    },
-    {
-      "seconds": 1.3030792019999353,
-      "bytes": 10375056,
-      "pages": 39,
-      "shards": 3
-    },
     {
       "seconds": 1.011828892000267,
       "bytes": 8292186,
@@ -394,6 +310,258 @@
       "bytes": 1187716,
       "pages": 35,
       "shards": 1
+    },
+    {
+      "seconds": 1.3033395869997548,
+      "bytes": 10049773,
+      "pages": 30,
+      "shards": 3
+    },
+    {
+      "seconds": 0.8627702979993046,
+      "bytes": 2455057,
+      "pages": 28,
+      "shards": 2
+    },
+    {
+      "seconds": 1.2129529189996902,
+      "bytes": 11955081,
+      "pages": 34,
+      "shards": 3
+    },
+    {
+      "seconds": 0.95688545899975,
+      "bytes": 1097352,
+      "pages": 30,
+      "shards": 1
+    },
+    {
+      "seconds": 1.40065100099946,
+      "bytes": 12237786,
+      "pages": 38,
+      "shards": 4
+    },
+    {
+      "seconds": 1.0810250989998167,
+      "bytes": 5909598,
+      "pages": 28,
+      "shards": 2
+    },
+    {
+      "seconds": 0.9096889889988233,
+      "bytes": 1202765,
+      "pages": 34,
+      "shards": 1
+    },
+    {
+      "seconds": 1.2456794589998026,
+      "bytes": 7629808,
+      "pages": 38,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9217846150004334,
+      "bytes": 6311419,
+      "pages": 21,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1030007570007,
+      "bytes": 11547889,
+      "pages": 39,
+      "shards": 4
+    },
+    {
+      "seconds": 1.05019463699864,
+      "bytes": 6120696,
+      "pages": 37,
+      "shards": 2
+    },
+    {
+      "seconds": 1.070229210999969,
+      "bytes": 1151046,
+      "pages": 34,
+      "shards": 1
+    },
+    {
+      "seconds": 1.0475356750011997,
+      "bytes": 11473932,
+      "pages": 33,
+      "shards": 4
+    },
+    {
+      "seconds": 1.2914608989995031,
+      "bytes": 6033641,
+      "pages": 36,
+      "shards": 2
+    },
+    {
+      "seconds": 1.115209474999574,
+      "bytes": 812365,
+      "pages": 23,
+      "shards": 1
+    },
+    {
+      "seconds": 1.1693048819997784,
+      "bytes": 7948420,
+      "pages": 20,
+      "shards": 3
+    },
+    {
+      "seconds": 1.2017747059999238,
+      "bytes": 5823273,
+      "pages": 30,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1922360469998239,
+      "bytes": 10997251,
+      "pages": 39,
+      "shards": 3
+    },
+    {
+      "seconds": 1.0726386229998752,
+      "bytes": 2419938,
+      "pages": 28,
+      "shards": 2
+    },
+    {
+      "seconds": 1.240178467000078,
+      "bytes": 6103593,
+      "pages": 36,
+      "shards": 2
+    },
+    {
+      "seconds": 0.8772162379991642,
+      "bytes": 2610307,
+      "pages": 37,
+      "shards": 2
+    },
+    {
+      "seconds": 1.183507477000603,
+      "bytes": 15672429,
+      "pages": 34,
+      "shards": 4
+    },
+    {
+      "seconds": 1.1264393319997907,
+      "bytes": 6331952,
+      "pages": 36,
+      "shards": 2
+    },
+    {
+      "seconds": 0.856032101000892,
+      "bytes": 2311911,
+      "pages": 29,
+      "shards": 2
+    },
+    {
+      "seconds": 1.5313736190000782,
+      "bytes": 6009990,
+      "pages": 37,
+      "shards": 3
+    },
+    {
+      "seconds": 1.0109308320006676,
+      "bytes": 5084502,
+      "pages": 28,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1290311129996553,
+      "bytes": 7603249,
+      "pages": 38,
+      "shards": 3
+    },
+    {
+      "seconds": 1.4725222399993072,
+      "bytes": 12055069,
+      "pages": 36,
+      "shards": 3
+    },
+    {
+      "seconds": 1.2178526319985394,
+      "bytes": 7463726,
+      "pages": 32,
+      "shards": 3
+    },
+    {
+      "seconds": 1.3964880730000004,
+      "bytes": 6495644,
+      "pages": 37,
+      "shards": 2
+    },
+    {
+      "seconds": 1.2645150809985353,
+      "bytes": 6000960,
+      "pages": 35,
+      "shards": 2
+    },
+    {
+      "seconds": 0.9461197429991444,
+      "bytes": 2470605,
+      "pages": 29,
+      "shards": 2
+    },
+    {
+      "seconds": 1.2941537009992317,
+      "bytes": 6225550,
+      "pages": 37,
+      "shards": 2
+    },
+    {
+      "seconds": 1.2335469089994149,
+      "bytes": 11703737,
+      "pages": 35,
+      "shards": 3
+    },
+    {
+      "seconds": 1.1459508749994711,
+      "bytes": 7841640,
+      "pages": 34,
+      "shards": 3
+    },
+    {
+      "seconds": 1.0024716899988562,
+      "bytes": 5917383,
+      "pages": 36,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1789036200007104,
+      "bytes": 2313669,
+      "pages": 29,
+      "shards": 2
+    },
+    {
+      "seconds": 1.5398991070014745,
+      "bytes": 10300581,
+      "pages": 40,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9979469910012995,
+      "bytes": 6506963,
+      "pages": 36,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1795985310000106,
+      "bytes": 8204299,
+      "pages": 31,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9598508790004416,
+      "bytes": 6041026,
+      "pages": 37,
+      "shards": 2
+    },
+    {
+      "seconds": 0.9930732779994287,
+      "bytes": 6062259,
+      "pages": 32,
+      "shards": 3
     }
   ],
   "cache_key": {

trainer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:057bc6789cc9af9e9681c23890813cd87d445ef5153b156a985027190827e3c4
-size 721770431
+oid sha256:b09148f8bdb03fa1cdc8767b6fe281fcc58f0012b4c4869a74f3162a6d4c4a19
+size 721760383