Qwen90M step 610000; complete resumable checkpoint

complete.json

@@ -1,18 +1,18 @@
 {
-  "step": 605000,
+  "step": 610000,
   "files": {
     "config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7",
     "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4",
-    "model.safetensors": "ae98fb2eb852ed9b2ef3d507885819a6d2ff073c8d6b5c3f2962d07318d92067",
+    "model.safetensors": "8b25da00cb0b2a963e4cb081606d08666f7bba3d0bf0aee4e1acf0666e72cf45",
     "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03",
     "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300",
     "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263",
     "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6",
-    "trainer.pt": "0d3af9a8da6b443eadd810e3c1064dd4eef75f364569d9e4096a54e70b5227d9",
+    "trainer.pt": "478e3feddd5e715e333af4d01c7c276cb17b2faf46c01877e15a16c415e9bb60",
     "training_config.json": "6c3c3ccff89dabb5098a22bc207c2f3abaee8b3bffec3f6cf6ad361a63b1a7fb",
     "performance.json": "539995c1e4203016bfc6259363445c7c25dfb51604d74cd18d2cc7ad51bfb71e",
-    "shard_scheduler.json": "73644496d4fc18122195c127b0dde4b315912feb170d8757f56b71ce52b9030d",
-    "coverage.json": "2533a345a29ab4b098939b907ba6cdee33e969c8784659c7ca62523942ef7752",
+    "shard_scheduler.json": "d2dd9104d6ecd28878b172aff3036091ad951fcff01930fc538e5377b692dfb4",
+    "coverage.json": "fdc409abc87a90ebef70a4935f32045cc301b48f97054e4d2a9d47db721dea80",
     "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"
   },
   "signature": "33a459803c8cffa8929a90def998b4ad2c7c52fd1245365a9ee946b5552d845a"

coverage.json

@@ -1,107 +1,107 @@
 {
-  "step": 605000,
+  "step": 610000,
   "stats": {
-    "climbmix/blocks": 21780000,
-    "climbmix/targets": 36408254077,
-    "climbmix/input_tokens": 36410561797,
-    "ultra_qa/blocks": 8167500,
-    "ultra_qa/targets": 13324585723,
-    "ultra_qa/input_tokens": 13324611432,
-    "cortex/blocks": 8167500,
-    "cortex/targets": 310565399,
-    "cortex/input_tokens": 12982796500,
-    "ultra_style/blocks": 8167500,
-    "ultra_style/targets": 12918082747,
-    "ultra_style/input_tokens": 12918760755,
-    "openmath/blocks": 5445000,
-    "openmath/targets": 8413669599,
-    "openmath/input_tokens": 8416472178,
-    "rewrite6/blocks": 2722500,
-    "rewrite6/targets": 4463218274,
-    "rewrite6/input_tokens": 4468111496
+    "climbmix/blocks": 21960000,
+    "climbmix/targets": 36709591885,
+    "climbmix/input_tokens": 36711918536,
+    "ultra_qa/blocks": 8235000,
+    "ultra_qa/targets": 13434677003,
+    "ultra_qa/input_tokens": 13434702938,
+    "cortex/blocks": 8235000,
+    "cortex/targets": 313126567,
+    "cortex/input_tokens": 13090163320,
+    "ultra_style/blocks": 8235000,
+    "ultra_style/targets": 13025170521,
+    "ultra_style/input_tokens": 13025854292,
+    "openmath/blocks": 5490000,
+    "openmath/targets": 8483160554,
+    "openmath/input_tokens": 8485986310,
+    "rewrite6/blocks": 2745000,
+    "rewrite6/targets": 4500059266,
+    "rewrite6/input_tokens": 4504992845
   },
   "sources": {
     "climbmix": {
-      "page": 2983,
-      "row": 912,
+      "page": 3007,
+      "row": 2901,
       "start": {
-        "file": 144,
+        "file": 145,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 30,
+          "row_group": 44,
           "row": 1024
         }
       },
       "epoch": 0,
-      "offset": 6144,
-      "documents": 12219280
+      "offset": 0,
+      "documents": 12319573
     },
     "rewrite6": {
-      "page": 47,
-      "row": 282,
+      "page": 10,
+      "row": 3924,
       "start": {
         "file": 0,
         "reader": {
           "format": "jsonl-row-v1",
-          "row": 192512
+          "row": 40960
         }
       },
-      "epoch": 33,
+      "epoch": 34,
       "offset": 0,
-      "documents": 6971126
+      "documents": 7028620
     },
     "openmath": {
-      "page": 1863,
-      "row": 3962,
+      "page": 1879,
+      "row": 1544,
       "start": {
         "file": 17,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 207,
-          "row": 798
+          "row_group": 273,
+          "row": 334
         }
       },
       "epoch": 0,
       "offset": 0,
-      "documents": 7634810
+      "documents": 7697928
     },
     "ultra_qa": {
-      "page": 806,
-      "row": 91,
+      "page": 812,
+      "row": 2805,
       "start": {
         "file": 6,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 2,
-          "row": 54627
+          "row_group": 3,
+          "row": 14647
         }
       },
       "epoch": 0,
-      "offset": 6144,
-      "documents": 3301467
+      "offset": 2048,
+      "documents": 3328757
     },
     "ultra_style": {
-      "page": 1230,
-      "row": 1900,
+      "page": 1240,
+      "row": 2528,
       "start": {
         "file": 7,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 2,
-          "row": 72502
+          "row_group": 3,
+          "row": 27699
         }
       },
       "epoch": 0,
-      "offset": 2048,
-      "documents": 5039980
+      "offset": 0,
+      "documents": 5081568
     },
     "cortex": {
-      "page": 158401,
-      "row": 33,
+      "page": 159712,
+      "row": 53,
       "start": null,
       "epoch": 0,
       "offset": 0,
-      "documents": 20275361
+      "documents": 20443189
     }
   }
 }

latest.json

@@ -1 +1 @@
-{"step": 605000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "ae98fb2eb852ed9b2ef3d507885819a6d2ff073c8d6b5c3f2962d07318d92067", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "0d3af9a8da6b443eadd810e3c1064dd4eef75f364569d9e4096a54e70b5227d9", "training_config.json": "6c3c3ccff89dabb5098a22bc207c2f3abaee8b3bffec3f6cf6ad361a63b1a7fb", "performance.json": "539995c1e4203016bfc6259363445c7c25dfb51604d74cd18d2cc7ad51bfb71e", "shard_scheduler.json": "73644496d4fc18122195c127b0dde4b315912feb170d8757f56b71ce52b9030d", "coverage.json": "2533a345a29ab4b098939b907ba6cdee33e969c8784659c7ca62523942ef7752", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "33a459803c8cffa8929a90def998b4ad2c7c52fd1245365a9ee946b5552d845a", "folder": "step-000000605000"}
+{"step": 610000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "8b25da00cb0b2a963e4cb081606d08666f7bba3d0bf0aee4e1acf0666e72cf45", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "478e3feddd5e715e333af4d01c7c276cb17b2faf46c01877e15a16c415e9bb60", "training_config.json": "6c3c3ccff89dabb5098a22bc207c2f3abaee8b3bffec3f6cf6ad361a63b1a7fb", "performance.json": "539995c1e4203016bfc6259363445c7c25dfb51604d74cd18d2cc7ad51bfb71e", "shard_scheduler.json": "d2dd9104d6ecd28878b172aff3036091ad951fcff01930fc538e5377b692dfb4", "coverage.json": "fdc409abc87a90ebef70a4935f32045cc301b48f97054e4d2a9d47db721dea80", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "33a459803c8cffa8929a90def998b4ad2c7c52fd1245365a9ee946b5552d845a", "folder": "step-000000610000"}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae98fb2eb852ed9b2ef3d507885819a6d2ff073c8d6b5c3f2962d07318d92067
+oid sha256:8b25da00cb0b2a963e4cb081606d08666f7bba3d0bf0aee4e1acf0666e72cf45
 size 360622744

shard_scheduler.json

@@ -22,8 +22,8 @@
       "tokens_per_second": 598315.063954628,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 77247.77051665467,
-      "required_pages_per_second_upper_estimate": 0.005344180003864155
+      "required_token_positions_per_second": 77013.47401247967,
+      "required_pages_per_second_upper_estimate": 0.00532797082806254
     },
     {
       "source": "rewrite6",
@@ -36,8 +36,8 @@
       "tokens_per_second": 8533.957213384612,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 9655.971314581833,
-      "required_pages_per_second_upper_estimate": 0.00404275354980516
+      "required_token_positions_per_second": 9626.684251559958,
+      "required_pages_per_second_upper_estimate": 0.004030491668101323
     },
     {
       "source": "openmath",
@@ -50,8 +50,8 @@
       "tokens_per_second": 380144.35761751566,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 19311.942629163666,
-      "required_pages_per_second_upper_estimate": 0.00402216835947938
+      "required_token_positions_per_second": 19253.368503119917,
+      "required_pages_per_second_upper_estimate": 0.004009968913728047
     },
     {
       "source": "ultra_qa",
@@ -64,8 +64,8 @@
       "tokens_per_second": 43742.16556775435,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28967.9139437455,
-      "required_pages_per_second_upper_estimate": 0.0017365822094840345
+      "required_token_positions_per_second": 28880.052754679873,
+      "required_pages_per_second_upper_estimate": 0.0017313150653558177
     },
     {
       "source": "ultra_style",
@@ -78,8 +78,8 @@
       "tokens_per_second": 22018.86276523721,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28967.9139437455,
-      "required_pages_per_second_upper_estimate": 0.0022521278927385533
+      "required_token_positions_per_second": 28880.052754679873,
+      "required_pages_per_second_upper_estimate": 0.002245297071749229
     },
     {
       "source": "cortex",
@@ -92,8 +92,8 @@
       "tokens_per_second": 910315.184299418,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 23,
-      "required_token_positions_per_second": 28967.9139437455,
-      "required_pages_per_second_upper_estimate": 0.37156929674767514
+      "required_token_positions_per_second": 28880.052754679873,
+      "required_pages_per_second_upper_estimate": 0.37044230775233605
     }
   ],
   "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
@@ -158,24 +158,24 @@
     "prefetch_batches": 63,
     "prefetch_capacity_limit": 512,
     "batch_memory_estimate_bytes": 720750,
-    "producer_batch_seconds": 0.06336448393750427,
-    "producer_p95_seconds": 0.12817057600000226,
-    "gpu_update_seconds": 0.9544353125907264,
-    "producer_batches_per_second": 15.781711423489057,
-    "gpu_batches_per_second": 1.0477399429884802,
-    "producer_headroom": 15.062622675694417,
+    "producer_batch_seconds": 0.05319347438283728,
+    "producer_p95_seconds": 0.16769830500015814,
+    "gpu_update_seconds": 0.9573389714643016,
+    "producer_batches_per_second": 18.799298440311073,
+    "gpu_batches_per_second": 1.0445620932682247,
+    "producer_headroom": 17.997301033097852,
     "sustainable": true,
-    "buffer_seconds": 60.12942469321577,
+    "buffer_seconds": 60.312355202251,
     "buffer_drain_seconds": null,
     "shard_target_mib": 16,
-    "upload_pages_per_commit": 55,
+    "upload_pages_per_commit": 82,
     "max_pending_pages": 4096,
-    "generation_bytes_per_second": 374353.52534401446,
-    "upload_bytes_per_second": 3622099.0301886583,
+    "generation_bytes_per_second": 489298.1172003974,
+    "upload_bytes_per_second": 5419125.96073942,
     "upload_sustainable": true,
     "logical_page_rows": 4096,
     "logical_cortex_page_rows": 128,
-    "estimated_full_shards_per_hour": 80.32755203476262,
+    "estimated_full_shards_per_hour": 104.9919856739897,
     "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
   },
   "upload_measurements": [
@@ -214,6 +214,252 @@
       "bytes": 6002577,
       "pages": 28,
       "shards": 2
+    },
+    {
+      "seconds": 1.2785650519999763,
+      "bytes": 7789202,
+      "pages": 19,
+      "shards": 3
+    },
+    {
+      "seconds": 1.1231035699997847,
+      "bytes": 6063645,
+      "pages": 36,
+      "shards": 2
+    },
+    {
+      "seconds": 0.8495643409999047,
+      "bytes": 825792,
+      "pages": 27,
+      "shards": 1
+    },
+    {
+      "seconds": 1.5516937659999712,
+      "bytes": 10821331,
+      "pages": 34,
+      "shards": 4
+    },
+    {
+      "seconds": 1.0081327930001862,
+      "bytes": 1129039,
+      "pages": 34,
+      "shards": 1
+    },
+    {
+      "seconds": 1.1534891419998985,
+      "bytes": 7764027,
+      "pages": 37,
+      "shards": 3
+    },
+    {
+      "seconds": 1.4264037609998468,
+      "bytes": 6430937,
+      "pages": 21,
+      "shards": 2
+    },
+    {
+      "seconds": 1.233521140999983,
+      "bytes": 10999051,
+      "pages": 45,
+      "shards": 3
+    },
+    {
+      "seconds": 1.1022731230000318,
+      "bytes": 7378325,
+      "pages": 29,
+      "shards": 3
+    },
+    {
+      "seconds": 0.7817219419998764,
+      "bytes": 1157795,
+      "pages": 35,
+      "shards": 1
+    },
+    {
+      "seconds": 1.0034240229997522,
+      "bytes": 6082079,
+      "pages": 28,
+      "shards": 2
+    },
+    {
+      "seconds": 1.0907009550001021,
+      "bytes": 7267703,
+      "pages": 37,
+      "shards": 3
+    },
+    {
+      "seconds": 1.059233770999981,
+      "bytes": 6001877,
+      "pages": 28,
+      "shards": 2
+    },
+    {
+      "seconds": 1.2980492899996534,
+      "bytes": 11801808,
+      "pages": 36,
+      "shards": 3
+    },
+    {
+      "seconds": 1.0429953859998022,
+      "bytes": 2436103,
+      "pages": 28,
+      "shards": 2
+    },
+    {
+      "seconds": 1.247582260999934,
+      "bytes": 10464030,
+      "pages": 39,
+      "shards": 3
+    },
+    {
+      "seconds": 1.128213264000351,
+      "bytes": 2552181,
+      "pages": 37,
+      "shards": 2
+    },
+    {
+      "seconds": 1.4455456679997951,
+      "bytes": 5746642,
+      "pages": 27,
+      "shards": 2
+    },
+    {
+      "seconds": 0.88688044599985,
+      "bytes": 6234900,
+      "pages": 36,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1657635070000651,
+      "bytes": 12844748,
+      "pages": 34,
+      "shards": 5
+    },
+    {
+      "seconds": 1.0483822090000103,
+      "bytes": 6048562,
+      "pages": 36,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1226819749999777,
+      "bytes": 7652419,
+      "pages": 39,
+      "shards": 3
+    },
+    {
+      "seconds": 1.3251570170000377,
+      "bytes": 1162985,
+      "pages": 33,
+      "shards": 1
+    },
+    {
+      "seconds": 1.4578549020002356,
+      "bytes": 10422176,
+      "pages": 33,
+      "shards": 3
+    },
+    {
+      "seconds": 1.1834071509997557,
+      "bytes": 2468071,
+      "pages": 31,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1340931629997613,
+      "bytes": 11786170,
+      "pages": 37,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9738086980005392,
+      "bytes": 6055828,
+      "pages": 36,
+      "shards": 2
+    },
+    {
+      "seconds": 1.397671879999507,
+      "bytes": 6213114,
+      "pages": 33,
+      "shards": 3
+    },
+    {
+      "seconds": 1.0006439850003517,
+      "bytes": 6195776,
+      "pages": 35,
+      "shards": 2
+    },
+    {
+      "seconds": 1.0036822970005232,
+      "bytes": 2451346,
+      "pages": 32,
+      "shards": 2
+    },
+    {
+      "seconds": 1.0700342689997342,
+      "bytes": 10599218,
+      "pages": 33,
+      "shards": 3
+    },
+    {
+      "seconds": 1.020371795999381,
+      "bytes": 12084005,
+      "pages": 40,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9868787969999175,
+      "bytes": 2703362,
+      "pages": 37,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1407470789999934,
+      "bytes": 6061047,
+      "pages": 27,
+      "shards": 2
+    },
+    {
+      "seconds": 1.7620159190000777,
+      "bytes": 5407000,
+      "pages": 32,
+      "shards": 2
+    },
+    {
+      "seconds": 1.0307093159999567,
+      "bytes": 7655434,
+      "pages": 39,
+      "shards": 3
+    },
+    {
+      "seconds": 1.0102191360001598,
+      "bytes": 6169276,
+      "pages": 31,
+      "shards": 2
+    },
+    {
+      "seconds": 1.2401527040001383,
+      "bytes": 8401681,
+      "pages": 39,
+      "shards": 3
+    },
+    {
+      "seconds": 1.6231297679996715,
+      "bytes": 9980497,
+      "pages": 35,
+      "shards": 3
+    },
+    {
+      "seconds": 1.1220608109997556,
+      "bytes": 7908917,
+      "pages": 38,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9769788749999861,
+      "bytes": 875600,
+      "pages": 26,
+      "shards": 1
     }
   ],
   "cache_key": {

trainer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0d3af9a8da6b443eadd810e3c1064dd4eef75f364569d9e4096a54e70b5227d9
-size 721766335
+oid sha256:478e3feddd5e715e333af4d01c7c276cb17b2faf46c01877e15a16c415e9bb60
+size 721778367