Qwen90M step 5000; complete resumable checkpoint

complete.json

@@ -1,18 +1,18 @@
 {
-  "step": 0,
+  "step": 5000,
   "files": {
     "config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7",
     "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4",
-    "model.safetensors": "1c688d2ef10bd45dcb2ed3ca6945c4cc383ba4cc356d90bbb93b725e427338c1",
+    "model.safetensors": "c8aa1507ab6f726c5ffd1b20a36284bd30c32cd1cf5c419d73cbc66d98ed6afd",
     "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03",
     "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300",
     "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263",
     "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6",
-    "trainer.pt": "21aa2cccd888f3918a44b02c1ad179a74c9924c4304886988affaf97c4132422",
-    "training_config.json": "528161bcd9687da465595131ceb9bfb223ac3e494a1c6d5fcfdbd97b88324bd8",
+    "trainer.pt": "bc1fa5664111a090e684ef00bc5753f106bfd58076a5047bfda30955ab3d9b43",
+    "training_config.json": "f08d2b46b36e5a060b47b42699c2a03ce255f88d1589e7e6a18a4b35ab59c4c7",
     "performance.json": "b95a1befd575f8d9a02b43fd0cc41c54243f5297127b93d2656e0292ba15e77e",
-    "shard_scheduler.json": "c16912023ae706fc358e0de73ec412ec549d6d440abc128e7900e41c5c63a6f3",
-    "coverage.json": "1db676f88d72dde495669f8f42a2a0104a6dc4ec5bfa810967263316d2335d79",
+    "shard_scheduler.json": "fe416c1fb82665d94c94be2f3fbdb92516402dd5bee1d4df92808e6efd9f97ab",
+    "coverage.json": "6a692152610fe0c0340ddcdb7db228594b1270a850305aaeacbb81308e2c0c68",
     "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"
   },
   "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063"

coverage.json

@@ -1,54 +1,107 @@
 {
-  "step": 0,
-  "stats": {},
+  "step": 5000,
+  "stats": {
+    "climbmix/blocks": 180000,
+    "climbmix/targets": 300287575,
+    "climbmix/input_tokens": 300306959,
+    "ultra_qa/blocks": 67500,
+    "ultra_qa/targets": 110133457,
+    "ultra_qa/input_tokens": 110133662,
+    "cortex/blocks": 67500,
+    "cortex/targets": 2571669,
+    "cortex/input_tokens": 107334831,
+    "ultra_style/blocks": 67500,
+    "ultra_style/targets": 106887961,
+    "ultra_style/input_tokens": 106893821,
+    "openmath/blocks": 45000,
+    "openmath/targets": 69552253,
+    "openmath/input_tokens": 69575649,
+    "rewrite6/blocks": 22500,
+    "rewrite6/targets": 36855636,
+    "rewrite6/input_tokens": 36896125
+  },
   "sources": {
     "climbmix": {
-      "page": 0,
-      "row": 0,
-      "start": null,
+      "page": 24,
+      "row": 3774,
+      "start": {
+        "file": 1,
+        "reader": {
+          "format": "parquet-row-v1",
+          "row_group": 11,
+          "row": 1024
+        }
+      },
       "epoch": 0,
-      "offset": 0,
-      "documents": 0
+      "offset": 2048,
+      "documents": 102078
     },
     "rewrite6": {
-      "page": 0,
-      "row": 0,
-      "start": null,
+      "page": 14,
+      "row": 277,
+      "start": {
+        "file": 0,
+        "reader": {
+          "format": "jsonl-row-v1",
+          "row": 57344
+        }
+      },
       "epoch": 0,
       "offset": 0,
-      "documents": 0
+      "documents": 57621
     },
     "openmath": {
-      "page": 0,
-      "row": 0,
-      "start": null,
+      "page": 15,
+      "row": 1897,
+      "start": {
+        "file": 0,
+        "reader": {
+          "format": "parquet-row-v1",
+          "row_group": 61,
+          "row": 440
+        }
+      },
       "epoch": 0,
       "offset": 0,
-      "documents": 0
+      "documents": 63337
     },
     "ultra_qa": {
-      "page": 0,
-      "row": 0,
-      "start": null,
+      "page": 6,
+      "row": 2707,
+      "start": {
+        "file": 0,
+        "reader": {
+          "format": "parquet-row-v1",
+          "row_group": 0,
+          "row": 24576
+        }
+      },
       "epoch": 0,
       "offset": 0,
-      "documents": 0
+      "documents": 27283
     },
     "ultra_style": {
-      "page": 0,
-      "row": 0,
-      "start": null,
+      "page": 10,
+      "row": 1000,
+      "start": {
+        "file": 0,
+        "reader": {
+          "format": "parquet-row-v1",
+          "row_group": 0,
+          "row": 40960
+        }
+      },
       "epoch": 0,
-      "offset": 0,
-      "documents": 0
+      "offset": 2048,
+      "documents": 41960
     },
     "cortex": {
-      "page": 0,
-      "row": 0,
+      "page": 1309,
+      "row": 72,
       "start": null,
       "epoch": 0,
       "offset": 0,
-      "documents": 0
+      "documents": 167624
     }
   }
 }

latest.json

@@ -1 +1 @@
-{"step": 0, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "1c688d2ef10bd45dcb2ed3ca6945c4cc383ba4cc356d90bbb93b725e427338c1", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "21aa2cccd888f3918a44b02c1ad179a74c9924c4304886988affaf97c4132422", "training_config.json": "528161bcd9687da465595131ceb9bfb223ac3e494a1c6d5fcfdbd97b88324bd8", "performance.json": "b95a1befd575f8d9a02b43fd0cc41c54243f5297127b93d2656e0292ba15e77e", "shard_scheduler.json": "c16912023ae706fc358e0de73ec412ec549d6d440abc128e7900e41c5c63a6f3", "coverage.json": "1db676f88d72dde495669f8f42a2a0104a6dc4ec5bfa810967263316d2335d79", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000000000"}
+{"step": 5000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "c8aa1507ab6f726c5ffd1b20a36284bd30c32cd1cf5c419d73cbc66d98ed6afd", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "bc1fa5664111a090e684ef00bc5753f106bfd58076a5047bfda30955ab3d9b43", "training_config.json": "f08d2b46b36e5a060b47b42699c2a03ce255f88d1589e7e6a18a4b35ab59c4c7", "performance.json": "b95a1befd575f8d9a02b43fd0cc41c54243f5297127b93d2656e0292ba15e77e", "shard_scheduler.json": "fe416c1fb82665d94c94be2f3fbdb92516402dd5bee1d4df92808e6efd9f97ab", "coverage.json": "6a692152610fe0c0340ddcdb7db228594b1270a850305aaeacbb81308e2c0c68", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000005000"}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c688d2ef10bd45dcb2ed3ca6945c4cc383ba4cc356d90bbb93b725e427338c1
+oid sha256:c8aa1507ab6f726c5ffd1b20a36284bd30c32cd1cf5c419d73cbc66d98ed6afd
 size 360622744

shard_scheduler.json

@@ -22,8 +22,8 @@
       "tokens_per_second": 1062967.6168812835,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 78853.96613762408,
-      "required_pages_per_second_upper_estimate": 0.00683486193125923
+      "required_token_positions_per_second": 74279.87317241088,
+      "required_pages_per_second_upper_estimate": 0.006438391145967182
     },
     {
       "source": "rewrite6",
@@ -36,8 +36,8 @@
       "tokens_per_second": 1206672.263377472,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 9856.74576720301,
-      "required_pages_per_second_upper_estimate": 0.003932003258019391
+      "required_token_positions_per_second": 9284.98414655136,
+      "required_pages_per_second_upper_estimate": 0.003703918998943418
     },
     {
       "source": "openmath",
@@ -50,8 +50,8 @@
       "tokens_per_second": 410042.7581134458,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 19713.49153440602,
-      "required_pages_per_second_upper_estimate": 0.004560369320883491
+      "required_token_positions_per_second": 18569.96829310272,
+      "required_pages_per_second_upper_estimate": 0.004295835344329655
     },
     {
       "source": "ultra_qa",
@@ -64,8 +64,8 @@
       "tokens_per_second": 2163486.5624068812,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 29570.23730160903,
-      "required_pages_per_second_upper_estimate": 0.0018006723086229548
+      "required_token_positions_per_second": 27854.95243965408,
+      "required_pages_per_second_upper_estimate": 0.0016962204599340583
     },
     {
       "source": "ultra_style",
@@ -78,8 +78,8 @@
       "tokens_per_second": 1546694.2737633225,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 29570.23730160903,
-      "required_pages_per_second_upper_estimate": 0.0022763987231337094
+      "required_token_positions_per_second": 27854.95243965408,
+      "required_pages_per_second_upper_estimate": 0.00214435134624802
     },
     {
       "source": "cortex",
@@ -91,9 +91,9 @@
       "read_seconds": 0.042780299000014566,
       "tokens_per_second": 1015177.8395898147,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
-      "estimated_pages_for_buffer": 23,
-      "required_token_positions_per_second": 29570.23730160903,
-      "required_pages_per_second_upper_estimate": 0.36716338206798155
+      "estimated_pages_for_buffer": 21,
+      "required_token_positions_per_second": 27854.95243965408,
+      "required_pages_per_second_upper_estimate": 0.3458652847716463
     }
   ],
   "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
@@ -155,27 +155,27 @@
   "shard_benchmark_scope": "Local packing/read measurements; upload capacity measured only on real data commits. Targets larger than available pages are sample-limited.",
   "estimated_fresh_generation_batch_seconds": 0.1434574083060457,
   "selected": {
-    "prefetch_batches": 65,
+    "prefetch_batches": 61,
     "prefetch_capacity_limit": 512,
     "batch_memory_estimate_bytes": 688946,
-    "producer_batch_seconds": 2.3582960475177965,
-    "producer_p95_seconds": 17.862166522000052,
-    "gpu_update_seconds": 0.9349941874999956,
-    "producer_batches_per_second": 0.4240349726458394,
-    "gpu_batches_per_second": 1.06952536536491,
-    "producer_headroom": 0.39647023472057946,
-    "sustainable": false,
-    "buffer_seconds": 60.774622187499716,
-    "buffer_drain_seconds": 100.69863274989008,
-    "shard_target_mib": 32,
-    "upload_pages_per_commit": 15,
-    "max_pending_pages": 1842,
-    "generation_bytes_per_second": 1035119.2626246791,
-    "upload_bytes_per_second": 6146691.738378885,
+    "producer_batch_seconds": 0.2317790046171737,
+    "producer_p95_seconds": 0.17695051500049885,
+    "gpu_update_seconds": 0.992570353867865,
+    "producer_batches_per_second": 4.314454631694042,
+    "gpu_batches_per_second": 1.007485258957396,
+    "producer_headroom": 4.2823997605274045,
+    "sustainable": true,
+    "buffer_seconds": 60.546791585939765,
+    "buffer_drain_seconds": null,
+    "shard_target_mib": 16,
+    "upload_pages_per_commit": 79,
+    "max_pending_pages": 4096,
+    "generation_bytes_per_second": 328113.0883383849,
+    "upload_bytes_per_second": 4268388.328580749,
     "upload_sustainable": true,
     "logical_page_rows": 4096,
     "logical_cortex_page_rows": 128,
-    "estimated_full_shards_per_hour": 111.0562487080349,
+    "estimated_full_shards_per_hour": 70.40543067563688,
     "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
   },
   "upload_measurements": [
@@ -184,6 +184,264 @@
       "bytes": 5047481,
       "pages": 1,
       "shards": 1
+    },
+    {
+      "seconds": 2.1621551530000716,
+      "bytes": 13914829,
+      "pages": 15,
+      "shards": 5
+    },
+    {
+      "seconds": 0.7754683500000965,
+      "bytes": 204111,
+      "pages": 6,
+      "shards": 1
+    },
+    {
+      "seconds": 1.1240084219999744,
+      "bytes": 42370,
+      "pages": 1,
+      "shards": 1
+    },
+    {
+      "seconds": 1.1000538829999869,
+      "bytes": 494707,
+      "pages": 15,
+      "shards": 1
+    },
+    {
+      "seconds": 0.9781999550000364,
+      "bytes": 5863999,
+      "pages": 27,
+      "shards": 2
+    },
+    {
+      "seconds": 0.9417934939999668,
+      "bytes": 2584042,
+      "pages": 34,
+      "shards": 2
+    },
+    {
+      "seconds": 0.966999364000003,
+      "bytes": 6885855,
+      "pages": 36,
+      "shards": 3
+    },
+    {
+      "seconds": 1.0705818949998047,
+      "bytes": 10769730,
+      "pages": 37,
+      "shards": 3
+    },
+    {
+      "seconds": 1.2535849739999776,
+      "bytes": 3640398,
+      "pages": 33,
+      "shards": 3
+    },
+    {
+      "seconds": 1.3395652879999034,
+      "bytes": 11637618,
+      "pages": 33,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9964154829999643,
+      "bytes": 2622281,
+      "pages": 35,
+      "shards": 2
+    },
+    {
+      "seconds": 1.0029198830000041,
+      "bytes": 11222438,
+      "pages": 33,
+      "shards": 4
+    },
+    {
+      "seconds": 1.2415246609998576,
+      "bytes": 5982796,
+      "pages": 34,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1530046430000311,
+      "bytes": 2613544,
+      "pages": 34,
+      "shards": 2
+    },
+    {
+      "seconds": 22.672299763999945,
+      "bytes": 7363447,
+      "pages": 38,
+      "shards": 3
+    },
+    {
+      "seconds": 1.0496788179998475,
+      "bytes": 15965043,
+      "pages": 34,
+      "shards": 4
+    },
+    {
+      "seconds": 0.8725824939997437,
+      "bytes": 2480897,
+      "pages": 35,
+      "shards": 2
+    },
+    {
+      "seconds": 1.156973391000065,
+      "bytes": 7369438,
+      "pages": 38,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9484031940000932,
+      "bytes": 2544896,
+      "pages": 35,
+      "shards": 2
+    },
+    {
+      "seconds": 1.0683630309999899,
+      "bytes": 9860015,
+      "pages": 31,
+      "shards": 3
+    },
+    {
+      "seconds": 0.8969114139999874,
+      "bytes": 7503017,
+      "pages": 38,
+      "shards": 3
+    },
+    {
+      "seconds": 0.8951566169998841,
+      "bytes": 8341517,
+      "pages": 32,
+      "shards": 3
+    },
+    {
+      "seconds": 1.104310605000137,
+      "bytes": 5895784,
+      "pages": 34,
+      "shards": 2
+    },
+    {
+      "seconds": 1.2573788659997263,
+      "bytes": 8080667,
+      "pages": 37,
+      "shards": 4
+    },
+    {
+      "seconds": 0.9144526599998244,
+      "bytes": 5532432,
+      "pages": 26,
+      "shards": 2
+    },
+    {
+      "seconds": 0.9253262060001362,
+      "bytes": 5975019,
+      "pages": 35,
+      "shards": 2
+    },
+    {
+      "seconds": 0.8811729529998047,
+      "bytes": 3757694,
+      "pages": 36,
+      "shards": 3
+    },
+    {
+      "seconds": 1.5062500680000994,
+      "bytes": 15040116,
+      "pages": 34,
+      "shards": 4
+    },
+    {
+      "seconds": 1.0638150449999557,
+      "bytes": 7164962,
+      "pages": 39,
+      "shards": 3
+    },
+    {
+      "seconds": 1.0547767480002221,
+      "bytes": 2359076,
+      "pages": 27,
+      "shards": 2
+    },
+    {
+      "seconds": 1.027329577999808,
+      "bytes": 11020380,
+      "pages": 34,
+      "shards": 3
+    },
+    {
+      "seconds": 1.2178119280006285,
+      "bytes": 3672261,
+      "pages": 37,
+      "shards": 3
+    },
+    {
+      "seconds": 1.0616448490000039,
+      "bytes": 5911843,
+      "pages": 30,
+      "shards": 2
+    },
+    {
+      "seconds": 1.2120231849994525,
+      "bytes": 11658602,
+      "pages": 33,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9783495700003186,
+      "bytes": 7342912,
+      "pages": 40,
+      "shards": 4
+    },
+    {
+      "seconds": 0.9914971539992621,
+      "bytes": 6238955,
+      "pages": 35,
+      "shards": 2
+    },
+    {
+      "seconds": 0.9600033409997195,
+      "bytes": 1092240,
+      "pages": 31,
+      "shards": 1
+    },
+    {
+      "seconds": 0.9772284549999313,
+      "bytes": 8526326,
+      "pages": 38,
+      "shards": 4
+    },
+    {
+      "seconds": 1.260473302999344,
+      "bytes": 10994776,
+      "pages": 35,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9115331740003967,
+      "bytes": 8005286,
+      "pages": 27,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9674303089996101,
+      "bytes": 7444249,
+      "pages": 41,
+      "shards": 3
+    },
+    {
+      "seconds": 0.8210835840000072,
+      "bytes": 900296,
+      "pages": 25,
+      "shards": 1
+    },
+    {
+      "seconds": 1.093376490999617,
+      "bytes": 11563576,
+      "pages": 39,
+      "shards": 4
     }
   ],
   "cache_key": {

trainer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21aa2cccd888f3918a44b02c1ad179a74c9924c4304886988affaf97c4132422
-size 340197
+oid sha256:bc1fa5664111a090e684ef00bc5753f106bfd58076a5047bfda30955ab3d9b43
+size 721764031

training_config.json

@@ -46,8 +46,8 @@
   "cpu_threads": 4,
   "page_rows": 4096,
   "cortex_page_rows": 128,
-  "prefetch_batches": 65,
-  "prefetch_low_water": 32,
+  "prefetch_batches": 61,
+  "prefetch_low_water": 30,
   "memory_pages": 12,
   "cache_gb": 20,
   "force_rebenchmark": false,