Qwen90M step 557164; complete resumable checkpoint

complete.json

@@ -1,18 +1,18 @@
 {
-  "step": 555000,
+  "step": 557164,
   "files": {
     "config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7",
     "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4",
-    "model.safetensors": "bcae9326cfbbc3209e19608c2763a0085cdbfb1dc180d042030dade3f1ff469b",
+    "model.safetensors": "9aeb84d48b1a3e7876ab13cf2a9db637ae3eea5a8e14c759a56dfd289a32ce86",
     "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03",
     "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300",
     "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263",
     "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6",
-    "trainer.pt": "4c3e5c3949b8f4716b37f94e501fd3bf9bfc15652bfd259b8ceeb75ba134374a",
+    "trainer.pt": "625680f99c72ef12793fea5968b6c72147fca5bf0986649c268a6a0699e0b367",
     "training_config.json": "c8bdf052510c2d9092e5a6824b119dd6584dc7697f8c4a8e9b7ecf3057c5bbb6",
     "performance.json": "3070ebbdf9b2b0fd6a0a237819043aa2c2a35c74a97edfa223e5c324471af4af",
-    "shard_scheduler.json": "80a4f3be32aeb96381dd50c3cd54fbb71414db41f241a6d17b6f8416b766ca04",
-    "coverage.json": "16acad3ee0fe0340416767bf5ff3a94ee49c9a3a058f03397d274c129f8e1bcf",
+    "shard_scheduler.json": "21879a49824847fe89e795ae96e1fd2dc476cf1541cf6501892fb7519e8c9036",
+    "coverage.json": "73df3c9242608e741c41ad8c77453ef20d1273e9c95e5fe1751e37fda2234e71",
     "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"
   },
   "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063"

coverage.json

@@ -1,107 +1,107 @@
 {
-  "step": 555000,
+  "step": 557164,
   "stats": {
-    "climbmix/blocks": 19980000,
-    "climbmix/targets": 33398363172,
-    "climbmix/input_tokens": 33400479609,
-    "ultra_qa/blocks": 7492500,
-    "ultra_qa/targets": 12223124541,
-    "ultra_qa/input_tokens": 12223148113,
-    "cortex/blocks": 7492500,
-    "cortex/targets": 284926292,
-    "cortex/input_tokens": 11909982208,
-    "ultra_style/blocks": 7492500,
-    "ultra_style/targets": 11850383023,
-    "ultra_style/input_tokens": 11851004388,
-    "openmath/blocks": 4995000,
-    "openmath/targets": 7718088789,
-    "openmath/input_tokens": 7720659344,
-    "rewrite6/blocks": 2497500,
-    "rewrite6/targets": 4094322151,
-    "rewrite6/input_tokens": 4098811046
+    "climbmix/blocks": 20057904,
+    "climbmix/targets": 33528559976,
+    "climbmix/input_tokens": 33530684873,
+    "ultra_qa/blocks": 7521714,
+    "ultra_qa/targets": 12270782181,
+    "ultra_qa/input_tokens": 12270805843,
+    "cortex/blocks": 7521714,
+    "cortex/targets": 286034884,
+    "cortex/input_tokens": 11956373662,
+    "ultra_style/blocks": 7521714,
+    "ultra_style/targets": 11896892314,
+    "ultra_style/input_tokens": 11897516222,
+    "openmath/blocks": 5014476,
+    "openmath/targets": 7748188395,
+    "openmath/input_tokens": 7750768917,
+    "rewrite6/blocks": 2507238,
+    "rewrite6/targets": 4110276401,
+    "rewrite6/input_tokens": 4114782547
   },
   "sources": {
     "climbmix": {
-      "page": 2736,
-      "row": 2897,
+      "page": 2747,
+      "row": 2075,
       "start": {
         "file": 132,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 33,
+          "row_group": 77,
           "row": 1024
         }
       },
       "epoch": 0,
-      "offset": 10240,
-      "documents": 11209553
+      "offset": 2048,
+      "documents": 11253787
     },
     "rewrite6": {
-      "page": 6,
-      "row": 2921,
+      "page": 12,
+      "row": 3016,
       "start": {
         "file": 0,
         "reader": {
           "format": "jsonl-row-v1",
-          "row": 24576
+          "row": 49152
         }
       },
       "epoch": 31,
       "offset": 0,
-      "documents": 6395021
+      "documents": 6419692
     },
     "openmath": {
-      "page": 1709,
-      "row": 3426,
+      "page": 1716,
+      "row": 2073,
       "start": {
         "file": 16,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 13,
-          "row": 664
+          "row_group": 42,
+          "row": 336
         }
       },
       "epoch": 0,
-      "offset": 2048,
-      "documents": 7003490
+      "offset": 0,
+      "documents": 7030809
     },
     "ultra_qa": {
-      "page": 739,
-      "row": 1676,
+      "page": 742,
+      "row": 1162,
       "start": {
         "file": 5,
         "reader": {
           "format": "parquet-row-v1",
           "row_group": 6,
-          "row": 41706
+          "row": 53994
         }
       },
       "epoch": 0,
       "offset": 2048,
-      "documents": 3028620
+      "documents": 3040394
     },
     "ultra_style": {
-      "page": 1128,
-      "row": 2292,
+      "page": 1132,
+      "row": 4042,
       "start": {
         "file": 6,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 5,
-          "row": 84382
+          "row_group": 6,
+          "row": 15848
         }
       },
       "epoch": 0,
-      "offset": 2048,
-      "documents": 4622580
+      "offset": 0,
+      "documents": 4640714
     },
     "cortex": {
-      "page": 145310,
-      "row": 26,
+      "page": 145876,
+      "row": 97,
       "start": null,
       "epoch": 0,
       "offset": 0,
-      "documents": 18599706
+      "documents": 18672225
     }
   }
 }

latest.json

@@ -1 +1 @@
-{"step": 555000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "bcae9326cfbbc3209e19608c2763a0085cdbfb1dc180d042030dade3f1ff469b", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "4c3e5c3949b8f4716b37f94e501fd3bf9bfc15652bfd259b8ceeb75ba134374a", "training_config.json": "c8bdf052510c2d9092e5a6824b119dd6584dc7697f8c4a8e9b7ecf3057c5bbb6", "performance.json": "3070ebbdf9b2b0fd6a0a237819043aa2c2a35c74a97edfa223e5c324471af4af", "shard_scheduler.json": "80a4f3be32aeb96381dd50c3cd54fbb71414db41f241a6d17b6f8416b766ca04", "coverage.json": "16acad3ee0fe0340416767bf5ff3a94ee49c9a3a058f03397d274c129f8e1bcf", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000555000"}
+{"step": 557164, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "9aeb84d48b1a3e7876ab13cf2a9db637ae3eea5a8e14c759a56dfd289a32ce86", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "625680f99c72ef12793fea5968b6c72147fca5bf0986649c268a6a0699e0b367", "training_config.json": "c8bdf052510c2d9092e5a6824b119dd6584dc7697f8c4a8e9b7ecf3057c5bbb6", "performance.json": "3070ebbdf9b2b0fd6a0a237819043aa2c2a35c74a97edfa223e5c324471af4af", "shard_scheduler.json": "21879a49824847fe89e795ae96e1fd2dc476cf1541cf6501892fb7519e8c9036", "coverage.json": "73df3c9242608e741c41ad8c77453ef20d1273e9c95e5fe1751e37fda2234e71", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000557164"}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bcae9326cfbbc3209e19608c2763a0085cdbfb1dc180d042030dade3f1ff469b
+oid sha256:9aeb84d48b1a3e7876ab13cf2a9db637ae3eea5a8e14c759a56dfd289a32ce86
 size 360622744

shard_scheduler.json

@@ -22,8 +22,8 @@
       "tokens_per_second": 598315.063954628,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 77005.1087248329,
-      "required_pages_per_second_upper_estimate": 0.005327392098052995
+      "required_token_positions_per_second": 76613.12993076043,
+      "required_pages_per_second_upper_estimate": 0.005300274095562953
     },
     {
       "source": "rewrite6",
@@ -36,8 +36,8 @@
       "tokens_per_second": 8533.957213384612,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 9625.638590604112,
-      "required_pages_per_second_upper_estimate": 0.004030053871694994
+      "required_token_positions_per_second": 9576.641241345054,
+      "required_pages_per_second_upper_estimate": 0.004009539704741229
     },
     {
       "source": "openmath",
@@ -50,8 +50,8 @@
       "tokens_per_second": 380144.35761751566,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 19251.277181208225,
-      "required_pages_per_second_upper_estimate": 0.004009533346525709
+      "required_token_positions_per_second": 19153.28248269011,
+      "required_pages_per_second_upper_estimate": 0.003989123635118372
     },
     {
       "source": "ultra_qa",
@@ -64,8 +64,8 @@
       "tokens_per_second": 43742.16556775435,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28876.915771812335,
-      "required_pages_per_second_upper_estimate": 0.0017311270080228043
+      "required_token_positions_per_second": 28729.923724035165,
+      "required_pages_per_second_upper_estimate": 0.0017223150592024243
     },
     {
       "source": "ultra_style",
@@ -78,8 +78,8 @@
       "tokens_per_second": 22018.86276523721,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28876.915771812335,
-      "required_pages_per_second_upper_estimate": 0.0022450531851294074
+      "required_token_positions_per_second": 28729.923724035165,
+      "required_pages_per_second_upper_estimate": 0.002233625199964421
     },
     {
       "source": "cortex",
@@ -92,8 +92,8 @@
       "tokens_per_second": 910315.184299418,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 23,
-      "required_token_positions_per_second": 28876.915771812335,
-      "required_pages_per_second_upper_estimate": 0.3704020699043411
+      "required_token_positions_per_second": 28729.923724035165,
+      "required_pages_per_second_upper_estimate": 0.36851661374321987
     }
   ],
   "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
@@ -158,129 +158,27 @@
     "prefetch_batches": 63,
     "prefetch_capacity_limit": 512,
     "batch_memory_estimate_bytes": 704566,
-    "producer_batch_seconds": 0.09249245312517473,
-    "producer_p95_seconds": 0.272625303001405,
-    "gpu_update_seconds": 0.9574429699652371,
-    "producer_batches_per_second": 10.811692913438561,
-    "gpu_batches_per_second": 1.0444486317929809,
-    "producer_headroom": 10.351579373394724,
+    "producer_batch_seconds": 0.09595589186756115,
+    "producer_p95_seconds": 0.29021746900252765,
+    "gpu_update_seconds": 0.9623415733912987,
+    "producer_batches_per_second": 10.421454905345525,
+    "gpu_batches_per_second": 1.0391320791390033,
+    "producer_headroom": 10.028999310636681,
     "sustainable": true,
-    "buffer_seconds": 60.31890710780994,
+    "buffer_seconds": 60.627519123651815,
     "buffer_drain_seconds": null,
     "shard_target_mib": 16,
-    "upload_pages_per_commit": 79,
+    "upload_pages_per_commit": 82,
     "max_pending_pages": 4096,
-    "generation_bytes_per_second": 264013.5722249477,
-    "upload_bytes_per_second": 3976907.631126424,
+    "generation_bytes_per_second": 370915.98441075865,
+    "upload_bytes_per_second": 5961453.864962328,
     "upload_sustainable": true,
     "logical_page_rows": 4096,
     "logical_cortex_page_rows": 128,
-    "estimated_full_shards_per_hour": 56.651166678059795,
+    "estimated_full_shards_per_hour": 79.58993577234335,
     "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
   },
   "upload_measurements": [
-    {
-      "seconds": 1.2181368159981503,
-      "bytes": 12003212,
-      "pages": 41,
-      "shards": 4
-    },
-    {
-      "seconds": 0.9120353730031638,
-      "bytes": 952497,
-      "pages": 31,
-      "shards": 1
-    },
-    {
-      "seconds": 1.6672031760026584,
-      "bytes": 13248105,
-      "pages": 37,
-      "shards": 4
-    },
-    {
-      "seconds": 1.4338108570009354,
-      "bytes": 5962058,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 1.2595222179988923,
-      "bytes": 5231936,
-      "pages": 41,
-      "shards": 2
-    },
-    {
-      "seconds": 1.146318651000911,
-      "bytes": 7213324,
-      "pages": 30,
-      "shards": 3
-    },
-    {
-      "seconds": 1.1155308010020235,
-      "bytes": 6292434,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 1.3688695919991005,
-      "bytes": 12940523,
-      "pages": 39,
-      "shards": 4
-    },
-    {
-      "seconds": 1.308492041000136,
-      "bytes": 6108009,
-      "pages": 32,
-      "shards": 2
-    },
-    {
-      "seconds": 0.922122173000389,
-      "bytes": 1095644,
-      "pages": 31,
-      "shards": 1
-    },
-    {
-      "seconds": 1.0743960049985617,
-      "bytes": 7509044,
-      "pages": 39,
-      "shards": 3
-    },
-    {
-      "seconds": 1.3776889159998973,
-      "bytes": 10102958,
-      "pages": 36,
-      "shards": 3
-    },
-    {
-      "seconds": 4.997759090001637,
-      "bytes": 838514,
-      "pages": 26,
-      "shards": 1
-    },
-    {
-      "seconds": 34.2643265609986,
-      "bytes": 13380891,
-      "pages": 44,
-      "shards": 4
-    },
-    {
-      "seconds": 1.1768819699973392,
-      "bytes": 10254247,
-      "pages": 45,
-      "shards": 3
-    },
-    {
-      "seconds": 2.0107650110003306,
-      "bytes": 2402302,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 1.039769940001861,
-      "bytes": 6252309,
-      "pages": 38,
-      "shards": 2
-    },
     {
       "seconds": 1.272125319999759,
       "bytes": 7866146,
@@ -562,6 +460,108 @@
       "bytes": 5775284,
       "pages": 29,
       "shards": 2
+    },
+    {
+      "seconds": 1.2868636950006476,
+      "bytes": 7532566,
+      "pages": 40,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9102224750022287,
+      "bytes": 989956,
+      "pages": 26,
+      "shards": 1
+    },
+    {
+      "seconds": 1.172385066998686,
+      "bytes": 10249964,
+      "pages": 40,
+      "shards": 3
+    },
+    {
+      "seconds": 1.0767950209992705,
+      "bytes": 2405647,
+      "pages": 29,
+      "shards": 2
+    },
+    {
+      "seconds": 1.3306630199986103,
+      "bytes": 11933752,
+      "pages": 33,
+      "shards": 3
+    },
+    {
+      "seconds": 1.1605643660004716,
+      "bytes": 6187685,
+      "pages": 38,
+      "shards": 2
+    },
+    {
+      "seconds": 1.1438976530007494,
+      "bytes": 6974681,
+      "pages": 33,
+      "shards": 3
+    },
+    {
+      "seconds": 0.9301952720015834,
+      "bytes": 6018270,
+      "pages": 37,
+      "shards": 2
+    },
+    {
+      "seconds": 1.0365287800013903,
+      "bytes": 2455252,
+      "pages": 29,
+      "shards": 2
+    },
+    {
+      "seconds": 1.3521412089976366,
+      "bytes": 6109192,
+      "pages": 37,
+      "shards": 2
+    },
+    {
+      "seconds": 1.5889534400012053,
+      "bytes": 16077810,
+      "pages": 39,
+      "shards": 4
+    },
+    {
+      "seconds": 0.8900750899993,
+      "bytes": 2373140,
+      "pages": 29,
+      "shards": 2
+    },
+    {
+      "seconds": 1.2026867530003074,
+      "bytes": 6250314,
+      "pages": 37,
+      "shards": 2
+    },
+    {
+      "seconds": 1.0975493779988028,
+      "bytes": 5901759,
+      "pages": 32,
+      "shards": 2
+    },
+    {
+      "seconds": 1.0364178640011232,
+      "bytes": 6898240,
+      "pages": 40,
+      "shards": 3
+    },
+    {
+      "seconds": 1.1201825170028314,
+      "bytes": 6019295,
+      "pages": 28,
+      "shards": 2
+    },
+    {
+      "seconds": 1.523898775998532,
+      "bytes": 8240555,
+      "pages": 36,
+      "shards": 3
     }
   ],
   "cache_key": {

trainer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4c3e5c3949b8f4716b37f94e501fd3bf9bfc15652bfd259b8ceeb75ba134374a
-size 721768511
+oid sha256:625680f99c72ef12793fea5968b6c72147fca5bf0986649c268a6a0699e0b367
+size 721775807