Qwen90M step 280000; complete resumable checkpoint

complete.json

@@ -1,18 +1,18 @@
 {
-  "step": 275000,
+  "step": 280000,
   "files": {
     "config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7",
     "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4",
-    "model.safetensors": "2af807d46a42c19f19c7b32b800e214d63c1433a37862b054d1767b88ee00e92",
+    "model.safetensors": "b202beb2e692c902003e4624a58ed61b492689af5528e336e78027d172cdc6a5",
     "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03",
     "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300",
     "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263",
     "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6",
-    "trainer.pt": "8f706fccdd6b87209ff5215abd019c5c04c63cfa5c3c1d339b8f5d3a1bdf7fe4",
+    "trainer.pt": "086c04c0e606c24ef7dab8710d39cde6eff9e0a7ec67fc90f9930de401842100",
     "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783",
-    "performance.json": "ac15a69eb79c2ac9ef8406edc12d9ae8abafb561b0f74b54b23d5920ee4fe3f7",
-    "shard_scheduler.json": "d4146d4949443885a89e71741ab422c603360627ea40e296539a0be0a218caee",
-    "coverage.json": "8b908eef3d6b156adf6b83ba5c2f7263f4abb751ab7df3899af5bff2595659e7",
+    "performance.json": "7ab839f3b8798329051e29b39e08bd08f8678f49ed79d09e3b70b9de78a4765c",
+    "shard_scheduler.json": "bb41f80e95e1faea1ad1243a8e7d9ee991b7eb64bf0e6b9b463722b029efa994",
+    "coverage.json": "c7677efe249bc40dea701da173ea31fe609c61d934d175645935757dc9bffe1c",
     "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"
   },
   "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063"

coverage.json

@@ -1,107 +1,107 @@
 {
-  "step": 275000,
+  "step": 280000,
   "stats": {
-    "climbmix/blocks": 9900000,
-    "climbmix/targets": 16549266942,
-    "climbmix/input_tokens": 16550315706,
-    "ultra_qa/blocks": 3712500,
-    "ultra_qa/targets": 6056418591,
-    "ultra_qa/input_tokens": 6056430195,
-    "cortex/blocks": 3712500,
-    "cortex/targets": 141226525,
-    "cortex/input_tokens": 5901276792,
-    "ultra_style/blocks": 3712500,
-    "ultra_style/targets": 5871811169,
-    "ultra_style/input_tokens": 5872119620,
-    "openmath/blocks": 2475000,
-    "openmath/targets": 3824405251,
-    "openmath/input_tokens": 3825680165,
-    "rewrite6/blocks": 1237500,
-    "rewrite6/targets": 2028675231,
-    "rewrite6/input_tokens": 2030899715
+    "climbmix/blocks": 10080000,
+    "climbmix/targets": 16850702549,
+    "climbmix/input_tokens": 16851770299,
+    "ultra_qa/blocks": 3780000,
+    "ultra_qa/targets": 6166762492,
+    "ultra_qa/input_tokens": 6166774323,
+    "cortex/blocks": 3780000,
+    "cortex/targets": 143784713,
+    "cortex/input_tokens": 6008633757,
+    "ultra_style/blocks": 3780000,
+    "ultra_style/targets": 5978866890,
+    "ultra_style/input_tokens": 5979181245,
+    "openmath/blocks": 2520000,
+    "openmath/targets": 3893894891,
+    "openmath/input_tokens": 3895193089,
+    "rewrite6/blocks": 1260000,
+    "rewrite6/targets": 2065551785,
+    "rewrite6/input_tokens": 2067816586
   },
   "sources": {
     "climbmix": {
-      "page": 1355,
-      "row": 3394,
+      "page": 1380,
+      "row": 1360,
       "start": {
-        "file": 65,
+        "file": 66,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 48,
+          "row_group": 66,
           "row": 1024
         }
       },
       "epoch": 0,
       "offset": 2048,
-      "documents": 5553474
+      "documents": 5653840
     },
     "rewrite6": {
-      "page": 21,
-      "row": 1930,
+      "page": 35,
+      "row": 2118,
       "start": {
         "file": 0,
         "reader": {
           "format": "jsonl-row-v1",
-          "row": 86016
+          "row": 143360
         }
       },
       "epoch": 15,
       "offset": 0,
-      "documents": 3169006
+      "documents": 3226538
     },
     "openmath": {
-      "page": 847,
-      "row": 2438,
+      "page": 863,
+      "row": 65,
       "start": {
-        "file": 7,
+        "file": 8,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 412,
-          "row": 762
+          "row_group": 41,
+          "row": 648
         }
       },
       "epoch": 0,
       "offset": 0,
-      "documents": 3471750
+      "documents": 3534913
     },
     "ultra_qa": {
-      "page": 366,
-      "row": 1530,
+      "page": 373,
+      "row": 122,
       "start": {
         "file": 2,
         "reader": {
           "format": "parquet-row-v1",
           "row_group": 7,
-          "row": 7993
+          "row": 36665
         }
       },
       "epoch": 0,
-      "offset": 2048,
-      "documents": 1500666
+      "offset": 4096,
+      "documents": 1527930
     },
     "ultra_style": {
-      "page": 559,
-      "row": 2401,
+      "page": 569,
+      "row": 3492,
       "start": {
         "file": 3,
         "reader": {
           "format": "parquet-row-v1",
-          "row_group": 2,
-          "row": 63989
+          "row_group": 3,
+          "row": 18427
         }
       },
       "epoch": 0,
       "offset": 2048,
-      "documents": 2292065
+      "documents": 2334116
     },
     "cortex": {
-      "page": 71999,
-      "row": 4,
+      "page": 73309,
+      "row": 69,
       "start": null,
       "epoch": 0,
       "offset": 0,
-      "documents": 9215876
+      "documents": 9383621
     }
   }
 }

latest.json

@@ -1 +1 @@
-{"step": 275000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "2af807d46a42c19f19c7b32b800e214d63c1433a37862b054d1767b88ee00e92", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "8f706fccdd6b87209ff5215abd019c5c04c63cfa5c3c1d339b8f5d3a1bdf7fe4", "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783", "performance.json": "ac15a69eb79c2ac9ef8406edc12d9ae8abafb561b0f74b54b23d5920ee4fe3f7", "shard_scheduler.json": "d4146d4949443885a89e71741ab422c603360627ea40e296539a0be0a218caee", "coverage.json": "8b908eef3d6b156adf6b83ba5c2f7263f4abb751ab7df3899af5bff2595659e7", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000275000"}
+{"step": 280000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "b202beb2e692c902003e4624a58ed61b492689af5528e336e78027d172cdc6a5", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "086c04c0e606c24ef7dab8710d39cde6eff9e0a7ec67fc90f9930de401842100", "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783", "performance.json": "7ab839f3b8798329051e29b39e08bd08f8678f49ed79d09e3b70b9de78a4765c", "shard_scheduler.json": "bb41f80e95e1faea1ad1243a8e7d9ee991b7eb64bf0e6b9b463722b029efa994", "coverage.json": "c7677efe249bc40dea701da173ea31fe609c61d934d175645935757dc9bffe1c", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000280000"}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2af807d46a42c19f19c7b32b800e214d63c1433a37862b054d1767b88ee00e92
+oid sha256:b202beb2e692c902003e4624a58ed61b492689af5528e336e78027d172cdc6a5
 size 360622744

performance.json

@@ -49,11 +49,11 @@
   "selected": {
     "microbatch": 86,
     "mode": "compiled",
-    "microbatch_seconds": 0.8988038599999868,
+    "microbatch_seconds": 0.8973423759999832,
     "peak_gb": 85.39913082122803,
     "fits_budget": true,
-    "estimated_update_seconds": 0.9552281219999941,
-    "input_tokens_per_second": 195958.21495470946
+    "estimated_update_seconds": 0.9528963840000415,
+    "input_tokens_per_second": 196277.36827175462
   },
   "measurements": [
     {

shard_scheduler.json

@@ -22,8 +22,8 @@
       "tokens_per_second": 1062967.6168812835,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 75917.17339470465,
-      "required_pages_per_second_upper_estimate": 0.006580308179536131
+      "required_token_positions_per_second": 75842.57462698415,
+      "required_pages_per_second_upper_estimate": 0.006573842147418966
     },
     {
       "source": "rewrite6",
@@ -36,8 +36,8 @@
       "tokens_per_second": 1206672.263377472,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 9489.64667433808,
-      "required_pages_per_second_upper_estimate": 0.0037855619412550186
+      "required_token_positions_per_second": 9480.321828373018,
+      "required_pages_per_second_upper_estimate": 0.0037818421207806837
     },
     {
       "source": "openmath",
@@ -50,8 +50,8 @@
       "tokens_per_second": 410042.7581134458,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 18979.29334867616,
-      "required_pages_per_second_upper_estimate": 0.004390525492061635
+      "required_token_positions_per_second": 18960.643656746037,
+      "required_pages_per_second_upper_estimate": 0.0043862112140569684
     },
     {
       "source": "ultra_qa",
@@ -64,8 +64,8 @@
       "tokens_per_second": 2163486.5624068812,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28468.940023014242,
-      "required_pages_per_second_upper_estimate": 0.001733609082416801
+      "required_token_positions_per_second": 28440.965485119053,
+      "required_pages_per_second_upper_estimate": 0.0017319055798300411
     },
     {
       "source": "ultra_style",
@@ -78,8 +78,8 @@
       "tokens_per_second": 1546694.2737633225,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28468.940023014242,
-      "required_pages_per_second_upper_estimate": 0.002191617810041497
+      "required_token_positions_per_second": 28440.965485119053,
+      "required_pages_per_second_upper_estimate": 0.002189464252675848
     },
     {
       "source": "cortex",
@@ -92,8 +92,8 @@
       "tokens_per_second": 1015177.8395898147,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 22,
-      "required_token_positions_per_second": 28468.940023014242,
-      "required_pages_per_second_upper_estimate": 0.35348895567272487
+      "required_token_positions_per_second": 28440.965485119053,
+      "required_pages_per_second_upper_estimate": 0.3531416055368222
     }
   ],
   "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
@@ -158,410 +158,38 @@
     "prefetch_batches": 62,
     "prefetch_capacity_limit": 512,
     "batch_memory_estimate_bytes": 690670,
-    "producer_batch_seconds": 0.10167211079664185,
-    "producer_p95_seconds": 0.38399030399887124,
-    "gpu_update_seconds": 0.9711636603838922,
-    "producer_batches_per_second": 9.835538892274373,
-    "gpu_batches_per_second": 1.0296925644898092,
-    "producer_headroom": 9.551917952469312,
+    "producer_batch_seconds": 0.080577971249987,
+    "producer_p95_seconds": 0.18257642799926543,
+    "gpu_update_seconds": 0.9721188971051651,
+    "producer_batches_per_second": 12.410339755236285,
+    "gpu_batches_per_second": 1.028680753946725,
+    "producer_headroom": 12.06432579556068,
     "sustainable": true,
-    "buffer_seconds": 60.212146943801315,
+    "buffer_seconds": 60.271371620520235,
     "buffer_drain_seconds": null,
-    "shard_target_mib": 16,
-    "upload_pages_per_commit": 79,
+    "shard_target_mib": 32,
+    "upload_pages_per_commit": 160,
     "max_pending_pages": 4096,
-    "generation_bytes_per_second": 298328.0392779603,
-    "upload_bytes_per_second": 5292592.716316272,
-    "upload_sustainable": true,
+    "generation_bytes_per_second": 905156.4916824569,
+    "upload_bytes_per_second": 607793.4295288165,
+    "upload_sustainable": false,
     "logical_page_rows": 4096,
     "logical_cortex_page_rows": 128,
-    "estimated_full_shards_per_hour": 64.01425250772579,
+    "estimated_full_shards_per_hour": 97.11275607516899,
     "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
   },
   "upload_measurements": [
     {
-      "seconds": 1.5463005599995086,
-      "bytes": 13315083,
-      "pages": 40,
-      "shards": 4
-    },
-    {
-      "seconds": 1.3599293949991988,
-      "bytes": 4987866,
-      "pages": 33,
-      "shards": 3
-    },
-    {
-      "seconds": 1.145182983000268,
-      "bytes": 6203180,
-      "pages": 32,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9317472719994839,
-      "bytes": 2642711,
-      "pages": 36,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9964947309999843,
-      "bytes": 5697842,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1368190850007522,
-      "bytes": 10318857,
-      "pages": 37,
-      "shards": 3
-    },
-    {
-      "seconds": 1.0222451929985255,
-      "bytes": 8375215,
-      "pages": 35,
-      "shards": 3
-    },
-    {
-      "seconds": 0.9998172479972709,
-      "bytes": 6024653,
-      "pages": 36,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9568293169977551,
-      "bytes": 2417450,
-      "pages": 29,
-      "shards": 2
-    },
-    {
-      "seconds": 1.2115873000002466,
-      "bytes": 10372111,
-      "pages": 33,
-      "shards": 3
-    },
-    {
-      "seconds": 1.1448558080010116,
-      "bytes": 6260573,
-      "pages": 36,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1358172180007386,
-      "bytes": 2620953,
-      "pages": 36,
-      "shards": 2
-    },
-    {
-      "seconds": 1.4711359560023993,
-      "bytes": 11859463,
-      "pages": 29,
-      "shards": 3
-    },
-    {
-      "seconds": 1.2465269989988883,
-      "bytes": 5535585,
-      "pages": 33,
-      "shards": 2
-    },
-    {
-      "seconds": 1.2969893279987446,
-      "bytes": 7578437,
-      "pages": 39,
-      "shards": 3
-    },
-    {
-      "seconds": 1.5002685959989321,
-      "bytes": 6193769,
-      "pages": 27,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1753300150012365,
-      "bytes": 2646901,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 1.4061265680029464,
-      "bytes": 9691299,
-      "pages": 28,
-      "shards": 3
-    },
-    {
-      "seconds": 1.0368577580011333,
-      "bytes": 6834427,
-      "pages": 31,
-      "shards": 2
-    },
-    {
-      "seconds": 1.4560243180021644,
-      "bytes": 7277270,
-      "pages": 30,
-      "shards": 3
-    },
-    {
-      "seconds": 1.4158115079990239,
-      "bytes": 6180346,
-      "pages": 38,
-      "shards": 2
-    },
-    {
-      "seconds": 1.5717087800003355,
-      "bytes": 560796,
-      "pages": 18,
-      "shards": 1
-    },
-    {
-      "seconds": 4.375384323997423,
-      "bytes": 11988381,
-      "pages": 43,
-      "shards": 4
-    },
-    {
-      "seconds": 0.9212507589982124,
-      "bytes": 677495,
-      "pages": 18,
-      "shards": 1
-    },
-    {
-      "seconds": 0.8941036469987012,
-      "bytes": 796711,
-      "pages": 25,
+      "seconds": 0.8291290090000984,
+      "bytes": 27860,
+      "pages": 1,
       "shards": 1
     },
     {
-      "seconds": 1.6776708719989983,
-      "bytes": 13189069,
-      "pages": 31,
-      "shards": 4
-    },
-    {
-      "seconds": 1.216554687001917,
-      "bytes": 10409429,
-      "pages": 33,
-      "shards": 3
-    },
-    {
-      "seconds": 1.1135386019959697,
-      "bytes": 1273006,
-      "pages": 34,
-      "shards": 1
-    },
-    {
-      "seconds": 1.2612176139955409,
-      "bytes": 7427910,
-      "pages": 31,
-      "shards": 3
-    },
-    {
-      "seconds": 0.7726261329953559,
-      "bytes": 1134173,
-      "pages": 35,
-      "shards": 1
-    },
-    {
-      "seconds": 1.5038225289972615,
-      "bytes": 10256540,
-      "pages": 38,
-      "shards": 3
-    },
-    {
-      "seconds": 1.727812716999324,
-      "bytes": 13149585,
-      "pages": 36,
-      "shards": 4
-    },
-    {
-      "seconds": 0.9765169189995504,
-      "bytes": 821604,
+      "seconds": 0.7566531879992908,
+      "bytes": 935968,
       "pages": 27,
       "shards": 1
-    },
-    {
-      "seconds": 1.1198572450011852,
-      "bytes": 7695575,
-      "pages": 38,
-      "shards": 4
-    },
-    {
-      "seconds": 1.1846314020003774,
-      "bytes": 10317171,
-      "pages": 36,
-      "shards": 3
-    },
-    {
-      "seconds": 1.0667465840015211,
-      "bytes": 820608,
-      "pages": 26,
-      "shards": 1
-    },
-    {
-      "seconds": 1.5008300169938593,
-      "bytes": 7497655,
-      "pages": 39,
-      "shards": 3
-    },
-    {
-      "seconds": 1.085039866004081,
-      "bytes": 745159,
-      "pages": 23,
-      "shards": 1
-    },
-    {
-      "seconds": 2.1224419440040947,
-      "bytes": 16141114,
-      "pages": 38,
-      "shards": 4
-    },
-    {
-      "seconds": 1.301262590000988,
-      "bytes": 7677650,
-      "pages": 39,
-      "shards": 3
-    },
-    {
-      "seconds": 0.976449344001594,
-      "bytes": 1139759,
-      "pages": 35,
-      "shards": 1
-    },
-    {
-      "seconds": 1.2381579889988643,
-      "bytes": 7253317,
-      "pages": 31,
-      "shards": 3
-    },
-    {
-      "seconds": 1.0107035210021422,
-      "bytes": 4758637,
-      "pages": 36,
-      "shards": 2
-    },
-    {
-      "seconds": 1.040413394002826,
-      "bytes": 5884125,
-      "pages": 26,
-      "shards": 2
-    },
-    {
-      "seconds": 1.2434632910008077,
-      "bytes": 13606244,
-      "pages": 40,
-      "shards": 4
-    },
-    {
-      "seconds": 0.7609897279980942,
-      "bytes": 952904,
-      "pages": 26,
-      "shards": 1
-    },
-    {
-      "seconds": 1.0394860849992256,
-      "bytes": 10827416,
-      "pages": 40,
-      "shards": 3
-    },
-    {
-      "seconds": 0.9459326559954206,
-      "bytes": 2314805,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 1.2841032439973787,
-      "bytes": 6126814,
-      "pages": 38,
-      "shards": 2
-    },
-    {
-      "seconds": 1.3457263079981203,
-      "bytes": 7394502,
-      "pages": 29,
-      "shards": 3
-    },
-    {
-      "seconds": 1.2961700559972087,
-      "bytes": 10630748,
-      "pages": 31,
-      "shards": 3
-    },
-    {
-      "seconds": 1.0071994079989963,
-      "bytes": 6082322,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 0.977721251998446,
-      "bytes": 2308590,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 1.0036859399988316,
-      "bytes": 6116236,
-      "pages": 36,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9474672509968514,
-      "bytes": 5614078,
-      "pages": 31,
-      "shards": 2
-    },
-    {
-      "seconds": 1.196396844003175,
-      "bytes": 7831133,
-      "pages": 38,
-      "shards": 3
-    },
-    {
-      "seconds": 1.1607214789983118,
-      "bytes": 11917502,
-      "pages": 33,
-      "shards": 3
-    },
-    {
-      "seconds": 1.2743855959997745,
-      "bytes": 2697194,
-      "pages": 36,
-      "shards": 2
-    },
-    {
-      "seconds": 1.4709027479984798,
-      "bytes": 10099926,
-      "pages": 34,
-      "shards": 3
-    },
-    {
-      "seconds": 0.8219810020018485,
-      "bytes": 940002,
-      "pages": 29,
-      "shards": 1
-    },
-    {
-      "seconds": 1.2411883530003252,
-      "bytes": 7506339,
-      "pages": 35,
-      "shards": 3
-    },
-    {
-      "seconds": 1.1381002670023008,
-      "bytes": 6063351,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1012217039969983,
-      "bytes": 10763585,
-      "pages": 27,
-      "shards": 3
-    },
-    {
-      "seconds": 1.2329660859977594,
-      "bytes": 7594797,
-      "pages": 31,
-      "shards": 3
     }
   ],
   "cache_key": {
@@ -606,7 +234,7 @@
   },
   "cache_reused": true,
   "resume_probe_seconds": [
-    2.844574209999962,
-    0.2503580399999805
+    3.215140269000017,
+    0.014588057999958437
   ]
 }

trainer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8f706fccdd6b87209ff5215abd019c5c04c63cfa5c3c1d339b8f5d3a1bdf7fe4
-size 721785471
+oid sha256:086c04c0e606c24ef7dab8710d39cde6eff9e0a7ec67fc90f9930de401842100
+size 721763967