Update TensorBoard training metrics

benchmark_cache/performance.json

@@ -44,16 +44,16 @@
     "source": "e6d135a4487a9fd1907e12903cda79e75fd61305fad149fda7ec5281279dc18b",
     "requested_microbatch_ceiling": 90
   },
-  "cache_reused": false,
+  "cache_reused": true,
   "attention": "PyTorch FlashAttention (forced; forward/backward verified)",
   "selected": {
     "microbatch": 86,
     "mode": "compiled",
-    "microbatch_seconds": 0.8818629400000191,
-    "peak_gb": 84.78368663787842,
+    "microbatch_seconds": 0.8974938420000171,
+    "peak_gb": 85.39913082122803,
     "fits_budget": true,
-    "estimated_update_seconds": 0.9349941874999956,
-    "input_tokens_per_second": 199722.64624250587
+    "estimated_update_seconds": 0.9533413040000482,
+    "input_tokens_per_second": 196244.2434228943
   },
   "measurements": [
     {
@@ -218,5 +218,6 @@
   "selection_policy": "max_batch",
   "budget_gb": 89.6964394569397,
   "reserve_gb": 2.1791439056396484,
-  "note": "Forward/backward throughput; excludes optimizer, data, logging and checkpoint time."
+  "note": "Forward/backward throughput; excludes optimizer, data, logging and checkpoint time.",
+  "legacy_seed_verified": false
 }

benchmark_cache/shard_scheduler.json

@@ -22,8 +22,8 @@
       "tokens_per_second": 1062967.6168812835,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 75162.62603138451,
-      "required_pages_per_second_upper_estimate": 0.0065149059264663495
+      "required_token_positions_per_second": 77336.41633972073,
+      "required_pages_per_second_upper_estimate": 0.0067033245609717665
     },
     {
       "source": "rewrite6",
@@ -36,8 +36,8 @@
       "tokens_per_second": 1206672.263377472,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 9395.328253923064,
-      "required_pages_per_second_upper_estimate": 0.0037479369131654156
+      "required_token_positions_per_second": 9667.052042465091,
+      "required_pages_per_second_upper_estimate": 0.003856331595047507
     },
     {
       "source": "openmath",
@@ -50,8 +50,8 @@
       "tokens_per_second": 410042.7581134458,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 18790.656507846128,
-      "required_pages_per_second_upper_estimate": 0.0043468876788306165
+      "required_token_positions_per_second": 19334.104084930183,
+      "required_pages_per_second_upper_estimate": 0.004472604711438319
     },
     {
       "source": "ultra_qa",
@@ -64,8 +64,8 @@
       "tokens_per_second": 2163486.5624068812,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28185.984761769192,
-      "required_pages_per_second_upper_estimate": 0.001716378591558501
+      "required_token_positions_per_second": 29001.156127395276,
+      "required_pages_per_second_upper_estimate": 0.001766018250851508
     },
     {
       "source": "ultra_style",
@@ -78,8 +78,8 @@
       "tokens_per_second": 1546694.2737633225,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28185.984761769192,
-      "required_pages_per_second_upper_estimate": 0.0021698351307605584
+      "required_token_positions_per_second": 29001.156127395276,
+      "required_pages_per_second_upper_estimate": 0.0022325892790252193
     },
     {
       "source": "cortex",
@@ -92,8 +92,8 @@
       "tokens_per_second": 1015177.8395898147,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 22,
-      "required_token_positions_per_second": 28185.984761769192,
-      "required_pages_per_second_upper_estimate": 0.34997559831840264
+      "required_token_positions_per_second": 29001.156127395276,
+      "required_pages_per_second_upper_estimate": 0.36009729847641797
     }
   ],
   "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
@@ -155,415 +155,30 @@
   "shard_benchmark_scope": "Local packing/read measurements; upload capacity measured only on real data commits. Targets larger than available pages are sample-limited.",
   "estimated_fresh_generation_batch_seconds": 0.1434574083060457,
   "selected": {
-    "prefetch_batches": 62,
+    "prefetch_batches": 63,
     "prefetch_capacity_limit": 512,
     "batch_memory_estimate_bytes": 688946,
-    "producer_batch_seconds": 0.1280048998668235,
-    "producer_p95_seconds": 0.4314359210038674,
-    "gpu_update_seconds": 0.9809130400688039,
-    "producer_batches_per_second": 7.812200947310623,
-    "gpu_batches_per_second": 1.0194583608857493,
-    "producer_headroom": 7.663089780854852,
-    "sustainable": true,
-    "buffer_seconds": 60.81660848426584,
-    "buffer_drain_seconds": null,
+    "producer_batch_seconds": 1.3367408980000164,
+    "producer_p95_seconds": 2.657314213999996,
+    "gpu_update_seconds": 0.9533413040000482,
+    "producer_batches_per_second": 0.7480881309879603,
+    "gpu_batches_per_second": 1.048942278913313,
+    "producer_headroom": 0.7131833143030211,
+    "sustainable": false,
+    "buffer_seconds": 60.06050215200304,
+    "buffer_drain_seconds": 209.40379394613316,
     "shard_target_mib": 16,
-    "upload_pages_per_commit": 83,
-    "max_pending_pages": 4096,
-    "generation_bytes_per_second": 456524.683563734,
-    "upload_bytes_per_second": 5056368.789854124,
-    "upload_sustainable": true,
+    "upload_pages_per_commit": 7,
+    "max_pending_pages": 1780,
+    "generation_bytes_per_second": 0,
+    "upload_bytes_per_second": null,
+    "upload_sustainable": null,
     "logical_page_rows": 4096,
     "logical_cortex_page_rows": 128,
-    "estimated_full_shards_per_hour": 97.95956974205032,
+    "estimated_full_shards_per_hour": 0.0,
     "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
   },
-  "upload_measurements": [
-    {
-      "seconds": 0.90664235100121,
-      "bytes": 2695308,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 0.7943546639944543,
-      "bytes": 258770,
-      "pages": 8,
-      "shards": 1
-    },
-    {
-      "seconds": 1.1189483790003578,
-      "bytes": 5921688,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 1.2605290909996256,
-      "bytes": 6824656,
-      "pages": 33,
-      "shards": 2
-    },
-    {
-      "seconds": 11.391802897000161,
-      "bytes": 12565360,
-      "pages": 37,
-      "shards": 4
-    },
-    {
-      "seconds": 0.941881225000543,
-      "bytes": 6023516,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9670609070017235,
-      "bytes": 2689085,
-      "pages": 35,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1650014270053362,
-      "bytes": 5886860,
-      "pages": 31,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1598776869941503,
-      "bytes": 10363400,
-      "pages": 37,
-      "shards": 3
-    },
-    {
-      "seconds": 1.1336681269967812,
-      "bytes": 8324153,
-      "pages": 31,
-      "shards": 3
-    },
-    {
-      "seconds": 1.0718341040046653,
-      "bytes": 6483305,
-      "pages": 36,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9573828110005707,
-      "bytes": 873310,
-      "pages": 27,
-      "shards": 1
-    },
-    {
-      "seconds": 1.2830225369980326,
-      "bytes": 12398676,
-      "pages": 40,
-      "shards": 4
-    },
-    {
-      "seconds": 1.2898951360039064,
-      "bytes": 6161152,
-      "pages": 36,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9979428110018489,
-      "bytes": 2470155,
-      "pages": 30,
-      "shards": 2
-    },
-    {
-      "seconds": 1.098158920998685,
-      "bytes": 11654949,
-      "pages": 31,
-      "shards": 3
-    },
-    {
-      "seconds": 1.0713990720032598,
-      "bytes": 5874732,
-      "pages": 36,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9352636539988453,
-      "bytes": 6076485,
-      "pages": 31,
-      "shards": 3
-    },
-    {
-      "seconds": 1.2692730029957602,
-      "bytes": 6086693,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 1.036868928000331,
-      "bytes": 2682212,
-      "pages": 38,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1587374009977793,
-      "bytes": 10973458,
-      "pages": 30,
-      "shards": 3
-    },
-    {
-      "seconds": 0.8646178900016821,
-      "bytes": 947338,
-      "pages": 27,
-      "shards": 1
-    },
-    {
-      "seconds": 1.0228583920033998,
-      "bytes": 13452885,
-      "pages": 43,
-      "shards": 4
-    },
-    {
-      "seconds": 1.0817572929954622,
-      "bytes": 6130820,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9214774369975203,
-      "bytes": 1243052,
-      "pages": 35,
-      "shards": 1
-    },
-    {
-      "seconds": 1.3106369909946807,
-      "bytes": 11452061,
-      "pages": 33,
-      "shards": 4
-    },
-    {
-      "seconds": 1.0961634519990184,
-      "bytes": 6066738,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9469924939985503,
-      "bytes": 890694,
-      "pages": 27,
-      "shards": 2
-    },
-    {
-      "seconds": 1.2476149669964798,
-      "bytes": 13021486,
-      "pages": 30,
-      "shards": 4
-    },
-    {
-      "seconds": 0.8867707609970239,
-      "bytes": 6143048,
-      "pages": 43,
-      "shards": 2
-    },
-    {
-      "seconds": 1.05023945299763,
-      "bytes": 7400872,
-      "pages": 31,
-      "shards": 3
-    },
-    {
-      "seconds": 1.1047442850031075,
-      "bytes": 6203278,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 1.0125872680000612,
-      "bytes": 1072664,
-      "pages": 31,
-      "shards": 1
-    },
-    {
-      "seconds": 1.132626735001395,
-      "bytes": 11459123,
-      "pages": 36,
-      "shards": 4
-    },
-    {
-      "seconds": 1.060930259001907,
-      "bytes": 6808205,
-      "pages": 34,
-      "shards": 2
-    },
-    {
-      "seconds": 3.3303450339953997,
-      "bytes": 5952538,
-      "pages": 29,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9529956519982079,
-      "bytes": 7196338,
-      "pages": 30,
-      "shards": 3
-    },
-    {
-      "seconds": 1.1464728400023887,
-      "bytes": 6003121,
-      "pages": 37,
-      "shards": 2
-    },
-    {
-      "seconds": 0.9798986519963364,
-      "bytes": 7781027,
-      "pages": 39,
-      "shards": 3
-    },
-    {
-      "seconds": 0.9489700769991032,
-      "bytes": 911408,
-      "pages": 26,
-      "shards": 1
-    },
-    {
-      "seconds": 1.3251626939963899,
-      "bytes": 11870118,
-      "pages": 36,
-      "shards": 3
-    },
-    {
-      "seconds": 1.1823667450007633,
-      "bytes": 10876587,
-      "pages": 32,
-      "shards": 4
-    },
-    {
-      "seconds": 0.8632450160002918,
-      "bytes": 820169,
-      "pages": 26,
-      "shards": 1
-    },
-    {
-      "seconds": 1.2373706080034026,
-      "bytes": 5701994,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 1.2111527899978682,
-      "bytes": 2603030,
-      "pages": 38,
-      "shards": 2
-    },
-    {
-      "seconds": 1.2956993760017212,
-      "bytes": 10664503,
-      "pages": 31,
-      "shards": 3
-    },
-    {
-      "seconds": 1.0216036510028061,
-      "bytes": 8317362,
-      "pages": 37,
-      "shards": 3
-    },
-    {
-      "seconds": 1.006012271005602,
-      "bytes": 5704285,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 1.041346000994963,
-      "bytes": 6267131,
-      "pages": 38,
-      "shards": 2
-    },
-    {
-      "seconds": 0.8457523769975523,
-      "bytes": 628978,
-      "pages": 18,
-      "shards": 1
-    },
-    {
-      "seconds": 1.3523500509982114,
-      "bytes": 11328528,
-      "pages": 40,
-      "shards": 4
-    },
-    {
-      "seconds": 0.9737502849966404,
-      "bytes": 1202420,
-      "pages": 35,
-      "shards": 1
-    },
-    {
-      "seconds": 1.281989146998967,
-      "bytes": 7415406,
-      "pages": 31,
-      "shards": 3
-    },
-    {
-      "seconds": 1.4836684109977796,
-      "bytes": 16710893,
-      "pages": 39,
-      "shards": 4
-    },
-    {
-      "seconds": 1.0522764270062908,
-      "bytes": 1123327,
-      "pages": 34,
-      "shards": 1
-    },
-    {
-      "seconds": 1.1691736089996994,
-      "bytes": 7423568,
-      "pages": 31,
-      "shards": 3
-    },
-    {
-      "seconds": 0.8287520000012591,
-      "bytes": 1267463,
-      "pages": 36,
-      "shards": 1
-    },
-    {
-      "seconds": 1.080831170998863,
-      "bytes": 11426717,
-      "pages": 33,
-      "shards": 4
-    },
-    {
-      "seconds": 0.9978844479992404,
-      "bytes": 5811681,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 1.0041134909988614,
-      "bytes": 6949586,
-      "pages": 36,
-      "shards": 3
-    },
-    {
-      "seconds": 1.203377494995948,
-      "bytes": 7585529,
-      "pages": 37,
-      "shards": 3
-    },
-    {
-      "seconds": 1.1198065829958068,
-      "bytes": 5440269,
-      "pages": 33,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1098419440022553,
-      "bytes": 5868008,
-      "pages": 28,
-      "shards": 2
-    },
-    {
-      "seconds": 1.1827127520009526,
-      "bytes": 8081279,
-      "pages": 39,
-      "shards": 3
-    }
-  ],
+  "upload_measurements": [],
   "cache_key": {
     "version": 1,
     "kind": "data",
@@ -604,5 +219,9 @@
     "cpu_count": 20,
     "microbatch": 86
   },
-  "cache_reused": false
+  "cache_reused": true,
+  "resume_probe_seconds": [
+    2.657314213999996,
+    0.016167582000036873
+  ]
 }

runs/void-byte-v1/events.out.tfevents.1790434090.372bd512-57cc-4928-b44a-3b375a0fa071-5spg5.451.0.1790434090210070261-a05fc3fb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f930f15e999fb437ffae5ff43bb10a8bef6677821fd507462d8d1f1d5ab87053
+size 165