Update TensorBoard training metrics

benchmark_cache/shard_scheduler.json

@@ -22,8 +22,8 @@
       "tokens_per_second": 1062967.6168812835,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 75691.65435691421,
-      "required_pages_per_second_upper_estimate": 0.006560760760913231
+      "required_token_positions_per_second": 75334.32265789765,
+      "required_pages_per_second_upper_estimate": 0.006529788154891387
     },
     {
       "source": "rewrite6",
@@ -36,8 +36,8 @@
       "tokens_per_second": 1206672.263377472,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 9461.456794614276,
-      "required_pages_per_second_upper_estimate": 0.0037743165767569315
+      "required_token_positions_per_second": 9416.790332237206,
+      "required_pages_per_second_upper_estimate": 0.0037564984570915934
     },
     {
       "source": "openmath",
@@ -50,8 +50,8 @@
       "tokens_per_second": 410042.7581134458,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 18922.913589228552,
-      "required_pages_per_second_upper_estimate": 0.004377483026963307
+      "required_token_positions_per_second": 18833.58066447441,
+      "required_pages_per_second_upper_estimate": 0.004356817427027215
     },
     {
       "source": "ultra_qa",
@@ -64,8 +64,8 @@
       "tokens_per_second": 2163486.5624068812,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28384.370383842826,
-      "required_pages_per_second_upper_estimate": 0.0017284592350938672
+      "required_token_positions_per_second": 28250.37099671162,
+      "required_pages_per_second_upper_estimate": 0.001720299375458027
     },
     {
       "source": "ultra_style",
@@ -78,8 +78,8 @@
       "tokens_per_second": 1546694.2737633225,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28384.370383842826,
-      "required_pages_per_second_upper_estimate": 0.002185107405114337
+      "required_token_positions_per_second": 28250.37099671162,
+      "required_pages_per_second_upper_estimate": 0.002174791761358932
     },
     {
       "source": "cortex",
@@ -92,8 +92,8 @@
       "tokens_per_second": 1015177.8395898147,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 22,
-      "required_token_positions_per_second": 28384.370383842826,
-      "required_pages_per_second_upper_estimate": 0.35243888379059096
+      "required_token_positions_per_second": 28250.37099671162,
+      "required_pages_per_second_upper_estimate": 0.35077505986952107
     }
   ],
   "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
@@ -158,33 +158,27 @@
     "prefetch_batches": 62,
     "prefetch_capacity_limit": 512,
     "batch_memory_estimate_bytes": 690670,
-    "producer_batch_seconds": 0.2555313578981213,
-    "producer_p95_seconds": 0.39806883900018875,
-    "gpu_update_seconds": 0.9740571880269012,
-    "producer_batches_per_second": 3.913414025681707,
-    "gpu_batches_per_second": 1.0266337667767227,
-    "producer_headroom": 3.811889061440559,
+    "producer_batch_seconds": 0.5958057715078553,
+    "producer_p95_seconds": 0.4404174180017435,
+    "gpu_update_seconds": 0.9786774128813481,
+    "producer_batches_per_second": 1.6783993170613583,
+    "gpu_batches_per_second": 1.021787145425044,
+    "producer_headroom": 1.6426115014034317,
     "sustainable": true,
-    "buffer_seconds": 60.391545657667876,
+    "buffer_seconds": 60.67799959864358,
     "buffer_drain_seconds": null,
     "shard_target_mib": 16,
-    "upload_pages_per_commit": 84,
+    "upload_pages_per_commit": 78,
     "max_pending_pages": 4096,
-    "generation_bytes_per_second": 445202.18390778836,
-    "upload_bytes_per_second": 3734321.560257869,
+    "generation_bytes_per_second": 404934.06527303037,
+    "upload_bytes_per_second": 3864373.591385076,
     "upload_sustainable": true,
     "logical_page_rows": 4096,
     "logical_cortex_page_rows": 128,
-    "estimated_full_shards_per_hour": 95.53002489018668,
+    "estimated_full_shards_per_hour": 86.88942402499374,
     "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
   },
   "upload_measurements": [
-    {
-      "seconds": 1.0449587049988622,
-      "bytes": 2671135,
-      "pages": 37,
-      "shards": 2
-    },
     {
       "seconds": 1.0467685680014256,
       "bytes": 5999640,
@@ -562,6 +556,12 @@
       "bytes": 1183539,
       "pages": 34,
       "shards": 1
+    },
+    {
+      "seconds": 1.2286420309974346,
+      "bytes": 17809213,
+      "pages": 32,
+      "shards": 5
     }
   ],
   "cache_key": {

metrics/void-byte-v1/1790738827394728502-c744ae20/train.jsonl

@@ -369,3 +369,4 @@
 {"step": 351900, "loss": 0.5837832074612379, "lr": 0.0003, "grad_norm": 0.15345544160138913, "recovery_retries": 0, "loss_scale": 1.0, "microbatch": 86, "input_bytes_and_specials": 51508829587, "tokens_per_second": 148419.20414938993, "data_wait_s": 1.3760000001639128e-05, "gpu_peak_gb": 85.46427917480469}
 {"step": 352000, "loss": 0.6193990055471659, "lr": 0.0003, "grad_norm": 0.16301558693260768, "recovery_retries": 0, "loss_scale": 1.0, "microbatch": 86, "input_bytes_and_specials": 51523348305, "tokens_per_second": 148386.81821898517, "data_wait_s": 1.5316996723413467e-05, "gpu_peak_gb": 85.46427917480469}
 {"step": 352100, "loss": 0.5566759584471583, "lr": 0.0003, "grad_norm": 0.1369592259047149, "recovery_retries": 0, "loss_scale": 1.0, "microbatch": 86, "input_bytes_and_specials": 51538001908, "tokens_per_second": 148347.85974753468, "data_wait_s": 1.5137004083953798e-05, "gpu_peak_gb": 85.46427917480469}
+{"step": 352200, "loss": 0.5887500774115324, "lr": 0.0003, "grad_norm": 0.14642585079011658, "recovery_retries": 0, "loss_scale": 1.0, "microbatch": 86, "input_bytes_and_specials": 51552662911, "tokens_per_second": 149100.3170076322, "data_wait_s": 1.491200237069279e-05, "gpu_peak_gb": 85.46427917480469}

runs/void-byte-v1/events.out.tfevents.1790738827.1a46e860-412e-40b5-8c01-42d24543f8c9-dfnph.319.0.1790738827394728502-c744ae20

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9acf247668d51102c18d9492d221eefefa1571dba3ae6857019f294a25473346
-size 791424
+oid sha256:ddf4a635bd860d1f97c4f367d54c02aea1de7cfd6aff00dd4932b7d9bcec1fef
+size 793542