Update TensorBoard training metrics

benchmark_cache/shard_scheduler.json

@@ -22,8 +22,8 @@
       "tokens_per_second": 1062967.6168812835,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 75594.9714394182,
-      "required_pages_per_second_upper_estimate": 0.00655238053066529
+      "required_token_positions_per_second": 75456.50727192067,
+      "required_pages_per_second_upper_estimate": 0.006540378807560829
     },
     {
       "source": "rewrite6",
@@ -36,8 +36,8 @@
       "tokens_per_second": 1206672.263377472,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 9449.371429927274,
-      "required_pages_per_second_upper_estimate": 0.003769495544090982
+      "required_token_positions_per_second": 9432.063408990083,
+      "required_pages_per_second_upper_estimate": 0.003762591115761163
     },
     {
       "source": "openmath",
@@ -50,8 +50,8 @@
       "tokens_per_second": 410042.7581134458,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 18898.74285985455,
-      "required_pages_per_second_upper_estimate": 0.00437189155411294
+      "required_token_positions_per_second": 18864.126817980166,
+      "required_pages_per_second_upper_estimate": 0.004363883742046831
     },
     {
       "source": "ultra_qa",
@@ -64,8 +64,8 @@
       "tokens_per_second": 2163486.5624068812,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28348.114289781824,
-      "required_pages_per_second_upper_estimate": 0.0017262514291865767
+      "required_token_positions_per_second": 28296.190226970248,
+      "required_pages_per_second_upper_estimate": 0.0017230895261858552
     },
     {
       "source": "ultra_style",
@@ -78,8 +78,8 @@
       "tokens_per_second": 1546694.2737633225,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 1,
-      "required_token_positions_per_second": 28348.114289781824,
-      "required_pages_per_second_upper_estimate": 0.002182316310630229
+      "required_token_positions_per_second": 28296.190226970248,
+      "required_pages_per_second_upper_estimate": 0.0021783190525399886
     },
     {
       "source": "cortex",
@@ -92,8 +92,8 @@
       "tokens_per_second": 1015177.8395898147,
       "note": "Bounded fresh sample; full-page latency is an extrapolation.",
       "estimated_pages_for_buffer": 22,
-      "required_token_positions_per_second": 28348.114289781824,
-      "required_pages_per_second_upper_estimate": 0.3519887044436945
+      "required_token_positions_per_second": 28296.190226970248,
+      "required_pages_per_second_upper_estimate": 0.3513439813622341
     }
   ],
   "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
@@ -158,17 +158,17 @@
     "prefetch_batches": 62,
     "prefetch_capacity_limit": 512,
     "batch_memory_estimate_bytes": 688946,
-    "producer_batch_seconds": 0.05659168946093818,
-    "producer_p95_seconds": 0.1505978339999956,
-    "gpu_update_seconds": 0.9753029678579297,
-    "producer_batches_per_second": 17.670439061379138,
-    "gpu_batches_per_second": 1.0253224207820393,
-    "producer_headroom": 17.23403165991576,
+    "producer_batch_seconds": 0.04893538489063154,
+    "producer_p95_seconds": 0.13259811100010666,
+    "gpu_update_seconds": 0.9770926678902366,
+    "producer_batches_per_second": 20.43511054904251,
+    "gpu_batches_per_second": 1.0234443803157642,
+    "producer_headroom": 19.966996684995863,
     "sustainable": true,
-    "buffer_seconds": 60.468784007191644,
+    "buffer_seconds": 60.57974540919467,
     "buffer_drain_seconds": null,
     "shard_target_mib": 16,
-    "upload_pages_per_commit": 33,
+    "upload_pages_per_commit": 41,
     "max_pending_pages": 4096,
     "generation_bytes_per_second": 0,
     "upload_bytes_per_second": null,

metrics/void-byte-v1/1790606918076636751-3952f1c5/train.jsonl

@@ -1,3 +1,4 @@
 {"step": 195100, "loss": 0.5759704150259495, "lr": 0.0003, "grad_norm": 0.14326037266666766, "recovery_retries": 0, "loss_scale": 1.0, "microbatch": 86, "input_bytes_and_specials": 28555098636, "tokens_per_second": 47929.59643830316, "data_wait_s": 1.7429000081392587e-05, "gpu_peak_gb": 85.46427917480469}
 {"step": 195200, "loss": 0.6118105687201023, "lr": 0.0003, "grad_norm": 0.13807380891249546, "recovery_retries": 0, "loss_scale": 1.0, "microbatch": 86, "input_bytes_and_specials": 28569686874, "tokens_per_second": 148719.4751744541, "data_wait_s": 1.639900006011885e-05, "gpu_peak_gb": 85.46427917480469}
 {"step": 195300, "loss": 0.5980571527034044, "lr": 0.0003, "grad_norm": 0.13369053890824728, "recovery_retries": 0, "loss_scale": 1.0, "microbatch": 86, "input_bytes_and_specials": 28584231055, "tokens_per_second": 148945.28831152062, "data_wait_s": 1.7738999986249837e-05, "gpu_peak_gb": 85.46427917480469}
+{"step": 195400, "loss": 0.599296510219574, "lr": 0.0003, "grad_norm": 0.13781984487626003, "recovery_retries": 0, "loss_scale": 1.0, "microbatch": 86, "input_bytes_and_specials": 28598726348, "tokens_per_second": 148718.05140297735, "data_wait_s": 1.5413000028274837e-05, "gpu_peak_gb": 85.46427917480469}

runs/void-byte-v1/events.out.tfevents.1790606918.0a7f8bdc-23d3-4ecb-bd0c-4a4bf460026e-rl2cx.1192.0.1790606918076636751-3952f1c5

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6345014fa6965218095170137a31c3efd60391dc8629b64f8fdd7f8a4c661ee
-size 6519
+oid sha256:1390f2d413741ca23548dfc6494ffa2eea3348db36c85a793f6de36bb51c6e00
+size 8637