diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -2,403 +2,6519 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.40384615384615385,
+  "epoch": 2.0,
   "eval_steps": 500,
-  "global_step": 126,
+  "global_step": 2500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0625,
-      "completions/max_length": 4096.0,
-      "completions/max_terminated_length": 2993.0,
-      "completions/mean_length": 842.1671875,
-      "completions/mean_terminated_length": 625.3808288574219,
-      "completions/min_length": 143.7,
-      "completions/min_terminated_length": 143.7,
-      "entropy": 0.18502369821071624,
-      "epoch": 0.03205128205128205,
-      "frac_reward_zero_std": 0.046875,
-      "grad_norm": 0.00025154745623815123,
-      "learning_rate": 1.4285714285714286e-06,
-      "loss": -0.0002,
-      "num_tokens": 2443746.0,
-      "reward": -0.22845993265509607,
-      "reward_std": 0.2702211543917656,
-      "rewards/grpo_reward_function/mean": -0.2284599334001541,
-      "rewards/grpo_reward_function/std": 0.7218511283397675,
-      "sampling/importance_sampling_ratio/max": 0.12355739073827862,
-      "sampling/importance_sampling_ratio/mean": 0.0029692337644519284,
-      "sampling/importance_sampling_ratio/min": 5.253219159649809e-30,
-      "sampling/sampling_logp_difference/max": 1.5544446587562561,
-      "sampling/sampling_logp_difference/mean": 0.03719679862260818,
+      "clip_ratio/high_max": 0.00017109453838202172,
+      "clip_ratio/high_mean": 4.4756848637916846e-05,
+      "clip_ratio/low_mean": 3.173092932229338e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 7.648777773283655e-05,
+      "completion_length": 4508.853125,
+      "completions/clipped_ratio": 0.2875,
+      "completions/max_length": 11627.6,
+      "completions/max_terminated_length": 11075.4,
+      "completions/mean_length": 4508.853125,
+      "completions/mean_terminated_length": 4391.1884765625,
+      "completions/min_length": 298.2,
+      "completions/min_terminated_length": 298.2,
+      "epoch": 0.008,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.04023393243551254,
+      "kl": 3.445962256591884e-05,
+      "learning_rate": 4.5e-07,
+      "loss": 0.0,
+      "num_tokens": 1756993.0,
+      "reward": 0.735617047548294,
+      "reward_std": 0.7832039952278137,
       "step": 10,
-      "step_time": 152.91457959786058
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.05625,
-      "completions/max_length": 4096.0,
-      "completions/max_terminated_length": 2889.4,
-      "completions/mean_length": 863.446875,
-      "completions/mean_terminated_length": 671.0787475585937,
-      "completions/min_length": 164.7,
-      "completions/min_terminated_length": 164.7,
-      "entropy": 0.1817332101985812,
-      "epoch": 0.0641025641025641,
-      "frac_reward_zero_std": 0.05,
-      "grad_norm": 1.9538725158444555e-05,
-      "learning_rate": 3.015873015873016e-06,
-      "loss": -0.0001,
-      "num_tokens": 4916654.0,
-      "reward": -0.18899608142673968,
-      "reward_std": 0.3027258589863777,
-      "rewards/grpo_reward_function/mean": -0.18899608589708805,
-      "rewards/grpo_reward_function/std": 0.7454370498657227,
-      "sampling/importance_sampling_ratio/max": 0.0923810960026458,
-      "sampling/importance_sampling_ratio/mean": 0.0023226177279866535,
-      "sampling/importance_sampling_ratio/min": 6.460355799993298e-36,
-      "sampling/sampling_logp_difference/max": 1.6125647306442261,
-      "sampling/sampling_logp_difference/mean": 0.03694682139903307,
+      "step_time": 248.331918213889
+    },
+    {
+      "clip_ratio/high_max": 0.00019471835221338552,
+      "clip_ratio/high_mean": 4.8914679518929914e-05,
+      "clip_ratio/low_mean": 4.743807050999749e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 9.635274900574587e-05,
+      "completion_length": 4459.30625,
+      "completions/clipped_ratio": 0.215625,
+      "completions/max_length": 12273.0,
+      "completions/max_terminated_length": 11036.6,
+      "completions/mean_length": 4459.30625,
+      "completions/mean_terminated_length": 4299.077197265625,
+      "completions/min_length": 408.6,
+      "completions/min_terminated_length": 408.6,
+      "epoch": 0.016,
+      "frac_reward_zero_std": 0.1125,
+      "grad_norm": 0.021025462076067924,
+      "kl": 0.0004306546949010226,
+      "learning_rate": 9.499999999999999e-07,
+      "loss": 0.0,
+      "num_tokens": 3534707.0,
+      "reward": 0.9525277614593506,
+      "reward_std": 0.4153731524944305,
       "step": 20,
-      "step_time": 159.88802388124168
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.046875,
-      "completions/max_length": 4096.0,
-      "completions/max_terminated_length": 2620.1,
-      "completions/mean_length": 832.69296875,
-      "completions/mean_terminated_length": 672.2784484863281,
-      "completions/min_length": 167.4,
-      "completions/min_terminated_length": 167.4,
-      "entropy": 0.19116276763379575,
-      "epoch": 0.09615384615384616,
-      "frac_reward_zero_std": 0.03125,
-      "grad_norm": 0.00031796028415218583,
-      "learning_rate": 4.603174603174604e-06,
-      "loss": -0.0005,
-      "num_tokens": 7342561.0,
-      "reward": -0.20435776934027672,
-      "reward_std": 0.2867868050932884,
-      "rewards/grpo_reward_function/mean": -0.20435777083039283,
-      "rewards/grpo_reward_function/std": 0.7296508550643921,
-      "sampling/importance_sampling_ratio/max": 0.06920737095642834,
-      "sampling/importance_sampling_ratio/mean": 0.0011224002329981886,
-      "sampling/importance_sampling_ratio/min": 5.1338851255929605e-34,
-      "sampling/sampling_logp_difference/max": 1.7805282711982726,
-      "sampling/sampling_logp_difference/mean": 0.03969721123576164,
+      "step_time": 267.5913352359086
+    },
+    {
+      "clip_ratio/high_max": 0.00028358621420920826,
+      "clip_ratio/high_mean": 7.155986404541181e-05,
+      "clip_ratio/low_mean": 8.107369249046314e-05,
+      "clip_ratio/low_min": 2.1462912627612242e-06,
+      "clip_ratio/region_mean": 0.0001526335558082792,
+      "completion_length": 4901.7625,
+      "completions/clipped_ratio": 0.23125,
+      "completions/max_length": 11967.2,
+      "completions/max_terminated_length": 11499.4,
+      "completions/mean_length": 4901.7625,
+      "completions/mean_terminated_length": 4801.2583984375,
+      "completions/min_length": 339.4,
+      "completions/min_terminated_length": 339.4,
+      "epoch": 0.024,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.02784157544374466,
+      "kl": 0.0047081467884709126,
+      "learning_rate": 1e-06,
+      "loss": 0.0001,
+      "num_tokens": 5446215.0,
+      "reward": 0.6667743384838104,
+      "reward_std": 0.6910888135433197,
       "step": 30,
-      "step_time": 153.14357568509877
-    },
-    {
-      "clip_ratio/high_max": 5.1770552090602e-06,
-      "clip_ratio/high_mean": 6.47131901132525e-07,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 6.47131901132525e-07,
-      "completions/clipped_ratio": 0.075,
-      "completions/max_length": 4096.0,
-      "completions/max_terminated_length": 2996.0,
-      "completions/mean_length": 908.88203125,
-      "completions/mean_terminated_length": 650.461572265625,
-      "completions/min_length": 172.5,
-      "completions/min_terminated_length": 172.5,
-      "entropy": 0.18940101601183415,
-      "epoch": 0.1282051282051282,
-      "frac_reward_zero_std": 0.0375,
-      "grad_norm": 0.00046653815964065254,
-      "learning_rate": 6.1904761904761914e-06,
-      "loss": -0.0007,
-      "num_tokens": 9897290.0,
-      "reward": -0.20948485173285009,
-      "reward_std": 0.3274365648627281,
-      "rewards/grpo_reward_function/mean": -0.20948485508561135,
-      "rewards/grpo_reward_function/std": 0.7691417872905731,
-      "sampling/importance_sampling_ratio/max": 0.12555439178831876,
-      "sampling/importance_sampling_ratio/mean": 0.0029983414759044537,
-      "sampling/importance_sampling_ratio/min": 7.312246668686746e-35,
-      "sampling/sampling_logp_difference/max": 1.8727038741111754,
-      "sampling/sampling_logp_difference/mean": 0.03728015031665564,
+      "step_time": 245.90135910734534
+    },
+    {
+      "clip_ratio/high_max": 0.0002775157414362184,
+      "clip_ratio/high_mean": 7.129447290026292e-05,
+      "clip_ratio/low_mean": 7.341240775531333e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.000144706879837031,
+      "completion_length": 4518.809375,
+      "completions/clipped_ratio": 0.26875,
+      "completions/max_length": 11747.8,
+      "completions/max_terminated_length": 10563.0,
+      "completions/mean_length": 4518.809375,
+      "completions/mean_terminated_length": 4361.417578125,
+      "completions/min_length": 371.6,
+      "completions/min_terminated_length": 444.6,
+      "epoch": 0.032,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.029684122651815414,
+      "kl": 0.008555429431726224,
+      "learning_rate": 1e-06,
+      "loss": 0.0001,
+      "num_tokens": 7240022.0,
+      "reward": 0.910753047466278,
+      "reward_std": 0.6723451852798462,
       "step": 40,
-      "step_time": 146.35395600683987
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0515625,
-      "completions/max_length": 4096.0,
-      "completions/max_terminated_length": 2913.9,
-      "completions/mean_length": 806.88984375,
-      "completions/mean_terminated_length": 627.2792419433594,
-      "completions/min_length": 123.4,
-      "completions/min_terminated_length": 123.4,
-      "entropy": 0.17744111455976963,
-      "epoch": 0.16025641025641027,
-      "frac_reward_zero_std": 0.05625,
-      "grad_norm": 9.61458607191603e-05,
-      "learning_rate": 7.77777777777778e-06,
-      "loss": 0.0003,
-      "num_tokens": 12302889.0,
-      "reward": -0.18009744100272657,
-      "reward_std": 0.28012245148420334,
-      "rewards/grpo_reward_function/mean": -0.18009743914008142,
-      "rewards/grpo_reward_function/std": 0.7488289535045624,
-      "sampling/importance_sampling_ratio/max": 0.17218044362962245,
-      "sampling/importance_sampling_ratio/mean": 0.004992885573301464,
-      "sampling/importance_sampling_ratio/min": 1.0306885153000913e-30,
-      "sampling/sampling_logp_difference/max": 1.6258580565452576,
-      "sampling/sampling_logp_difference/mean": 0.03607845064252615,
+      "step_time": 259.3002842463553
+    },
+    {
+      "clip_ratio/high_max": 0.0003546871921571437,
+      "clip_ratio/high_mean": 0.00010078994046125445,
+      "clip_ratio/low_mean": 3.091837692181798e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0001317083166213706,
+      "completion_length": 4327.0875,
+      "completions/clipped_ratio": 0.315625,
+      "completions/max_length": 11880.4,
+      "completions/max_terminated_length": 11294.8,
+      "completions/mean_length": 4327.0875,
+      "completions/mean_terminated_length": 4127.9142578125,
+      "completions/min_length": 319.2,
+      "completions/min_terminated_length": 319.2,
+      "epoch": 0.04,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.02914806827902794,
+      "kl": 0.013973764760885387,
+      "learning_rate": 1e-06,
+      "loss": 0.0004,
+      "num_tokens": 8958490.0,
+      "reward": 0.9430972099304199,
+      "reward_std": 0.6455870628356933,
       "step": 50,
-      "step_time": 147.80184614248574
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.03984375,
-      "completions/max_length": 4096.0,
-      "completions/max_terminated_length": 2814.3,
-      "completions/mean_length": 809.65078125,
-      "completions/mean_terminated_length": 673.7666534423828,
-      "completions/min_length": 141.0,
-      "completions/min_terminated_length": 141.0,
-      "entropy": 0.18477382734417916,
-      "epoch": 0.19230769230769232,
-      "frac_reward_zero_std": 0.034375,
-      "grad_norm": 0.0005422528630453429,
-      "learning_rate": 9.365079365079366e-06,
-      "loss": -0.0002,
-      "num_tokens": 14762678.0,
-      "reward": -0.18410459933802487,
-      "reward_std": 0.3047805741429329,
-      "rewards/grpo_reward_function/mean": -0.18410460744053125,
-      "rewards/grpo_reward_function/std": 0.7665571451187134,
-      "sampling/importance_sampling_ratio/max": 0.10517687269020826,
-      "sampling/importance_sampling_ratio/mean": 0.0030420748858887236,
-      "sampling/importance_sampling_ratio/min": 3.9674455768173534e-27,
-      "sampling/sampling_logp_difference/max": 1.571552813053131,
-      "sampling/sampling_logp_difference/mean": 0.03778362385928631,
+      "step_time": 247.06374224312603
+    },
+    {
+      "clip_ratio/high_max": 0.00033122178556368456,
+      "clip_ratio/high_mean": 9.912220411933959e-05,
+      "clip_ratio/low_mean": 2.439793472603924e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00012352013850431832,
+      "completion_length": 4520.571875,
+      "completions/clipped_ratio": 0.36875,
+      "completions/max_length": 12350.0,
+      "completions/max_terminated_length": 11248.6,
+      "completions/mean_length": 4520.571875,
+      "completions/mean_terminated_length": 4284.872705078125,
+      "completions/min_length": 354.6,
+      "completions/min_terminated_length": 354.6,
+      "epoch": 0.048,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.03457050397992134,
+      "kl": 0.016152070835232735,
+      "learning_rate": 1e-06,
+      "loss": 0.0003,
+      "num_tokens": 10737417.0,
+      "reward": 0.6907579690217972,
+      "reward_std": 0.827730929851532,
       "step": 60,
-      "step_time": 144.8840892329812
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.059375,
-      "completions/max_length": 4096.0,
-      "completions/max_terminated_length": 2730.5,
-      "completions/mean_length": 846.08828125,
-      "completions/mean_terminated_length": 641.0483703613281,
-      "completions/min_length": 134.2,
-      "completions/min_terminated_length": 134.2,
-      "entropy": 0.18578412756323814,
-      "epoch": 0.22435897435897437,
-      "frac_reward_zero_std": 0.03125,
-      "grad_norm": 0.00012190250477276098,
-      "learning_rate": 9.985680226398261e-06,
-      "loss": -0.0002,
-      "num_tokens": 17196835.0,
-      "reward": -0.23204001784324646,
-      "reward_std": 0.3367327839136124,
-      "rewards/grpo_reward_function/mean": -0.23204001784324646,
-      "rewards/grpo_reward_function/std": 0.7099093854427337,
-      "sampling/importance_sampling_ratio/max": 0.04683403689414263,
-      "sampling/importance_sampling_ratio/mean": 0.0009577752702170983,
-      "sampling/importance_sampling_ratio/min": 1.4199457666368224e-36,
-      "sampling/sampling_logp_difference/max": 1.5709129929542542,
-      "sampling/sampling_logp_difference/mean": 0.037703079357743266,
+      "step_time": 267.7415327694267
+    },
+    {
+      "clip_ratio/high_max": 0.0002710134916924289,
+      "clip_ratio/high_mean": 6.441019427256833e-05,
+      "clip_ratio/low_mean": 4.6787399742242994e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00011119759619759861,
+      "completion_length": 4166.43125,
+      "completions/clipped_ratio": 0.425,
+      "completions/max_length": 12149.6,
+      "completions/max_terminated_length": 11088.0,
+      "completions/mean_length": 4166.43125,
+      "completions/mean_terminated_length": 4071.306884765625,
+      "completions/min_length": 251.4,
+      "completions/min_terminated_length": 251.4,
+      "epoch": 0.056,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.03421972319483757,
+      "kl": 0.012524099717848004,
+      "learning_rate": 1e-06,
+      "loss": 0.0003,
+      "num_tokens": 12407279.0,
+      "reward": 0.8804079532623291,
+      "reward_std": 0.6892471015453339,
       "step": 70,
-      "step_time": 156.71906763464213
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0421875,
-      "completions/max_length": 4096.0,
-      "completions/max_terminated_length": 2464.4,
-      "completions/mean_length": 817.75546875,
-      "completions/mean_terminated_length": 674.2539978027344,
-      "completions/min_length": 149.3,
-      "completions/min_terminated_length": 149.3,
-      "entropy": 0.18018180541694165,
-      "epoch": 0.2564102564102564,
-      "frac_reward_zero_std": 0.05,
-      "grad_norm": 5.762501312773685e-05,
-      "learning_rate": 9.89846735808731e-06,
-      "loss": -0.0,
-      "num_tokens": 19646058.0,
-      "reward": -0.1727930422872305,
-      "reward_std": 0.31547281742095945,
-      "rewards/grpo_reward_function/mean": -0.1727930411696434,
-      "rewards/grpo_reward_function/std": 0.7881518006324768,
-      "sampling/importance_sampling_ratio/max": 0.13909520097076894,
-      "sampling/importance_sampling_ratio/mean": 0.004160506054176949,
-      "sampling/importance_sampling_ratio/min": 6.660712543136218e-25,
-      "sampling/sampling_logp_difference/max": 1.5812078952789306,
-      "sampling/sampling_logp_difference/mean": 0.03748476393520832,
+      "step_time": 268.9941872537136
+    },
+    {
+      "clip_ratio/high_max": 0.00032792381061881316,
+      "clip_ratio/high_mean": 8.613915451860521e-05,
+      "clip_ratio/low_mean": 3.0676640176352524e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00011681579180731206,
+      "completion_length": 5040.99375,
+      "completions/clipped_ratio": 0.5,
+      "completions/max_length": 12350.0,
+      "completions/max_terminated_length": 11060.8,
+      "completions/mean_length": 5040.99375,
+      "completions/mean_terminated_length": 5145.896142578125,
+      "completions/min_length": 392.6,
+      "completions/min_terminated_length": 392.6,
+      "epoch": 0.064,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.021355072036385536,
+      "kl": 0.01887360216351226,
+      "learning_rate": 1e-06,
+      "loss": 0.0004,
+      "num_tokens": 14376173.0,
+      "reward": 0.7827508926391602,
+      "reward_std": 0.7515387535095215,
       "step": 80,
-      "step_time": 151.56825386658312
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.04609375,
-      "completions/max_length": 4096.0,
-      "completions/max_terminated_length": 2902.4,
-      "completions/mean_length": 803.97109375,
-      "completions/mean_terminated_length": 644.9527221679688,
-      "completions/min_length": 159.1,
-      "completions/min_terminated_length": 159.1,
-      "entropy": 0.18126422837376593,
-      "epoch": 0.28846153846153844,
-      "frac_reward_zero_std": 0.071875,
-      "grad_norm": 1.7400093375084389e-06,
-      "learning_rate": 9.733381816303395e-06,
-      "loss": 0.0001,
-      "num_tokens": 22062549.0,
-      "reward": -0.1791908085346222,
-      "reward_std": 0.283918160200119,
-      "rewards/grpo_reward_function/mean": -0.1791908085346222,
-      "rewards/grpo_reward_function/std": 0.7836591958999634,
-      "sampling/importance_sampling_ratio/max": 0.15381892090663313,
-      "sampling/importance_sampling_ratio/mean": 0.004421875764819561,
-      "sampling/importance_sampling_ratio/min": 3.314493692446194e-30,
-      "sampling/sampling_logp_difference/max": 1.59914892911911,
-      "sampling/sampling_logp_difference/mean": 0.03814994990825653,
+      "step_time": 273.59297795519234
+    },
+    {
+      "clip_ratio/high_max": 0.0003735334103112109,
+      "clip_ratio/high_mean": 0.00011695282519212924,
+      "clip_ratio/low_mean": 4.054330054259481e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0001574961279402487,
+      "completion_length": 4363.28125,
+      "completions/clipped_ratio": 0.565625,
+      "completions/max_length": 11940.8,
+      "completions/max_terminated_length": 11177.0,
+      "completions/mean_length": 4363.28125,
+      "completions/mean_terminated_length": 4615.284765625,
+      "completions/min_length": 414.0,
+      "completions/min_terminated_length": 414.0,
+      "epoch": 0.072,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.037766724824905396,
+      "kl": 0.021469853981398047,
+      "learning_rate": 1e-06,
+      "loss": 0.0004,
+      "num_tokens": 16128431.0,
+      "reward": 0.7956950902938843,
+      "reward_std": 0.722217059135437,
       "step": 90,
-      "step_time": 143.37383015677332
-    },
-    {
-      "clip_ratio/high_max": 2.9168124456191437e-06,
-      "clip_ratio/high_mean": 3.6460155570239296e-07,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 3.6460155570239296e-07,
-      "completions/clipped_ratio": 0.06015625,
-      "completions/max_length": 4096.0,
-      "completions/max_terminated_length": 3118.0,
-      "completions/mean_length": 833.84453125,
-      "completions/mean_terminated_length": 625.6610107421875,
-      "completions/min_length": 146.9,
-      "completions/min_terminated_length": 146.9,
-      "entropy": 0.17976893596351146,
-      "epoch": 0.32051282051282054,
-      "frac_reward_zero_std": 0.053125,
-      "grad_norm": 1.5939340167787748e-05,
-      "learning_rate": 9.493048024473413e-06,
-      "loss": 0.0002,
-      "num_tokens": 24458486.0,
-      "reward": -0.188871893286705,
-      "reward_std": 0.31348400712013247,
-      "rewards/grpo_reward_function/mean": -0.18887189254164696,
-      "rewards/grpo_reward_function/std": 0.7829755127429963,
-      "sampling/importance_sampling_ratio/max": 0.16449878164567053,
-      "sampling/importance_sampling_ratio/mean": 0.005532666263025021,
-      "sampling/importance_sampling_ratio/min": 3.4595329788052715e-35,
-      "sampling/sampling_logp_difference/max": 1.61622656583786,
-      "sampling/sampling_logp_difference/mean": 0.036300059966742994,
+      "step_time": 244.99433429948985
+    },
+    {
+      "clip_ratio/high_max": 0.00025482733217359056,
+      "clip_ratio/high_mean": 7.33067777218821e-05,
+      "clip_ratio/low_mean": 2.154921501187346e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 9.485599275649292e-05,
+      "completion_length": 4830.125,
+      "completions/clipped_ratio": 0.515625,
+      "completions/max_length": 12298.2,
+      "completions/max_terminated_length": 11621.2,
+      "completions/mean_length": 4830.125,
+      "completions/mean_terminated_length": 4941.6603515625,
+      "completions/min_length": 360.8,
+      "completions/min_terminated_length": 360.8,
+      "epoch": 0.08,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.02395726554095745,
+      "kl": 0.021513055684044956,
+      "learning_rate": 1e-06,
+      "loss": 0.0005,
+      "num_tokens": 18023079.0,
+      "reward": 0.9237602233886719,
+      "reward_std": 0.5612543225288391,
       "step": 100,
-      "step_time": 150.7187235403806
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.05,
-      "completions/max_length": 4096.0,
-      "completions/max_terminated_length": 2412.8,
-      "completions/mean_length": 794.35859375,
-      "completions/mean_terminated_length": 620.5320068359375,
-      "completions/min_length": 128.8,
-      "completions/min_terminated_length": 128.8,
-      "entropy": 0.1772497620433569,
-      "epoch": 0.3525641025641026,
-      "frac_reward_zero_std": 0.046875,
-      "grad_norm": 4.817197293953718e-06,
-      "learning_rate": 9.18128665415186e-06,
-      "loss": -0.0001,
-      "num_tokens": 26866049.0,
-      "reward": -0.22932868972420692,
-      "reward_std": 0.31401748955249786,
-      "rewards/grpo_reward_function/mean": -0.22932869493961333,
-      "rewards/grpo_reward_function/std": 0.7706308960914612,
-      "sampling/importance_sampling_ratio/max": 0.1050915487576276,
-      "sampling/importance_sampling_ratio/mean": 0.0028572272043675185,
-      "sampling/importance_sampling_ratio/min": 5.94164918536593e-23,
-      "sampling/sampling_logp_difference/max": 1.725569200515747,
-      "sampling/sampling_logp_difference/mean": 0.03586800619959831,
+      "step_time": 272.47559492252765
+    },
+    {
+      "clip_ratio/high_max": 0.0002424940677883569,
+      "clip_ratio/high_mean": 6.159636550364666e-05,
+      "clip_ratio/low_mean": 3.111228581929026e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 9.270864893551334e-05,
+      "completion_length": 4095.61875,
+      "completions/clipped_ratio": 0.571875,
+      "completions/max_length": 12088.6,
+      "completions/max_terminated_length": 10257.2,
+      "completions/mean_length": 4095.61875,
+      "completions/mean_terminated_length": 3609.52900390625,
+      "completions/min_length": 266.8,
+      "completions/min_terminated_length": 266.8,
+      "epoch": 0.088,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.038928303867578506,
+      "kl": 0.02800684408284724,
+      "learning_rate": 1e-06,
+      "loss": 0.0006,
+      "num_tokens": 19671753.0,
+      "reward": 0.977106761932373,
+      "reward_std": 0.5293310284614563,
       "step": 110,
-      "step_time": 141.95229457244278
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.04921875,
-      "completions/max_length": 4096.0,
-      "completions/max_terminated_length": 2462.2,
-      "completions/mean_length": 793.1078125,
-      "completions/mean_terminated_length": 622.2306030273437,
-      "completions/min_length": 117.5,
-      "completions/min_terminated_length": 117.5,
-      "entropy": 0.17735922262072562,
-      "epoch": 0.38461538461538464,
-      "frac_reward_zero_std": 0.053125,
-      "grad_norm": 9.213808267741939e-05,
-      "learning_rate": 8.803053886449644e-06,
-      "loss": -0.0001,
-      "num_tokens": 29246979.0,
-      "reward": -0.2611034892499447,
-      "reward_std": 0.28704351782798765,
-      "rewards/grpo_reward_function/mean": -0.26110349521040915,
-      "rewards/grpo_reward_function/std": 0.7350896656513214,
-      "sampling/importance_sampling_ratio/max": 0.084448746079579,
-      "sampling/importance_sampling_ratio/mean": 0.001967475106357597,
-      "sampling/importance_sampling_ratio/min": 3.0353923760337493e-31,
-      "sampling/sampling_logp_difference/max": 1.6334303975105287,
-      "sampling/sampling_logp_difference/mean": 0.03619050718843937,
+      "step_time": 238.56932506002485
+    },
+    {
+      "clip_ratio/high_max": 0.00017560835531185148,
+      "clip_ratio/high_mean": 3.749609584247082e-05,
+      "clip_ratio/low_mean": 8.9413937757854e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00012691003303189063,
+      "completion_length": 4334.653125,
+      "completions/clipped_ratio": 0.553125,
+      "completions/max_length": 11209.2,
+      "completions/max_terminated_length": 10787.4,
+      "completions/mean_length": 4334.653125,
+      "completions/mean_terminated_length": 4386.7771484375,
+      "completions/min_length": 273.4,
+      "completions/min_terminated_length": 273.4,
+      "epoch": 0.096,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.03180592134594917,
+      "kl": 0.03545058385934681,
+      "learning_rate": 1e-06,
+      "loss": 0.0007,
+      "num_tokens": 21407610.0,
+      "reward": 1.05890212059021,
+      "reward_std": 0.5852488875389099,
       "step": 120,
-      "step_time": 144.3923004835844
+      "step_time": 240.1034249998629
+    },
+    {
+      "clip_ratio/high_max": 0.0002589082570921164,
+      "clip_ratio/high_mean": 6.931500292921555e-05,
+      "clip_ratio/low_mean": 2.6596866814543318e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 9.591187117621302e-05,
+      "completion_length": 4778.421875,
+      "completions/clipped_ratio": 0.615625,
+      "completions/max_length": 12145.8,
+      "completions/max_terminated_length": 11531.6,
+      "completions/mean_length": 4778.421875,
+      "completions/mean_terminated_length": 4792.2806640625,
+      "completions/min_length": 451.8,
+      "completions/min_terminated_length": 476.0,
+      "epoch": 0.104,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.021660316735506058,
+      "kl": 0.035527407703921196,
+      "learning_rate": 1e-06,
+      "loss": 0.0008,
+      "num_tokens": 23282777.0,
+      "reward": 0.8782649159431457,
+      "reward_std": 0.5700801074504852,
+      "step": 130,
+      "step_time": 259.071363786608
+    },
+    {
+      "clip_ratio/high_max": 0.00014997779398981948,
+      "clip_ratio/high_mean": 4.131667119509075e-05,
+      "clip_ratio/low_mean": 7.407385214719397e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00011539052375155733,
+      "completion_length": 4513.66875,
+      "completions/clipped_ratio": 0.521875,
+      "completions/max_length": 11222.6,
+      "completions/max_terminated_length": 11141.8,
+      "completions/mean_length": 4513.66875,
+      "completions/mean_terminated_length": 5039.8330078125,
+      "completions/min_length": 305.4,
+      "completions/min_terminated_length": 305.4,
+      "epoch": 0.112,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.02563977614045143,
+      "kl": 0.037015831121243535,
+      "learning_rate": 1e-06,
+      "loss": 0.0008,
+      "num_tokens": 25067987.0,
+      "reward": 0.8918760657310486,
+      "reward_std": 0.641742593050003,
+      "step": 140,
+      "step_time": 252.8746670950204
+    },
+    {
+      "clip_ratio/high_max": 0.0005541605991311371,
+      "clip_ratio/high_mean": 0.00017582649088581092,
+      "clip_ratio/low_mean": 2.9667073204109328e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00020549356468109182,
+      "completion_length": 4256.33125,
+      "completions/clipped_ratio": 0.59375,
+      "completions/max_length": 12401.8,
+      "completions/max_terminated_length": 10912.0,
+      "completions/mean_length": 4256.33125,
+      "completions/mean_terminated_length": 4269.922607421875,
+      "completions/min_length": 478.4,
+      "completions/min_terminated_length": 629.0,
+      "epoch": 0.12,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.0503266416490078,
+      "kl": 0.06640126733109356,
+      "learning_rate": 1e-06,
+      "loss": 0.0016,
+      "num_tokens": 26775077.0,
+      "reward": 0.8434090971946716,
+      "reward_std": 0.6193858087062836,
+      "step": 150,
+      "step_time": 250.72402255497872
+    },
+    {
+      "clip_ratio/high_max": 0.00034618185381987134,
+      "clip_ratio/high_mean": 8.284584082502988e-05,
+      "clip_ratio/low_mean": 5.385545400713454e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0001367012970149517,
+      "completion_length": 4253.425,
+      "completions/clipped_ratio": 0.55625,
+      "completions/max_length": 12108.6,
+      "completions/max_terminated_length": 10451.0,
+      "completions/mean_length": 4253.425,
+      "completions/mean_terminated_length": 4368.098193359375,
+      "completions/min_length": 337.8,
+      "completions/min_terminated_length": 337.8,
+      "epoch": 0.128,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.020152408629655838,
+      "kl": 0.03334178449586034,
+      "learning_rate": 1e-06,
+      "loss": 0.0007,
+      "num_tokens": 28468729.0,
+      "reward": 0.9704797983169555,
+      "reward_std": 0.6033674567937851,
+      "step": 160,
+      "step_time": 256.1296712331474
+    },
+    {
+      "clip_ratio/high_max": 0.0002411140318145044,
+      "clip_ratio/high_mean": 5.5890660632940126e-05,
+      "clip_ratio/low_mean": 0.00011550156973498815,
+      "clip_ratio/low_min": 1.5271838492481037e-06,
+      "clip_ratio/region_mean": 0.00017139223127742297,
+      "completion_length": 4348.09375,
+      "completions/clipped_ratio": 0.56875,
+      "completions/max_length": 11772.2,
+      "completions/max_terminated_length": 11365.8,
+      "completions/mean_length": 4348.09375,
+      "completions/mean_terminated_length": 4047.23994140625,
+      "completions/min_length": 213.2,
+      "completions/min_terminated_length": 213.2,
+      "epoch": 0.136,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.023590492084622383,
+      "kl": 0.03905970910564065,
+      "learning_rate": 1e-06,
+      "loss": 0.0008,
+      "num_tokens": 30197847.0,
+      "reward": 1.0630458116531372,
+      "reward_std": 0.6647608757019043,
+      "step": 170,
+      "step_time": 260.2357140336186
+    },
+    {
+      "clip_ratio/high_max": 0.0004132027068408206,
+      "clip_ratio/high_mean": 0.0001175258098555787,
+      "clip_ratio/low_mean": 5.0041659585531305e-05,
+      "clip_ratio/low_min": 1.6107209376059472e-06,
+      "clip_ratio/region_mean": 0.00016756747227191225,
+      "completion_length": 4600.15625,
+      "completions/clipped_ratio": 0.6375,
+      "completions/max_length": 12165.2,
+      "completions/max_terminated_length": 11660.4,
+      "completions/mean_length": 4600.15625,
+      "completions/mean_terminated_length": 4767.691357421875,
+      "completions/min_length": 397.0,
+      "completions/min_terminated_length": 397.0,
+      "epoch": 0.144,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.027969539165496826,
+      "kl": 0.0629050656221807,
+      "learning_rate": 1e-06,
+      "loss": 0.0017,
+      "num_tokens": 32001281.0,
+      "reward": 0.7982242465019226,
+      "reward_std": 0.6176994144916534,
+      "step": 180,
+      "step_time": 266.1079626072198
+    },
+    {
+      "clip_ratio/high_max": 0.0003143023199299932,
+      "clip_ratio/high_mean": 8.408852108914289e-05,
+      "clip_ratio/low_mean": 3.088964660946658e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00011497816813061945,
+      "completion_length": 4165.0125,
+      "completions/clipped_ratio": 0.584375,
+      "completions/max_length": 10773.8,
+      "completions/max_terminated_length": 9282.8,
+      "completions/mean_length": 4165.0125,
+      "completions/mean_terminated_length": 3493.499365234375,
+      "completions/min_length": 372.4,
+      "completions/min_terminated_length": 372.4,
+      "epoch": 0.152,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.022769423201680183,
+      "kl": 0.04423899878747761,
+      "learning_rate": 1e-06,
+      "loss": 0.0008,
+      "num_tokens": 33682949.0,
+      "reward": 1.1953443527221679,
+      "reward_std": 0.5240781962871551,
+      "step": 190,
+      "step_time": 226.94223432950676
+    },
+    {
+      "clip_ratio/high_max": 0.00027104075634269974,
+      "clip_ratio/high_mean": 8.54599676131329e-05,
+      "clip_ratio/low_mean": 4.205289217225072e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00012751285976264626,
+      "completion_length": 4271.071875,
+      "completions/clipped_ratio": 0.58125,
+      "completions/max_length": 12365.8,
+      "completions/max_terminated_length": 11339.2,
+      "completions/mean_length": 4271.071875,
+      "completions/mean_terminated_length": 5041.990576171875,
+      "completions/min_length": 366.2,
+      "completions/min_terminated_length": 366.2,
+      "epoch": 0.16,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.02732943743467331,
+      "kl": 0.05689977160654962,
+      "learning_rate": 1e-06,
+      "loss": 0.0013,
+      "num_tokens": 35376336.0,
+      "reward": 0.7087696433067322,
+      "reward_std": 0.4853620886802673,
+      "step": 200,
+      "step_time": 260.072031204775
+    },
+    {
+      "clip_ratio/high_max": 0.00011647099672700279,
+      "clip_ratio/high_mean": 2.99047061616875e-05,
+      "clip_ratio/low_mean": 6.795050112486933e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 9.785520978766726e-05,
+      "completion_length": 4562.984375,
+      "completions/clipped_ratio": 0.440625,
+      "completions/max_length": 11690.0,
+      "completions/max_terminated_length": 10887.2,
+      "completions/mean_length": 4562.984375,
+      "completions/mean_terminated_length": 4700.125927734375,
+      "completions/min_length": 641.0,
+      "completions/min_terminated_length": 641.0,
+      "epoch": 0.168,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.037929717451334,
+      "kl": 0.06010220586322248,
+      "learning_rate": 1e-06,
+      "loss": 0.0013,
+      "num_tokens": 37190419.0,
+      "reward": 1.0412225484848023,
+      "reward_std": 0.5834640264511108,
+      "step": 210,
+      "step_time": 250.99351372420787
+    },
+    {
+      "clip_ratio/high_max": 0.00021369188398239202,
+      "clip_ratio/high_mean": 5.54367934455513e-05,
+      "clip_ratio/low_mean": 3.861530169615435e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 9.405209439137253e-05,
+      "completion_length": 4373.7125,
+      "completions/clipped_ratio": 0.44375,
+      "completions/max_length": 12020.2,
+      "completions/max_terminated_length": 11092.0,
+      "completions/mean_length": 4373.7125,
+      "completions/mean_terminated_length": 4283.738134765625,
+      "completions/min_length": 453.2,
+      "completions/min_terminated_length": 453.2,
+      "epoch": 0.176,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.03626369684934616,
+      "kl": 0.049935971619561315,
+      "learning_rate": 1e-06,
+      "loss": 0.0011,
+      "num_tokens": 38932643.0,
+      "reward": 1.0245176434516907,
+      "reward_std": 0.6660697817802429,
+      "step": 220,
+      "step_time": 250.5279387306422
+    },
+    {
+      "clip_ratio/high_max": 0.00022633985463471618,
+      "clip_ratio/high_mean": 5.4864864750925335e-05,
+      "clip_ratio/low_mean": 4.1018057640940245e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 9.588292332409764e-05,
+      "completion_length": 4248.484375,
+      "completions/clipped_ratio": 0.48125,
+      "completions/max_length": 11881.6,
+      "completions/max_terminated_length": 11083.8,
+      "completions/mean_length": 4248.484375,
+      "completions/mean_terminated_length": 4365.04306640625,
+      "completions/min_length": 396.8,
+      "completions/min_terminated_length": 396.8,
+      "epoch": 0.184,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.0313262976706028,
+      "kl": 0.04819903369061649,
+      "learning_rate": 1e-06,
+      "loss": 0.001,
+      "num_tokens": 40629210.0,
+      "reward": 1.162195372581482,
+      "reward_std": 0.49698888063430785,
+      "step": 230,
+      "step_time": 258.92632155828176
+    },
+    {
+      "clip_ratio/high_max": 0.000382890687615145,
+      "clip_ratio/high_mean": 6.794390196773747e-05,
+      "clip_ratio/low_mean": 9.703520649964048e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0001649791061936412,
+      "completion_length": 4839.2625,
+      "completions/clipped_ratio": 0.44375,
+      "completions/max_length": 11599.0,
+      "completions/max_terminated_length": 11011.4,
+      "completions/mean_length": 4839.2625,
+      "completions/mean_terminated_length": 4754.535595703125,
+      "completions/min_length": 463.4,
+      "completions/min_terminated_length": 463.4,
+      "epoch": 0.192,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03009759820997715,
+      "kl": 0.03415646129287779,
+      "learning_rate": 1e-06,
+      "loss": 0.0006,
+      "num_tokens": 42534570.0,
+      "reward": 1.3234744548797608,
+      "reward_std": 0.4837889313697815,
+      "step": 240,
+      "step_time": 246.19934557192028
+    },
+    {
+      "clip_ratio/high_max": 0.0002871942226192914,
+      "clip_ratio/high_mean": 7.994032857823185e-05,
+      "clip_ratio/low_mean": 7.656777952433914e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0001565081089211162,
+      "completion_length": 4670.725,
+      "completions/clipped_ratio": 0.43125,
+      "completions/max_length": 11417.6,
+      "completions/max_terminated_length": 11006.4,
+      "completions/mean_length": 4670.725,
+      "completions/mean_terminated_length": 5211.94130859375,
+      "completions/min_length": 376.2,
+      "completions/min_terminated_length": 376.2,
+      "epoch": 0.2,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.02753828465938568,
+      "kl": 0.05637350436300039,
+      "learning_rate": 1e-06,
+      "loss": 0.0014,
+      "num_tokens": 44354614.0,
+      "reward": 0.8665350317955017,
+      "reward_std": 0.5523547768592835,
+      "step": 250,
+      "step_time": 240.42417666763066
+    },
+    {
+      "clip_ratio/high_max": 0.00040627521157148293,
+      "clip_ratio/high_mean": 0.00010838812293059163,
+      "clip_ratio/low_mean": 6.683943574898876e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00017522755697427784,
+      "completion_length": 4684.696875,
+      "completions/clipped_ratio": 0.53125,
+      "completions/max_length": 12004.4,
+      "completions/max_terminated_length": 11613.8,
+      "completions/mean_length": 4684.696875,
+      "completions/mean_terminated_length": 4769.5064453125,
+      "completions/min_length": 289.6,
+      "completions/min_terminated_length": 289.6,
+      "epoch": 0.208,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.03328331559896469,
+      "kl": 0.05595397977158427,
+      "learning_rate": 1e-06,
+      "loss": 0.0011,
+      "num_tokens": 46198121.0,
+      "reward": 0.9549756288528443,
+      "reward_std": 0.5750404953956604,
+      "step": 260,
+      "step_time": 261.9464150454849
+    },
+    {
+      "clip_ratio/high_max": 0.00022607493065152084,
+      "clip_ratio/high_mean": 6.600209994758188e-05,
+      "clip_ratio/low_mean": 0.0001434656996934791,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00020946780168742407,
+      "completion_length": 4123.875,
+      "completions/clipped_ratio": 0.5375,
+      "completions/max_length": 12045.2,
+      "completions/max_terminated_length": 11739.8,
+      "completions/mean_length": 4123.875,
+      "completions/mean_terminated_length": 4613.94541015625,
+      "completions/min_length": 305.6,
+      "completions/min_terminated_length": 305.6,
+      "epoch": 0.216,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.03498842194676399,
+      "kl": 0.05647929436527192,
+      "learning_rate": 1e-06,
+      "loss": 0.0011,
+      "num_tokens": 47836653.0,
+      "reward": 0.9957808256149292,
+      "reward_std": 0.44771295189857485,
+      "step": 270,
+      "step_time": 229.83147338069975
+    },
+    {
+      "clip_ratio/high_max": 0.000279573153511592,
+      "clip_ratio/high_mean": 6.148562068801766e-05,
+      "clip_ratio/low_mean": 0.0001419806900230469,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00020346631135907954,
+      "completion_length": 3967.25,
+      "completions/clipped_ratio": 0.45,
+      "completions/max_length": 10765.4,
+      "completions/max_terminated_length": 10571.8,
+      "completions/mean_length": 3967.25,
+      "completions/mean_terminated_length": 3932.38408203125,
+      "completions/min_length": 421.6,
+      "completions/min_terminated_length": 421.6,
+      "epoch": 0.224,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.035050343722105026,
+      "kl": 0.04583937788847834,
+      "learning_rate": 1e-06,
+      "loss": 0.0007,
+      "num_tokens": 49437953.0,
+      "reward": 1.2781912565231324,
+      "reward_std": 0.4175270140171051,
+      "step": 280,
+      "step_time": 224.67504201419652
+    },
+    {
+      "clip_ratio/high_max": 0.00030530111653206406,
+      "clip_ratio/high_mean": 7.985553465914564e-05,
+      "clip_ratio/low_mean": 6.674134313016112e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0001465968776756199,
+      "completion_length": 4506.98125,
+      "completions/clipped_ratio": 0.421875,
+      "completions/max_length": 11506.0,
+      "completions/max_terminated_length": 11325.6,
+      "completions/mean_length": 4506.98125,
+      "completions/mean_terminated_length": 4571.89404296875,
+      "completions/min_length": 226.8,
+      "completions/min_terminated_length": 226.8,
+      "epoch": 0.232,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.028526220470666885,
+      "kl": 0.04422827670350671,
+      "learning_rate": 1e-06,
+      "loss": 0.001,
+      "num_tokens": 51205051.0,
+      "reward": 1.2004335880279542,
+      "reward_std": 0.49139217734336854,
+      "step": 290,
+      "step_time": 229.9489072304219
+    },
+    {
+      "clip_ratio/high_max": 0.0005340036273992155,
+      "clip_ratio/high_mean": 0.00013940151584392879,
+      "clip_ratio/low_mean": 0.0001237586952811398,
+      "clip_ratio/low_min": 1.2678770872298629e-06,
+      "clip_ratio/region_mean": 0.00026316020703234246,
+      "completion_length": 4461.890625,
+      "completions/clipped_ratio": 0.46875,
+      "completions/max_length": 11333.2,
+      "completions/max_terminated_length": 11049.8,
+      "completions/mean_length": 4461.890625,
+      "completions/mean_terminated_length": 4622.53955078125,
+      "completions/min_length": 428.2,
+      "completions/min_terminated_length": 428.2,
+      "epoch": 0.24,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03177066147327423,
+      "kl": 0.05946307005360722,
+      "learning_rate": 1e-06,
+      "loss": 0.0012,
+      "num_tokens": 52961416.0,
+      "reward": 1.0947697520256043,
+      "reward_std": 0.48844210505485536,
+      "step": 300,
+      "step_time": 230.99577718563378
+    },
+    {
+      "clip_ratio/high_max": 0.00030668704366689783,
+      "clip_ratio/high_mean": 6.90227852373937e-05,
+      "clip_ratio/low_mean": 0.0001242063490280998,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00019322913540236185,
+      "completion_length": 4146.29375,
+      "completions/clipped_ratio": 0.46875,
+      "completions/max_length": 11417.4,
+      "completions/max_terminated_length": 10862.8,
+      "completions/mean_length": 4146.29375,
+      "completions/mean_terminated_length": 4350.4654296875,
+      "completions/min_length": 433.8,
+      "completions/min_terminated_length": 433.8,
+      "epoch": 0.248,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03460133820772171,
+      "kl": 0.04774801931343973,
+      "learning_rate": 1e-06,
+      "loss": 0.0009,
+      "num_tokens": 54628606.0,
+      "reward": 1.2949848651885987,
+      "reward_std": 0.4347102284431458,
+      "step": 310,
+      "step_time": 237.50362341850996
+    },
+    {
+      "clip_ratio/high_max": 0.00042981846308975946,
+      "clip_ratio/high_mean": 0.00011620506666076835,
+      "clip_ratio/low_mean": 5.608592448425043e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00017229099212272557,
+      "completion_length": 4964.096875,
+      "completions/clipped_ratio": 0.340625,
+      "completions/max_length": 11447.8,
+      "completions/max_terminated_length": 11083.2,
+      "completions/mean_length": 4964.096875,
+      "completions/mean_terminated_length": 5135.3416015625,
+      "completions/min_length": 373.6,
+      "completions/min_terminated_length": 373.6,
+      "epoch": 0.256,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.04397926107048988,
+      "kl": 0.06573723931796849,
+      "learning_rate": 1e-06,
+      "loss": 0.0013,
+      "num_tokens": 56579853.0,
+      "reward": 1.1206733226776122,
+      "reward_std": 0.5718165636062622,
+      "step": 320,
+      "step_time": 249.2268256921321
+    },
+    {
+      "clip_ratio/high_max": 0.00037124209902685836,
+      "clip_ratio/high_mean": 8.100136890334397e-05,
+      "clip_ratio/low_mean": 7.194726995294332e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00015294864097086246,
+      "completion_length": 4675.753125,
+      "completions/clipped_ratio": 0.31875,
+      "completions/max_length": 11971.0,
+      "completions/max_terminated_length": 11669.4,
+      "completions/mean_length": 4675.753125,
+      "completions/mean_terminated_length": 4907.5275390625,
+      "completions/min_length": 259.0,
+      "completions/min_terminated_length": 259.0,
+      "epoch": 0.264,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03300746902823448,
+      "kl": 0.06565800551325082,
+      "learning_rate": 1e-06,
+      "loss": 0.0014,
+      "num_tokens": 58418254.0,
+      "reward": 1.11955064535141,
+      "reward_std": 0.5300369203090668,
+      "step": 330,
+      "step_time": 245.15777591653168
+    },
+    {
+      "clip_ratio/high_max": 0.0005809355869132559,
+      "clip_ratio/high_mean": 0.0001689938762865495,
+      "clip_ratio/low_mean": 4.94793175448649e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00021847319294465706,
+      "completion_length": 4694.05625,
+      "completions/clipped_ratio": 0.359375,
+      "completions/max_length": 12041.0,
+      "completions/max_terminated_length": 11219.4,
+      "completions/mean_length": 4694.05625,
+      "completions/mean_terminated_length": 4816.820947265625,
+      "completions/min_length": 290.4,
+      "completions/min_terminated_length": 290.4,
+      "epoch": 0.272,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.027289442718029022,
+      "kl": 0.07141233496367931,
+      "learning_rate": 1e-06,
+      "loss": 0.0016,
+      "num_tokens": 60251536.0,
+      "reward": 1.0798933386802674,
+      "reward_std": 0.5533171415328979,
+      "step": 340,
+      "step_time": 240.81128605864942
+    },
+    {
+      "clip_ratio/high_max": 0.000545687766498304,
+      "clip_ratio/high_mean": 0.00015871557816353742,
+      "clip_ratio/low_mean": 4.358888336355449e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00020230446207278874,
+      "completion_length": 4427.746875,
+      "completions/clipped_ratio": 0.325,
+      "completions/max_length": 11681.6,
+      "completions/max_terminated_length": 11304.2,
+      "completions/mean_length": 4427.746875,
+      "completions/mean_terminated_length": 4570.8802734375,
+      "completions/min_length": 325.8,
+      "completions/min_terminated_length": 325.8,
+      "epoch": 0.28,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03146129101514816,
+      "kl": 0.05891165290959179,
+      "learning_rate": 1e-06,
+      "loss": 0.0012,
+      "num_tokens": 61994659.0,
+      "reward": 1.0934993386268617,
+      "reward_std": 0.47097904682159425,
+      "step": 350,
+      "step_time": 246.4419607952237
+    },
+    {
+      "clip_ratio/high_max": 0.00042810856120922836,
+      "clip_ratio/high_mean": 9.242635810551292e-05,
+      "clip_ratio/low_mean": 7.2325302153331e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00016475166030431864,
+      "completion_length": 4509.628125,
+      "completions/clipped_ratio": 0.2875,
+      "completions/max_length": 11863.6,
+      "completions/max_terminated_length": 11559.0,
+      "completions/mean_length": 4509.628125,
+      "completions/mean_terminated_length": 4623.6330078125,
+      "completions/min_length": 261.0,
+      "completions/min_terminated_length": 261.0,
+      "epoch": 0.288,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03730464354157448,
+      "kl": 0.05299770068377256,
+      "learning_rate": 1e-06,
+      "loss": 0.0011,
+      "num_tokens": 63763760.0,
+      "reward": 1.2220880508422851,
+      "reward_std": 0.47905906438827517,
+      "step": 360,
+      "step_time": 234.6107139196247
+    },
+    {
+      "clip_ratio/high_max": 0.0005190593132283538,
+      "clip_ratio/high_mean": 0.00011336807592670084,
+      "clip_ratio/low_mean": 0.00014939533775759628,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00026276341486664023,
+      "completion_length": 4117.878125,
+      "completions/clipped_ratio": 0.24375,
+      "completions/max_length": 11531.2,
+      "completions/max_terminated_length": 11211.6,
+      "completions/mean_length": 4117.878125,
+      "completions/mean_terminated_length": 4104.486376953125,
+      "completions/min_length": 283.0,
+      "completions/min_terminated_length": 283.0,
+      "epoch": 0.296,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.041058458387851715,
+      "kl": 0.07122893151827156,
+      "learning_rate": 1e-06,
+      "loss": 0.0013,
+      "num_tokens": 65434033.0,
+      "reward": 1.2355949521064757,
+      "reward_std": 0.4991305351257324,
+      "step": 370,
+      "step_time": 246.6441868480295
+    },
+    {
+      "clip_ratio/high_max": 0.0005346895028196741,
+      "clip_ratio/high_mean": 0.0001276340078220528,
+      "clip_ratio/low_mean": 6.609286824641459e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00019372687602299267,
+      "completion_length": 4294.95,
+      "completions/clipped_ratio": 0.290625,
+      "completions/max_length": 10511.2,
+      "completions/max_terminated_length": 10220.6,
+      "completions/mean_length": 4294.95,
+      "completions/mean_terminated_length": 4219.465576171875,
+      "completions/min_length": 296.0,
+      "completions/min_terminated_length": 296.0,
+      "epoch": 0.304,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 1.1131203174591064,
+      "kl": 0.07513927523978055,
+      "learning_rate": 1e-06,
+      "loss": 0.0015,
+      "num_tokens": 67155545.0,
+      "reward": 1.231179451942444,
+      "reward_std": 0.5423740983009339,
+      "step": 380,
+      "step_time": 239.1368755325675
+    },
+    {
+      "clip_ratio/high_max": 0.0005402129640060593,
+      "clip_ratio/high_mean": 0.0001354157960122393,
+      "clip_ratio/low_mean": 5.617468443688267e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00019159048279107082,
+      "completion_length": 4757.9375,
+      "completions/clipped_ratio": 0.375,
+      "completions/max_length": 11389.4,
+      "completions/max_terminated_length": 11389.4,
+      "completions/mean_length": 4757.9375,
+      "completions/mean_terminated_length": 4970.34443359375,
+      "completions/min_length": 317.4,
+      "completions/min_terminated_length": 317.4,
+      "epoch": 0.312,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.038911446928977966,
+      "kl": 0.06953863021917642,
+      "learning_rate": 1e-06,
+      "loss": 0.0015,
+      "num_tokens": 69015061.0,
+      "reward": 1.2488256216049194,
+      "reward_std": 0.5137399554252624,
+      "step": 390,
+      "step_time": 239.8175220735371
+    },
+    {
+      "clip_ratio/high_max": 0.0004766742502397392,
+      "clip_ratio/high_mean": 0.0001258964039152488,
+      "clip_ratio/low_mean": 6.187823919390212e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0001877746428363025,
+      "completion_length": 4531.934375,
+      "completions/clipped_ratio": 0.3,
+      "completions/max_length": 11814.0,
+      "completions/max_terminated_length": 11241.4,
+      "completions/mean_length": 4531.934375,
+      "completions/mean_terminated_length": 4565.529638671875,
+      "completions/min_length": 536.4,
+      "completions/min_terminated_length": 536.4,
+      "epoch": 0.32,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.04457750543951988,
+      "kl": 0.07946720817126333,
+      "learning_rate": 1e-06,
+      "loss": 0.0018,
+      "num_tokens": 70787284.0,
+      "reward": 0.9880791902542114,
+      "reward_std": 0.5113428354263305,
+      "step": 400,
+      "step_time": 257.0787096582353
+    },
+    {
+      "clip_ratio/high_max": 0.0003801339269557502,
+      "clip_ratio/high_mean": 0.00010227670595668315,
+      "clip_ratio/low_mean": 0.0001341289223091735,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00023640562485525152,
+      "completion_length": 4902.3875,
+      "completions/clipped_ratio": 0.3375,
+      "completions/max_length": 11321.8,
+      "completions/max_terminated_length": 11079.6,
+      "completions/mean_length": 4902.3875,
+      "completions/mean_terminated_length": 5045.71689453125,
+      "completions/min_length": 591.2,
+      "completions/min_terminated_length": 591.2,
+      "epoch": 0.328,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.04044875502586365,
+      "kl": 0.0734462249558419,
+      "learning_rate": 1e-06,
+      "loss": 0.0015,
+      "num_tokens": 72719108.0,
+      "reward": 1.245608401298523,
+      "reward_std": 0.5725983142852783,
+      "step": 410,
+      "step_time": 254.89171202145516
+    },
+    {
+      "clip_ratio/high_max": 0.0015263679924828466,
+      "clip_ratio/high_mean": 0.0003602910978770524,
+      "clip_ratio/low_mean": 0.0001467544160050238,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0005070455110626426,
+      "completion_length": 4545.9375,
+      "completions/clipped_ratio": 0.259375,
+      "completions/max_length": 12461.8,
+      "completions/max_terminated_length": 11428.2,
+      "completions/mean_length": 4545.9375,
+      "completions/mean_terminated_length": 4409.2998046875,
+      "completions/min_length": 269.6,
+      "completions/min_terminated_length": 269.6,
+      "epoch": 0.336,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.0473661832511425,
+      "kl": 0.0833735628053546,
+      "learning_rate": 1e-06,
+      "loss": 0.0015,
+      "num_tokens": 74511884.0,
+      "reward": 1.2157001733779906,
+      "reward_std": 0.5033408522605896,
+      "step": 420,
+      "step_time": 254.41874387748538
+    },
+    {
+      "clip_ratio/high_max": 0.0010654171423084336,
+      "clip_ratio/high_mean": 0.00018638188930708566,
+      "clip_ratio/low_mean": 0.00018482315081200796,
+      "clip_ratio/low_min": 2.1714584363508037e-06,
+      "clip_ratio/region_mean": 0.0003712050398462452,
+      "completion_length": 4527.1375,
+      "completions/clipped_ratio": 0.29375,
+      "completions/max_length": 11731.6,
+      "completions/max_terminated_length": 10628.6,
+      "completions/mean_length": 4527.1375,
+      "completions/mean_terminated_length": 4291.91865234375,
+      "completions/min_length": 236.0,
+      "completions/min_terminated_length": 236.0,
+      "epoch": 0.344,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.0517924502491951,
+      "kl": 0.07865471374243498,
+      "learning_rate": 1e-06,
+      "loss": 0.0012,
+      "num_tokens": 76284760.0,
+      "reward": 1.3334136247634887,
+      "reward_std": 0.559002012014389,
+      "step": 430,
+      "step_time": 253.32570067867636
+    },
+    {
+      "clip_ratio/high_max": 0.0006989054323639721,
+      "clip_ratio/high_mean": 0.00018430034688208253,
+      "clip_ratio/low_mean": 4.059004072587413e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00022489039292850066,
+      "completion_length": 4738.33125,
+      "completions/clipped_ratio": 0.240625,
+      "completions/max_length": 12441.2,
+      "completions/max_terminated_length": 11288.0,
+      "completions/mean_length": 4738.33125,
+      "completions/mean_terminated_length": 4666.544287109375,
+      "completions/min_length": 360.0,
+      "completions/min_terminated_length": 360.0,
+      "epoch": 0.352,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.05862632393836975,
+      "kl": 0.0983058319427073,
+      "learning_rate": 1e-06,
+      "loss": 0.002,
+      "num_tokens": 78147046.0,
+      "reward": 1.1325592517852783,
+      "reward_std": 0.5911938667297363,
+      "step": 440,
+      "step_time": 245.0912904150784
+    },
+    {
+      "clip_ratio/high_max": 0.0006878788452013395,
+      "clip_ratio/high_mean": 0.00018554991283963317,
+      "clip_ratio/low_mean": 0.00015036590839372365,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0003359158203238621,
+      "completion_length": 4581.9125,
+      "completions/clipped_ratio": 0.253125,
+      "completions/max_length": 12019.0,
+      "completions/max_terminated_length": 11817.6,
+      "completions/mean_length": 4581.9125,
+      "completions/mean_terminated_length": 4664.235888671875,
+      "completions/min_length": 285.2,
+      "completions/min_terminated_length": 285.2,
+      "epoch": 0.36,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.04850015044212341,
+      "kl": 0.10464066509157419,
+      "learning_rate": 1e-06,
+      "loss": 0.002,
+      "num_tokens": 79948250.0,
+      "reward": 1.2726510524749757,
+      "reward_std": 0.5423218309879303,
+      "step": 450,
+      "step_time": 246.19276090674103
+    },
+    {
+      "clip_ratio/high_max": 0.0009510355652309954,
+      "clip_ratio/high_mean": 0.00029086956456012556,
+      "clip_ratio/low_mean": 5.8237574421582394e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00034910713802673856,
+      "completion_length": 4714.2125,
+      "completions/clipped_ratio": 0.275,
+      "completions/max_length": 12193.6,
+      "completions/max_terminated_length": 12126.2,
+      "completions/mean_length": 4714.2125,
+      "completions/mean_terminated_length": 4948.716796875,
+      "completions/min_length": 458.0,
+      "completions/min_terminated_length": 458.0,
+      "epoch": 0.368,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.036279428750276566,
+      "kl": 0.10845831176266074,
+      "learning_rate": 1e-06,
+      "loss": 0.0022,
+      "num_tokens": 81792042.0,
+      "reward": 1.0608064770698546,
+      "reward_std": 0.4727827250957489,
+      "step": 460,
+      "step_time": 255.36917145513
+    },
+    {
+      "clip_ratio/high_max": 0.000247496422889526,
+      "clip_ratio/high_mean": 5.9935277022304945e-05,
+      "clip_ratio/low_mean": 0.00011364564270479604,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00017358091918140417,
+      "completion_length": 5343.984375,
+      "completions/clipped_ratio": 0.31875,
+      "completions/max_length": 12238.0,
+      "completions/max_terminated_length": 11210.4,
+      "completions/mean_length": 5343.984375,
+      "completions/mean_terminated_length": 5192.0314453125,
+      "completions/min_length": 301.8,
+      "completions/min_terminated_length": 301.8,
+      "epoch": 0.376,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.04100991040468216,
+      "kl": 0.07249630820006132,
+      "learning_rate": 1e-06,
+      "loss": 0.0013,
+      "num_tokens": 83858121.0,
+      "reward": 1.3011021375656129,
+      "reward_std": 0.568158221244812,
+      "step": 470,
+      "step_time": 258.0305543400347
+    },
+    {
+      "clip_ratio/high_max": 0.000352592545641528,
+      "clip_ratio/high_mean": 8.989804132397694e-05,
+      "clip_ratio/low_mean": 0.001060699175013724,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0011505972039230982,
+      "completion_length": 4293.228125,
+      "completions/clipped_ratio": 0.284375,
+      "completions/max_length": 11049.8,
+      "completions/max_terminated_length": 11049.8,
+      "completions/mean_length": 4293.228125,
+      "completions/mean_terminated_length": 4412.972509765625,
+      "completions/min_length": 293.8,
+      "completions/min_terminated_length": 293.8,
+      "epoch": 0.384,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.05517025291919708,
+      "kl": 0.13858566973358394,
+      "learning_rate": 1e-06,
+      "loss": 0.0025,
+      "num_tokens": 85608198.0,
+      "reward": 1.27374324798584,
+      "reward_std": 0.5257342696189881,
+      "step": 480,
+      "step_time": 224.68164771050215
+    },
+    {
+      "clip_ratio/high_max": 0.0011510542281030212,
+      "clip_ratio/high_mean": 0.0002912246545747621,
+      "clip_ratio/low_mean": 0.0014171333015838173,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0017083579252357595,
+      "completion_length": 4264.5875,
+      "completions/clipped_ratio": 0.35,
+      "completions/max_length": 11768.6,
+      "completions/max_terminated_length": 11314.2,
+      "completions/mean_length": 4264.5875,
+      "completions/mean_terminated_length": 4047.234521484375,
+      "completions/min_length": 331.4,
+      "completions/min_terminated_length": 331.4,
+      "epoch": 0.392,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1340084969997406,
+      "kl": 0.30975929675623776,
+      "learning_rate": 1e-06,
+      "loss": 0.0052,
+      "num_tokens": 87322254.0,
+      "reward": 1.2258143186569215,
+      "reward_std": 0.594356369972229,
+      "step": 490,
+      "step_time": 238.40311096683143
+    },
+    {
+      "clip_ratio/high_max": 0.008202312963476288,
+      "clip_ratio/high_mean": 0.0018977287676534616,
+      "clip_ratio/low_mean": 0.0004962101493219961,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0023939389298902824,
+      "completion_length": 4813.09375,
+      "completions/clipped_ratio": 0.35625,
+      "completions/max_length": 11470.4,
+      "completions/max_terminated_length": 11251.8,
+      "completions/mean_length": 4813.09375,
+      "completions/mean_terminated_length": 4771.305322265625,
+      "completions/min_length": 506.4,
+      "completions/min_terminated_length": 566.6,
+      "epoch": 0.4,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.07679229974746704,
+      "kl": 0.22180975526571273,
+      "learning_rate": 1e-06,
+      "loss": 0.004,
+      "num_tokens": 89217948.0,
+      "reward": 1.0258091926574706,
+      "reward_std": 0.5038924098014832,
+      "step": 500,
+      "step_time": 230.99808970950545
+    },
+    {
+      "clip_ratio/high_max": 0.0004784486485732486,
+      "clip_ratio/high_mean": 0.00011152361953463697,
+      "clip_ratio/low_mean": 0.00047115937049966303,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0005826830041769426,
+      "completion_length": 4992.96875,
+      "completions/clipped_ratio": 0.35,
+      "completions/max_length": 11788.4,
+      "completions/max_terminated_length": 11490.4,
+      "completions/mean_length": 4992.96875,
+      "completions/mean_terminated_length": 4921.0212890625,
+      "completions/min_length": 320.8,
+      "completions/min_terminated_length": 320.8,
+      "epoch": 0.408,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.07539121061563492,
+      "kl": 0.21761196088045837,
+      "learning_rate": 1e-06,
+      "loss": 0.0037,
+      "num_tokens": 91167574.0,
+      "reward": 1.20388503074646,
+      "reward_std": 0.5358129501342773,
+      "step": 510,
+      "step_time": 242.51621155813336
+    },
+    {
+      "clip_ratio/high_max": 0.004420937015675008,
+      "clip_ratio/high_mean": 0.0007193908625595213,
+      "clip_ratio/low_mean": 0.0009956757531654147,
+      "clip_ratio/low_min": 1.4214929251465946e-05,
+      "clip_ratio/region_mean": 0.0017150666457382614,
+      "completion_length": 4858.653125,
+      "completions/clipped_ratio": 0.390625,
+      "completions/max_length": 10925.4,
+      "completions/max_terminated_length": 10723.6,
+      "completions/mean_length": 4858.653125,
+      "completions/mean_terminated_length": 4865.331640625,
+      "completions/min_length": 313.4,
+      "completions/min_terminated_length": 313.4,
+      "epoch": 0.416,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3061217665672302,
+      "kl": 0.4010165734216571,
+      "learning_rate": 1e-06,
+      "loss": 0.0076,
+      "num_tokens": 93062167.0,
+      "reward": 1.15363609790802,
+      "reward_std": 0.5805089712142945,
+      "step": 520,
+      "step_time": 238.53153089359404
+    },
+    {
+      "clip_ratio/high_max": 0.0017592069169040769,
+      "clip_ratio/high_mean": 0.0003917591035133228,
+      "clip_ratio/low_mean": 0.00350101253052344,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.003892771749633539,
+      "completion_length": 4645.453125,
+      "completions/clipped_ratio": 0.38125,
+      "completions/max_length": 11893.4,
+      "completions/max_terminated_length": 10984.6,
+      "completions/mean_length": 4645.453125,
+      "completions/mean_terminated_length": 4399.831201171875,
+      "completions/min_length": 323.4,
+      "completions/min_terminated_length": 323.4,
+      "epoch": 0.424,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22139529883861542,
+      "kl": 0.22714131269603968,
+      "learning_rate": 1e-06,
+      "loss": 0.0037,
+      "num_tokens": 94904596.0,
+      "reward": 0.9871150135993958,
+      "reward_std": 0.6627055644989014,
+      "step": 530,
+      "step_time": 254.1042493838817
+    },
+    {
+      "clip_ratio/high_max": 0.0010623440881317946,
+      "clip_ratio/high_mean": 0.0002860782171410392,
+      "clip_ratio/low_mean": 0.00011166805140874203,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0003977462627517525,
+      "completion_length": 4897.04375,
+      "completions/clipped_ratio": 0.44375,
+      "completions/max_length": 12029.4,
+      "completions/max_terminated_length": 10975.8,
+      "completions/mean_length": 4897.04375,
+      "completions/mean_terminated_length": 4650.61689453125,
+      "completions/min_length": 441.0,
+      "completions/min_terminated_length": 594.0,
+      "epoch": 0.432,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22648373246192932,
+      "kl": 0.25860741632059214,
+      "learning_rate": 1e-06,
+      "loss": 0.0047,
+      "num_tokens": 96829138.0,
+      "reward": 0.9951073884963989,
+      "reward_std": 0.564429247379303,
+      "step": 540,
+      "step_time": 251.64892664775252
+    },
+    {
+      "clip_ratio/high_max": 0.00558230672540958,
+      "clip_ratio/high_mean": 0.00104510458149889,
+      "clip_ratio/low_mean": 0.00025361020116179136,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0012987148507818346,
+      "completion_length": 4926.740625,
+      "completions/clipped_ratio": 0.496875,
+      "completions/max_length": 11611.6,
+      "completions/max_terminated_length": 11085.8,
+      "completions/mean_length": 4926.740625,
+      "completions/mean_terminated_length": 4802.2626953125,
+      "completions/min_length": 408.8,
+      "completions/min_terminated_length": 408.8,
+      "epoch": 0.44,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.36984431743621826,
+      "kl": 0.36771030593663456,
+      "learning_rate": 1e-06,
+      "loss": 0.006,
+      "num_tokens": 98734755.0,
+      "reward": 0.9801221489906311,
+      "reward_std": 0.8593629360198974,
+      "step": 550,
+      "step_time": 230.4229119028896
+    },
+    {
+      "clip_ratio/high_max": 0.0006551830358148436,
+      "clip_ratio/high_mean": 0.00011234816256546764,
+      "clip_ratio/low_mean": 0.006395395010622451,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.006507743133988697,
+      "completion_length": 5226.790625,
+      "completions/clipped_ratio": 0.459375,
+      "completions/max_length": 11713.2,
+      "completions/max_terminated_length": 11713.2,
+      "completions/mean_length": 5226.790625,
+      "completions/mean_terminated_length": 4749.200830078125,
+      "completions/min_length": 466.8,
+      "completions/min_terminated_length": 466.8,
+      "epoch": 0.448,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.212530717253685,
+      "kl": 0.43885244242846966,
+      "learning_rate": 1e-06,
+      "loss": 0.004,
+      "num_tokens": 100746984.0,
+      "reward": 0.9360185027122497,
+      "reward_std": 0.7851440191268921,
+      "step": 560,
+      "step_time": 252.5003844358027
+    },
+    {
+      "clip_ratio/high_max": 0.0014559835672116606,
+      "clip_ratio/high_mean": 0.00022575238588160574,
+      "clip_ratio/low_mean": 0.023684275292362146,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.023910027825331782,
+      "completion_length": 5231.196875,
+      "completions/clipped_ratio": 0.525,
+      "completions/max_length": 11815.2,
+      "completions/max_terminated_length": 10757.4,
+      "completions/mean_length": 5231.196875,
+      "completions/mean_terminated_length": 4093.40732421875,
+      "completions/min_length": 225.4,
+      "completions/min_terminated_length": 225.4,
+      "epoch": 0.456,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2536648213863373,
+      "kl": 0.6717401832342148,
+      "learning_rate": 1e-06,
+      "loss": 0.0062,
+      "num_tokens": 102792051.0,
+      "reward": 0.6574952661991119,
+      "reward_std": 0.8662473678588867,
+      "step": 570,
+      "step_time": 274.9575475856662
+    },
+    {
+      "clip_ratio/high_max": 0.0008674728404002962,
+      "clip_ratio/high_mean": 0.00014791082539886702,
+      "clip_ratio/low_mean": 0.020389114872909887,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.020537025111843833,
+      "completion_length": 5314.496875,
+      "completions/clipped_ratio": 0.63125,
+      "completions/max_length": 12972.6,
+      "completions/max_terminated_length": 12352.4,
+      "completions/mean_length": 5314.496875,
+      "completions/mean_terminated_length": 4657.326611328125,
+      "completions/min_length": 306.4,
+      "completions/min_terminated_length": 306.4,
+      "epoch": 0.464,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.4102652668952942,
+      "kl": 1.6274013640359044,
+      "learning_rate": 1e-06,
+      "loss": 0.0203,
+      "num_tokens": 104845794.0,
+      "reward": 0.5376813858747482,
+      "reward_std": 1.0272823452949524,
+      "step": 580,
+      "step_time": 265.2071466587484
+    },
+    {
+      "clip_ratio/high_max": 0.0017352815350022866,
+      "clip_ratio/high_mean": 0.00024971098614514633,
+      "clip_ratio/low_mean": 0.004629414569171786,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004879125578008825,
+      "completion_length": 4916.3875,
+      "completions/clipped_ratio": 0.621875,
+      "completions/max_length": 11932.6,
+      "completions/max_terminated_length": 10954.8,
+      "completions/mean_length": 4916.3875,
+      "completions/mean_terminated_length": 3707.5984375,
+      "completions/min_length": 258.4,
+      "completions/min_terminated_length": 258.4,
+      "epoch": 0.472,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.22192338109016418,
+      "kl": 4.016752981441096,
+      "learning_rate": 1e-06,
+      "loss": 0.0613,
+      "num_tokens": 106798966.0,
+      "reward": 0.7549613535404205,
+      "reward_std": 0.8964547753334046,
+      "step": 590,
+      "step_time": 251.62146447189153
+    },
+    {
+      "clip_ratio/high_max": 0.046285361977788854,
+      "clip_ratio/high_mean": 0.008274853558805261,
+      "clip_ratio/low_mean": 0.006437226842467681,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0147120808181171,
+      "completion_length": 5916.165625,
+      "completions/clipped_ratio": 0.790625,
+      "completions/max_length": 12685.0,
+      "completions/max_terminated_length": 9897.2,
+      "completions/mean_length": 5916.165625,
+      "completions/mean_terminated_length": 4525.33154296875,
+      "completions/min_length": 385.4,
+      "completions/min_terminated_length": 385.4,
+      "epoch": 0.48,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.5054551959037781,
+      "kl": 6.486891683936119,
+      "learning_rate": 1e-06,
+      "loss": 0.1084,
+      "num_tokens": 109047027.0,
+      "reward": 0.3534734547138214,
+      "reward_std": 1.0079365611076354,
+      "step": 600,
+      "step_time": 284.38370757475496
+    },
+    {
+      "clip_ratio/high_max": 0.013970322842214954,
+      "clip_ratio/high_mean": 0.0017682650514871056,
+      "clip_ratio/low_mean": 0.0010710656190781264,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.002839330586721189,
+      "completion_length": 6401.953125,
+      "completions/clipped_ratio": 0.8125,
+      "completions/max_length": 13704.8,
+      "completions/max_terminated_length": 10002.8,
+      "completions/mean_length": 6401.953125,
+      "completions/mean_terminated_length": 4352.1337890625,
+      "completions/min_length": 432.6,
+      "completions/min_terminated_length": 432.6,
+      "epoch": 0.488,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.45525485277175903,
+      "kl": 5.254763677716255,
+      "learning_rate": 1e-06,
+      "loss": 0.0887,
+      "num_tokens": 111428340.0,
+      "reward": 0.4791525512933731,
+      "reward_std": 0.8741311192512512,
+      "step": 610,
+      "step_time": 287.25211645402015
+    },
+    {
+      "clip_ratio/high_max": 0.00019130092805426103,
+      "clip_ratio/high_mean": 3.9704555638309104e-05,
+      "clip_ratio/low_mean": 0.0023979359820714306,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0024376405516704836,
+      "completion_length": 6213.634375,
+      "completions/clipped_ratio": 0.834375,
+      "completions/max_length": 12994.4,
+      "completions/max_terminated_length": 8991.8,
+      "completions/mean_length": 6213.634375,
+      "completions/mean_terminated_length": 2916.3680908203123,
+      "completions/min_length": 505.6,
+      "completions/min_terminated_length": 847.8,
+      "epoch": 0.496,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.6184539198875427,
+      "kl": 8.150065049529076,
+      "learning_rate": 1e-06,
+      "loss": 0.1321,
+      "num_tokens": 113785795.0,
+      "reward": 0.28945698738098147,
+      "reward_std": 0.8822868824005127,
+      "step": 620,
+      "step_time": 284.815426626429
+    },
+    {
+      "clip_ratio/high_max": 0.03065611421916401,
+      "clip_ratio/high_mean": 0.005156366467235785,
+      "clip_ratio/low_mean": 1.6292886175506283e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.005172659454365203,
+      "completion_length": 5815.29375,
+      "completions/clipped_ratio": 0.803125,
+      "completions/max_length": 12807.8,
+      "completions/max_terminated_length": 9526.2,
+      "completions/mean_length": 5815.29375,
+      "completions/mean_terminated_length": 2708.931396484375,
+      "completions/min_length": 279.0,
+      "completions/min_terminated_length": 279.0,
+      "epoch": 0.504,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.3669980466365814,
+      "kl": 6.478772513568401,
+      "learning_rate": 1e-06,
+      "loss": 0.1064,
+      "num_tokens": 115973525.0,
+      "reward": 0.48847304582595824,
+      "reward_std": 0.9684848070144654,
+      "step": 630,
+      "step_time": 285.0067568361759
+    },
+    {
+      "clip_ratio/high_max": 0.027629933714524667,
+      "clip_ratio/high_mean": 0.004163694726685208,
+      "clip_ratio/low_mean": 1.4767896198009112e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004178462701474928,
+      "completion_length": 5612.215625,
+      "completions/clipped_ratio": 0.815625,
+      "completions/max_length": 12719.0,
+      "completions/max_terminated_length": 9276.6,
+      "completions/mean_length": 5612.215625,
+      "completions/mean_terminated_length": 2254.1920654296873,
+      "completions/min_length": 336.8,
+      "completions/min_terminated_length": 336.8,
+      "epoch": 0.512,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.6951866149902344,
+      "kl": 6.111414223909378,
+      "learning_rate": 1e-06,
+      "loss": 0.0958,
+      "num_tokens": 118090874.0,
+      "reward": 0.5856490015983582,
+      "reward_std": 0.8794823169708252,
+      "step": 640,
+      "step_time": 287.24285530894997
+    },
+    {
+      "clip_ratio/high_max": 0.00649556765338275,
+      "clip_ratio/high_mean": 0.0008382659107724066,
+      "clip_ratio/low_mean": 0.00033291406082298637,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0011711799729027916,
+      "completion_length": 5465.80625,
+      "completions/clipped_ratio": 0.84375,
+      "completions/max_length": 12979.0,
+      "completions/max_terminated_length": 6644.6,
+      "completions/mean_length": 5465.80625,
+      "completions/mean_terminated_length": 2300.868017578125,
+      "completions/min_length": 332.6,
+      "completions/min_terminated_length": 332.6,
+      "epoch": 0.52,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.33075660467147827,
+      "kl": 5.029376268386841,
+      "learning_rate": 1e-06,
+      "loss": 0.0752,
+      "num_tokens": 120214060.0,
+      "reward": 0.6074592411518097,
+      "reward_std": 0.924161410331726,
+      "step": 650,
+      "step_time": 282.2009522151202
+    },
+    {
+      "clip_ratio/high_max": 0.0009886611755064224,
+      "clip_ratio/high_mean": 0.0001391213447050177,
+      "clip_ratio/low_mean": 0.003190721172040867,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.003329842483799439,
+      "completion_length": 5675.05625,
+      "completions/clipped_ratio": 0.83125,
+      "completions/max_length": 13808.6,
+      "completions/max_terminated_length": 8415.8,
+      "completions/mean_length": 5675.05625,
+      "completions/mean_terminated_length": 3089.78935546875,
+      "completions/min_length": 389.2,
+      "completions/min_terminated_length": 389.2,
+      "epoch": 0.528,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.13812169432640076,
+      "kl": 5.344193743681535,
+      "learning_rate": 1e-06,
+      "loss": 0.08,
+      "num_tokens": 122364066.0,
+      "reward": 0.8582603931427002,
+      "reward_std": 0.7798061966896057,
+      "step": 660,
+      "step_time": 278.39258024953307
+    },
+    {
+      "clip_ratio/high_max": 0.008314772717858432,
+      "clip_ratio/high_mean": 0.0010547058868951354,
+      "clip_ratio/low_mean": 0.0001259349507108709,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0011806408541815473,
+      "completion_length": 6072.66875,
+      "completions/clipped_ratio": 0.85,
+      "completions/max_length": 13622.0,
+      "completions/max_terminated_length": 9543.4,
+      "completions/mean_length": 6072.66875,
+      "completions/mean_terminated_length": 3586.5512451171876,
+      "completions/min_length": 342.8,
+      "completions/min_terminated_length": 430.4,
+      "epoch": 0.536,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 1.6785002946853638,
+      "kl": 8.283730751276016,
+      "learning_rate": 1e-06,
+      "loss": 0.145,
+      "num_tokens": 124663208.0,
+      "reward": 0.38076815009117126,
+      "reward_std": 0.9819870591163635,
+      "step": 670,
+      "step_time": 289.20823122859
+    },
+    {
+      "clip_ratio/high_max": 0.046133012961672645,
+      "clip_ratio/high_mean": 0.007587127906265323,
+      "clip_ratio/low_mean": 0.0024964586295027403,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.01008358645503904,
+      "completion_length": 5168.353125,
+      "completions/clipped_ratio": 0.7625,
+      "completions/max_length": 12340.6,
+      "completions/max_terminated_length": 8658.2,
+      "completions/mean_length": 5168.353125,
+      "completions/mean_terminated_length": 2016.5061645507812,
+      "completions/min_length": 268.6,
+      "completions/min_terminated_length": 268.6,
+      "epoch": 0.544,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.3794982135295868,
+      "kl": 8.002963012456894,
+      "learning_rate": 1e-06,
+      "loss": 0.129,
+      "num_tokens": 126667545.0,
+      "reward": 0.6942238539457322,
+      "reward_std": 0.9573667883872986,
+      "step": 680,
+      "step_time": 269.1268596865237
+    },
+    {
+      "clip_ratio/high_max": 0.0010048973315861076,
+      "clip_ratio/high_mean": 0.00016022471363612566,
+      "clip_ratio/low_mean": 0.0011764754735622772,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0013367001325605087,
+      "completion_length": 5706.590625,
+      "completions/clipped_ratio": 0.84375,
+      "completions/max_length": 12593.6,
+      "completions/max_terminated_length": 9800.4,
+      "completions/mean_length": 5706.590625,
+      "completions/mean_terminated_length": 3760.7462890625,
+      "completions/min_length": 466.6,
+      "completions/min_terminated_length": 466.6,
+      "epoch": 0.552,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.2747977375984192,
+      "kl": 4.467666064202786,
+      "learning_rate": 1e-06,
+      "loss": 0.0676,
+      "num_tokens": 128841862.0,
+      "reward": 0.6847242653369904,
+      "reward_std": 0.6977497220039368,
+      "step": 690,
+      "step_time": 283.7122025307268
+    },
+    {
+      "clip_ratio/high_max": 0.00789058599548298,
+      "clip_ratio/high_mean": 0.0010111700751167518,
+      "clip_ratio/low_mean": 0.007297157552375211,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.008308327868098786,
+      "completion_length": 5376.49375,
+      "completions/clipped_ratio": 0.859375,
+      "completions/max_length": 12573.4,
+      "completions/max_terminated_length": 7850.0,
+      "completions/mean_length": 5376.49375,
+      "completions/mean_terminated_length": 2859.6035400390624,
+      "completions/min_length": 370.8,
+      "completions/min_terminated_length": 370.8,
+      "epoch": 0.56,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.22591447830200195,
+      "kl": 4.129212433286011,
+      "learning_rate": 1e-06,
+      "loss": 0.0592,
+      "num_tokens": 130896504.0,
+      "reward": 0.808956629037857,
+      "reward_std": 0.7688298106193543,
+      "step": 700,
+      "step_time": 264.2991053637117
+    },
+    {
+      "clip_ratio/high_max": 0.06855156899910071,
+      "clip_ratio/high_mean": 0.009158036535700376,
+      "clip_ratio/low_mean": 3.734665184538244e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.009195383218138887,
+      "completion_length": 6033.39375,
+      "completions/clipped_ratio": 0.8875,
+      "completions/max_length": 13078.8,
+      "completions/max_terminated_length": 2358.2,
+      "completions/mean_length": 6033.39375,
+      "completions/mean_terminated_length": 1079.7886352539062,
+      "completions/min_length": 321.2,
+      "completions/min_terminated_length": 321.2,
+      "epoch": 0.568,
+      "frac_reward_zero_std": 0.075,
+      "grad_norm": 0.3750945031642914,
+      "kl": 7.744498783349991,
+      "learning_rate": 1e-06,
+      "loss": 0.1303,
+      "num_tokens": 133156810.0,
+      "reward": 0.18387374877929688,
+      "reward_std": 0.972456407546997,
+      "step": 710,
+      "step_time": 290.7822148200124
+    },
+    {
+      "clip_ratio/high_max": 0.04844408968674543,
+      "clip_ratio/high_mean": 0.006084340314828296,
+      "clip_ratio/low_mean": 0.00025215936792619684,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.006336499662938877,
+      "completion_length": 5165.69375,
+      "completions/clipped_ratio": 0.85,
+      "completions/max_length": 13154.2,
+      "completions/max_terminated_length": 6251.6,
+      "completions/mean_length": 5165.69375,
+      "completions/mean_terminated_length": 1839.8445678710937,
+      "completions/min_length": 415.8,
+      "completions/min_terminated_length": 415.8,
+      "epoch": 0.576,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.393494188785553,
+      "kl": 5.751070095319301,
+      "learning_rate": 1e-06,
+      "loss": 0.0887,
+      "num_tokens": 135162820.0,
+      "reward": 0.6741469085216523,
+      "reward_std": 0.8685160994529724,
+      "step": 720,
+      "step_time": 275.98039374127984
+    },
+    {
+      "clip_ratio/high_max": 0.0003100098518189043,
+      "clip_ratio/high_mean": 6.665145456281607e-05,
+      "clip_ratio/low_mean": 2.9909206682532384e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 9.656066285970155e-05,
+      "completion_length": 5722.003125,
+      "completions/clipped_ratio": 0.86875,
+      "completions/max_length": 12639.8,
+      "completions/max_terminated_length": 4289.2,
+      "completions/mean_length": 5722.003125,
+      "completions/mean_terminated_length": 992.5607177734375,
+      "completions/min_length": 289.2,
+      "completions/min_terminated_length": 289.2,
+      "epoch": 0.584,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.576591968536377,
+      "kl": 8.910334551334381,
+      "learning_rate": 1e-06,
+      "loss": 0.1359,
+      "num_tokens": 137342057.0,
+      "reward": 0.4476357713341713,
+      "reward_std": 1.0780439615249633,
+      "step": 730,
+      "step_time": 283.4746839959174
+    },
+    {
+      "clip_ratio/high_max": 0.08911914828277076,
+      "clip_ratio/high_mean": 0.013512145449158196,
+      "clip_ratio/low_mean": 5.0309093342093544e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.013562454467137285,
+      "completion_length": 6080.546875,
+      "completions/clipped_ratio": 0.88125,
+      "completions/max_length": 13401.8,
+      "completions/max_terminated_length": 6814.2,
+      "completions/mean_length": 6080.546875,
+      "completions/mean_terminated_length": 2217.181005859375,
+      "completions/min_length": 218.8,
+      "completions/min_terminated_length": 218.8,
+      "epoch": 0.592,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.7265051007270813,
+      "kl": 8.19523628950119,
+      "learning_rate": 1e-06,
+      "loss": 0.1365,
+      "num_tokens": 139630348.0,
+      "reward": 0.41782193183898925,
+      "reward_std": 0.8201668381690979,
+      "step": 740,
+      "step_time": 295.31291244886813
+    },
+    {
+      "clip_ratio/high_max": 0.003131780230614822,
+      "clip_ratio/high_mean": 0.00041973076481554016,
+      "clip_ratio/low_mean": 1.7058133857972278e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00043678890988303466,
+      "completion_length": 5908.475,
+      "completions/clipped_ratio": 0.915625,
+      "completions/max_length": 13056.0,
+      "completions/max_terminated_length": 4425.4,
+      "completions/mean_length": 5908.475,
+      "completions/mean_terminated_length": 1618.715594482422,
+      "completions/min_length": 304.0,
+      "completions/min_terminated_length": 304.0,
+      "epoch": 0.6,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.42771846055984497,
+      "kl": 4.577450105547905,
+      "learning_rate": 1e-06,
+      "loss": 0.0739,
+      "num_tokens": 141855656.0,
+      "reward": 0.3339735597372055,
+      "reward_std": 0.7836308121681214,
+      "step": 750,
+      "step_time": 280.91214460879564
+    },
+    {
+      "clip_ratio/high_max": 0.0011050502536818386,
+      "clip_ratio/high_mean": 0.00016600425851720502,
+      "clip_ratio/low_mean": 0.006519388711149076,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.006685392962026526,
+      "completion_length": 5358.71875,
+      "completions/clipped_ratio": 0.840625,
+      "completions/max_length": 12723.8,
+      "completions/max_terminated_length": 6033.2,
+      "completions/mean_length": 5358.71875,
+      "completions/mean_terminated_length": 1335.1285888671875,
+      "completions/min_length": 404.2,
+      "completions/min_terminated_length": 404.2,
+      "epoch": 0.608,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.38392481207847595,
+      "kl": 5.269394595734775,
+      "learning_rate": 1e-06,
+      "loss": 0.0701,
+      "num_tokens": 143936010.0,
+      "reward": 0.9060750603675842,
+      "reward_std": 0.9165179491043091,
+      "step": 760,
+      "step_time": 285.19950570464135
+    },
+    {
+      "clip_ratio/high_max": 0.04125237505995756,
+      "clip_ratio/high_mean": 0.005186066534474776,
+      "clip_ratio/low_mean": 9.788106376618089e-06,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00519585482520597,
+      "completion_length": 5389.5375,
+      "completions/clipped_ratio": 0.871875,
+      "completions/max_length": 12969.8,
+      "completions/max_terminated_length": 7054.8,
+      "completions/mean_length": 5389.5375,
+      "completions/mean_terminated_length": 1502.188916015625,
+      "completions/min_length": 357.8,
+      "completions/min_terminated_length": 357.8,
+      "epoch": 0.616,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.8449738025665283,
+      "kl": 7.533557245135308,
+      "learning_rate": 1e-06,
+      "loss": 0.116,
+      "num_tokens": 146010370.0,
+      "reward": 0.6727673232555389,
+      "reward_std": 0.8307777285575867,
+      "step": 770,
+      "step_time": 275.70242310278115
+    },
+    {
+      "clip_ratio/high_max": 0.0393053182924632,
+      "clip_ratio/high_mean": 0.0049453356771209656,
+      "clip_ratio/low_mean": 3.684269100858728e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00498217858512362,
+      "completion_length": 5449.40625,
+      "completions/clipped_ratio": 0.86875,
+      "completions/max_length": 13829.0,
+      "completions/max_terminated_length": 5621.0,
+      "completions/mean_length": 5449.40625,
+      "completions/mean_terminated_length": 1356.31787109375,
+      "completions/min_length": 569.2,
+      "completions/min_terminated_length": 319.0,
+      "epoch": 0.624,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.7409809827804565,
+      "kl": 5.487593057379127,
+      "learning_rate": 1e-06,
+      "loss": 0.0834,
+      "num_tokens": 148088560.0,
+      "reward": 0.6035884141921997,
+      "reward_std": 0.7359590947628021,
+      "step": 780,
+      "step_time": 285.02171131893994
+    },
+    {
+      "clip_ratio/high_max": 0.0014547745035088156,
+      "clip_ratio/high_mean": 0.00022145010402709885,
+      "clip_ratio/low_mean": 0.00039898897614421003,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0006204390816492378,
+      "completion_length": 5633.9,
+      "completions/clipped_ratio": 0.8875,
+      "completions/max_length": 12286.4,
+      "completions/max_terminated_length": 2003.8,
+      "completions/mean_length": 5633.9,
+      "completions/mean_terminated_length": 700.6364379882813,
+      "completions/min_length": 315.4,
+      "completions/min_terminated_length": 315.4,
+      "epoch": 0.632,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.19570334255695343,
+      "kl": 6.794969582185149,
+      "learning_rate": 1e-06,
+      "loss": 0.1033,
+      "num_tokens": 150249848.0,
+      "reward": 0.5970386147499085,
+      "reward_std": 0.8987948536872864,
+      "step": 790,
+      "step_time": 284.52562670484184
+    },
+    {
+      "clip_ratio/high_max": 0.0036940005516953533,
+      "clip_ratio/high_mean": 0.0004877435465459712,
+      "clip_ratio/low_mean": 0.00020428227956017508,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.000692025873786406,
+      "completion_length": 5042.6375,
+      "completions/clipped_ratio": 0.846875,
+      "completions/max_length": 11240.2,
+      "completions/max_terminated_length": 4922.4,
+      "completions/mean_length": 5042.6375,
+      "completions/mean_terminated_length": 873.9874145507813,
+      "completions/min_length": 454.0,
+      "completions/min_terminated_length": 169.8,
+      "epoch": 0.64,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 1.1369402408599854,
+      "kl": 6.715023368503898,
+      "learning_rate": 1e-06,
+      "loss": 0.1099,
+      "num_tokens": 152207752.0,
+      "reward": 0.7560559496283531,
+      "reward_std": 0.7173129498958588,
+      "step": 800,
+      "step_time": 271.5230449665338
+    },
+    {
+      "clip_ratio/high_max": 0.03752983724261867,
+      "clip_ratio/high_mean": 0.005046068698425188,
+      "clip_ratio/low_mean": 0.0022608103692618895,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.007306879234420194,
+      "completion_length": 5583.371875,
+      "completions/clipped_ratio": 0.921875,
+      "completions/max_length": 12738.0,
+      "completions/max_terminated_length": 684.4,
+      "completions/mean_length": 5583.371875,
+      "completions/mean_terminated_length": 359.9100036621094,
+      "completions/min_length": 417.0,
+      "completions/min_terminated_length": 198.8,
+      "epoch": 0.648,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.30459651350975037,
+      "kl": 4.85752305239439,
+      "learning_rate": 1e-06,
+      "loss": 0.0772,
+      "num_tokens": 154353791.0,
+      "reward": 0.547912859916687,
+      "reward_std": 0.703293788433075,
+      "step": 810,
+      "step_time": 301.0829934667796
+    },
+    {
+      "clip_ratio/high_max": 0.0006389279991708463,
+      "clip_ratio/high_mean": 0.00011830468756670597,
+      "clip_ratio/low_mean": 0.0009571294550482889,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0010754340663879703,
+      "completion_length": 5380.953125,
+      "completions/clipped_ratio": 0.8875,
+      "completions/max_length": 12690.2,
+      "completions/max_terminated_length": 3702.2,
+      "completions/mean_length": 5380.953125,
+      "completions/mean_terminated_length": 1176.0460388183594,
+      "completions/min_length": 229.8,
+      "completions/min_terminated_length": 229.8,
+      "epoch": 0.656,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.7789612412452698,
+      "kl": 5.40329829454422,
+      "learning_rate": 1e-06,
+      "loss": 0.0827,
+      "num_tokens": 156389888.0,
+      "reward": 0.7733067989349365,
+      "reward_std": 0.7118594527244568,
+      "step": 820,
+      "step_time": 268.0118765439838
+    },
+    {
+      "clip_ratio/high_max": 0.0016143538267897385,
+      "clip_ratio/high_mean": 0.00022825244263913191,
+      "clip_ratio/low_mean": 0.00026090854946687615,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0004891609856713331,
+      "completion_length": 5892.171875,
+      "completions/clipped_ratio": 0.88125,
+      "completions/max_length": 12998.2,
+      "completions/max_terminated_length": 1175.2,
+      "completions/mean_length": 5892.171875,
+      "completions/mean_terminated_length": 711.39208984375,
+      "completions/min_length": 418.6,
+      "completions/min_terminated_length": 418.6,
+      "epoch": 0.664,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.3663407266139984,
+      "kl": 7.701748874783516,
+      "learning_rate": 1e-06,
+      "loss": 0.1223,
+      "num_tokens": 158610611.0,
+      "reward": 0.4492041528224945,
+      "reward_std": 1.1021975994110107,
+      "step": 830,
+      "step_time": 281.78603946492075
+    },
+    {
+      "clip_ratio/high_max": 0.00597827873971255,
+      "clip_ratio/high_mean": 0.0007777050503591454,
+      "clip_ratio/low_mean": 1.602187546723144e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0007937268945170217,
+      "completion_length": 5961.584375,
+      "completions/clipped_ratio": 0.9125,
+      "completions/max_length": 13834.4,
+      "completions/max_terminated_length": 4696.0,
+      "completions/mean_length": 5961.584375,
+      "completions/mean_terminated_length": 1012.24365234375,
+      "completions/min_length": 483.2,
+      "completions/min_terminated_length": 246.0,
+      "epoch": 0.672,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.21859169006347656,
+      "kl": 6.3625454908236865,
+      "learning_rate": 1e-06,
+      "loss": 0.1003,
+      "num_tokens": 160865202.0,
+      "reward": 0.37939960658550265,
+      "reward_std": 0.8259820461273193,
+      "step": 840,
+      "step_time": 302.0347501628101
+    },
+    {
+      "clip_ratio/high_max": 0.00024534441254218107,
+      "clip_ratio/high_mean": 5.61226994250319e-05,
+      "clip_ratio/low_mean": 0.0046444120071214455,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004700534656785748,
+      "completion_length": 5556.325,
+      "completions/clipped_ratio": 0.884375,
+      "completions/max_length": 12403.4,
+      "completions/max_terminated_length": 5045.4,
+      "completions/mean_length": 5556.325,
+      "completions/mean_terminated_length": 1776.9342956542969,
+      "completions/min_length": 336.0,
+      "completions/min_terminated_length": 336.0,
+      "epoch": 0.68,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.20565854012966156,
+      "kl": 5.322125254105776,
+      "learning_rate": 1e-06,
+      "loss": 0.0814,
+      "num_tokens": 162999602.0,
+      "reward": 0.47903812527656553,
+      "reward_std": 0.835130387544632,
+      "step": 850,
+      "step_time": 278.87317293696105
+    },
+    {
+      "clip_ratio/high_max": 0.0030428783313254824,
+      "clip_ratio/high_mean": 0.00040509605864826883,
+      "clip_ratio/low_mean": 6.493205305559968e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00047002810533740557,
+      "completion_length": 6198.734375,
+      "completions/clipped_ratio": 0.8625,
+      "completions/max_length": 13112.8,
+      "completions/max_terminated_length": 3219.8,
+      "completions/mean_length": 6198.734375,
+      "completions/mean_terminated_length": 663.8349487304688,
+      "completions/min_length": 282.0,
+      "completions/min_terminated_length": 282.0,
+      "epoch": 0.688,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.18980222940444946,
+      "kl": 7.659272170066833,
+      "learning_rate": 1e-06,
+      "loss": 0.1269,
+      "num_tokens": 165333109.0,
+      "reward": 0.4767775356769562,
+      "reward_std": 0.768990671634674,
+      "step": 860,
+      "step_time": 270.46701884008945
+    },
+    {
+      "clip_ratio/high_max": 0.0010170941095566377,
+      "clip_ratio/high_mean": 0.0001506442631352911,
+      "clip_ratio/low_mean": 2.226838878414128e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00017291265203311923,
+      "completion_length": 5310.6375,
+      "completions/clipped_ratio": 0.865625,
+      "completions/max_length": 12770.0,
+      "completions/max_terminated_length": 911.4,
+      "completions/mean_length": 5310.6375,
+      "completions/mean_terminated_length": 615.451171875,
+      "completions/min_length": 327.8,
+      "completions/min_terminated_length": 327.8,
+      "epoch": 0.696,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.21837736666202545,
+      "kl": 7.995494793355465,
+      "learning_rate": 1e-06,
+      "loss": 0.1177,
+      "num_tokens": 167392045.0,
+      "reward": 0.578833955526352,
+      "reward_std": 0.9156623005867004,
+      "step": 870,
+      "step_time": 283.2357983257622
+    },
+    {
+      "clip_ratio/high_max": 0.004631627641538216,
+      "clip_ratio/high_mean": 0.000647384279545804,
+      "clip_ratio/low_mean": 0.0027991519861188864,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0034465362263290445,
+      "completion_length": 5920.7375,
+      "completions/clipped_ratio": 0.9125,
+      "completions/max_length": 13779.2,
+      "completions/max_terminated_length": 2449.4,
+      "completions/mean_length": 5920.7375,
+      "completions/mean_terminated_length": 1144.940167236328,
+      "completions/min_length": 516.4,
+      "completions/min_terminated_length": 516.4,
+      "epoch": 0.704,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.27441316843032837,
+      "kl": 5.103387340717018,
+      "learning_rate": 1e-06,
+      "loss": 0.0783,
+      "num_tokens": 169638117.0,
+      "reward": 0.2579709529876709,
+      "reward_std": 0.8399613499641418,
+      "step": 880,
+      "step_time": 312.4003202129155
+    },
+    {
+      "clip_ratio/high_max": 0.02108584188390523,
+      "clip_ratio/high_mean": 0.002671135628747834,
+      "clip_ratio/low_mean": 0.0016337981513743216,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0043049337409684085,
+      "completion_length": 6999.978125,
+      "completions/clipped_ratio": 0.91875,
+      "completions/max_length": 14874.8,
+      "completions/max_terminated_length": 2305.0,
+      "completions/mean_length": 6999.978125,
+      "completions/mean_terminated_length": 782.1055786132813,
+      "completions/min_length": 359.6,
+      "completions/min_terminated_length": 359.6,
+      "epoch": 0.712,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.5956175923347473,
+      "kl": 9.077560269832611,
+      "learning_rate": 1e-06,
+      "loss": 0.1553,
+      "num_tokens": 172244762.0,
+      "reward": -0.0006461262702941895,
+      "reward_std": 0.9863644480705261,
+      "step": 890,
+      "step_time": 300.8817716598511
+    },
+    {
+      "clip_ratio/high_max": 0.007002430693319184,
+      "clip_ratio/high_mean": 0.0009952093887477531,
+      "clip_ratio/low_mean": 2.7229509396420325e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0010224388917777106,
+      "completion_length": 6289.590625,
+      "completions/clipped_ratio": 0.875,
+      "completions/max_length": 13894.0,
+      "completions/max_terminated_length": 6021.6,
+      "completions/mean_length": 6289.590625,
+      "completions/mean_terminated_length": 1873.5143310546875,
+      "completions/min_length": 272.0,
+      "completions/min_terminated_length": 272.0,
+      "epoch": 0.72,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.206376314163208,
+      "kl": 5.696395242214203,
+      "learning_rate": 1e-06,
+      "loss": 0.0876,
+      "num_tokens": 174601863.0,
+      "reward": 0.39101632237434386,
+      "reward_std": 0.8918306231498718,
+      "step": 900,
+      "step_time": 290.529536973685
+    },
+    {
+      "clip_ratio/high_max": 0.0002361544560699258,
+      "clip_ratio/high_mean": 5.661724217134179e-05,
+      "clip_ratio/low_mean": 0.0005228795069115222,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0005794967328256462,
+      "completion_length": 5588.2625,
+      "completions/clipped_ratio": 0.878125,
+      "completions/max_length": 12936.2,
+      "completions/max_terminated_length": 2996.2,
+      "completions/mean_length": 5588.2625,
+      "completions/mean_terminated_length": 1100.7228759765626,
+      "completions/min_length": 492.4,
+      "completions/min_terminated_length": 593.2,
+      "epoch": 0.728,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.5276018381118774,
+      "kl": 5.388607639819384,
+      "learning_rate": 1e-06,
+      "loss": 0.0773,
+      "num_tokens": 176743183.0,
+      "reward": 0.6213108956813812,
+      "reward_std": 0.7233340680599213,
+      "step": 910,
+      "step_time": 268.2470346078277
+    },
+    {
+      "clip_ratio/high_max": 0.025876993390193094,
+      "clip_ratio/high_mean": 0.003342244123268756,
+      "clip_ratio/low_mean": 0.0005985347108435235,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.003940779038384789,
+      "completion_length": 5608.509375,
+      "completions/clipped_ratio": 0.85,
+      "completions/max_length": 12542.0,
+      "completions/max_terminated_length": 2180.8,
+      "completions/mean_length": 5608.509375,
+      "completions/mean_terminated_length": 531.8144653320312,
+      "completions/min_length": 182.6,
+      "completions/min_terminated_length": 182.6,
+      "epoch": 0.736,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.18233880400657654,
+      "kl": 4.931325871963054,
+      "learning_rate": 1e-06,
+      "loss": 0.0757,
+      "num_tokens": 178888242.0,
+      "reward": 0.650401520729065,
+      "reward_std": 0.6106677830219269,
+      "step": 920,
+      "step_time": 273.93614843897524
+    },
+    {
+      "clip_ratio/high_max": 0.006613020508757473,
+      "clip_ratio/high_mean": 0.000864742542910335,
+      "clip_ratio/low_mean": 0.0003621207220248834,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0012268632528503077,
+      "completion_length": 5506.69375,
+      "completions/clipped_ratio": 0.878125,
+      "completions/max_length": 13172.0,
+      "completions/max_terminated_length": 1828.4,
+      "completions/mean_length": 5506.69375,
+      "completions/mean_terminated_length": 730.2668273925781,
+      "completions/min_length": 248.4,
+      "completions/min_terminated_length": 248.4,
+      "epoch": 0.744,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.18033641576766968,
+      "kl": 5.955277198553086,
+      "learning_rate": 1e-06,
+      "loss": 0.0882,
+      "num_tokens": 180981528.0,
+      "reward": 0.6884328842163085,
+      "reward_std": 0.806658923625946,
+      "step": 930,
+      "step_time": 281.1714373987168
+    },
+    {
+      "clip_ratio/high_max": 0.0004885086647846038,
+      "clip_ratio/high_mean": 0.0001015300282233511,
+      "clip_ratio/low_mean": 0.0008287591308544506,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0009302891263359925,
+      "completion_length": 5848.190625,
+      "completions/clipped_ratio": 0.890625,
+      "completions/max_length": 13171.4,
+      "completions/max_terminated_length": 2146.0,
+      "completions/mean_length": 5848.190625,
+      "completions/mean_terminated_length": 707.4535766601563,
+      "completions/min_length": 401.0,
+      "completions/min_terminated_length": 461.8,
+      "epoch": 0.752,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.1716427057981491,
+      "kl": 6.372476962208748,
+      "learning_rate": 1e-06,
+      "loss": 0.0986,
+      "num_tokens": 183176009.0,
+      "reward": 0.5971156597137451,
+      "reward_std": 0.8174049496650696,
+      "step": 940,
+      "step_time": 270.3288380548358
+    },
+    {
+      "clip_ratio/high_max": 0.015033719808889146,
+      "clip_ratio/high_mean": 0.001926349865470911,
+      "clip_ratio/low_mean": 0.001952208310831338,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0038785580535659393,
+      "completion_length": 5401.428125,
+      "completions/clipped_ratio": 0.821875,
+      "completions/max_length": 13483.2,
+      "completions/max_terminated_length": 2672.4,
+      "completions/mean_length": 5401.428125,
+      "completions/mean_terminated_length": 952.7174133300781,
+      "completions/min_length": 362.8,
+      "completions/min_terminated_length": 362.8,
+      "epoch": 0.76,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.404834508895874,
+      "kl": 6.688159626722336,
+      "learning_rate": 1e-06,
+      "loss": 0.1041,
+      "num_tokens": 185258350.0,
+      "reward": 0.6181340456008911,
+      "reward_std": 0.9163050651550293,
+      "step": 950,
+      "step_time": 273.49380723014474
+    },
+    {
+      "clip_ratio/high_max": 0.01149220318220614,
+      "clip_ratio/high_mean": 0.0019651008982691565,
+      "clip_ratio/low_mean": 0.00044536748168866325,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0024104683145196757,
+      "completion_length": 5798.015625,
+      "completions/clipped_ratio": 0.834375,
+      "completions/max_length": 13954.6,
+      "completions/max_terminated_length": 1513.4,
+      "completions/mean_length": 5798.015625,
+      "completions/mean_terminated_length": 642.6238464355469,
+      "completions/min_length": 316.2,
+      "completions/min_terminated_length": 316.2,
+      "epoch": 0.768,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.46848994493484497,
+      "kl": 5.954007020592689,
+      "learning_rate": 1e-06,
+      "loss": 0.091,
+      "num_tokens": 187467031.0,
+      "reward": 0.46877015829086305,
+      "reward_std": 0.8111052274703979,
+      "step": 960,
+      "step_time": 296.83381325267254
+    },
+    {
+      "clip_ratio/high_max": 0.022685668736085064,
+      "clip_ratio/high_mean": 0.0038915716541850997,
+      "clip_ratio/low_mean": 0.003070029996217727,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.006961601929288008,
+      "completion_length": 6276.540625,
+      "completions/clipped_ratio": 0.86875,
+      "completions/max_length": 13087.4,
+      "completions/max_terminated_length": 1151.2,
+      "completions/mean_length": 6276.540625,
+      "completions/mean_terminated_length": 643.8334716796875,
+      "completions/min_length": 379.8,
+      "completions/min_terminated_length": 379.8,
+      "epoch": 0.776,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.5902615785598755,
+      "kl": 9.856386515498162,
+      "learning_rate": 1e-06,
+      "loss": 0.1604,
+      "num_tokens": 189810364.0,
+      "reward": 0.29494223892688753,
+      "reward_std": 1.012243962287903,
+      "step": 970,
+      "step_time": 273.57245545461774
+    },
+    {
+      "clip_ratio/high_max": 0.03790558946238889,
+      "clip_ratio/high_mean": 0.005965857012733977,
+      "clip_ratio/low_mean": 0.00027457670706780845,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.006240433637049137,
+      "completion_length": 5711.653125,
+      "completions/clipped_ratio": 0.853125,
+      "completions/max_length": 12898.0,
+      "completions/max_terminated_length": 1106.4,
+      "completions/mean_length": 5711.653125,
+      "completions/mean_terminated_length": 657.5742614746093,
+      "completions/min_length": 319.6,
+      "completions/min_terminated_length": 319.6,
+      "epoch": 0.784,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.5094711184501648,
+      "kl": 6.392683005332946,
+      "learning_rate": 1e-06,
+      "loss": 0.0962,
+      "num_tokens": 191973425.0,
+      "reward": 0.29337393045425414,
+      "reward_std": 1.0867493629455567,
+      "step": 980,
+      "step_time": 298.35278257094325
+    },
+    {
+      "clip_ratio/high_max": 0.04529184505900048,
+      "clip_ratio/high_mean": 0.006695482004261066,
+      "clip_ratio/low_mean": 0.0053869577694513286,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.012082439642472308,
+      "completion_length": 6158.11875,
+      "completions/clipped_ratio": 0.90625,
+      "completions/max_length": 13422.4,
+      "completions/max_terminated_length": 2353.4,
+      "completions/mean_length": 6158.11875,
+      "completions/mean_terminated_length": 1008.2076293945313,
+      "completions/min_length": 349.4,
+      "completions/min_terminated_length": 349.4,
+      "epoch": 0.792,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.2965131103992462,
+      "kl": 11.391275238990783,
+      "learning_rate": 1e-06,
+      "loss": 0.1711,
+      "num_tokens": 194306919.0,
+      "reward": -0.01198197603225708,
+      "reward_std": 1.2059163331985474,
+      "step": 990,
+      "step_time": 291.684672626853
+    },
+    {
+      "clip_ratio/high_max": 0.01816512215627881,
+      "clip_ratio/high_mean": 0.0026342601271608147,
+      "clip_ratio/low_mean": 0.002743410276298164,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.005377670466623385,
+      "completion_length": 5674.753125,
+      "completions/clipped_ratio": 0.86875,
+      "completions/max_length": 13665.2,
+      "completions/max_terminated_length": 1892.0,
+      "completions/mean_length": 5674.753125,
+      "completions/mean_terminated_length": 553.2566619873047,
+      "completions/min_length": 320.4,
+      "completions/min_terminated_length": 320.4,
+      "epoch": 0.8,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.37527522444725037,
+      "kl": 5.525828441232443,
+      "learning_rate": 1e-06,
+      "loss": 0.0923,
+      "num_tokens": 196456008.0,
+      "reward": 0.4234964370727539,
+      "reward_std": 0.7747700929641723,
+      "step": 1000,
+      "step_time": 277.086211149022
+    },
+    {
+      "clip_ratio/high_max": 0.03981928765097109,
+      "clip_ratio/high_mean": 0.006004743749326735,
+      "clip_ratio/low_mean": 0.0015710417625882656,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.007575785858171003,
+      "completion_length": 6036.396875,
+      "completions/clipped_ratio": 0.921875,
+      "completions/max_length": 13660.8,
+      "completions/max_terminated_length": 739.0,
+      "completions/mean_length": 6036.396875,
+      "completions/mean_terminated_length": 389.88096313476564,
+      "completions/min_length": 220.0,
+      "completions/min_terminated_length": 220.0,
+      "epoch": 0.808,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.9124040603637695,
+      "kl": 5.480291825532913,
+      "learning_rate": 1e-06,
+      "loss": 0.0864,
+      "num_tokens": 198706999.0,
+      "reward": 0.21880382746458055,
+      "reward_std": 0.8379449248313904,
+      "step": 1010,
+      "step_time": 282.54359353147447
+    },
+    {
+      "clip_ratio/high_max": 0.014635300190820998,
+      "clip_ratio/high_mean": 0.002417540353667391,
+      "clip_ratio/low_mean": 0.0006898103696357794,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0031073508573172147,
+      "completion_length": 5603.18125,
+      "completions/clipped_ratio": 0.878125,
+      "completions/max_length": 13103.0,
+      "completions/max_terminated_length": 1018.6,
+      "completions/mean_length": 5603.18125,
+      "completions/mean_terminated_length": 570.5734619140625,
+      "completions/min_length": 318.0,
+      "completions/min_terminated_length": 318.0,
+      "epoch": 0.816,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.26409414410591125,
+      "kl": 7.749452966451645,
+      "learning_rate": 1e-06,
+      "loss": 0.1169,
+      "num_tokens": 200833473.0,
+      "reward": 0.38421454429626467,
+      "reward_std": 1.059171938896179,
+      "step": 1020,
+      "step_time": 279.65719040483236
+    },
+    {
+      "clip_ratio/high_max": 0.026264496512521874,
+      "clip_ratio/high_mean": 0.004170728239296295,
+      "clip_ratio/low_mean": 0.0007857010263478514,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004956429285812192,
+      "completion_length": 6121.2125,
+      "completions/clipped_ratio": 0.834375,
+      "completions/max_length": 14152.8,
+      "completions/max_terminated_length": 997.4,
+      "completions/mean_length": 6121.2125,
+      "completions/mean_terminated_length": 511.15715942382815,
+      "completions/min_length": 311.0,
+      "completions/min_terminated_length": 311.0,
+      "epoch": 0.824,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.4613886773586273,
+      "kl": 9.4208127617836,
+      "learning_rate": 1e-06,
+      "loss": 0.1372,
+      "num_tokens": 203134393.0,
+      "reward": 0.6404453933238983,
+      "reward_std": 0.9427806615829468,
+      "step": 1030,
+      "step_time": 304.64545446000994
+    },
+    {
+      "clip_ratio/high_max": 0.02121812974128261,
+      "clip_ratio/high_mean": 0.0032299092125640527,
+      "clip_ratio/low_mean": 0.0005214747876607362,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.003751384012684866,
+      "completion_length": 6295.003125,
+      "completions/clipped_ratio": 0.859375,
+      "completions/max_length": 13696.0,
+      "completions/max_terminated_length": 803.0,
+      "completions/mean_length": 6295.003125,
+      "completions/mean_terminated_length": 440.6103118896484,
+      "completions/min_length": 233.6,
+      "completions/min_terminated_length": 233.6,
+      "epoch": 0.832,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.25060251355171204,
+      "kl": 8.619005155563354,
+      "learning_rate": 1e-06,
+      "loss": 0.1424,
+      "num_tokens": 205510490.0,
+      "reward": 0.34280063211917877,
+      "reward_std": 1.0963841676712036,
+      "step": 1040,
+      "step_time": 300.40770224444566
+    },
+    {
+      "clip_ratio/high_max": 0.009756845778065326,
+      "clip_ratio/high_mean": 0.0012488235522823744,
+      "clip_ratio/low_mean": 0.0006655195924395229,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0019143431552038237,
+      "completion_length": 5729.671875,
+      "completions/clipped_ratio": 0.85625,
+      "completions/max_length": 13109.0,
+      "completions/max_terminated_length": 842.4,
+      "completions/mean_length": 5729.671875,
+      "completions/mean_terminated_length": 449.9298095703125,
+      "completions/min_length": 212.0,
+      "completions/min_terminated_length": 212.0,
+      "epoch": 0.84,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.29876866936683655,
+      "kl": 8.124797195196152,
+      "learning_rate": 1e-06,
+      "loss": 0.1218,
+      "num_tokens": 207696025.0,
+      "reward": 0.4881739318370819,
+      "reward_std": 1.0822341918945313,
+      "step": 1050,
+      "step_time": 293.27442999258636
+    },
+    {
+      "clip_ratio/high_max": 0.012549413445503888,
+      "clip_ratio/high_mean": 0.0018336864915909246,
+      "clip_ratio/low_mean": 0.003277436838141057,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.005111123312599375,
+      "completion_length": 6297.19375,
+      "completions/clipped_ratio": 0.909375,
+      "completions/max_length": 13557.8,
+      "completions/max_terminated_length": 882.6,
+      "completions/mean_length": 6297.19375,
+      "completions/mean_terminated_length": 629.1555541992187,
+      "completions/min_length": 461.6,
+      "completions/min_terminated_length": 461.6,
+      "epoch": 0.848,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.6523057222366333,
+      "kl": 8.930494999885559,
+      "learning_rate": 1e-06,
+      "loss": 0.1478,
+      "num_tokens": 210054103.0,
+      "reward": 0.06715324521064758,
+      "reward_std": 0.9533425450325013,
+      "step": 1060,
+      "step_time": 296.7576637730002
+    },
+    {
+      "clip_ratio/high_max": 0.013247115068770654,
+      "clip_ratio/high_mean": 0.0017769423098798142,
+      "clip_ratio/low_mean": 5.091709094813268e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.001827859399782028,
+      "completion_length": 6713.146875,
+      "completions/clipped_ratio": 0.86875,
+      "completions/max_length": 13902.0,
+      "completions/max_terminated_length": 833.6,
+      "completions/mean_length": 6713.146875,
+      "completions/mean_terminated_length": 558.729833984375,
+      "completions/min_length": 431.8,
+      "completions/min_terminated_length": 431.8,
+      "epoch": 0.856,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.423062264919281,
+      "kl": 5.562233930826187,
+      "learning_rate": 1e-06,
+      "loss": 0.0946,
+      "num_tokens": 212537954.0,
+      "reward": -0.0638770580291748,
+      "reward_std": 0.8862400650978088,
+      "step": 1070,
+      "step_time": 296.67767008803787
+    },
+    {
+      "clip_ratio/high_max": 0.023606011657648196,
+      "clip_ratio/high_mean": 0.003014383661911779,
+      "clip_ratio/low_mean": 0.0006893456631587469,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.003703729040444159,
+      "completion_length": 6096.271875,
+      "completions/clipped_ratio": 0.871875,
+      "completions/max_length": 14072.6,
+      "completions/max_terminated_length": 811.2,
+      "completions/mean_length": 6096.271875,
+      "completions/mean_terminated_length": 475.67084350585935,
+      "completions/min_length": 324.0,
+      "completions/min_terminated_length": 324.0,
+      "epoch": 0.864,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.3318747878074646,
+      "kl": 6.561303415894509,
+      "learning_rate": 1e-06,
+      "loss": 0.1054,
+      "num_tokens": 214828537.0,
+      "reward": 0.27624769806861876,
+      "reward_std": 0.9645497083663941,
+      "step": 1080,
+      "step_time": 280.5619152456522
+    },
+    {
+      "clip_ratio/high_max": 0.018787590960528178,
+      "clip_ratio/high_mean": 0.0023815736062715587,
+      "clip_ratio/low_mean": 0.002419464008289651,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004801037690413068,
+      "completion_length": 6759.95625,
+      "completions/clipped_ratio": 0.9,
+      "completions/max_length": 13997.4,
+      "completions/max_terminated_length": 691.2,
+      "completions/mean_length": 6759.95625,
+      "completions/mean_terminated_length": 540.0283447265625,
+      "completions/min_length": 437.4,
+      "completions/min_terminated_length": 437.4,
+      "epoch": 0.872,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.7714992761611938,
+      "kl": 7.222531634569168,
+      "learning_rate": 1e-06,
+      "loss": 0.1213,
+      "num_tokens": 217338827.0,
+      "reward": -0.08061599135398864,
+      "reward_std": 0.9236310362815857,
+      "step": 1090,
+      "step_time": 310.4243446025997
+    },
+    {
+      "clip_ratio/high_max": 0.008008336583225173,
+      "clip_ratio/high_mean": 0.001024207373598074,
+      "clip_ratio/low_mean": 0.0017953175767615904,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.002819524997357803,
+      "completion_length": 6966.65625,
+      "completions/clipped_ratio": 0.921875,
+      "completions/max_length": 14411.6,
+      "completions/max_terminated_length": 785.8,
+      "completions/mean_length": 6966.65625,
+      "completions/mean_terminated_length": 533.7500122070312,
+      "completions/min_length": 598.8,
+      "completions/min_terminated_length": 329.4,
+      "epoch": 0.88,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.6073609590530396,
+      "kl": 6.270501172542572,
+      "learning_rate": 1e-06,
+      "loss": 0.1039,
+      "num_tokens": 219907461.0,
+      "reward": -0.28439933955669405,
+      "reward_std": 0.902522599697113,
+      "step": 1100,
+      "step_time": 337.0149421799928
+    },
+    {
+      "clip_ratio/high_max": 0.01877592925357021,
+      "clip_ratio/high_mean": 0.002749286360790393,
+      "clip_ratio/low_mean": 0.0013581480993934748,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00410743454031035,
+      "completion_length": 6837.10625,
+      "completions/clipped_ratio": 0.840625,
+      "completions/max_length": 14178.4,
+      "completions/max_terminated_length": 1165.6,
+      "completions/mean_length": 6837.10625,
+      "completions/mean_terminated_length": 688.6991027832031,
+      "completions/min_length": 403.0,
+      "completions/min_terminated_length": 403.0,
+      "epoch": 0.888,
+      "frac_reward_zero_std": 0.075,
+      "grad_norm": 0.3872028887271881,
+      "kl": 5.760408401489258,
+      "learning_rate": 1e-06,
+      "loss": 0.0972,
+      "num_tokens": 222472331.0,
+      "reward": -0.0003803849220275879,
+      "reward_std": 0.8155203342437745,
+      "step": 1110,
+      "step_time": 319.58374512679876
+    },
+    {
+      "clip_ratio/high_max": 0.008775160694131045,
+      "clip_ratio/high_mean": 0.0011167728966029245,
+      "clip_ratio/low_mean": 0.0007240017559524859,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0018407746740194853,
+      "completion_length": 5472.675,
+      "completions/clipped_ratio": 0.8375,
+      "completions/max_length": 13164.2,
+      "completions/max_terminated_length": 1345.6,
+      "completions/mean_length": 5472.675,
+      "completions/mean_terminated_length": 568.0773864746094,
+      "completions/min_length": 320.2,
+      "completions/min_terminated_length": 320.2,
+      "epoch": 0.896,
+      "frac_reward_zero_std": 0.075,
+      "grad_norm": 1.2009961605072021,
+      "kl": 4.859293684363365,
+      "learning_rate": 1e-06,
+      "loss": 0.074,
+      "num_tokens": 224553879.0,
+      "reward": 0.5265417963266372,
+      "reward_std": 0.6715977668762207,
+      "step": 1120,
+      "step_time": 296.8402795359492
+    },
+    {
+      "clip_ratio/high_max": 0.018852601882827003,
+      "clip_ratio/high_mean": 0.003166215562941943,
+      "clip_ratio/low_mean": 0.0001349339607259026,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0033011496123435792,
+      "completion_length": 6570.596875,
+      "completions/clipped_ratio": 0.865625,
+      "completions/max_length": 13621.2,
+      "completions/max_terminated_length": 819.4,
+      "completions/mean_length": 6570.596875,
+      "completions/mean_terminated_length": 384.5136413574219,
+      "completions/min_length": 495.0,
+      "completions/min_terminated_length": 164.6,
+      "epoch": 0.904,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.427226185798645,
+      "kl": 6.8142201662063595,
+      "learning_rate": 1e-06,
+      "loss": 0.1139,
+      "num_tokens": 227011326.0,
+      "reward": 0.09996834099292755,
+      "reward_std": 0.7323241293430328,
+      "step": 1130,
+      "step_time": 331.18104233555493
+    },
+    {
+      "clip_ratio/high_max": 0.0034632282939128347,
+      "clip_ratio/high_mean": 0.00046680087079948863,
+      "clip_ratio/low_mean": 0.0036836089600001285,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004150409929388843,
+      "completion_length": 5414.471875,
+      "completions/clipped_ratio": 0.878125,
+      "completions/max_length": 13177.6,
+      "completions/max_terminated_length": 886.2,
+      "completions/mean_length": 5414.471875,
+      "completions/mean_terminated_length": 526.9681945800781,
+      "completions/min_length": 765.2,
+      "completions/min_terminated_length": 388.4,
+      "epoch": 0.912,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.18849848210811615,
+      "kl": 5.050818152725697,
+      "learning_rate": 1e-06,
+      "loss": 0.0732,
+      "num_tokens": 229073417.0,
+      "reward": 0.4277604937553406,
+      "reward_std": 0.9787284970283509,
+      "step": 1140,
+      "step_time": 291.63691817708315
+    },
+    {
+      "clip_ratio/high_max": 0.04213488262430474,
+      "clip_ratio/high_mean": 0.006554762130645031,
+      "clip_ratio/low_mean": 0.0002588785070201993,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00681364067040704,
+      "completion_length": 5363.4125,
+      "completions/clipped_ratio": 0.85,
+      "completions/max_length": 13187.0,
+      "completions/max_terminated_length": 933.8,
+      "completions/mean_length": 5363.4125,
+      "completions/mean_terminated_length": 468.4944641113281,
+      "completions/min_length": 296.2,
+      "completions/min_terminated_length": 296.2,
+      "epoch": 0.92,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5877802968025208,
+      "kl": 8.33590975701809,
+      "learning_rate": 1e-06,
+      "loss": 0.1286,
+      "num_tokens": 231115673.0,
+      "reward": 0.6187662124633789,
+      "reward_std": 0.795666491985321,
+      "step": 1150,
+      "step_time": 292.8479825194925
+    },
+    {
+      "clip_ratio/high_max": 0.02313367812494107,
+      "clip_ratio/high_mean": 0.0029735829068158635,
+      "clip_ratio/low_mean": 0.0011590449278855886,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004132627930812305,
+      "completion_length": 6096.56875,
+      "completions/clipped_ratio": 0.9,
+      "completions/max_length": 12940.4,
+      "completions/max_terminated_length": 2896.0,
+      "completions/mean_length": 6096.56875,
+      "completions/mean_terminated_length": 1246.1752319335938,
+      "completions/min_length": 715.2,
+      "completions/min_terminated_length": 715.2,
+      "epoch": 0.928,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.3248451352119446,
+      "kl": 7.249865186214447,
+      "learning_rate": 1e-06,
+      "loss": 0.1169,
+      "num_tokens": 233432595.0,
+      "reward": 0.2131304532289505,
+      "reward_std": 0.9439968466758728,
+      "step": 1160,
+      "step_time": 293.4663334593177
+    },
+    {
+      "clip_ratio/high_max": 0.02019488452915539,
+      "clip_ratio/high_mean": 0.003011522550104928,
+      "clip_ratio/low_mean": 0.0008108435110898427,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.003822366013810097,
+      "completion_length": 6209.06875,
+      "completions/clipped_ratio": 0.853125,
+      "completions/max_length": 13179.2,
+      "completions/max_terminated_length": 871.4,
+      "completions/mean_length": 6209.06875,
+      "completions/mean_terminated_length": 569.7732666015625,
+      "completions/min_length": 374.4,
+      "completions/min_terminated_length": 374.4,
+      "epoch": 0.936,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.3624282479286194,
+      "kl": 8.34157618880272,
+      "learning_rate": 1e-06,
+      "loss": 0.1358,
+      "num_tokens": 235765877.0,
+      "reward": 0.22831922471523286,
+      "reward_std": 0.7948000073432923,
+      "step": 1170,
+      "step_time": 296.3901917222887
+    },
+    {
+      "clip_ratio/high_max": 0.012562487991272065,
+      "clip_ratio/high_mean": 0.0019445388491931225,
+      "clip_ratio/low_mean": 0.004173541884165388,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00611808084686345,
+      "completion_length": 6293.571875,
+      "completions/clipped_ratio": 0.85,
+      "completions/max_length": 14067.8,
+      "completions/max_terminated_length": 1126.2,
+      "completions/mean_length": 6293.571875,
+      "completions/mean_terminated_length": 470.57810668945314,
+      "completions/min_length": 188.4,
+      "completions/min_terminated_length": 188.4,
+      "epoch": 0.944,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.4447183907032013,
+      "kl": 6.042785331606865,
+      "learning_rate": 1e-06,
+      "loss": 0.0912,
+      "num_tokens": 238094240.0,
+      "reward": 0.2554441273212433,
+      "reward_std": 0.8412348031997681,
+      "step": 1180,
+      "step_time": 309.75795306526123
+    },
+    {
+      "clip_ratio/high_max": 0.0038944089885262654,
+      "clip_ratio/high_mean": 0.0006425480004281781,
+      "clip_ratio/low_mean": 0.0017261221296621444,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.002368670111559368,
+      "completion_length": 6340.2,
+      "completions/clipped_ratio": 0.8625,
+      "completions/max_length": 13684.8,
+      "completions/max_terminated_length": 905.2,
+      "completions/mean_length": 6340.2,
+      "completions/mean_terminated_length": 569.667724609375,
+      "completions/min_length": 231.2,
+      "completions/min_terminated_length": 231.2,
+      "epoch": 0.952,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.5602854490280151,
+      "kl": 7.093182611465454,
+      "learning_rate": 1e-06,
+      "loss": 0.1123,
+      "num_tokens": 240451212.0,
+      "reward": 0.2567012131214142,
+      "reward_std": 0.7290396332740784,
+      "step": 1190,
+      "step_time": 314.5449781946838
+    },
+    {
+      "clip_ratio/high_max": 0.02071729673061782,
+      "clip_ratio/high_mean": 0.002950307414539566,
+      "clip_ratio/low_mean": 0.0004041533088752658,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00335446065528231,
+      "completion_length": 6742.7125,
+      "completions/clipped_ratio": 0.903125,
+      "completions/max_length": 14254.8,
+      "completions/max_terminated_length": 627.6,
+      "completions/mean_length": 6742.7125,
+      "completions/mean_terminated_length": 470.81666870117186,
+      "completions/min_length": 348.0,
+      "completions/min_terminated_length": 366.8,
+      "epoch": 0.96,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.27792564034461975,
+      "kl": 5.798007053136826,
+      "learning_rate": 1e-06,
+      "loss": 0.0953,
+      "num_tokens": 242929384.0,
+      "reward": -0.29501100778579714,
+      "reward_std": 0.8618368148803711,
+      "step": 1200,
+      "step_time": 315.47993235588075
+    },
+    {
+      "clip_ratio/high_max": 0.006161564569265465,
+      "clip_ratio/high_mean": 0.0008106029246164326,
+      "clip_ratio/low_mean": 0.0006769363412104212,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0014875392742396798,
+      "completion_length": 6583.79375,
+      "completions/clipped_ratio": 0.859375,
+      "completions/max_length": 13476.2,
+      "completions/max_terminated_length": 1165.4,
+      "completions/mean_length": 6583.79375,
+      "completions/mean_terminated_length": 849.8432678222656,
+      "completions/min_length": 499.8,
+      "completions/min_terminated_length": 608.8,
+      "epoch": 0.968,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.4840505123138428,
+      "kl": 6.946047046780587,
+      "learning_rate": 1e-06,
+      "loss": 0.1045,
+      "num_tokens": 245391942.0,
+      "reward": 0.0006704270839691162,
+      "reward_std": 0.9802399039268493,
+      "step": 1210,
+      "step_time": 338.2759423546493
+    },
+    {
+      "clip_ratio/high_max": 0.09983646621803928,
+      "clip_ratio/high_mean": 0.018934413978604424,
+      "clip_ratio/low_mean": 0.0001308560610141285,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.019065270088685794,
+      "completion_length": 6503.665625,
+      "completions/clipped_ratio": 0.85625,
+      "completions/max_length": 14221.6,
+      "completions/max_terminated_length": 847.2,
+      "completions/mean_length": 6503.665625,
+      "completions/mean_terminated_length": 433.45635986328125,
+      "completions/min_length": 347.4,
+      "completions/min_terminated_length": 159.4,
+      "epoch": 0.976,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.3807828724384308,
+      "kl": 8.599406844377517,
+      "learning_rate": 1e-06,
+      "loss": 0.1423,
+      "num_tokens": 247806635.0,
+      "reward": -0.024665796756744386,
+      "reward_std": 0.7261250138282775,
+      "step": 1220,
+      "step_time": 328.0090681761503
+    },
+    {
+      "clip_ratio/high_max": 0.0010887240639931405,
+      "clip_ratio/high_mean": 0.00015852728815843876,
+      "clip_ratio/low_mean": 6.214564720039562e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0002206729307886235,
+      "completion_length": 6740.54375,
+      "completions/clipped_ratio": 0.884375,
+      "completions/max_length": 13935.0,
+      "completions/max_terminated_length": 1472.2,
+      "completions/mean_length": 6740.54375,
+      "completions/mean_terminated_length": 835.1389038085938,
+      "completions/min_length": 415.8,
+      "completions/min_terminated_length": 552.8,
+      "epoch": 0.984,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.7836365103721619,
+      "kl": 5.074967938661575,
+      "learning_rate": 1e-06,
+      "loss": 0.083,
+      "num_tokens": 250299109.0,
+      "reward": -0.2541376054286957,
+      "reward_std": 0.9162593960762024,
+      "step": 1230,
+      "step_time": 320.4685521025211
+    },
+    {
+      "clip_ratio/high_max": 0.0013122542464770958,
+      "clip_ratio/high_mean": 0.00018187519312959922,
+      "clip_ratio/low_mean": 0.00045183536470858596,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0006337105427974166,
+      "completion_length": 6756.228125,
+      "completions/clipped_ratio": 0.878125,
+      "completions/max_length": 14498.8,
+      "completions/max_terminated_length": 920.0,
+      "completions/mean_length": 6756.228125,
+      "completions/mean_terminated_length": 661.6540344238281,
+      "completions/min_length": 412.0,
+      "completions/min_terminated_length": 412.0,
+      "epoch": 0.992,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.461943119764328,
+      "kl": 6.330149835348129,
+      "learning_rate": 1e-06,
+      "loss": 0.0992,
+      "num_tokens": 252799386.0,
+      "reward": -0.19658967107534409,
+      "reward_std": 0.9254164338111878,
+      "step": 1240,
+      "step_time": 346.76764199063183
+    },
+    {
+      "clip_ratio/high_max": 0.0030583876879973102,
+      "clip_ratio/high_mean": 0.0004197893070340797,
+      "clip_ratio/low_mean": 0.0002909168464384493,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0007107061337592313,
+      "completion_length": 6792.60625,
+      "completions/clipped_ratio": 0.90625,
+      "completions/max_length": 14249.4,
+      "completions/max_terminated_length": 2649.8,
+      "completions/mean_length": 6792.60625,
+      "completions/mean_terminated_length": 783.3814392089844,
+      "completions/min_length": 375.4,
+      "completions/min_terminated_length": 375.4,
+      "epoch": 1.0,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.9175572991371155,
+      "kl": 7.089669048786163,
+      "learning_rate": 1e-06,
+      "loss": 0.1188,
+      "num_tokens": 255319168.0,
+      "reward": -0.08971141874790192,
+      "reward_std": 0.8509580373764039,
+      "step": 1250,
+      "step_time": 296.31806163191794
+    },
+    {
+      "clip_ratio/high_max": 0.06331478241918376,
+      "clip_ratio/high_mean": 0.017041940525541577,
+      "clip_ratio/low_mean": 0.0017875447764254203,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.018829486197682856,
+      "completion_length": 6555.584375,
+      "completions/clipped_ratio": 0.85625,
+      "completions/max_length": 13789.4,
+      "completions/max_terminated_length": 593.4,
+      "completions/mean_length": 6555.584375,
+      "completions/mean_terminated_length": 403.6272506713867,
+      "completions/min_length": 301.2,
+      "completions/min_terminated_length": 301.2,
+      "epoch": 1.008,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.5270373821258545,
+      "kl": 5.429290795326233,
+      "learning_rate": 1e-06,
+      "loss": 0.0904,
+      "num_tokens": 257757319.0,
+      "reward": -0.1491591215133667,
+      "reward_std": 0.8710783839225769,
+      "step": 1260,
+      "step_time": 301.0354870621115
+    },
+    {
+      "clip_ratio/high_max": 0.0015826046908841817,
+      "clip_ratio/high_mean": 0.00021522579916108953,
+      "clip_ratio/low_mean": 0.0007693763136444431,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0009846021005614602,
+      "completion_length": 6022.0,
+      "completions/clipped_ratio": 0.85,
+      "completions/max_length": 12558.8,
+      "completions/max_terminated_length": 3774.2,
+      "completions/mean_length": 6022.0,
+      "completions/mean_terminated_length": 983.7999877929688,
+      "completions/min_length": 433.4,
+      "completions/min_terminated_length": 433.4,
+      "epoch": 1.016,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.30265286564826965,
+      "kl": 4.875985875725746,
+      "learning_rate": 1e-06,
+      "loss": 0.0763,
+      "num_tokens": 260028443.0,
+      "reward": 0.40891923308372496,
+      "reward_std": 0.6646989822387696,
+      "step": 1270,
+      "step_time": 279.7881353031844
+    },
+    {
+      "clip_ratio/high_max": 0.00973056090188038,
+      "clip_ratio/high_mean": 0.0013696761425762815,
+      "clip_ratio/low_mean": 0.00014194257407780243,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0015116187698367867,
+      "completion_length": 7153.640625,
+      "completions/clipped_ratio": 0.909375,
+      "completions/max_length": 14647.8,
+      "completions/max_terminated_length": 2632.8,
+      "completions/mean_length": 7153.640625,
+      "completions/mean_terminated_length": 1295.95615234375,
+      "completions/min_length": 453.0,
+      "completions/min_terminated_length": 241.4,
+      "epoch": 1.024,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.549114465713501,
+      "kl": 5.445976492762566,
+      "learning_rate": 1e-06,
+      "loss": 0.0899,
+      "num_tokens": 262663964.0,
+      "reward": -0.08243993818759918,
+      "reward_std": 0.7162402153015137,
+      "step": 1280,
+      "step_time": 297.6551021475345
+    },
+    {
+      "clip_ratio/high_max": 0.0007519308774135425,
+      "clip_ratio/high_mean": 0.00010810922681230295,
+      "clip_ratio/low_mean": 0.0013422108902432228,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0014503201217848983,
+      "completion_length": 6293.15,
+      "completions/clipped_ratio": 0.8375,
+      "completions/max_length": 14389.6,
+      "completions/max_terminated_length": 1053.2,
+      "completions/mean_length": 6293.15,
+      "completions/mean_terminated_length": 469.5631958007813,
+      "completions/min_length": 296.2,
+      "completions/min_terminated_length": 296.2,
+      "epoch": 1.032,
+      "frac_reward_zero_std": 0.075,
+      "grad_norm": 0.5712789297103882,
+      "kl": 7.148627138137817,
+      "learning_rate": 1e-06,
+      "loss": 0.1148,
+      "num_tokens": 265022840.0,
+      "reward": -0.07614173591136933,
+      "reward_std": 0.9813422977924346,
+      "step": 1290,
+      "step_time": 299.8684740707278
+    },
+    {
+      "clip_ratio/high_max": 0.009607324418084318,
+      "clip_ratio/high_mean": 0.0012292004256323707,
+      "clip_ratio/low_mean": 0.0018529474596334695,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0030821478708276118,
+      "completion_length": 6184.609375,
+      "completions/clipped_ratio": 0.828125,
+      "completions/max_length": 12752.6,
+      "completions/max_terminated_length": 2089.6,
+      "completions/mean_length": 6184.609375,
+      "completions/mean_terminated_length": 556.9234680175781,
+      "completions/min_length": 328.2,
+      "completions/min_terminated_length": 328.2,
+      "epoch": 1.04,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.7816324234008789,
+      "kl": 4.917503276467324,
+      "learning_rate": 1e-06,
+      "loss": 0.0767,
+      "num_tokens": 267343091.0,
+      "reward": 0.28854849338531496,
+      "reward_std": 0.6657142341136932,
+      "step": 1300,
+      "step_time": 260.09778020493684
+    },
+    {
+      "clip_ratio/high_max": 0.00860492365736718,
+      "clip_ratio/high_mean": 0.0013212844631880216,
+      "clip_ratio/low_mean": 0.0001922703831041872,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0015135548268517597,
+      "completion_length": 7458.08125,
+      "completions/clipped_ratio": 0.875,
+      "completions/max_length": 14229.0,
+      "completions/max_terminated_length": 1080.2,
+      "completions/mean_length": 7458.08125,
+      "completions/mean_terminated_length": 618.8072875976562,
+      "completions/min_length": 299.4,
+      "completions/min_terminated_length": 299.4,
+      "epoch": 1.048,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.5447946786880493,
+      "kl": 6.905389970541,
+      "learning_rate": 1e-06,
+      "loss": 0.1092,
+      "num_tokens": 270080797.0,
+      "reward": -0.2932537078857422,
+      "reward_std": 0.8897507905960083,
+      "step": 1310,
+      "step_time": 324.0993477422744
+    },
+    {
+      "clip_ratio/high_max": 0.0017108621217630572,
+      "clip_ratio/high_mean": 0.00023435080597664638,
+      "clip_ratio/low_mean": 0.0010592916048267399,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0012936423696373821,
+      "completion_length": 5995.565625,
+      "completions/clipped_ratio": 0.871875,
+      "completions/max_length": 13099.4,
+      "completions/max_terminated_length": 776.8,
+      "completions/mean_length": 5995.565625,
+      "completions/mean_terminated_length": 399.3552978515625,
+      "completions/min_length": 502.8,
+      "completions/min_terminated_length": 214.6,
+      "epoch": 1.056,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.5155627131462097,
+      "kl": 6.561501860618591,
+      "learning_rate": 1e-06,
+      "loss": 0.1025,
+      "num_tokens": 272345370.0,
+      "reward": 0.14361725449562074,
+      "reward_std": 0.9727361798286438,
+      "step": 1320,
+      "step_time": 297.0127417188138
+    },
+    {
+      "clip_ratio/high_max": 0.03395840809243964,
+      "clip_ratio/high_mean": 0.00490884905520943,
+      "clip_ratio/low_mean": 0.002795874337130044,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.007704723431197635,
+      "completion_length": 6322.384375,
+      "completions/clipped_ratio": 0.76875,
+      "completions/max_length": 14055.8,
+      "completions/max_terminated_length": 781.4,
+      "completions/mean_length": 6322.384375,
+      "completions/mean_terminated_length": 455.48919677734375,
+      "completions/min_length": 227.2,
+      "completions/min_terminated_length": 227.2,
+      "epoch": 1.064,
+      "frac_reward_zero_std": 0.075,
+      "grad_norm": 2.2143149375915527,
+      "kl": 8.452388608455658,
+      "learning_rate": 1e-06,
+      "loss": 0.1322,
+      "num_tokens": 274723081.0,
+      "reward": 0.12068977952003479,
+      "reward_std": 0.8671477019786835,
+      "step": 1330,
+      "step_time": 320.14473057202997
+    },
+    {
+      "clip_ratio/high_max": 0.008818195551793907,
+      "clip_ratio/high_mean": 0.001228650620669214,
+      "clip_ratio/low_mean": 4.203654460752659e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0012706871390946617,
+      "completion_length": 6861.09375,
+      "completions/clipped_ratio": 0.83125,
+      "completions/max_length": 14112.2,
+      "completions/max_terminated_length": 493.2,
+      "completions/mean_length": 6861.09375,
+      "completions/mean_terminated_length": 351.92333984375,
+      "completions/min_length": 232.8,
+      "completions/min_terminated_length": 232.8,
+      "epoch": 1.072,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 0.4763873815536499,
+      "kl": 5.826183921098709,
+      "learning_rate": 1e-06,
+      "loss": 0.095,
+      "num_tokens": 277272847.0,
+      "reward": -0.1976471424102783,
+      "reward_std": 0.7249846637248993,
+      "step": 1340,
+      "step_time": 319.05854049362244
+    },
+    {
+      "clip_ratio/high_max": 0.011285549855892895,
+      "clip_ratio/high_mean": 0.0014538088653580417,
+      "clip_ratio/low_mean": 0.0045533335471986906,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.006007142591920455,
+      "completion_length": 7021.309375,
+      "completions/clipped_ratio": 0.834375,
+      "completions/max_length": 14484.8,
+      "completions/max_terminated_length": 840.0,
+      "completions/mean_length": 7021.309375,
+      "completions/mean_terminated_length": 553.4257507324219,
+      "completions/min_length": 234.8,
+      "completions/min_terminated_length": 234.8,
+      "epoch": 1.08,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.8348678350448608,
+      "kl": 7.649586719274521,
+      "learning_rate": 1e-06,
+      "loss": 0.127,
+      "num_tokens": 279872842.0,
+      "reward": -0.5506071716547012,
+      "reward_std": 0.9008312165737152,
+      "step": 1350,
+      "step_time": 268.14785116277636
+    },
+    {
+      "clip_ratio/high_max": 0.02259904375059705,
+      "clip_ratio/high_mean": 0.004251178922913823,
+      "clip_ratio/low_mean": 0.0011239594841754297,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.005375138385988976,
+      "completion_length": 6991.90625,
+      "completions/clipped_ratio": 0.8625,
+      "completions/max_length": 14036.6,
+      "completions/max_terminated_length": 832.2,
+      "completions/mean_length": 6991.90625,
+      "completions/mean_terminated_length": 604.1214416503906,
+      "completions/min_length": 381.2,
+      "completions/min_terminated_length": 381.2,
+      "epoch": 1.088,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.33938491344451904,
+      "kl": 6.754237234592438,
+      "learning_rate": 1e-06,
+      "loss": 0.1137,
+      "num_tokens": 282447788.0,
+      "reward": -0.474853178858757,
+      "reward_std": 0.7467346787452698,
+      "step": 1360,
+      "step_time": 325.8671242542565
+    },
+    {
+      "clip_ratio/high_max": 0.01206368842485972,
+      "clip_ratio/high_mean": 0.0016831803827244585,
+      "clip_ratio/low_mean": 0.001954715311660493,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0036378956205226133,
+      "completion_length": 6690.55625,
+      "completions/clipped_ratio": 0.803125,
+      "completions/max_length": 13978.8,
+      "completions/max_terminated_length": 1065.4,
+      "completions/mean_length": 6690.55625,
+      "completions/mean_terminated_length": 642.2342102050782,
+      "completions/min_length": 277.6,
+      "completions/min_terminated_length": 277.6,
+      "epoch": 1.096,
+      "frac_reward_zero_std": 0.075,
+      "grad_norm": 0.4850001931190491,
+      "kl": 5.152865344285965,
+      "learning_rate": 1e-06,
+      "loss": 0.0776,
+      "num_tokens": 284913526.0,
+      "reward": -0.38518447875976564,
+      "reward_std": 0.7817031145095825,
+      "step": 1370,
+      "step_time": 316.62638979032636
+    },
+    {
+      "clip_ratio/high_max": 0.014483596660284093,
+      "clip_ratio/high_mean": 0.002148332479009696,
+      "clip_ratio/low_mean": 0.0029272636041696386,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0050755961003233095,
+      "completion_length": 6963.70625,
+      "completions/clipped_ratio": 0.875,
+      "completions/max_length": 14028.4,
+      "completions/max_terminated_length": 743.0,
+      "completions/mean_length": 6963.70625,
+      "completions/mean_terminated_length": 479.2916625976562,
+      "completions/min_length": 313.0,
+      "completions/min_terminated_length": 313.0,
+      "epoch": 1.104,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.9713339805603027,
+      "kl": 6.490580981969833,
+      "learning_rate": 1e-06,
+      "loss": 0.1095,
+      "num_tokens": 287466512.0,
+      "reward": -0.4649930775165558,
+      "reward_std": 0.9429963231086731,
+      "step": 1380,
+      "step_time": 313.35793677940967
+    },
+    {
+      "clip_ratio/high_max": 0.018218386419903254,
+      "clip_ratio/high_mean": 0.002577951976127224,
+      "clip_ratio/low_mean": 0.005079276807828137,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00765722867945442,
+      "completion_length": 7159.24375,
+      "completions/clipped_ratio": 0.85625,
+      "completions/max_length": 13459.2,
+      "completions/max_terminated_length": 562.8,
+      "completions/mean_length": 7159.24375,
+      "completions/mean_terminated_length": 419.16309814453126,
+      "completions/min_length": 287.2,
+      "completions/min_terminated_length": 287.2,
+      "epoch": 1.112,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.9339843392372131,
+      "kl": 4.823704534769059,
+      "learning_rate": 1e-06,
+      "loss": 0.0818,
+      "num_tokens": 290084894.0,
+      "reward": -0.5641295999288559,
+      "reward_std": 0.7534005880355835,
+      "step": 1390,
+      "step_time": 305.3198899012059
+    },
+    {
+      "clip_ratio/high_max": 0.059659806568743076,
+      "clip_ratio/high_mean": 0.010064624485812601,
+      "clip_ratio/low_mean": 0.0025443085289225563,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.012608932614057267,
+      "completion_length": 6942.16875,
+      "completions/clipped_ratio": 0.815625,
+      "completions/max_length": 13270.2,
+      "completions/max_terminated_length": 1514.4,
+      "completions/mean_length": 6942.16875,
+      "completions/mean_terminated_length": 626.9309326171875,
+      "completions/min_length": 266.0,
+      "completions/min_terminated_length": 266.0,
+      "epoch": 1.12,
+      "frac_reward_zero_std": 0.075,
+      "grad_norm": 1.0880489349365234,
+      "kl": 5.2463163793087,
+      "learning_rate": 1e-06,
+      "loss": 0.0881,
+      "num_tokens": 292642500.0,
+      "reward": -0.4800167828798294,
+      "reward_std": 0.7427855730056763,
+      "step": 1400,
+      "step_time": 282.1937792759389
+    },
+    {
+      "clip_ratio/high_max": 0.04458362648101684,
+      "clip_ratio/high_mean": 0.006630092494253859,
+      "clip_ratio/low_mean": 0.0009642676280009254,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.007594360200528172,
+      "completion_length": 8244.68125,
+      "completions/clipped_ratio": 0.884375,
+      "completions/max_length": 13846.6,
+      "completions/max_terminated_length": 942.2,
+      "completions/mean_length": 8244.68125,
+      "completions/mean_terminated_length": 613.3642944335937,
+      "completions/min_length": 372.6,
+      "completions/min_terminated_length": 372.6,
+      "epoch": 1.1280000000000001,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.4903317987918854,
+      "kl": 5.025724387168884,
+      "learning_rate": 1e-06,
+      "loss": 0.0916,
+      "num_tokens": 295619846.0,
+      "reward": -1.2492106437683106,
+      "reward_std": 0.6370679199695587,
+      "step": 1410,
+      "step_time": 310.49520882442596
+    },
+    {
+      "clip_ratio/high_max": 0.008771005711969338,
+      "clip_ratio/high_mean": 0.0011150768715197045,
+      "clip_ratio/low_mean": 0.0006917667670677474,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0018068437035253736,
+      "completion_length": 7823.059375,
+      "completions/clipped_ratio": 0.80625,
+      "completions/max_length": 14505.4,
+      "completions/max_terminated_length": 806.6,
+      "completions/mean_length": 7823.059375,
+      "completions/mean_terminated_length": 523.5938232421875,
+      "completions/min_length": 281.8,
+      "completions/min_terminated_length": 281.8,
+      "epoch": 1.1360000000000001,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.4781282842159271,
+      "kl": 3.4560777723789213,
+      "learning_rate": 1e-06,
+      "loss": 0.0606,
+      "num_tokens": 298468125.0,
+      "reward": -0.8919565379619598,
+      "reward_std": 0.6289015650749207,
+      "step": 1420,
+      "step_time": 314.4893631014973
+    },
+    {
+      "clip_ratio/high_max": 0.0215093705012805,
+      "clip_ratio/high_mean": 0.0027202213535474583,
+      "clip_ratio/low_mean": 1.2716831213310797e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.002732938354529324,
+      "completion_length": 8430.78125,
+      "completions/clipped_ratio": 0.8375,
+      "completions/max_length": 14001.2,
+      "completions/max_terminated_length": 734.8,
+      "completions/mean_length": 8430.78125,
+      "completions/mean_terminated_length": 484.84730834960936,
+      "completions/min_length": 316.4,
+      "completions/min_terminated_length": 316.4,
+      "epoch": 1.144,
+      "frac_reward_zero_std": 0.1,
+      "grad_norm": 0.3892126977443695,
+      "kl": 2.4982973873615264,
+      "learning_rate": 1e-06,
+      "loss": 0.0436,
+      "num_tokens": 301496771.0,
+      "reward": -1.4001689195632934,
+      "reward_std": 0.3681447505950928,
+      "step": 1430,
+      "step_time": 296.2141377706081
+    },
+    {
+      "clip_ratio/high_max": 0.007695864518063899,
+      "clip_ratio/high_mean": 0.001064059616214763,
+      "clip_ratio/low_mean": 0.010107422303394741,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.011171482213967465,
+      "completion_length": 8590.3875,
+      "completions/clipped_ratio": 0.871875,
+      "completions/max_length": 14487.2,
+      "completions/max_terminated_length": 784.4,
+      "completions/mean_length": 8590.3875,
+      "completions/mean_terminated_length": 559.3938537597656,
+      "completions/min_length": 260.8,
+      "completions/min_terminated_length": 260.8,
+      "epoch": 1.152,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.7115932703018188,
+      "kl": 3.1174581050872803,
+      "learning_rate": 1e-06,
+      "loss": 0.0478,
+      "num_tokens": 304594959.0,
+      "reward": -1.3695315599441529,
+      "reward_std": 0.6895758867263794,
+      "step": 1440,
+      "step_time": 317.56537790857254
+    },
+    {
+      "clip_ratio/high_max": 0.05393229034343676,
+      "clip_ratio/high_mean": 0.009469474181059923,
+      "clip_ratio/low_mean": 0.00031330420518997925,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.009782778202225017,
+      "completion_length": 7545.321875,
+      "completions/clipped_ratio": 0.8375,
+      "completions/max_length": 14330.0,
+      "completions/max_terminated_length": 1209.2,
+      "completions/mean_length": 7545.321875,
+      "completions/mean_terminated_length": 687.7175903320312,
+      "completions/min_length": 391.2,
+      "completions/min_terminated_length": 391.2,
+      "epoch": 1.16,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.6682623028755188,
+      "kl": 4.785371509194374,
+      "learning_rate": 1e-06,
+      "loss": 0.0845,
+      "num_tokens": 307352754.0,
+      "reward": -0.8678668528795243,
+      "reward_std": 0.765156877040863,
+      "step": 1450,
+      "step_time": 307.3786625292152
+    },
+    {
+      "clip_ratio/high_max": 0.004673573118452623,
+      "clip_ratio/high_mean": 0.0006075203566751952,
+      "clip_ratio/low_mean": 0.003566812149733778,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004174332359980326,
+      "completion_length": 7367.2125,
+      "completions/clipped_ratio": 0.846875,
+      "completions/max_length": 13439.2,
+      "completions/max_terminated_length": 921.4,
+      "completions/mean_length": 7367.2125,
+      "completions/mean_terminated_length": 532.4515625,
+      "completions/min_length": 286.4,
+      "completions/min_terminated_length": 286.4,
+      "epoch": 1.168,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.3403703272342682,
+      "kl": 3.685047924518585,
+      "learning_rate": 1e-06,
+      "loss": 0.0587,
+      "num_tokens": 310063554.0,
+      "reward": -0.6264693170785904,
+      "reward_std": 0.7679219603538513,
+      "step": 1460,
+      "step_time": 303.9987796962261
+    },
+    {
+      "clip_ratio/high_max": 0.019599513894354458,
+      "clip_ratio/high_mean": 0.0028967360664182706,
+      "clip_ratio/low_mean": 0.00514739912970299,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.008044135397858554,
+      "completion_length": 6869.0,
+      "completions/clipped_ratio": 0.7875,
+      "completions/max_length": 14164.0,
+      "completions/max_terminated_length": 1067.4,
+      "completions/mean_length": 6869.0,
+      "completions/mean_terminated_length": 545.4647827148438,
+      "completions/min_length": 283.2,
+      "completions/min_terminated_length": 283.2,
+      "epoch": 1.176,
+      "frac_reward_zero_std": 0.1375,
+      "grad_norm": 3.339172124862671,
+      "kl": 3.5414348036050796,
+      "learning_rate": 1e-06,
+      "loss": 0.0612,
+      "num_tokens": 312607230.0,
+      "reward": -0.3908490777015686,
+      "reward_std": 0.514220517873764,
+      "step": 1470,
+      "step_time": 306.77851864248515
+    },
+    {
+      "clip_ratio/high_max": 0.024134045062965015,
+      "clip_ratio/high_mean": 0.0030528913965326863,
+      "clip_ratio/low_mean": 0.020031159874099556,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.023084050847501203,
+      "completion_length": 8233.646875,
+      "completions/clipped_ratio": 0.9,
+      "completions/max_length": 14255.6,
+      "completions/max_terminated_length": 676.8,
+      "completions/mean_length": 8233.646875,
+      "completions/mean_terminated_length": 452.0185913085937,
+      "completions/min_length": 510.0,
+      "completions/min_terminated_length": 293.8,
+      "epoch": 1.184,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.6161352396011353,
+      "kl": 2.986080950498581,
+      "learning_rate": 1e-06,
+      "loss": 0.0472,
+      "num_tokens": 315587641.0,
+      "reward": -1.0295475840568542,
+      "reward_std": 0.8488625288009644,
+      "step": 1480,
+      "step_time": 325.14691910259427
+    },
+    {
+      "clip_ratio/high_max": 0.049144736468770135,
+      "clip_ratio/high_mean": 0.006986489323116985,
+      "clip_ratio/low_mean": 0.0038267810123898014,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.010813270109747464,
+      "completion_length": 7443.825,
+      "completions/clipped_ratio": 0.884375,
+      "completions/max_length": 13231.0,
+      "completions/max_terminated_length": 2525.2,
+      "completions/mean_length": 7443.825,
+      "completions/mean_terminated_length": 540.3030151367187,
+      "completions/min_length": 481.6,
+      "completions/min_terminated_length": 175.0,
+      "epoch": 1.192,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.613201916217804,
+      "kl": 4.23599865436554,
+      "learning_rate": 1e-06,
+      "loss": 0.071,
+      "num_tokens": 318313937.0,
+      "reward": -0.6151518881320953,
+      "reward_std": 0.6845939517021179,
+      "step": 1490,
+      "step_time": 297.6175458163023
+    },
+    {
+      "clip_ratio/high_max": 0.06837600921799095,
+      "clip_ratio/high_mean": 0.014579909735778074,
+      "clip_ratio/low_mean": 0.02437086595468827,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.03895077579727513,
+      "completion_length": 7360.99375,
+      "completions/clipped_ratio": 0.853125,
+      "completions/max_length": 13325.8,
+      "completions/max_terminated_length": 1361.6,
+      "completions/mean_length": 7360.99375,
+      "completions/mean_terminated_length": 843.6401733398437,
+      "completions/min_length": 323.2,
+      "completions/min_terminated_length": 323.2,
+      "epoch": 1.2,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.9797989726066589,
+      "kl": 4.196526461839676,
+      "learning_rate": 1e-06,
+      "loss": 0.0708,
+      "num_tokens": 321018611.0,
+      "reward": -0.7194792687892914,
+      "reward_std": 0.7643240332603455,
+      "step": 1500,
+      "step_time": 300.3483832065016
+    },
+    {
+      "clip_ratio/high_max": 0.03759765217855602,
+      "clip_ratio/high_mean": 0.005149026537515056,
+      "clip_ratio/low_mean": 0.0022692163714395975,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.007418243181439266,
+      "completion_length": 7351.44375,
+      "completions/clipped_ratio": 0.8875,
+      "completions/max_length": 14389.6,
+      "completions/max_terminated_length": 793.6,
+      "completions/mean_length": 7351.44375,
+      "completions/mean_terminated_length": 488.0387939453125,
+      "completions/min_length": 581.4,
+      "completions/min_terminated_length": 213.2,
+      "epoch": 1.208,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 1.1288779973983765,
+      "kl": 4.597329115867614,
+      "learning_rate": 1e-06,
+      "loss": 0.0746,
+      "num_tokens": 323708761.0,
+      "reward": -0.4730029582977295,
+      "reward_std": 0.7795473098754883,
+      "step": 1510,
+      "step_time": 307.8148294389248
+    },
+    {
+      "clip_ratio/high_max": 0.035038114325288916,
+      "clip_ratio/high_mean": 0.004405222202899494,
+      "clip_ratio/low_mean": 0.0027728557738555537,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.007178077994740306,
+      "completion_length": 7848.75625,
+      "completions/clipped_ratio": 0.884375,
+      "completions/max_length": 14450.2,
+      "completions/max_terminated_length": 978.6,
+      "completions/mean_length": 7848.75625,
+      "completions/mean_terminated_length": 578.1466735839844,
+      "completions/min_length": 339.2,
+      "completions/min_terminated_length": 339.2,
+      "epoch": 1.216,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.44890928268432617,
+      "kl": 4.332034492492676,
+      "learning_rate": 1e-06,
+      "loss": 0.075,
+      "num_tokens": 326581039.0,
+      "reward": -0.9139844536781311,
+      "reward_std": 0.6369929909706116,
+      "step": 1520,
+      "step_time": 339.742920647189
+    },
+    {
+      "clip_ratio/high_max": 0.034550899828445836,
+      "clip_ratio/high_mean": 0.004448481359543166,
+      "clip_ratio/low_mean": 0.012137696501963547,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.016586177824865444,
+      "completion_length": 7270.253125,
+      "completions/clipped_ratio": 0.803125,
+      "completions/max_length": 14302.6,
+      "completions/max_terminated_length": 972.4,
+      "completions/mean_length": 7270.253125,
+      "completions/mean_terminated_length": 441.5677429199219,
+      "completions/min_length": 165.8,
+      "completions/min_terminated_length": 165.8,
+      "epoch": 1.224,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.6899687051773071,
+      "kl": 3.8735065937042235,
+      "learning_rate": 1e-06,
+      "loss": 0.063,
+      "num_tokens": 329253040.0,
+      "reward": -0.5632274687290192,
+      "reward_std": 0.718042540550232,
+      "step": 1530,
+      "step_time": 300.0597866803408
+    },
+    {
+      "clip_ratio/high_max": 0.07408902343959198,
+      "clip_ratio/high_mean": 0.015771999812432113,
+      "clip_ratio/low_mean": 0.0070456424785334095,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.022817642444033483,
+      "completion_length": 7886.8375,
+      "completions/clipped_ratio": 0.85,
+      "completions/max_length": 14211.2,
+      "completions/max_terminated_length": 2501.8,
+      "completions/mean_length": 7886.8375,
+      "completions/mean_terminated_length": 614.1438659667969,
+      "completions/min_length": 323.6,
+      "completions/min_terminated_length": 323.6,
+      "epoch": 1.232,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 1.6699937582015991,
+      "kl": 5.0570512294769285,
+      "learning_rate": 1e-06,
+      "loss": 0.0901,
+      "num_tokens": 332113532.0,
+      "reward": -0.9353869915008545,
+      "reward_std": 0.6918673396110535,
+      "step": 1540,
+      "step_time": 323.14392309784887
+    },
+    {
+      "clip_ratio/high_max": 0.005868705319880973,
+      "clip_ratio/high_mean": 0.0007524862891614247,
+      "clip_ratio/low_mean": 0.005878954011086535,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.006631440243290853,
+      "completion_length": 8349.45625,
+      "completions/clipped_ratio": 0.9125,
+      "completions/max_length": 14398.6,
+      "completions/max_terminated_length": 787.8,
+      "completions/mean_length": 8349.45625,
+      "completions/mean_terminated_length": 642.0123962402344,
+      "completions/min_length": 511.0,
+      "completions/min_terminated_length": 511.0,
+      "epoch": 1.24,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.5437561869621277,
+      "kl": 3.8968759536743165,
+      "learning_rate": 1e-06,
+      "loss": 0.0664,
+      "num_tokens": 335143810.0,
+      "reward": -1.1195544004440308,
+      "reward_std": 0.6576746284961701,
+      "step": 1550,
+      "step_time": 327.956759692356
+    },
+    {
+      "clip_ratio/high_max": 0.01633510329229466,
+      "clip_ratio/high_mean": 0.0025136648622719805,
+      "clip_ratio/low_mean": 0.020487385496335265,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.02300105070989957,
+      "completion_length": 7800.00625,
+      "completions/clipped_ratio": 0.83125,
+      "completions/max_length": 14040.6,
+      "completions/max_terminated_length": 1394.0,
+      "completions/mean_length": 7800.00625,
+      "completions/mean_terminated_length": 608.679345703125,
+      "completions/min_length": 301.8,
+      "completions/min_terminated_length": 301.8,
+      "epoch": 1.248,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.7677727937698364,
+      "kl": 5.157671511173248,
+      "learning_rate": 1e-06,
+      "loss": 0.0856,
+      "num_tokens": 338008236.0,
+      "reward": -0.9923810482025146,
+      "reward_std": 0.7874501168727874,
+      "step": 1560,
+      "step_time": 331.0990063320845
+    },
+    {
+      "clip_ratio/high_max": 0.08447018024480712,
+      "clip_ratio/high_mean": 0.013777695731857876,
+      "clip_ratio/low_mean": 0.003472857877113711,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.017250553564008442,
+      "completion_length": 7574.840625,
+      "completions/clipped_ratio": 0.865625,
+      "completions/max_length": 13780.0,
+      "completions/max_terminated_length": 595.0,
+      "completions/mean_length": 7574.840625,
+      "completions/mean_terminated_length": 412.3604309082031,
+      "completions/min_length": 262.6,
+      "completions/min_terminated_length": 262.6,
+      "epoch": 1.256,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.6850536465644836,
+      "kl": 3.3793764114379883,
+      "learning_rate": 1e-06,
+      "loss": 0.0585,
+      "num_tokens": 340773321.0,
+      "reward": -0.9019905716180802,
+      "reward_std": 0.5519898712635041,
+      "step": 1570,
+      "step_time": 304.26330440044404
+    },
+    {
+      "clip_ratio/high_max": 0.015825910789499176,
+      "clip_ratio/high_mean": 0.0022924330067553454,
+      "clip_ratio/low_mean": 0.011829062127617362,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.014121495039307774,
+      "completion_length": 8030.284375,
+      "completions/clipped_ratio": 0.85625,
+      "completions/max_length": 14523.8,
+      "completions/max_terminated_length": 1084.8,
+      "completions/mean_length": 8030.284375,
+      "completions/mean_terminated_length": 454.9821533203125,
+      "completions/min_length": 223.6,
+      "completions/min_terminated_length": 223.6,
+      "epoch": 1.264,
+      "frac_reward_zero_std": 0.0875,
+      "grad_norm": 0.6004147529602051,
+      "kl": 3.437529918551445,
+      "learning_rate": 1e-06,
+      "loss": 0.0564,
+      "num_tokens": 343701144.0,
+      "reward": -0.8397439926862716,
+      "reward_std": 0.7164801239967347,
+      "step": 1580,
+      "step_time": 316.99658858627083
+    },
+    {
+      "clip_ratio/high_max": 0.041982113188078075,
+      "clip_ratio/high_mean": 0.011347536940047576,
+      "clip_ratio/low_mean": 0.0010816721987794153,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.012429209357560467,
+      "completion_length": 8332.228125,
+      "completions/clipped_ratio": 0.865625,
+      "completions/max_length": 14141.8,
+      "completions/max_terminated_length": 641.4,
+      "completions/mean_length": 8332.228125,
+      "completions/mean_terminated_length": 407.5039428710937,
+      "completions/min_length": 640.4,
+      "completions/min_terminated_length": 239.4,
+      "epoch": 1.272,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.9972069263458252,
+      "kl": 4.374789208173752,
+      "learning_rate": 1e-06,
+      "loss": 0.0762,
+      "num_tokens": 346713753.0,
+      "reward": -1.031822407245636,
+      "reward_std": 0.49489548802375793,
+      "step": 1590,
+      "step_time": 313.5276669666171
+    },
+    {
+      "clip_ratio/high_max": 0.007640167550744081,
+      "clip_ratio/high_mean": 0.0010085220250857674,
+      "clip_ratio/low_mean": 1.7917773027420482e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00102643976406398,
+      "completion_length": 8112.9375,
+      "completions/clipped_ratio": 0.9375,
+      "completions/max_length": 14436.0,
+      "completions/max_terminated_length": 869.8,
+      "completions/mean_length": 8112.9375,
+      "completions/mean_terminated_length": 552.5762084960937,
+      "completions/min_length": 740.8,
+      "completions/min_terminated_length": 365.6,
+      "epoch": 1.28,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.5571445822715759,
+      "kl": 3.7177543163299562,
+      "learning_rate": 1e-06,
+      "loss": 0.0613,
+      "num_tokens": 349652461.0,
+      "reward": -1.1822019815444946,
+      "reward_std": 0.6629145979881287,
+      "step": 1600,
+      "step_time": 305.0861327838153
+    },
+    {
+      "clip_ratio/high_max": 0.07279175572111854,
+      "clip_ratio/high_mean": 0.01696916231489922,
+      "clip_ratio/low_mean": 0.004030338939764988,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.020999501320875424,
+      "completion_length": 8624.84375,
+      "completions/clipped_ratio": 0.91875,
+      "completions/max_length": 14869.6,
+      "completions/max_terminated_length": 586.8,
+      "completions/mean_length": 8624.84375,
+      "completions/mean_terminated_length": 471.5066772460938,
+      "completions/min_length": 360.2,
+      "completions/min_terminated_length": 360.2,
+      "epoch": 1.288,
+      "frac_reward_zero_std": 0.075,
+      "grad_norm": 1.3507874011993408,
+      "kl": 4.65111243724823,
+      "learning_rate": 1e-06,
+      "loss": 0.0836,
+      "num_tokens": 352754367.0,
+      "reward": -1.2586896061897277,
+      "reward_std": 0.7051358819007874,
+      "step": 1610,
+      "step_time": 325.74232677668334
+    },
+    {
+      "clip_ratio/high_max": 0.008155842824089632,
+      "clip_ratio/high_mean": 0.0016740183506499307,
+      "clip_ratio/low_mean": 5.861139784428815e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.001732629642538086,
+      "completion_length": 7508.975,
+      "completions/clipped_ratio": 0.86875,
+      "completions/max_length": 14157.0,
+      "completions/max_terminated_length": 426.0,
+      "completions/mean_length": 7508.975,
+      "completions/mean_terminated_length": 353.4790496826172,
+      "completions/min_length": 286.4,
+      "completions/min_terminated_length": 286.4,
+      "epoch": 1.296,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.3239416480064392,
+      "kl": 3.877475821971893,
+      "learning_rate": 1e-06,
+      "loss": 0.0624,
+      "num_tokens": 355502047.0,
+      "reward": -0.7997602164745331,
+      "reward_std": 0.6530973076820373,
+      "step": 1620,
+      "step_time": 312.35967790111897
+    },
+    {
+      "clip_ratio/high_max": 0.02986444603347991,
+      "clip_ratio/high_mean": 0.005354888591523377,
+      "clip_ratio/low_mean": 0.017150319920665423,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.02250520839479577,
+      "completion_length": 7552.80625,
+      "completions/clipped_ratio": 0.875,
+      "completions/max_length": 14579.2,
+      "completions/max_terminated_length": 664.2,
+      "completions/mean_length": 7552.80625,
+      "completions/mean_terminated_length": 404.3164367675781,
+      "completions/min_length": 233.4,
+      "completions/min_terminated_length": 233.4,
+      "epoch": 1.304,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.363732248544693,
+      "kl": 4.63218954205513,
+      "learning_rate": 1e-06,
+      "loss": 0.0801,
+      "num_tokens": 358224261.0,
+      "reward": -0.8780550241470337,
+      "reward_std": 0.7232318341732025,
+      "step": 1630,
+      "step_time": 288.8766934532672
+    },
+    {
+      "clip_ratio/high_max": 0.00895103060065594,
+      "clip_ratio/high_mean": 0.0011446045909906388,
+      "clip_ratio/low_mean": 0.004344335527753174,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.005488940137365716,
+      "completion_length": 6529.73125,
+      "completions/clipped_ratio": 0.796875,
+      "completions/max_length": 12670.4,
+      "completions/max_terminated_length": 724.2,
+      "completions/mean_length": 6529.73125,
+      "completions/mean_terminated_length": 504.81251220703126,
+      "completions/min_length": 259.8,
+      "completions/min_terminated_length": 259.8,
+      "epoch": 1.312,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.5185719132423401,
+      "kl": 3.362174725532532,
+      "learning_rate": 1e-06,
+      "loss": 0.0544,
+      "num_tokens": 360665275.0,
+      "reward": -0.23201373517513274,
+      "reward_std": 0.6880876660346985,
+      "step": 1640,
+      "step_time": 297.43593421801927
+    },
+    {
+      "clip_ratio/high_max": 0.005803733316497528,
+      "clip_ratio/high_mean": 0.0007698185747813113,
+      "clip_ratio/low_mean": 0.0001269581464384828,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0008967767036665464,
+      "completion_length": 6839.359375,
+      "completions/clipped_ratio": 0.859375,
+      "completions/max_length": 14096.2,
+      "completions/max_terminated_length": 899.6,
+      "completions/mean_length": 6839.359375,
+      "completions/mean_terminated_length": 507.9411865234375,
+      "completions/min_length": 317.4,
+      "completions/min_terminated_length": 317.4,
+      "epoch": 1.32,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.4414424002170563,
+      "kl": 5.096695286035538,
+      "learning_rate": 1e-06,
+      "loss": 0.0813,
+      "num_tokens": 363182682.0,
+      "reward": -0.45134063363075255,
+      "reward_std": 0.7799474716186523,
+      "step": 1650,
+      "step_time": 331.86532141827047
+    },
+    {
+      "clip_ratio/high_max": 0.005730935118481284,
+      "clip_ratio/high_mean": 0.0008386010791582521,
+      "clip_ratio/low_mean": 0.0007680615142248825,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.001606662633093947,
+      "completion_length": 7354.5,
+      "completions/clipped_ratio": 0.8625,
+      "completions/max_length": 13751.6,
+      "completions/max_terminated_length": 785.6,
+      "completions/mean_length": 7354.5,
+      "completions/mean_terminated_length": 466.1900146484375,
+      "completions/min_length": 292.0,
+      "completions/min_terminated_length": 292.0,
+      "epoch": 1.328,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.48892730474472046,
+      "kl": 5.362243953347206,
+      "learning_rate": 1e-06,
+      "loss": 0.0885,
+      "num_tokens": 365870174.0,
+      "reward": -0.5352614939212799,
+      "reward_std": 0.8570351243019104,
+      "step": 1660,
+      "step_time": 311.73420796841384
+    },
+    {
+      "clip_ratio/high_max": 0.0006089264337788336,
+      "clip_ratio/high_mean": 9.789929308681166e-05,
+      "clip_ratio/low_mean": 0.00019914621532279853,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.000297045511615579,
+      "completion_length": 7375.725,
+      "completions/clipped_ratio": 0.890625,
+      "completions/max_length": 14048.6,
+      "completions/max_terminated_length": 741.4,
+      "completions/mean_length": 7375.725,
+      "completions/mean_terminated_length": 513.3997741699219,
+      "completions/min_length": 381.6,
+      "completions/min_terminated_length": 381.6,
+      "epoch": 1.336,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.335811048746109,
+      "kl": 15.922817754745484,
+      "learning_rate": 1e-06,
+      "loss": 0.2995,
+      "num_tokens": 368613614.0,
+      "reward": -0.7102296352386475,
+      "reward_std": 0.8344632029533386,
+      "step": 1670,
+      "step_time": 334.38565318249164
+    },
+    {
+      "clip_ratio/high_max": 0.011796917468018364,
+      "clip_ratio/high_mean": 0.0015835313663956185,
+      "clip_ratio/low_mean": 4.751531942019938e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.001631046691636584,
+      "completion_length": 7190.796875,
+      "completions/clipped_ratio": 0.8875,
+      "completions/max_length": 14338.0,
+      "completions/max_terminated_length": 2314.8,
+      "completions/mean_length": 7190.796875,
+      "completions/mean_terminated_length": 654.4600128173828,
+      "completions/min_length": 241.4,
+      "completions/min_terminated_length": 241.4,
+      "epoch": 1.3439999999999999,
+      "frac_reward_zero_std": 0.1,
+      "grad_norm": 0.42243579030036926,
+      "kl": 7.53945170044899,
+      "learning_rate": 1e-06,
+      "loss": 43380.8,
+      "num_tokens": 371252653.0,
+      "reward": -0.6648498177528381,
+      "reward_std": 0.5828911900520325,
+      "step": 1680,
+      "step_time": 313.04961595647035
+    },
+    {
+      "clip_ratio/high_max": 0.0015191264037639485,
+      "clip_ratio/high_mean": 0.00021538833693739433,
+      "clip_ratio/low_mean": 0.0014850610619873805,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0017004494404318394,
+      "completion_length": 6759.7375,
+      "completions/clipped_ratio": 0.8625,
+      "completions/max_length": 14198.4,
+      "completions/max_terminated_length": 2476.8,
+      "completions/mean_length": 6759.7375,
+      "completions/mean_terminated_length": 656.6333435058593,
+      "completions/min_length": 258.2,
+      "completions/min_terminated_length": 337.0,
+      "epoch": 1.3519999999999999,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.492504745721817,
+      "kl": 4.80416579246521,
+      "learning_rate": 1e-06,
+      "loss": 0.0729,
+      "num_tokens": 373742845.0,
+      "reward": -0.35131359100341797,
+      "reward_std": 0.8569494128227234,
+      "step": 1690,
+      "step_time": 289.81545091718436
+    },
+    {
+      "clip_ratio/high_max": 0.027142747341531503,
+      "clip_ratio/high_mean": 0.0037980864915880376,
+      "clip_ratio/low_mean": 0.0007874563136510915,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00458554281922261,
+      "completion_length": 7333.26875,
+      "completions/clipped_ratio": 0.825,
+      "completions/max_length": 14641.6,
+      "completions/max_terminated_length": 884.8,
+      "completions/mean_length": 7333.26875,
+      "completions/mean_terminated_length": 548.0993347167969,
+      "completions/min_length": 350.6,
+      "completions/min_terminated_length": 350.6,
+      "epoch": 1.3599999999999999,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.34325432777404785,
+      "kl": 6.043759661912918,
+      "learning_rate": 1e-06,
+      "loss": 0.1037,
+      "num_tokens": 376441695.0,
+      "reward": -0.5454029619693757,
+      "reward_std": 0.7791022181510925,
+      "step": 1700,
+      "step_time": 328.68022905178367
+    },
+    {
+      "clip_ratio/high_max": 0.026626317079171714,
+      "clip_ratio/high_mean": 0.004588376164565489,
+      "clip_ratio/low_mean": 0.0012291551792714018,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.005817531294997025,
+      "completion_length": 6740.36875,
+      "completions/clipped_ratio": 0.834375,
+      "completions/max_length": 13794.2,
+      "completions/max_terminated_length": 844.8,
+      "completions/mean_length": 6740.36875,
+      "completions/mean_terminated_length": 505.4720397949219,
+      "completions/min_length": 287.2,
+      "completions/min_terminated_length": 287.2,
+      "epoch": 1.3679999999999999,
+      "frac_reward_zero_std": 0.075,
+      "grad_norm": 0.275851309299469,
+      "kl": 5.9093164026737215,
+      "learning_rate": 1e-06,
+      "loss": 0.0963,
+      "num_tokens": 378946481.0,
+      "reward": -0.3904070258140564,
+      "reward_std": 0.7841648101806641,
+      "step": 1710,
+      "step_time": 304.61850380860267
+    },
+    {
+      "clip_ratio/high_max": 0.0027612033467448783,
+      "clip_ratio/high_mean": 0.0003679330885461241,
+      "clip_ratio/low_mean": 0.0009494084622701848,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0013173415128221676,
+      "completion_length": 7608.75625,
+      "completions/clipped_ratio": 0.86875,
+      "completions/max_length": 14854.2,
+      "completions/max_terminated_length": 800.0,
+      "completions/mean_length": 7608.75625,
+      "completions/mean_terminated_length": 339.40227661132815,
+      "completions/min_length": 403.2,
+      "completions/min_terminated_length": 145.8,
+      "epoch": 1.376,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.7556061148643494,
+      "kl": 6.153275060653686,
+      "learning_rate": 1e-06,
+      "loss": 0.1,
+      "num_tokens": 381724671.0,
+      "reward": -0.8117992162704468,
+      "reward_std": 0.7739890575408935,
+      "step": 1720,
+      "step_time": 303.8462565366179
+    },
+    {
+      "clip_ratio/high_max": 0.1028171381502034,
+      "clip_ratio/high_mean": 0.01712444812906142,
+      "clip_ratio/low_mean": 0.0024003707304359524,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.019524819315120113,
+      "completion_length": 7011.6875,
+      "completions/clipped_ratio": 0.840625,
+      "completions/max_length": 13518.4,
+      "completions/max_terminated_length": 864.4,
+      "completions/mean_length": 7011.6875,
+      "completions/mean_terminated_length": 578.8002471923828,
+      "completions/min_length": 375.4,
+      "completions/min_terminated_length": 375.4,
+      "epoch": 1.384,
+      "frac_reward_zero_std": 0.0875,
+      "grad_norm": 0.42641180753707886,
+      "kl": 4.855829492211342,
+      "learning_rate": 1e-06,
+      "loss": 0.0837,
+      "num_tokens": 384298379.0,
+      "reward": -0.6525255978107453,
+      "reward_std": 0.7849682211875916,
+      "step": 1730,
+      "step_time": 308.7172281138599
+    },
+    {
+      "clip_ratio/high_max": 0.0301689045429157,
+      "clip_ratio/high_mean": 0.006348099837759946,
+      "clip_ratio/low_mean": 0.0020652898729963455,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.008413389488850953,
+      "completion_length": 7300.859375,
+      "completions/clipped_ratio": 0.815625,
+      "completions/max_length": 14408.2,
+      "completions/max_terminated_length": 1061.6,
+      "completions/mean_length": 7300.859375,
+      "completions/mean_terminated_length": 469.22026977539065,
+      "completions/min_length": 172.4,
+      "completions/min_terminated_length": 172.4,
+      "epoch": 1.392,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.3406514525413513,
+      "kl": 5.613585871458054,
+      "learning_rate": 1e-06,
+      "loss": 0.0952,
+      "num_tokens": 386977222.0,
+      "reward": -0.8445294499397278,
+      "reward_std": 0.7044556498527527,
+      "step": 1740,
+      "step_time": 308.65482194535434
+    },
+    {
+      "clip_ratio/high_max": 0.04071989728327026,
+      "clip_ratio/high_mean": 0.005147536941694853,
+      "clip_ratio/low_mean": 0.00019320666774547135,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0053407435292683655,
+      "completion_length": 8056.2,
+      "completions/clipped_ratio": 0.8875,
+      "completions/max_length": 14465.0,
+      "completions/max_terminated_length": 665.8,
+      "completions/mean_length": 8056.2,
+      "completions/mean_terminated_length": 305.0016723632813,
+      "completions/min_length": 465.8,
+      "completions/min_terminated_length": 204.0,
+      "epoch": 1.4,
+      "frac_reward_zero_std": 0.075,
+      "grad_norm": 0.835193395614624,
+      "kl": 4.910688358545303,
+      "learning_rate": 1e-06,
+      "loss": 0.0825,
+      "num_tokens": 389885418.0,
+      "reward": -1.1295666217803955,
+      "reward_std": 0.6768316566944123,
+      "step": 1750,
+      "step_time": 314.17429485172033
+    },
+    {
+      "clip_ratio/high_max": 0.05983588649069134,
+      "clip_ratio/high_mean": 0.012524210256742663,
+      "clip_ratio/low_mean": 0.0015352505709188336,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.014059461034048582,
+      "completion_length": 7581.115625,
+      "completions/clipped_ratio": 0.8875,
+      "completions/max_length": 14089.8,
+      "completions/max_terminated_length": 772.6,
+      "completions/mean_length": 7581.115625,
+      "completions/mean_terminated_length": 539.6441711425781,
+      "completions/min_length": 425.6,
+      "completions/min_terminated_length": 425.6,
+      "epoch": 1.408,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.49229419231414795,
+      "kl": 4.8703700572252275,
+      "learning_rate": 1e-06,
+      "loss": 0.0877,
+      "num_tokens": 392659575.0,
+      "reward": -0.8754526257514954,
+      "reward_std": 0.8820981025695801,
+      "step": 1760,
+      "step_time": 333.6646990183741
+    },
+    {
+      "clip_ratio/high_max": 0.008199390890877113,
+      "clip_ratio/high_mean": 0.0010584976750124043,
+      "clip_ratio/low_mean": 0.0012310153416819959,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0022895130518008956,
+      "completion_length": 7458.728125,
+      "completions/clipped_ratio": 0.8375,
+      "completions/max_length": 14657.8,
+      "completions/max_terminated_length": 909.8,
+      "completions/mean_length": 7458.728125,
+      "completions/mean_terminated_length": 488.05585327148435,
+      "completions/min_length": 212.8,
+      "completions/min_terminated_length": 212.8,
+      "epoch": 1.416,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.45208966732025146,
+      "kl": 7.141014528274536,
+      "learning_rate": 1e-06,
+      "loss": 0.1119,
+      "num_tokens": 395367560.0,
+      "reward": -0.7720985412597656,
+      "reward_std": 0.9184301018714904,
+      "step": 1770,
+      "step_time": 323.9591739594936
+    },
+    {
+      "clip_ratio/high_max": 0.03660275523616292,
+      "clip_ratio/high_mean": 0.006282128062321135,
+      "clip_ratio/low_mean": 0.0001722534187706515,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0064543813847194546,
+      "completion_length": 6857.1625,
+      "completions/clipped_ratio": 0.7875,
+      "completions/max_length": 14000.4,
+      "completions/max_terminated_length": 1351.2,
+      "completions/mean_length": 6857.1625,
+      "completions/mean_terminated_length": 625.5552307128906,
+      "completions/min_length": 230.8,
+      "completions/min_terminated_length": 230.8,
+      "epoch": 1.424,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.46720412373542786,
+      "kl": 4.935144352912903,
+      "learning_rate": 1e-06,
+      "loss": 0.0883,
+      "num_tokens": 397901128.0,
+      "reward": -0.3955793440341949,
+      "reward_std": 0.6523785710334777,
+      "step": 1780,
+      "step_time": 293.18738994561136
+    },
+    {
+      "clip_ratio/high_max": 0.00018550316171968008,
+      "clip_ratio/high_mean": 3.4164505257194834e-05,
+      "clip_ratio/low_mean": 0.0016243605525005477,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0016585250826665287,
+      "completion_length": 7895.48125,
+      "completions/clipped_ratio": 0.865625,
+      "completions/max_length": 14594.6,
+      "completions/max_terminated_length": 924.8,
+      "completions/mean_length": 7895.48125,
+      "completions/mean_terminated_length": 598.0382568359375,
+      "completions/min_length": 365.0,
+      "completions/min_terminated_length": 365.0,
+      "epoch": 1.432,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.36277657747268677,
+      "kl": 4.268905574083329,
+      "learning_rate": 1e-06,
+      "loss": 0.0692,
+      "num_tokens": 400769270.0,
+      "reward": -0.8184961378574371,
+      "reward_std": 0.760488498210907,
+      "step": 1790,
+      "step_time": 333.2015529088676
+    },
+    {
+      "clip_ratio/high_max": 0.018982572125423758,
+      "clip_ratio/high_mean": 0.0024804975744700643,
+      "clip_ratio/low_mean": 0.000667473782664274,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0031479713828957757,
+      "completion_length": 7391.95,
+      "completions/clipped_ratio": 0.871875,
+      "completions/max_length": 12992.4,
+      "completions/max_terminated_length": 608.4,
+      "completions/mean_length": 7391.95,
+      "completions/mean_terminated_length": 428.74755859375,
+      "completions/min_length": 318.8,
+      "completions/min_terminated_length": 318.8,
+      "epoch": 1.44,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.4370053708553314,
+      "kl": 4.412891730666161,
+      "learning_rate": 1e-06,
+      "loss": 0.076,
+      "num_tokens": 403465174.0,
+      "reward": -0.6571730256080628,
+      "reward_std": 0.5761035084724426,
+      "step": 1800,
+      "step_time": 311.13074304759505
+    },
+    {
+      "clip_ratio/high_max": 0.00772299240115899,
+      "clip_ratio/high_mean": 0.0009866765331821626,
+      "clip_ratio/low_mean": 9.07150817852198e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0010773915870231575,
+      "completion_length": 7489.3625,
+      "completions/clipped_ratio": 0.878125,
+      "completions/max_length": 13605.4,
+      "completions/max_terminated_length": 1179.6,
+      "completions/mean_length": 7489.3625,
+      "completions/mean_terminated_length": 900.1838562011719,
+      "completions/min_length": 675.8,
+      "completions/min_terminated_length": 675.8,
+      "epoch": 1.448,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.5730636715888977,
+      "kl": 4.087243703007698,
+      "learning_rate": 1e-06,
+      "loss": 0.0685,
+      "num_tokens": 406220338.0,
+      "reward": -0.6476436078548431,
+      "reward_std": 0.6274641394615174,
+      "step": 1810,
+      "step_time": 319.57065968737004
+    },
+    {
+      "clip_ratio/high_max": 0.023189625009581504,
+      "clip_ratio/high_mean": 0.0029430420753215002,
+      "clip_ratio/low_mean": 0.015247932069087256,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.01819097423222047,
+      "completion_length": 7035.45,
+      "completions/clipped_ratio": 0.775,
+      "completions/max_length": 14534.8,
+      "completions/max_terminated_length": 1528.8,
+      "completions/mean_length": 7035.45,
+      "completions/mean_terminated_length": 903.783349609375,
+      "completions/min_length": 313.0,
+      "completions/min_terminated_length": 313.0,
+      "epoch": 1.456,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.37114107608795166,
+      "kl": 5.25685955286026,
+      "learning_rate": 1e-06,
+      "loss": 0.075,
+      "num_tokens": 408837438.0,
+      "reward": -0.5768455624580383,
+      "reward_std": 0.7342843294143677,
+      "step": 1820,
+      "step_time": 321.14152909070253
+    },
+    {
+      "clip_ratio/high_max": 0.015128975209700002,
+      "clip_ratio/high_mean": 0.002321737056854545,
+      "clip_ratio/low_mean": 0.0004063514888002828,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.002728088483308966,
+      "completion_length": 7770.3375,
+      "completions/clipped_ratio": 0.896875,
+      "completions/max_length": 14121.6,
+      "completions/max_terminated_length": 677.8,
+      "completions/mean_length": 7770.3375,
+      "completions/mean_terminated_length": 477.9166687011719,
+      "completions/min_length": 286.2,
+      "completions/min_terminated_length": 286.2,
+      "epoch": 1.464,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 1.0560393333435059,
+      "kl": 7.414034223556518,
+      "learning_rate": 1e-06,
+      "loss": 0.1211,
+      "num_tokens": 411651786.0,
+      "reward": -0.9982760548591614,
+      "reward_std": 0.9028495073318481,
+      "step": 1830,
+      "step_time": 317.79503725804386
+    },
+    {
+      "clip_ratio/high_max": 0.03826671198976328,
+      "clip_ratio/high_mean": 0.005034505412186263,
+      "clip_ratio/low_mean": 0.0010659833242470996,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00610048879043461,
+      "completion_length": 7745.053125,
+      "completions/clipped_ratio": 0.88125,
+      "completions/max_length": 14487.2,
+      "completions/max_terminated_length": 720.0,
+      "completions/mean_length": 7745.053125,
+      "completions/mean_terminated_length": 402.4125,
+      "completions/min_length": 235.4,
+      "completions/min_terminated_length": 235.4,
+      "epoch": 1.472,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.6105852723121643,
+      "kl": 6.400105988979339,
+      "learning_rate": 1e-06,
+      "loss": 0.1108,
+      "num_tokens": 414459731.0,
+      "reward": -0.990148514509201,
+      "reward_std": 0.7274603068828582,
+      "step": 1840,
+      "step_time": 317.37720348350706
+    },
+    {
+      "clip_ratio/high_max": 0.030974355795297014,
+      "clip_ratio/high_mean": 0.00394210687454688,
+      "clip_ratio/low_mean": 0.0014551245151778857,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00539723155034153,
+      "completion_length": 7509.59375,
+      "completions/clipped_ratio": 0.80625,
+      "completions/max_length": 14531.0,
+      "completions/max_terminated_length": 1090.2,
+      "completions/mean_length": 7509.59375,
+      "completions/mean_terminated_length": 624.4446533203125,
+      "completions/min_length": 347.6,
+      "completions/min_terminated_length": 347.6,
+      "epoch": 1.48,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 1.4150880575180054,
+      "kl": 6.810839539766311,
+      "learning_rate": 1e-06,
+      "loss": 0.1098,
+      "num_tokens": 417202617.0,
+      "reward": -0.7293478369712829,
+      "reward_std": 0.8159699499607086,
+      "step": 1850,
+      "step_time": 324.10161459222434
+    },
+    {
+      "clip_ratio/high_max": 0.058021416367046184,
+      "clip_ratio/high_mean": 0.008905231307994654,
+      "clip_ratio/low_mean": 0.00011163612964537606,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.009016867537684447,
+      "completion_length": 7698.534375,
+      "completions/clipped_ratio": 0.85,
+      "completions/max_length": 14550.4,
+      "completions/max_terminated_length": 1102.8,
+      "completions/mean_length": 7698.534375,
+      "completions/mean_terminated_length": 569.2155700683594,
+      "completions/min_length": 301.4,
+      "completions/min_terminated_length": 301.4,
+      "epoch": 1.488,
+      "frac_reward_zero_std": 0.0875,
+      "grad_norm": 0.5497668385505676,
+      "kl": 7.199986171722412,
+      "learning_rate": 1e-06,
+      "loss": 0.1226,
+      "num_tokens": 420012456.0,
+      "reward": -0.9498097777366639,
+      "reward_std": 0.6990577220916748,
+      "step": 1860,
+      "step_time": 318.34451083429155
+    },
+    {
+      "clip_ratio/high_max": 0.002702668162419286,
+      "clip_ratio/high_mean": 0.00044607967706724596,
+      "clip_ratio/low_mean": 0.0028728104520723717,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0033188902621986926,
+      "completion_length": 7716.565625,
+      "completions/clipped_ratio": 0.90625,
+      "completions/max_length": 14435.4,
+      "completions/max_terminated_length": 579.8,
+      "completions/mean_length": 7716.565625,
+      "completions/mean_terminated_length": 383.7211181640625,
+      "completions/min_length": 294.6,
+      "completions/min_terminated_length": 294.6,
+      "epoch": 1.496,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.5675954818725586,
+      "kl": 5.245901289582252,
+      "learning_rate": 1e-06,
+      "loss": 0.0871,
+      "num_tokens": 422796009.0,
+      "reward": -1.0870455503463745,
+      "reward_std": 0.7128821969032287,
+      "step": 1870,
+      "step_time": 313.8221114560962
+    },
+    {
+      "clip_ratio/high_max": 0.01378825720948953,
+      "clip_ratio/high_mean": 0.0017524628139653942,
+      "clip_ratio/low_mean": 4.144016561440367e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0017939029656190541,
+      "completion_length": 7643.80625,
+      "completions/clipped_ratio": 0.809375,
+      "completions/max_length": 14374.8,
+      "completions/max_terminated_length": 1099.6,
+      "completions/mean_length": 7643.80625,
+      "completions/mean_terminated_length": 626.4723266601562,
+      "completions/min_length": 296.8,
+      "completions/min_terminated_length": 296.8,
+      "epoch": 1.504,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.40422794222831726,
+      "kl": 4.8005776643753055,
+      "learning_rate": 1e-06,
+      "loss": 0.0828,
+      "num_tokens": 425593027.0,
+      "reward": -0.6814701616764068,
+      "reward_std": 0.7322932183742523,
+      "step": 1880,
+      "step_time": 321.37334411516787
+    },
+    {
+      "clip_ratio/high_max": 0.015270614092332834,
+      "clip_ratio/high_mean": 0.0020990254593471038,
+      "clip_ratio/low_mean": 4.4343673766888966e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.002143369148234342,
+      "completion_length": 8308.271875,
+      "completions/clipped_ratio": 0.896875,
+      "completions/max_length": 14475.2,
+      "completions/max_terminated_length": 1285.2,
+      "completions/mean_length": 8308.271875,
+      "completions/mean_terminated_length": 587.5418273925782,
+      "completions/min_length": 374.2,
+      "completions/min_terminated_length": 374.2,
+      "epoch": 1.512,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.5707919001579285,
+      "kl": 4.140536543726921,
+      "learning_rate": 1e-06,
+      "loss": 0.0717,
+      "num_tokens": 428614890.0,
+      "reward": -1.2861475467681884,
+      "reward_std": 0.7546603679656982,
+      "step": 1890,
+      "step_time": 296.81686701737345
+    },
+    {
+      "clip_ratio/high_max": 0.014154868550576793,
+      "clip_ratio/high_mean": 0.0018373189697285853,
+      "clip_ratio/low_mean": 0.0016325820673841918,
+      "clip_ratio/low_min": 3.4792964470398146e-06,
+      "clip_ratio/region_mean": 0.0034699011082921062,
+      "completion_length": 8078.11875,
+      "completions/clipped_ratio": 0.89375,
+      "completions/max_length": 14729.6,
+      "completions/max_terminated_length": 689.0,
+      "completions/mean_length": 8078.11875,
+      "completions/mean_terminated_length": 467.95416870117185,
+      "completions/min_length": 318.6,
+      "completions/min_terminated_length": 318.6,
+      "epoch": 1.52,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5620991587638855,
+      "kl": 6.075963377952576,
+      "learning_rate": 1e-06,
+      "loss": 0.099,
+      "num_tokens": 431530536.0,
+      "reward": -1.079388928413391,
+      "reward_std": 0.8224254846572876,
+      "step": 1900,
+      "step_time": 320.80998531207445
+    },
+    {
+      "clip_ratio/high_max": 0.03340993466717919,
+      "clip_ratio/high_mean": 0.004625561394095712,
+      "clip_ratio/low_mean": 1.6519264659109466e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0046420807742833855,
+      "completion_length": 8312.040625,
+      "completions/clipped_ratio": 0.8875,
+      "completions/max_length": 14518.2,
+      "completions/max_terminated_length": 1206.4,
+      "completions/mean_length": 8312.040625,
+      "completions/mean_terminated_length": 624.4714538574219,
+      "completions/min_length": 291.0,
+      "completions/min_terminated_length": 291.0,
+      "epoch": 1.528,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.4965921640396118,
+      "kl": 5.391463947296143,
+      "learning_rate": 1e-06,
+      "loss": 0.0953,
+      "num_tokens": 434544721.0,
+      "reward": -1.2239267587661744,
+      "reward_std": 0.7613436937332153,
+      "step": 1910,
+      "step_time": 335.9257302522659
+    },
+    {
+      "clip_ratio/high_max": 0.0005516917732165894,
+      "clip_ratio/high_mean": 9.483651592745446e-05,
+      "clip_ratio/low_mean": 0.015631945750874364,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.015726781863668292,
+      "completion_length": 7628.29375,
+      "completions/clipped_ratio": 0.93125,
+      "completions/max_length": 14249.8,
+      "completions/max_terminated_length": 648.0,
+      "completions/mean_length": 7628.29375,
+      "completions/mean_terminated_length": 467.821435546875,
+      "completions/min_length": 356.0,
+      "completions/min_terminated_length": 356.0,
+      "epoch": 1.536,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.8421487212181091,
+      "kl": 2.4635415256023405,
+      "learning_rate": 1e-06,
+      "loss": 0.039,
+      "num_tokens": 437331467.0,
+      "reward": -1.027067983150482,
+      "reward_std": 0.5949046552181244,
+      "step": 1920,
+      "step_time": 307.5789606437087
+    },
+    {
+      "clip_ratio/high_max": 0.0016680978526892431,
+      "clip_ratio/high_mean": 0.00023202180003636385,
+      "clip_ratio/low_mean": 4.552214246587027e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0002775439411834668,
+      "completion_length": 6649.8875,
+      "completions/clipped_ratio": 0.79375,
+      "completions/max_length": 14626.2,
+      "completions/max_terminated_length": 979.4,
+      "completions/mean_length": 6649.8875,
+      "completions/mean_terminated_length": 494.03441162109374,
+      "completions/min_length": 239.8,
+      "completions/min_terminated_length": 239.8,
+      "epoch": 1.544,
+      "frac_reward_zero_std": 0.075,
+      "grad_norm": 0.5468311309814453,
+      "kl": 4.812614178657531,
+      "learning_rate": 1e-06,
+      "loss": 0.0786,
+      "num_tokens": 439792003.0,
+      "reward": -0.4039729624986649,
+      "reward_std": 0.6889181971549988,
+      "step": 1930,
+      "step_time": 313.0702241566032
+    },
+    {
+      "clip_ratio/high_max": 0.0017313227286649634,
+      "clip_ratio/high_mean": 0.0002510984820219164,
+      "clip_ratio/low_mean": 0.00016244665353042364,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.000413545141054783,
+      "completion_length": 7002.63125,
+      "completions/clipped_ratio": 0.88125,
+      "completions/max_length": 14163.2,
+      "completions/max_terminated_length": 968.6,
+      "completions/mean_length": 7002.63125,
+      "completions/mean_terminated_length": 557.230322265625,
+      "completions/min_length": 332.4,
+      "completions/min_terminated_length": 332.4,
+      "epoch": 1.552,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.5152363777160645,
+      "kl": 6.2673690736293795,
+      "learning_rate": 1e-06,
+      "loss": 0.1015,
+      "num_tokens": 442372421.0,
+      "reward": -0.7051892280578613,
+      "reward_std": 0.8250641047954559,
+      "step": 1940,
+      "step_time": 311.96340665481983
+    },
+    {
+      "clip_ratio/high_max": 0.0068930494022424686,
+      "clip_ratio/high_mean": 0.0009225711411772863,
+      "clip_ratio/low_mean": 0.0022498531226119666,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0031724242257041625,
+      "completion_length": 7196.1625,
+      "completions/clipped_ratio": 0.921875,
+      "completions/max_length": 13647.4,
+      "completions/max_terminated_length": 428.2,
+      "completions/mean_length": 7196.1625,
+      "completions/mean_terminated_length": 282.8642883300781,
+      "completions/min_length": 576.8,
+      "completions/min_terminated_length": 203.4,
+      "epoch": 1.56,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.7045295238494873,
+      "kl": 5.592794871330261,
+      "learning_rate": 1e-06,
+      "loss": 0.0908,
+      "num_tokens": 445013589.0,
+      "reward": -0.5821906805038453,
+      "reward_std": 0.9834616541862488,
+      "step": 1950,
+      "step_time": 323.1513558320701
+    },
+    {
+      "clip_ratio/high_max": 0.00568473559578706,
+      "clip_ratio/high_mean": 0.0007821084434908698,
+      "clip_ratio/low_mean": 0.00019670500354322938,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.000978813472283946,
+      "completion_length": 6207.309375,
+      "completions/clipped_ratio": 0.81875,
+      "completions/max_length": 12608.4,
+      "completions/max_terminated_length": 795.8,
+      "completions/mean_length": 6207.309375,
+      "completions/mean_terminated_length": 565.8800109863281,
+      "completions/min_length": 366.2,
+      "completions/min_terminated_length": 366.2,
+      "epoch": 1.568,
+      "frac_reward_zero_std": 0.1125,
+      "grad_norm": 0.3099040687084198,
+      "kl": 4.988259103894234,
+      "learning_rate": 1e-06,
+      "loss": 0.0773,
+      "num_tokens": 447346172.0,
+      "reward": 0.12790471613407134,
+      "reward_std": 0.6847935140132904,
+      "step": 1960,
+      "step_time": 299.92682937979697
+    },
+    {
+      "clip_ratio/high_max": 0.01672947196966561,
+      "clip_ratio/high_mean": 0.0021450976130836354,
+      "clip_ratio/low_mean": 0.0010071704254187353,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0031522680090347423,
+      "completion_length": 6904.5375,
+      "completions/clipped_ratio": 0.921875,
+      "completions/max_length": 13711.2,
+      "completions/max_terminated_length": 837.0,
+      "completions/mean_length": 6904.5375,
+      "completions/mean_terminated_length": 737.74833984375,
+      "completions/min_length": 561.2,
+      "completions/min_terminated_length": 561.2,
+      "epoch": 1.576,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 1.1253758668899536,
+      "kl": 7.068167769908905,
+      "learning_rate": 1e-06,
+      "loss": 0.1149,
+      "num_tokens": 449905388.0,
+      "reward": -0.09115150272846222,
+      "reward_std": 0.7453244209289551,
+      "step": 1970,
+      "step_time": 306.71967142857613
+    },
+    {
+      "clip_ratio/high_max": 0.04397606839247601,
+      "clip_ratio/high_mean": 0.007157517294308491,
+      "clip_ratio/low_mean": 0.0018437671388937815,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.009001284174337343,
+      "completion_length": 6791.25625,
+      "completions/clipped_ratio": 0.89375,
+      "completions/max_length": 13866.6,
+      "completions/max_terminated_length": 784.6,
+      "completions/mean_length": 6791.25625,
+      "completions/mean_terminated_length": 502.6849304199219,
+      "completions/min_length": 358.2,
+      "completions/min_terminated_length": 358.2,
+      "epoch": 1.584,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.41604554653167725,
+      "kl": 6.8964033126831055,
+      "learning_rate": 1e-06,
+      "loss": 0.1148,
+      "num_tokens": 452429486.0,
+      "reward": -0.455812481045723,
+      "reward_std": 1.066281294822693,
+      "step": 1980,
+      "step_time": 300.6431626223028
+    },
+    {
+      "clip_ratio/high_max": 0.013903072644097847,
+      "clip_ratio/high_mean": 0.0018689936263399432,
+      "clip_ratio/low_mean": 0.004102108562346984,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.005971102190233069,
+      "completion_length": 7582.84375,
+      "completions/clipped_ratio": 0.9,
+      "completions/max_length": 14305.0,
+      "completions/max_terminated_length": 638.0,
+      "completions/mean_length": 7582.84375,
+      "completions/mean_terminated_length": 436.8814392089844,
+      "completions/min_length": 761.6,
+      "completions/min_terminated_length": 327.6,
+      "epoch": 1.592,
+      "frac_reward_zero_std": 0.0875,
+      "grad_norm": 0.8129948377609253,
+      "kl": 6.641011536121368,
+      "learning_rate": 1e-06,
+      "loss": 0.1136,
+      "num_tokens": 455193292.0,
+      "reward": -0.5087529361248017,
+      "reward_std": 0.7114526033401489,
+      "step": 1990,
+      "step_time": 306.53483033925295
+    },
+    {
+      "clip_ratio/high_max": 0.013703205476303992,
+      "clip_ratio/high_mean": 0.001839947919665974,
+      "clip_ratio/low_mean": 0.0003786525691339193,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0022186005297044177,
+      "completion_length": 6637.69375,
+      "completions/clipped_ratio": 0.853125,
+      "completions/max_length": 14275.6,
+      "completions/max_terminated_length": 927.4,
+      "completions/mean_length": 6637.69375,
+      "completions/mean_terminated_length": 531.0743774414062,
+      "completions/min_length": 269.8,
+      "completions/min_terminated_length": 269.8,
+      "epoch": 1.6,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.57973313331604,
+      "kl": 5.227190855145454,
+      "learning_rate": 1e-06,
+      "loss": 0.0876,
+      "num_tokens": 457654498.0,
+      "reward": -0.11092881560325622,
+      "reward_std": 0.7461041450500489,
+      "step": 2000,
+      "step_time": 307.4663611661643
+    },
+    {
+      "clip_ratio/high_max": 0.0038877414423041045,
+      "clip_ratio/high_mean": 0.0005486597392518888,
+      "clip_ratio/low_mean": 0.0007066724900482768,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.001255332270557119,
+      "completion_length": 6507.415625,
+      "completions/clipped_ratio": 0.9,
+      "completions/max_length": 13535.4,
+      "completions/max_terminated_length": 1483.8,
+      "completions/mean_length": 6507.415625,
+      "completions/mean_terminated_length": 1020.3436218261719,
+      "completions/min_length": 743.6,
+      "completions/min_terminated_length": 743.6,
+      "epoch": 1.608,
+      "frac_reward_zero_std": 0.075,
+      "grad_norm": 0.6102015972137451,
+      "kl": 6.4442802488803865,
+      "learning_rate": 1e-06,
+      "loss": 0.108,
+      "num_tokens": 460098943.0,
+      "reward": -0.4810569226741791,
+      "reward_std": 0.8459999442100525,
+      "step": 2010,
+      "step_time": 313.3716059781611
+    },
+    {
+      "clip_ratio/high_max": 0.016146921278050286,
+      "clip_ratio/high_mean": 0.0026539746577327605,
+      "clip_ratio/low_mean": 0.0018596557295040839,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004513630310611916,
+      "completion_length": 7005.74375,
+      "completions/clipped_ratio": 0.8875,
+      "completions/max_length": 13989.4,
+      "completions/max_terminated_length": 807.2,
+      "completions/mean_length": 7005.74375,
+      "completions/mean_terminated_length": 426.1073883056641,
+      "completions/min_length": 260.4,
+      "completions/min_terminated_length": 260.4,
+      "epoch": 1.616,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.3269818127155304,
+      "kl": 7.21044859290123,
+      "learning_rate": 1e-06,
+      "loss": 0.1161,
+      "num_tokens": 462671509.0,
+      "reward": -0.39448970556259155,
+      "reward_std": 0.6996575653553009,
+      "step": 2020,
+      "step_time": 314.45139493979514
+    },
+    {
+      "clip_ratio/high_max": 0.006285289960942464,
+      "clip_ratio/high_mean": 0.0008353622011782136,
+      "clip_ratio/low_mean": 0.0011586164584628023,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0019939786063332575,
+      "completion_length": 6574.3125,
+      "completions/clipped_ratio": 0.896875,
+      "completions/max_length": 13867.4,
+      "completions/max_terminated_length": 734.8,
+      "completions/mean_length": 6574.3125,
+      "completions/mean_terminated_length": 433.02454833984376,
+      "completions/min_length": 623.0,
+      "completions/min_terminated_length": 304.8,
+      "epoch": 1.624,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.36072203516960144,
+      "kl": 5.755641269683838,
+      "learning_rate": 1e-06,
+      "loss": 0.0931,
+      "num_tokens": 465107317.0,
+      "reward": -0.13747229874134065,
+      "reward_std": 0.8628092527389526,
+      "step": 2030,
+      "step_time": 316.7421080123633
+    },
+    {
+      "clip_ratio/high_max": 0.015530208135714929,
+      "clip_ratio/high_mean": 0.002022932600743843,
+      "clip_ratio/low_mean": 0.0014555061443729755,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.003478438723323052,
+      "completion_length": 6657.409375,
+      "completions/clipped_ratio": 0.86875,
+      "completions/max_length": 14535.4,
+      "completions/max_terminated_length": 663.4,
+      "completions/mean_length": 6657.409375,
+      "completions/mean_terminated_length": 489.4221618652344,
+      "completions/min_length": 322.4,
+      "completions/min_terminated_length": 322.4,
+      "epoch": 1.6320000000000001,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.5114694833755493,
+      "kl": 6.795010271668434,
+      "learning_rate": 1e-06,
+      "loss": 0.1126,
+      "num_tokens": 467574824.0,
+      "reward": 0.0513498842716217,
+      "reward_std": 0.955954623222351,
+      "step": 2040,
+      "step_time": 300.5062588863075
+    },
+    {
+      "clip_ratio/high_max": 0.005368124054803048,
+      "clip_ratio/high_mean": 0.0007172408697442734,
+      "clip_ratio/low_mean": 0.00492365905411134,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.005640899958234513,
+      "completion_length": 6322.45,
+      "completions/clipped_ratio": 0.871875,
+      "completions/max_length": 14390.4,
+      "completions/max_terminated_length": 1493.2,
+      "completions/mean_length": 6322.45,
+      "completions/mean_terminated_length": 829.098193359375,
+      "completions/min_length": 493.0,
+      "completions/min_terminated_length": 493.0,
+      "epoch": 1.6400000000000001,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.4913092851638794,
+      "kl": 8.04332650899887,
+      "learning_rate": 1e-06,
+      "loss": 0.1266,
+      "num_tokens": 469930876.0,
+      "reward": 0.060240551829338074,
+      "reward_std": 0.95547434091568,
+      "step": 2050,
+      "step_time": 312.9805041179061
+    },
+    {
+      "clip_ratio/high_max": 0.027355587054898933,
+      "clip_ratio/high_mean": 0.0037832170596061587,
+      "clip_ratio/low_mean": 0.0014067084622183756,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.00518992544075445,
+      "completion_length": 7098.8125,
+      "completions/clipped_ratio": 0.896875,
+      "completions/max_length": 13734.4,
+      "completions/max_terminated_length": 907.2,
+      "completions/mean_length": 7098.8125,
+      "completions/mean_terminated_length": 642.4111145019531,
+      "completions/min_length": 388.2,
+      "completions/min_terminated_length": 460.2,
+      "epoch": 1.6480000000000001,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.5205160975456238,
+      "kl": 8.380744779109955,
+      "learning_rate": 1e-06,
+      "loss": 0.1476,
+      "num_tokens": 472578572.0,
+      "reward": -0.5677948147058487,
+      "reward_std": 0.8875736713409423,
+      "step": 2060,
+      "step_time": 335.54505507610736
+    },
+    {
+      "clip_ratio/high_max": 0.04555327326233964,
+      "clip_ratio/high_mean": 0.006263616582918985,
+      "clip_ratio/low_mean": 0.00014125025719522455,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.006404866955836042,
+      "completion_length": 7282.6375,
+      "completions/clipped_ratio": 0.8875,
+      "completions/max_length": 14656.8,
+      "completions/max_terminated_length": 1009.4,
+      "completions/mean_length": 7282.6375,
+      "completions/mean_terminated_length": 816.7316772460938,
+      "completions/min_length": 660.4,
+      "completions/min_terminated_length": 660.4,
+      "epoch": 1.6560000000000001,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.5146597027778625,
+      "kl": 6.647422128915787,
+      "learning_rate": 1e-06,
+      "loss": 0.112,
+      "num_tokens": 475271044.0,
+      "reward": -0.5734635949134826,
+      "reward_std": 0.9648894786834716,
+      "step": 2070,
+      "step_time": 332.4960925482213
+    },
+    {
+      "clip_ratio/high_max": 0.01569050053403771,
+      "clip_ratio/high_mean": 0.0022207770128261474,
+      "clip_ratio/low_mean": 0.0006236558325099395,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0028444328090699855,
+      "completion_length": 6379.5,
+      "completions/clipped_ratio": 0.846875,
+      "completions/max_length": 14265.6,
+      "completions/max_terminated_length": 1245.0,
+      "completions/mean_length": 6379.5,
+      "completions/mean_terminated_length": 762.1740234375,
+      "completions/min_length": 392.0,
+      "completions/min_terminated_length": 392.0,
+      "epoch": 1.6640000000000001,
+      "frac_reward_zero_std": 0.0875,
+      "grad_norm": 0.8587217330932617,
+      "kl": 5.887396210432053,
+      "learning_rate": 1e-06,
+      "loss": 0.091,
+      "num_tokens": 477639476.0,
+      "reward": 0.09067589342594147,
+      "reward_std": 0.7809307217597962,
+      "step": 2080,
+      "step_time": 301.4870352927595
+    },
+    {
+      "clip_ratio/high_max": 0.01760388964357844,
+      "clip_ratio/high_mean": 0.0025960214072256347,
+      "clip_ratio/low_mean": 0.00021120944677477382,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0028072309400158702,
+      "completion_length": 6494.9125,
+      "completions/clipped_ratio": 0.871875,
+      "completions/max_length": 13420.8,
+      "completions/max_terminated_length": 2733.8,
+      "completions/mean_length": 6494.9125,
+      "completions/mean_terminated_length": 765.1151794433594,
+      "completions/min_length": 338.2,
+      "completions/min_terminated_length": 338.2,
+      "epoch": 1.6720000000000002,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.7346658706665039,
+      "kl": 5.626915913820267,
+      "learning_rate": 1e-06,
+      "loss": 0.0964,
+      "num_tokens": 480069544.0,
+      "reward": 0.07122343182563781,
+      "reward_std": 0.8187567114830017,
+      "step": 2090,
+      "step_time": 309.41073113679886
+    },
+    {
+      "clip_ratio/high_max": 0.027121048453955153,
+      "clip_ratio/high_mean": 0.004206985528844598,
+      "clip_ratio/low_mean": 2.7086456429969986e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004234071988503274,
+      "completion_length": 7281.1625,
+      "completions/clipped_ratio": 0.859375,
+      "completions/max_length": 13885.8,
+      "completions/max_terminated_length": 997.4,
+      "completions/mean_length": 7281.1625,
+      "completions/mean_terminated_length": 625.7767761230468,
+      "completions/min_length": 340.8,
+      "completions/min_terminated_length": 340.8,
+      "epoch": 1.6800000000000002,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.5700187683105469,
+      "kl": 5.6874337792396545,
+      "learning_rate": 1e-06,
+      "loss": 0.0927,
+      "num_tokens": 482762020.0,
+      "reward": -0.15894966423511506,
+      "reward_std": 0.8645217537879943,
+      "step": 2100,
+      "step_time": 316.42906203269956
+    },
+    {
+      "clip_ratio/high_max": 0.009361148513198714,
+      "clip_ratio/high_mean": 0.0013682607679129434,
+      "clip_ratio/low_mean": 0.008058730578704853,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.009426991218788316,
+      "completion_length": 6268.5625,
+      "completions/clipped_ratio": 0.821875,
+      "completions/max_length": 12931.2,
+      "completions/max_terminated_length": 816.0,
+      "completions/mean_length": 6268.5625,
+      "completions/mean_terminated_length": 550.0989990234375,
+      "completions/min_length": 379.4,
+      "completions/min_terminated_length": 379.4,
+      "epoch": 1.688,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.4148063361644745,
+      "kl": 6.779343357682228,
+      "learning_rate": 1e-06,
+      "loss": 0.1088,
+      "num_tokens": 485112340.0,
+      "reward": 0.14209796786308287,
+      "reward_std": 1.0344609498977662,
+      "step": 2110,
+      "step_time": 297.300151553005
+    },
+    {
+      "clip_ratio/high_max": 0.003908122167740657,
+      "clip_ratio/high_mean": 0.0005263265949906782,
+      "clip_ratio/low_mean": 0.003091011712933778,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0036173384112771602,
+      "completion_length": 6085.403125,
+      "completions/clipped_ratio": 0.859375,
+      "completions/max_length": 13081.6,
+      "completions/max_terminated_length": 1066.2,
+      "completions/mean_length": 6085.403125,
+      "completions/mean_terminated_length": 658.1070556640625,
+      "completions/min_length": 385.4,
+      "completions/min_terminated_length": 385.4,
+      "epoch": 1.696,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.6745482683181763,
+      "kl": 5.709767267107964,
+      "learning_rate": 1e-06,
+      "loss": 0.0962,
+      "num_tokens": 487398649.0,
+      "reward": 0.21472315192222596,
+      "reward_std": 0.6410177707672119,
+      "step": 2120,
+      "step_time": 293.38834251798687
+    },
+    {
+      "clip_ratio/high_max": 0.004804341078852303,
+      "clip_ratio/high_mean": 0.000627009575100601,
+      "clip_ratio/low_mean": 3.270579250056471e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0006597153495022212,
+      "completion_length": 6332.91875,
+      "completions/clipped_ratio": 0.8625,
+      "completions/max_length": 13652.2,
+      "completions/max_terminated_length": 934.8,
+      "completions/mean_length": 6332.91875,
+      "completions/mean_terminated_length": 543.2166748046875,
+      "completions/min_length": 251.2,
+      "completions/min_terminated_length": 251.2,
+      "epoch": 1.704,
+      "frac_reward_zero_std": 0.075,
+      "grad_norm": 0.35498422384262085,
+      "kl": 5.174889194965362,
+      "learning_rate": 1e-06,
+      "loss": 0.0839,
+      "num_tokens": 489782259.0,
+      "reward": 0.09973653554916381,
+      "reward_std": 0.7610346972942352,
+      "step": 2130,
+      "step_time": 275.8011051889509
+    },
+    {
+      "clip_ratio/high_max": 0.006427811557296081,
+      "clip_ratio/high_mean": 0.000828521346738853,
+      "clip_ratio/low_mean": 0.0016644572531731682,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.002492978652298916,
+      "completion_length": 6383.953125,
+      "completions/clipped_ratio": 0.809375,
+      "completions/max_length": 14208.4,
+      "completions/max_terminated_length": 1377.4,
+      "completions/mean_length": 6383.953125,
+      "completions/mean_terminated_length": 580.3758422851563,
+      "completions/min_length": 273.4,
+      "completions/min_terminated_length": 273.4,
+      "epoch": 1.712,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.43306076526641846,
+      "kl": 6.89620326757431,
+      "learning_rate": 1e-06,
+      "loss": 0.1128,
+      "num_tokens": 492195132.0,
+      "reward": -0.24029440581798553,
+      "reward_std": 0.7896882057189941,
+      "step": 2140,
+      "step_time": 319.00788057409227
+    },
+    {
+      "clip_ratio/high_max": 0.013189089658408194,
+      "clip_ratio/high_mean": 0.001826762795826653,
+      "clip_ratio/low_mean": 0.0005279180500338043,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.002354680804455711,
+      "completion_length": 7191.034375,
+      "completions/clipped_ratio": 0.88125,
+      "completions/max_length": 14389.6,
+      "completions/max_terminated_length": 600.2,
+      "completions/mean_length": 7191.034375,
+      "completions/mean_terminated_length": 447.7100036621094,
+      "completions/min_length": 279.6,
+      "completions/min_terminated_length": 279.6,
+      "epoch": 1.72,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.5097222924232483,
+      "kl": 6.525178366899491,
+      "learning_rate": 1e-06,
+      "loss": 0.1092,
+      "num_tokens": 494842615.0,
+      "reward": -0.22211894989013672,
+      "reward_std": 0.792072880268097,
+      "step": 2150,
+      "step_time": 327.9798742134124
+    },
+    {
+      "clip_ratio/high_max": 0.0029673849625396543,
+      "clip_ratio/high_mean": 0.0005816298829813604,
+      "clip_ratio/low_mean": 0.010161044023107024,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.010742673746062793,
+      "completion_length": 6576.859375,
+      "completions/clipped_ratio": 0.859375,
+      "completions/max_length": 13481.0,
+      "completions/max_terminated_length": 728.0,
+      "completions/mean_length": 6576.859375,
+      "completions/mean_terminated_length": 481.97130126953124,
+      "completions/min_length": 298.8,
+      "completions/min_terminated_length": 298.8,
+      "epoch": 1.728,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.2659173905849457,
+      "kl": 5.207599338889122,
+      "learning_rate": 1e-06,
+      "loss": 0.0841,
+      "num_tokens": 497281042.0,
+      "reward": -0.18097659051418305,
+      "reward_std": 0.8722228646278382,
+      "step": 2160,
+      "step_time": 303.44200913831594
+    },
+    {
+      "clip_ratio/high_max": 0.021955761154094942,
+      "clip_ratio/high_mean": 0.0028564377397742647,
+      "clip_ratio/low_mean": 0.001177138681487122,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004033576498841285,
+      "completion_length": 7244.059375,
+      "completions/clipped_ratio": 0.853125,
+      "completions/max_length": 14199.8,
+      "completions/max_terminated_length": 836.6,
+      "completions/mean_length": 7244.059375,
+      "completions/mean_terminated_length": 568.07421875,
+      "completions/min_length": 246.0,
+      "completions/min_terminated_length": 246.0,
+      "epoch": 1.736,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 1.4090313911437988,
+      "kl": 10.152984827756882,
+      "learning_rate": 1e-06,
+      "loss": 0.1574,
+      "num_tokens": 499933613.0,
+      "reward": -0.485373055934906,
+      "reward_std": 0.8234812617301941,
+      "step": 2170,
+      "step_time": 312.06590765714645
+    },
+    {
+      "clip_ratio/high_max": 0.050612439154065214,
+      "clip_ratio/high_mean": 0.0072527266953329676,
+      "clip_ratio/low_mean": 0.002120999518672306,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.009373726272315253,
+      "completion_length": 6789.80625,
+      "completions/clipped_ratio": 0.8625,
+      "completions/max_length": 13749.2,
+      "completions/max_terminated_length": 671.0,
+      "completions/mean_length": 6789.80625,
+      "completions/mean_terminated_length": 474.9792236328125,
+      "completions/min_length": 331.6,
+      "completions/min_terminated_length": 331.6,
+      "epoch": 1.744,
+      "frac_reward_zero_std": 0.0875,
+      "grad_norm": 0.7718347311019897,
+      "kl": 5.741251057386398,
+      "learning_rate": 1e-06,
+      "loss": 0.0996,
+      "num_tokens": 502430367.0,
+      "reward": -0.26149832308292387,
+      "reward_std": 0.7426448345184327,
+      "step": 2180,
+      "step_time": 304.73370529860256
+    },
+    {
+      "clip_ratio/high_max": 0.040165082055409584,
+      "clip_ratio/high_mean": 0.007682050271660046,
+      "clip_ratio/low_mean": 3.635791804299515e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.007718407744596334,
+      "completion_length": 7680.9375,
+      "completions/clipped_ratio": 0.825,
+      "completions/max_length": 14187.8,
+      "completions/max_terminated_length": 1520.8,
+      "completions/mean_length": 7680.9375,
+      "completions/mean_terminated_length": 736.2744140625,
+      "completions/min_length": 392.2,
+      "completions/min_terminated_length": 392.2,
+      "epoch": 1.752,
+      "frac_reward_zero_std": 0.1,
+      "grad_norm": 0.5506429076194763,
+      "kl": 3.834122496843338,
+      "learning_rate": 1e-06,
+      "loss": 0.0676,
+      "num_tokens": 505217991.0,
+      "reward": -0.7733753651380539,
+      "reward_std": 0.5388049304485321,
+      "step": 2190,
+      "step_time": 296.4476759292185
+    },
+    {
+      "clip_ratio/high_max": 0.008866971005045344,
+      "clip_ratio/high_mean": 0.0012439140255537495,
+      "clip_ratio/low_mean": 0.008203260842628879,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.009447175011700892,
+      "completion_length": 7818.775,
+      "completions/clipped_ratio": 0.8625,
+      "completions/max_length": 13854.6,
+      "completions/max_terminated_length": 853.6,
+      "completions/mean_length": 7818.775,
+      "completions/mean_terminated_length": 504.87334594726565,
+      "completions/min_length": 888.6,
+      "completions/min_terminated_length": 321.6,
+      "epoch": 1.76,
+      "frac_reward_zero_std": 0.075,
+      "grad_norm": 0.6452924609184265,
+      "kl": 4.339898377656937,
+      "learning_rate": 1e-06,
+      "loss": 0.0682,
+      "num_tokens": 508049647.0,
+      "reward": -0.6028796136379242,
+      "reward_std": 0.7801291584968567,
+      "step": 2200,
+      "step_time": 316.9260287415236
+    },
+    {
+      "clip_ratio/high_max": 0.009704191555010766,
+      "clip_ratio/high_mean": 0.0012543942358092864,
+      "clip_ratio/low_mean": 0.0026852894032799667,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.003939683516819059,
+      "completion_length": 7545.25,
+      "completions/clipped_ratio": 0.85625,
+      "completions/max_length": 13610.6,
+      "completions/max_terminated_length": 937.4,
+      "completions/mean_length": 7545.25,
+      "completions/mean_terminated_length": 597.4659606933594,
+      "completions/min_length": 427.6,
+      "completions/min_terminated_length": 427.6,
+      "epoch": 1.768,
+      "frac_reward_zero_std": 0.1,
+      "grad_norm": 0.5285654067993164,
+      "kl": 5.156833165884018,
+      "learning_rate": 1e-06,
+      "loss": 0.0848,
+      "num_tokens": 510787951.0,
+      "reward": -0.6228217363357544,
+      "reward_std": 0.7529417753219605,
+      "step": 2210,
+      "step_time": 312.52594977505504
+    },
+    {
+      "clip_ratio/high_max": 0.029655835621633743,
+      "clip_ratio/high_mean": 0.00403772029267202,
+      "clip_ratio/low_mean": 9.514446634284468e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0041328647448608535,
+      "completion_length": 6226.784375,
+      "completions/clipped_ratio": 0.80625,
+      "completions/max_length": 13780.4,
+      "completions/max_terminated_length": 1015.4,
+      "completions/mean_length": 6226.784375,
+      "completions/mean_terminated_length": 591.8316650390625,
+      "completions/min_length": 398.2,
+      "completions/min_terminated_length": 398.2,
+      "epoch": 1.776,
+      "frac_reward_zero_std": 0.075,
+      "grad_norm": 0.26575204730033875,
+      "kl": 6.301667976379394,
+      "learning_rate": 1e-06,
+      "loss": 0.0928,
+      "num_tokens": 513146022.0,
+      "reward": 0.12814039885997772,
+      "reward_std": 0.7725299239158631,
+      "step": 2220,
+      "step_time": 323.3315018944442
+    },
+    {
+      "clip_ratio/high_max": 0.005153245686960873,
+      "clip_ratio/high_mean": 0.0006790971553982672,
+      "clip_ratio/low_mean": 0.003933415139545105,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004612512448329653,
+      "completion_length": 6476.190625,
+      "completions/clipped_ratio": 0.8625,
+      "completions/max_length": 13528.4,
+      "completions/max_terminated_length": 1038.8,
+      "completions/mean_length": 6476.190625,
+      "completions/mean_terminated_length": 641.66240234375,
+      "completions/min_length": 380.4,
+      "completions/min_terminated_length": 380.4,
+      "epoch": 1.784,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.9479169249534607,
+      "kl": 5.67539764046669,
+      "learning_rate": 1e-06,
+      "loss": 0.0885,
+      "num_tokens": 515546919.0,
+      "reward": -0.06376125514507294,
+      "reward_std": 0.8095497608184814,
+      "step": 2230,
+      "step_time": 320.3889760892838
+    },
+    {
+      "clip_ratio/high_max": 0.043207791220629586,
+      "clip_ratio/high_mean": 0.005644996891714982,
+      "clip_ratio/low_mean": 0.00046986210284103434,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.006114859121498739,
+      "completion_length": 6778.059375,
+      "completions/clipped_ratio": 0.853125,
+      "completions/max_length": 14219.4,
+      "completions/max_terminated_length": 683.2,
+      "completions/mean_length": 6778.059375,
+      "completions/mean_terminated_length": 427.89263305664065,
+      "completions/min_length": 477.8,
+      "completions/min_terminated_length": 204.4,
+      "epoch": 1.792,
+      "frac_reward_zero_std": 0.0875,
+      "grad_norm": 0.4773111343383789,
+      "kl": 7.072042518854142,
+      "learning_rate": 1e-06,
+      "loss": 0.1196,
+      "num_tokens": 518085458.0,
+      "reward": -0.1907612919807434,
+      "reward_std": 0.7674532294273376,
+      "step": 2240,
+      "step_time": 345.7024728007615
+    },
+    {
+      "clip_ratio/high_max": 0.07397744312511349,
+      "clip_ratio/high_mean": 0.0106183896852599,
+      "clip_ratio/low_mean": 0.008931090195437719,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.019549479829402118,
+      "completion_length": 7133.834375,
+      "completions/clipped_ratio": 0.953125,
+      "completions/max_length": 13451.6,
+      "completions/max_terminated_length": 466.6,
+      "completions/mean_length": 7133.834375,
+      "completions/mean_terminated_length": 412.0722229003906,
+      "completions/min_length": 954.4,
+      "completions/min_terminated_length": 380.2,
+      "epoch": 1.8,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.4856284260749817,
+      "kl": 5.3610006153583525,
+      "learning_rate": 1e-06,
+      "loss": 0.0892,
+      "num_tokens": 520695273.0,
+      "reward": -0.5927276968955993,
+      "reward_std": 0.5936337083578109,
+      "step": 2250,
+      "step_time": 320.227502861619
+    },
+    {
+      "clip_ratio/high_max": 0.014333658152463614,
+      "clip_ratio/high_mean": 0.0018277670097631925,
+      "clip_ratio/low_mean": 0.013230280246807525,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.015058047568891198,
+      "completion_length": 5903.515625,
+      "completions/clipped_ratio": 0.84375,
+      "completions/max_length": 13707.6,
+      "completions/max_terminated_length": 4120.0,
+      "completions/mean_length": 5903.515625,
+      "completions/mean_terminated_length": 992.5334228515625,
+      "completions/min_length": 719.4,
+      "completions/min_terminated_length": 421.8,
+      "epoch": 1.808,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.7192886471748352,
+      "kl": 2.9445707239210606,
+      "learning_rate": 1e-06,
+      "loss": 0.0455,
+      "num_tokens": 522932802.0,
+      "reward": 0.3182583853602409,
+      "reward_std": 0.7300848722457886,
+      "step": 2260,
+      "step_time": 301.5019235692918
+    },
+    {
+      "clip_ratio/high_max": 0.020961728682596003,
+      "clip_ratio/high_mean": 0.0026503923204018063,
+      "clip_ratio/low_mean": 0.0006444326380915299,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0032948250338449723,
+      "completion_length": 6187.278125,
+      "completions/clipped_ratio": 0.8125,
+      "completions/max_length": 13850.0,
+      "completions/max_terminated_length": 1120.2,
+      "completions/mean_length": 6187.278125,
+      "completions/mean_terminated_length": 685.2111938476562,
+      "completions/min_length": 380.4,
+      "completions/min_terminated_length": 380.4,
+      "epoch": 1.8159999999999998,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.3279896378517151,
+      "kl": 6.154584142565727,
+      "learning_rate": 1e-06,
+      "loss": 0.0989,
+      "num_tokens": 525272395.0,
+      "reward": 0.008310723304748534,
+      "reward_std": 0.8006688535213471,
+      "step": 2270,
+      "step_time": 297.8387157525867
+    },
+    {
+      "clip_ratio/high_max": 0.036910661480942505,
+      "clip_ratio/high_mean": 0.005439730591206171,
+      "clip_ratio/low_mean": 0.00017012677212733252,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.005609857556373754,
+      "completion_length": 6382.2625,
+      "completions/clipped_ratio": 0.8,
+      "completions/max_length": 14635.0,
+      "completions/max_terminated_length": 1401.8,
+      "completions/mean_length": 6382.2625,
+      "completions/mean_terminated_length": 640.4230651855469,
+      "completions/min_length": 269.8,
+      "completions/min_terminated_length": 269.8,
+      "epoch": 1.8239999999999998,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.37194252014160156,
+      "kl": 5.981203085184097,
+      "learning_rate": 1e-06,
+      "loss": 0.0987,
+      "num_tokens": 527671471.0,
+      "reward": -0.10259532332420349,
+      "reward_std": 0.8162549734115601,
+      "step": 2280,
+      "step_time": 305.1188583061099
+    },
+    {
+      "clip_ratio/high_max": 0.013082735770149156,
+      "clip_ratio/high_mean": 0.0019771280799432134,
+      "clip_ratio/low_mean": 0.0001518776348348183,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0021290057427904687,
+      "completion_length": 7132.621875,
+      "completions/clipped_ratio": 0.9,
+      "completions/max_length": 14055.8,
+      "completions/max_terminated_length": 2698.6,
+      "completions/mean_length": 7132.621875,
+      "completions/mean_terminated_length": 905.1066925048829,
+      "completions/min_length": 421.0,
+      "completions/min_terminated_length": 421.0,
+      "epoch": 1.8319999999999999,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.3711191415786743,
+      "kl": 5.991676145792008,
+      "learning_rate": 1e-06,
+      "loss": 0.1002,
+      "num_tokens": 530304090.0,
+      "reward": -0.153307044506073,
+      "reward_std": 0.8692207336425781,
+      "step": 2290,
+      "step_time": 313.85635548196734
+    },
+    {
+      "clip_ratio/high_max": 0.005615949754428584,
+      "clip_ratio/high_mean": 0.0007425795844937966,
+      "clip_ratio/low_mean": 0.0019165310031439731,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0026591105832267205,
+      "completion_length": 5848.2375,
+      "completions/clipped_ratio": 0.8125,
+      "completions/max_length": 12335.2,
+      "completions/max_terminated_length": 906.4,
+      "completions/mean_length": 5848.2375,
+      "completions/mean_terminated_length": 538.8931640625,
+      "completions/min_length": 334.8,
+      "completions/min_terminated_length": 334.8,
+      "epoch": 1.8399999999999999,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.7779859900474548,
+      "kl": 3.760913848876953,
+      "learning_rate": 1e-06,
+      "loss": 0.0572,
+      "num_tokens": 532493358.0,
+      "reward": 0.24520628154277802,
+      "reward_std": 0.6538963139057159,
+      "step": 2300,
+      "step_time": 264.3694315161556
+    },
+    {
+      "clip_ratio/high_max": 0.003398410814406816,
+      "clip_ratio/high_mean": 0.0004513353481797822,
+      "clip_ratio/low_mean": 0.002540027555915003,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0029913629417023914,
+      "completion_length": 6186.18125,
+      "completions/clipped_ratio": 0.903125,
+      "completions/max_length": 12865.2,
+      "completions/max_terminated_length": 690.2,
+      "completions/mean_length": 6186.18125,
+      "completions/mean_terminated_length": 535.2866638183593,
+      "completions/min_length": 376.0,
+      "completions/min_terminated_length": 376.0,
+      "epoch": 1.8479999999999999,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 1.4688665866851807,
+      "kl": 7.289272135496139,
+      "learning_rate": 1e-06,
+      "loss": 0.1127,
+      "num_tokens": 534822948.0,
+      "reward": -0.13389126360416412,
+      "reward_std": 0.7947637438774109,
+      "step": 2310,
+      "step_time": 295.26516236402097
+    },
+    {
+      "clip_ratio/high_max": 0.05927158997146762,
+      "clip_ratio/high_mean": 0.00953923851966465,
+      "clip_ratio/low_mean": 0.0007618127499426919,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.010301051087662928,
+      "completion_length": 6882.63125,
+      "completions/clipped_ratio": 0.86875,
+      "completions/max_length": 14415.2,
+      "completions/max_terminated_length": 1041.8,
+      "completions/mean_length": 6882.63125,
+      "completions/mean_terminated_length": 635.753173828125,
+      "completions/min_length": 367.4,
+      "completions/min_terminated_length": 367.4,
+      "epoch": 1.8559999999999999,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.4180378317832947,
+      "kl": 7.819206011295319,
+      "learning_rate": 1e-06,
+      "loss": 0.1387,
+      "num_tokens": 537370726.0,
+      "reward": -0.29386329352855683,
+      "reward_std": 0.8454349160194397,
+      "step": 2320,
+      "step_time": 307.54919648729265
+    },
+    {
+      "clip_ratio/high_max": 0.01315740517075028,
+      "clip_ratio/high_mean": 0.0017706962563352135,
+      "clip_ratio/low_mean": 0.00022463770901595125,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0019953339522089664,
+      "completion_length": 6708.040625,
+      "completions/clipped_ratio": 0.84375,
+      "completions/max_length": 13584.0,
+      "completions/max_terminated_length": 1048.6,
+      "completions/mean_length": 6708.040625,
+      "completions/mean_terminated_length": 714.9666809082031,
+      "completions/min_length": 327.2,
+      "completions/min_terminated_length": 327.2,
+      "epoch": 1.8639999999999999,
+      "frac_reward_zero_std": 0.075,
+      "grad_norm": 0.3589797616004944,
+      "kl": 6.711132669448853,
+      "learning_rate": 1e-06,
+      "loss": 0.1124,
+      "num_tokens": 539846931.0,
+      "reward": -0.38727874159812925,
+      "reward_std": 0.9495174288749695,
+      "step": 2330,
+      "step_time": 312.2706393614411
+    },
+    {
+      "clip_ratio/high_max": 0.019468771108586226,
+      "clip_ratio/high_mean": 0.0024713833504392824,
+      "clip_ratio/low_mean": 0.0004231399322975449,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.002894523342001776,
+      "completion_length": 6688.35625,
+      "completions/clipped_ratio": 0.809375,
+      "completions/max_length": 13454.2,
+      "completions/max_terminated_length": 839.4,
+      "completions/mean_length": 6688.35625,
+      "completions/mean_terminated_length": 542.8548828125,
+      "completions/min_length": 287.8,
+      "completions/min_terminated_length": 287.8,
+      "epoch": 1.8719999999999999,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 0.5898558497428894,
+      "kl": 5.718288552761078,
+      "learning_rate": 1e-06,
+      "loss": 0.0952,
+      "num_tokens": 542335665.0,
+      "reward": -0.2967193841934204,
+      "reward_std": 0.8061963200569153,
+      "step": 2340,
+      "step_time": 305.2458208475262
+    },
+    {
+      "clip_ratio/high_max": 0.06324758578957698,
+      "clip_ratio/high_mean": 0.00804160315919944,
+      "clip_ratio/low_mean": 6.030804165675363e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.008101911203448253,
+      "completion_length": 6231.653125,
+      "completions/clipped_ratio": 0.815625,
+      "completions/max_length": 13286.0,
+      "completions/max_terminated_length": 1114.2,
+      "completions/mean_length": 6231.653125,
+      "completions/mean_terminated_length": 608.0856689453125,
+      "completions/min_length": 295.6,
+      "completions/min_terminated_length": 295.6,
+      "epoch": 1.88,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.3573845326900482,
+      "kl": 4.1242213428020476,
+      "learning_rate": 1e-06,
+      "loss": 0.0607,
+      "num_tokens": 544667518.0,
+      "reward": 0.16186422109603882,
+      "reward_std": 0.6919510215520859,
+      "step": 2350,
+      "step_time": 287.6100121814758
+    },
+    {
+      "clip_ratio/high_max": 0.005102483431437576,
+      "clip_ratio/high_mean": 0.0006666288423730294,
+      "clip_ratio/low_mean": 0.0036616890049799623,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.004328317779254576,
+      "completion_length": 6443.8625,
+      "completions/clipped_ratio": 0.84375,
+      "completions/max_length": 13792.4,
+      "completions/max_terminated_length": 1012.4,
+      "completions/mean_length": 6443.8625,
+      "completions/mean_terminated_length": 548.508349609375,
+      "completions/min_length": 301.4,
+      "completions/min_terminated_length": 301.4,
+      "epoch": 1.888,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.49083212018013,
+      "kl": 5.573210525512695,
+      "learning_rate": 1e-06,
+      "loss": 0.0852,
+      "num_tokens": 547075082.0,
+      "reward": 0.20654874742031099,
+      "reward_std": 0.8982063293457031,
+      "step": 2360,
+      "step_time": 309.90280218422413
+    },
+    {
+      "clip_ratio/high_max": 0.0024434430535620775,
+      "clip_ratio/high_mean": 0.000356975007980509,
+      "clip_ratio/low_mean": 0.00011230256666294736,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0004692775679359329,
+      "completion_length": 6014.384375,
+      "completions/clipped_ratio": 0.828125,
+      "completions/max_length": 13961.8,
+      "completions/max_terminated_length": 747.8,
+      "completions/mean_length": 6014.384375,
+      "completions/mean_terminated_length": 486.44683227539065,
+      "completions/min_length": 306.8,
+      "completions/min_terminated_length": 306.8,
+      "epoch": 1.896,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.4221237897872925,
+      "kl": 6.8200868427753445,
+      "learning_rate": 1e-06,
+      "loss": 0.1066,
+      "num_tokens": 549322753.0,
+      "reward": 0.11510170698165893,
+      "reward_std": 0.9175688683986664,
+      "step": 2370,
+      "step_time": 285.32596833482387
+    },
+    {
+      "clip_ratio/high_max": 0.008617430153026361,
+      "clip_ratio/high_mean": 0.0011065068349125796,
+      "clip_ratio/low_mean": 0.0007661993313604398,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0018727062224570545,
+      "completion_length": 6294.859375,
+      "completions/clipped_ratio": 0.865625,
+      "completions/max_length": 13517.6,
+      "completions/max_terminated_length": 597.0,
+      "completions/mean_length": 6294.859375,
+      "completions/mean_terminated_length": 376.0332275390625,
+      "completions/min_length": 229.0,
+      "completions/min_terminated_length": 229.0,
+      "epoch": 1.904,
+      "frac_reward_zero_std": 0.075,
+      "grad_norm": 0.8705013990402222,
+      "kl": 8.748363888263702,
+      "learning_rate": 1e-06,
+      "loss": 0.1359,
+      "num_tokens": 551667488.0,
+      "reward": -0.049513399600982666,
+      "reward_std": 0.7226118743419647,
+      "step": 2380,
+      "step_time": 289.822543182224
+    },
+    {
+      "clip_ratio/high_max": 0.061678221509282594,
+      "clip_ratio/high_mean": 0.01016918606201216,
+      "clip_ratio/low_mean": 1.4283552445704117e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.010183469294315729,
+      "completion_length": 6908.265625,
+      "completions/clipped_ratio": 0.865625,
+      "completions/max_length": 13683.6,
+      "completions/max_terminated_length": 729.0,
+      "completions/mean_length": 6908.265625,
+      "completions/mean_terminated_length": 525.4583374023438,
+      "completions/min_length": 411.2,
+      "completions/min_terminated_length": 411.2,
+      "epoch": 1.912,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.5795671343803406,
+      "kl": 5.254349167644977,
+      "learning_rate": 1e-06,
+      "loss": 0.0866,
+      "num_tokens": 554230845.0,
+      "reward": -0.37392993867397306,
+      "reward_std": 0.5164877355098725,
+      "step": 2390,
+      "step_time": 308.44501181580125
+    },
+    {
+      "clip_ratio/high_max": 0.011800866048361058,
+      "clip_ratio/high_mean": 0.0015051665154260264,
+      "clip_ratio/low_mean": 0.005142204931280503,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.006647371616054443,
+      "completion_length": 6933.871875,
+      "completions/clipped_ratio": 0.85,
+      "completions/max_length": 13644.2,
+      "completions/max_terminated_length": 935.0,
+      "completions/mean_length": 6933.871875,
+      "completions/mean_terminated_length": 532.5076293945312,
+      "completions/min_length": 338.6,
+      "completions/min_terminated_length": 338.6,
+      "epoch": 1.92,
+      "frac_reward_zero_std": 0.075,
+      "grad_norm": 0.6896202564239502,
+      "kl": 4.139239095151424,
+      "learning_rate": 1e-06,
+      "loss": 0.0661,
+      "num_tokens": 556798784.0,
+      "reward": -0.2477202147245407,
+      "reward_std": 0.7392874002456665,
+      "step": 2400,
+      "step_time": 292.43805152475835
+    },
+    {
+      "clip_ratio/high_max": 0.002832604536342842,
+      "clip_ratio/high_mean": 0.00039869583861218416,
+      "clip_ratio/low_mean": 0.009279681989391974,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.009678377641102997,
+      "completion_length": 5890.86875,
+      "completions/clipped_ratio": 0.85625,
+      "completions/max_length": 13498.2,
+      "completions/max_terminated_length": 1036.6,
+      "completions/mean_length": 5890.86875,
+      "completions/mean_terminated_length": 566.9150024414063,
+      "completions/min_length": 367.6,
+      "completions/min_terminated_length": 367.6,
+      "epoch": 1.928,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.549519956111908,
+      "kl": 4.1933613359928135,
+      "learning_rate": 1e-06,
+      "loss": 0.0594,
+      "num_tokens": 559013170.0,
+      "reward": 0.3459840893745422,
+      "reward_std": 0.7832869410514831,
+      "step": 2410,
+      "step_time": 302.41714767850937
+    },
+    {
+      "clip_ratio/high_max": 0.012216634216383682,
+      "clip_ratio/high_mean": 0.001652602920330537,
+      "clip_ratio/low_mean": 6.974892909283881e-05,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0017223518298123963,
+      "completion_length": 5921.85,
+      "completions/clipped_ratio": 0.8375,
+      "completions/max_length": 14015.8,
+      "completions/max_terminated_length": 909.2,
+      "completions/mean_length": 5921.85,
+      "completions/mean_terminated_length": 563.033203125,
+      "completions/min_length": 331.8,
+      "completions/min_terminated_length": 331.8,
+      "epoch": 1.936,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.2364901453256607,
+      "kl": 4.496628057956696,
+      "learning_rate": 1e-06,
+      "loss": 0.075,
+      "num_tokens": 561263342.0,
+      "reward": 0.51417336165905,
+      "reward_std": 0.5833874225616456,
+      "step": 2420,
+      "step_time": 281.15176695175467
+    },
+    {
+      "clip_ratio/high_max": 0.008311627488365048,
+      "clip_ratio/high_mean": 0.0010779233535686216,
+      "clip_ratio/low_mean": 0.000912609460465319,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0019905328151935465,
+      "completion_length": 5825.809375,
+      "completions/clipped_ratio": 0.88125,
+      "completions/max_length": 11747.6,
+      "completions/max_terminated_length": 911.6,
+      "completions/mean_length": 5825.809375,
+      "completions/mean_terminated_length": 512.2433349609375,
+      "completions/min_length": 309.8,
+      "completions/min_terminated_length": 309.8,
+      "epoch": 1.944,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.7613126039505005,
+      "kl": 5.580350521206856,
+      "learning_rate": 1e-06,
+      "loss": 0.0888,
+      "num_tokens": 563485509.0,
+      "reward": 0.16537492871284484,
+      "reward_std": 0.752752423286438,
+      "step": 2430,
+      "step_time": 276.9983322199434
+    },
+    {
+      "clip_ratio/high_max": 0.06976447441284109,
+      "clip_ratio/high_mean": 0.01159836501949485,
+      "clip_ratio/low_mean": 0.0006001963055496162,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.01219856101761252,
+      "completion_length": 6798.1625,
+      "completions/clipped_ratio": 0.875,
+      "completions/max_length": 13295.4,
+      "completions/max_terminated_length": 746.6,
+      "completions/mean_length": 6798.1625,
+      "completions/mean_terminated_length": 474.7562316894531,
+      "completions/min_length": 361.8,
+      "completions/min_terminated_length": 361.8,
+      "epoch": 1.952,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.4147081673145294,
+      "kl": 5.530451095104217,
+      "learning_rate": 1e-06,
+      "loss": 0.0984,
+      "num_tokens": 566017265.0,
+      "reward": -0.31506057977676394,
+      "reward_std": 0.6809921026229858,
+      "step": 2440,
+      "step_time": 301.3668847329915
+    },
+    {
+      "clip_ratio/high_max": 0.018801409114348643,
+      "clip_ratio/high_mean": 0.002579252344685301,
+      "clip_ratio/low_mean": 0.0015596423200804565,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0041388945206563225,
+      "completion_length": 6608.384375,
+      "completions/clipped_ratio": 0.871875,
+      "completions/max_length": 13803.8,
+      "completions/max_terminated_length": 641.8,
+      "completions/mean_length": 6608.384375,
+      "completions/mean_terminated_length": 392.2200073242187,
+      "completions/min_length": 267.8,
+      "completions/min_terminated_length": 267.8,
+      "epoch": 1.96,
+      "frac_reward_zero_std": 0.0875,
+      "grad_norm": 0.5155025124549866,
+      "kl": 5.799087914824486,
+      "learning_rate": 1e-06,
+      "loss": 0.0894,
+      "num_tokens": 568464772.0,
+      "reward": -0.18597653806209563,
+      "reward_std": 0.7143688559532165,
+      "step": 2450,
+      "step_time": 296.4705908089876
+    },
+    {
+      "clip_ratio/high_max": 0.06358991034267092,
+      "clip_ratio/high_mean": 0.008092962325008557,
+      "clip_ratio/low_mean": 0.006483247059190944,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.01457620928722463,
+      "completion_length": 6965.203125,
+      "completions/clipped_ratio": 0.846875,
+      "completions/max_length": 13718.8,
+      "completions/max_terminated_length": 1100.0,
+      "completions/mean_length": 6965.203125,
+      "completions/mean_terminated_length": 638.8133422851563,
+      "completions/min_length": 279.6,
+      "completions/min_terminated_length": 279.6,
+      "epoch": 1.968,
+      "frac_reward_zero_std": 0.0875,
+      "grad_norm": 0.6565930247306824,
+      "kl": 5.400943659245968,
+      "learning_rate": 1e-06,
+      "loss": 0.0833,
+      "num_tokens": 571029249.0,
+      "reward": -0.15108705312013626,
+      "reward_std": 0.8721365094184875,
+      "step": 2460,
+      "step_time": 297.3470008134842
+    },
+    {
+      "clip_ratio/high_max": 0.03568386041042686,
+      "clip_ratio/high_mean": 0.0045534338184097575,
+      "clip_ratio/low_mean": 0.002532727589334627,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.007086161449160499,
+      "completion_length": 6368.0875,
+      "completions/clipped_ratio": 0.875,
+      "completions/max_length": 13431.4,
+      "completions/max_terminated_length": 908.6,
+      "completions/mean_length": 6368.0875,
+      "completions/mean_terminated_length": 603.8839782714844,
+      "completions/min_length": 377.8,
+      "completions/min_terminated_length": 377.8,
+      "epoch": 1.976,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.49820491671562195,
+      "kl": 5.867856433987617,
+      "learning_rate": 1e-06,
+      "loss": 0.089,
+      "num_tokens": 573398137.0,
+      "reward": 0.0031133264303207396,
+      "reward_std": 0.6515501856803894,
+      "step": 2470,
+      "step_time": 301.59280881807206
+    },
+    {
+      "clip_ratio/high_max": 0.019649311208377186,
+      "clip_ratio/high_mean": 0.0024752861468186893,
+      "clip_ratio/low_mean": 0.0005164007315215713,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0029916868495547533,
+      "completion_length": 6597.1125,
+      "completions/clipped_ratio": 0.871875,
+      "completions/max_length": 13323.2,
+      "completions/max_terminated_length": 989.0,
+      "completions/mean_length": 6597.1125,
+      "completions/mean_terminated_length": 586.1579467773438,
+      "completions/min_length": 319.8,
+      "completions/min_terminated_length": 319.8,
+      "epoch": 1.984,
+      "frac_reward_zero_std": 0.05,
+      "grad_norm": 0.5008276700973511,
+      "kl": 5.805088073015213,
+      "learning_rate": 1e-06,
+      "loss": 0.0915,
+      "num_tokens": 575865101.0,
+      "reward": -0.002735450863838196,
+      "reward_std": 0.8002071380615234,
+      "step": 2480,
+      "step_time": 301.16595053635535
+    },
+    {
+      "clip_ratio/high_max": 0.014997787121137663,
+      "clip_ratio/high_mean": 0.002138797930115288,
+      "clip_ratio/low_mean": 0.00011812337122591998,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.002256921278603841,
+      "completion_length": 6670.409375,
+      "completions/clipped_ratio": 0.86875,
+      "completions/max_length": 13726.4,
+      "completions/max_terminated_length": 866.6,
+      "completions/mean_length": 6670.409375,
+      "completions/mean_terminated_length": 552.16513671875,
+      "completions/min_length": 331.0,
+      "completions/min_terminated_length": 331.0,
+      "epoch": 1.992,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.7289444804191589,
+      "kl": 7.294751286506653,
+      "learning_rate": 1e-06,
+      "loss": 0.1199,
+      "num_tokens": 578353012.0,
+      "reward": -0.2304362416267395,
+      "reward_std": 0.769736647605896,
+      "step": 2490,
+      "step_time": 306.3768429461867
+    },
+    {
+      "clip_ratio/high_max": 0.05473219842078834,
+      "clip_ratio/high_mean": 0.007481279596595414,
+      "clip_ratio/low_mean": 0.004122421985630354,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.01160370146699279,
+      "completion_length": 6147.984375,
+      "completions/clipped_ratio": 0.7875,
+      "completions/max_length": 14079.2,
+      "completions/max_terminated_length": 1159.8,
+      "completions/mean_length": 6147.984375,
+      "completions/mean_terminated_length": 580.0676879882812,
+      "completions/min_length": 318.6,
+      "completions/min_terminated_length": 318.6,
+      "epoch": 2.0,
+      "frac_reward_zero_std": 0.0375,
+      "grad_norm": 0.43284502625465393,
+      "kl": 6.553870472311973,
+      "learning_rate": 1e-06,
+      "loss": 0.1048,
+      "num_tokens": 580679415.0,
+      "reward": 0.08529881834983825,
+      "reward_std": 0.9655373930931092,
+      "step": 2500,
+      "step_time": 319.6675663843751
     }
   ],
   "logging_steps": 10,
-  "max_steps": 312,
-  "num_input_tokens_seen": 30685228,
-  "num_train_epochs": 1,
-  "save_steps": 63,
+  "max_steps": 2500,
+  "num_input_tokens_seen": 580679415,
+  "num_train_epochs": 2,
+  "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
@@ -406,13 +6522,13 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 4,
+  "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null
 }