Upload 18 files

model-00012-of-00013.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c5d622ca2669bc838f21156719505317ab48e00a63b7a247d871113ccf52382d
+oid sha256:f42e6df7b88b29742aa11ff2be8641a391e91b1ed3835543672877b92c394bb2
 size 4997743184

model-00013-of-00013.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75327d6c4949a66304ac865fde693349f88fb16df05c171de188b81bbab5a86c
+oid sha256:d3cd9f27caa30661a9a1d05bc2aca8253ba8332e44bf711127ceb70439afa3d1
 size 1094220288

rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4298c35ea36b83389d7d2655789aea76011636e31676d63169ea4b845ecd6456
-size 15877
+oid sha256:4eb4e67bcdc97985adbe5050384e314f0cd3e4e74bbbc97d1d73f5cdc94d9318
+size 16389

rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7bddd410d6bbaca01c2f12a598504e4746f0cc7e0dcea1b8eed9f872e8a3d9ac
-size 15877
+oid sha256:52c217fa87f200773681fbfc4dd7dd1261fcee84826d53a92d46327d823d83fc
+size 16389

rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47088a30d9582204be59ec249c10e30ee9aed11571a1c30b2656bc258babd292
-size 15941
+oid sha256:5e3e339fc2adedf9d769963c6f2c0d15de97174a4fc0a0f023b6eeaad295aabe
+size 16389

rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d7702901a424c5e325ea5ebc7cac53fde02b109b9000ed2759e159299279a5d
-size 15941
+oid sha256:81724b81ddb544c711c4b34ad8f2808de5e2dbeb8b8e75333310342ea996919a
+size 16389

rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:58e7d321ca8cfb0e9cfb10c69ee10a4905ed4713ee69ff62710973bfabca29b0
-size 15877
+oid sha256:29548c025559f395a5c54d1c1be54c5706ec9227615af3487de8bfff25146662
+size 16389

rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4b1827d395743f1e283fb8efd86b46e34091125196a1c4608507aadb53a68d89
-size 15877
+oid sha256:e7c44b5bb8855faf98166542aa05e8cf705532d662affe03390e282189221cf8
+size 16389

rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe432ea51f83d8a13a0b1a7e79fe1cd7cdaf0aedcb40dc02f4689c5e775c37bf
+size 16389

rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b50646369cd2ff2faa923bdc4c72f1fc8412d817f2d5344013686f4eb053913b
+size 16325

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:264812b5a239dbdf942c022a091d9288545dfab1d91c4ad424dec811e23c3912
+oid sha256:720c73500cbe673af6f33b912b8b891a04b4ade61484e95b540fb2d69fc838dd
 size 1465

tokenizer_config.json

@@ -233,9 +233,8 @@
   "extra_special_tokens": {},
   "model_max_length": 1048576,
   "pad_token": "<|endoftext|>",
-  "padding_side": "left",
+  "padding_side": "right",
   "split_special_tokens": false,
   "tokenizer_class": "Qwen2Tokenizer",
-  "truncation_side": "left",
   "unk_token": null
 }

trainer_state.json

@@ -2,531 +2,1163 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.20048019207683074,
+  "epoch": 1.4134615384615383,
   "eval_steps": 500,
-  "global_step": 167,
+  "global_step": 441,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.012500000186264515,
-      "completions/max_length": 2954.7,
-      "completions/max_terminated_length": 1951.6,
-      "completions/mean_length": 545.102099609375,
-      "completions/mean_terminated_length": 500.0404846191406,
-      "completions/min_length": 117.8,
-      "completions/min_terminated_length": 117.8,
-      "entropy": 0.1633674878627062,
-      "epoch": 0.012004801920768308,
-      "frac_reward_zero_std": 0.05000000149011612,
-      "grad_norm": 0.0819886103272438,
-      "learning_rate": 5.389221556886228e-07,
-      "loss": 0.0538,
-      "num_tokens": 1060997.0,
-      "reward": -0.349642014503479,
-      "reward_std": 0.18912948295474052,
-      "rewards/grpo_reward_function/mean": -0.3496420085430145,
-      "rewards/grpo_reward_function/std": 0.4486300081014633,
-      "sampling/importance_sampling_ratio/max": 2.3219674229621887,
-      "sampling/importance_sampling_ratio/mean": 0.3698740124702454,
-      "sampling/importance_sampling_ratio/min": 1.1996005980563495e-06,
-      "sampling/sampling_logp_difference/max": 2.5826863408088685,
-      "sampling/sampling_logp_difference/mean": 0.019079525023698807,
+      "clip_ratio/high_max": 0.004660501377657056,
+      "clip_ratio/high_mean": 0.002407502007554285,
+      "clip_ratio/low_mean": 0.0017146819620393217,
+      "clip_ratio/low_min": 0.0003550909183104523,
+      "clip_ratio/region_mean": 0.00412218397250399,
+      "completion_length": 2894.4234375,
+      "completions/clipped_ratio": 0.0078125,
+      "completions/max_length": 11733.4,
+      "completions/max_terminated_length": 10766.0,
+      "completions/mean_length": 2894.4234375,
+      "completions/mean_terminated_length": 2838.155126953125,
+      "completions/min_length": 502.0,
+      "completions/min_terminated_length": 502.0,
+      "epoch": 0.03205128205128205,
+      "frac_reward_zero_std": 0.021875,
+      "grad_norm": 0.5731685161590576,
+      "kl": 1.0286681566096376,
+      "learning_rate": 4.5e-07,
+      "loss": 0.2717,
+      "num_tokens": 5144718.0,
+      "reward": -0.1477112166583538,
+      "reward_std": 0.4637692987918854,
       "step": 10,
-      "step_time": 591.6221411965787
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.01250000037252903,
-      "completions/max_length": 2285.6,
-      "completions/max_terminated_length": 1834.2,
-      "completions/mean_length": 525.1979339599609,
-      "completions/mean_terminated_length": 481.30433044433596,
-      "completions/min_length": 111.4,
-      "completions/min_terminated_length": 111.4,
-      "entropy": 0.15604820642620326,
-      "epoch": 0.024009603841536616,
-      "frac_reward_zero_std": 0.0416666679084301,
-      "grad_norm": 0.09551126509904861,
-      "learning_rate": 1.1377245508982037e-06,
-      "loss": -0.0139,
-      "num_tokens": 2123212.0,
-      "reward": -0.298923921585083,
-      "reward_std": 0.21090517602860928,
-      "rewards/grpo_reward_function/mean": -0.2989239178597927,
-      "rewards/grpo_reward_function/std": 0.4665490254759789,
-      "sampling/importance_sampling_ratio/max": 2.135484504699707,
-      "sampling/importance_sampling_ratio/mean": 0.40152732133865354,
-      "sampling/importance_sampling_ratio/min": 2.7301041336613706e-05,
-      "sampling/sampling_logp_difference/max": 2.5806180000305177,
-      "sampling/sampling_logp_difference/mean": 0.019163084402680396,
+      "step_time": 1357.2852862857283
+    },
+    {
+      "clip_ratio/high_max": 0.006074614630779252,
+      "clip_ratio/high_mean": 0.0029590918275061995,
+      "clip_ratio/low_mean": 0.0020704118942376226,
+      "clip_ratio/low_min": 0.0003676083892059978,
+      "clip_ratio/region_mean": 0.005029503698460757,
+      "completion_length": 2867.24296875,
+      "completions/clipped_ratio": 0.01328125,
+      "completions/max_length": 12252.8,
+      "completions/max_terminated_length": 10853.6,
+      "completions/mean_length": 2867.24296875,
+      "completions/mean_terminated_length": 2775.650732421875,
+      "completions/min_length": 423.4,
+      "completions/min_terminated_length": 423.4,
+      "epoch": 0.0641025641025641,
+      "frac_reward_zero_std": 0.028125,
+      "grad_norm": 1.41015625,
+      "kl": 0.8294610816927162,
+      "learning_rate": 9.499999999999999e-07,
+      "loss": 0.0172,
+      "num_tokens": 10310201.0,
+      "reward": -0.13689552899450064,
+      "reward_std": 0.4330836236476898,
       "step": 20,
-      "step_time": 554.5896356501617
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.01666666716337204,
-      "completions/max_length": 3269.3,
-      "completions/max_terminated_length": 2072.6,
-      "completions/mean_length": 556.327099609375,
-      "completions/mean_terminated_length": 496.14097595214844,
-      "completions/min_length": 132.3,
-      "completions/min_terminated_length": 132.3,
-      "entropy": 0.18481314480304717,
-      "epoch": 0.03601440576230492,
-      "frac_reward_zero_std": 0.05000000074505806,
-      "grad_norm": 0.11139781028032303,
-      "learning_rate": 1.7365269461077847e-06,
-      "loss": 0.0256,
-      "num_tokens": 3227189.0,
-      "reward": -0.409818297624588,
-      "reward_std": 0.22780176997184753,
-      "rewards/grpo_reward_function/mean": -0.4098182961344719,
-      "rewards/grpo_reward_function/std": 0.5334153980016708,
-      "sampling/importance_sampling_ratio/max": 2.277985179424286,
-      "sampling/importance_sampling_ratio/mean": 0.3370798110961914,
-      "sampling/importance_sampling_ratio/min": 2.6906073216806556e-05,
-      "sampling/sampling_logp_difference/max": 2.5000483632087707,
-      "sampling/sampling_logp_difference/mean": 0.02049510907381773,
+      "step_time": 1400.939316440746
+    },
+    {
+      "clip_ratio/high_max": 0.006338656169828027,
+      "clip_ratio/high_mean": 0.003108998734387569,
+      "clip_ratio/low_mean": 0.001976575414300896,
+      "clip_ratio/low_min": 0.0003502949675748823,
+      "clip_ratio/region_mean": 0.0050855741370469335,
+      "completion_length": 2983.815625,
+      "completions/clipped_ratio": 0.01015625,
+      "completions/max_length": 11952.0,
+      "completions/max_terminated_length": 10907.2,
+      "completions/mean_length": 2983.815625,
+      "completions/mean_terminated_length": 2907.595947265625,
+      "completions/min_length": 410.0,
+      "completions/min_terminated_length": 410.0,
+      "epoch": 0.09615384615384616,
+      "frac_reward_zero_std": 0.025,
+      "grad_norm": 0.15793567895889282,
+      "kl": 0.3357926928496454,
+      "learning_rate": 1e-06,
+      "loss": 0.0379,
+      "num_tokens": 15630909.0,
+      "reward": -0.16447794139385224,
+      "reward_std": 0.49801658391952514,
       "step": 30,
-      "step_time": 555.5971007851883
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.01041666679084301,
-      "completions/max_length": 2135.6,
-      "completions/max_terminated_length": 1478.5,
-      "completions/mean_length": 508.5041778564453,
-      "completions/mean_terminated_length": 472.1120574951172,
-      "completions/min_length": 147.4,
-      "completions/min_terminated_length": 147.4,
-      "entropy": 0.1670758031308651,
-      "epoch": 0.04801920768307323,
-      "frac_reward_zero_std": 0.0416666679084301,
-      "grad_norm": 0.06704321503639221,
-      "learning_rate": 2.3353293413173654e-06,
-      "loss": -0.0127,
-      "num_tokens": 4318559.0,
-      "reward": -0.2258751168847084,
-      "reward_std": 0.16097248084843158,
-      "rewards/grpo_reward_function/mean": -0.22587510757148266,
-      "rewards/grpo_reward_function/std": 0.49552616477012634,
-      "sampling/importance_sampling_ratio/max": 2.3126969814300535,
-      "sampling/importance_sampling_ratio/mean": 0.35644740611314774,
-      "sampling/importance_sampling_ratio/min": 2.7391963689638034e-05,
-      "sampling/sampling_logp_difference/max": 2.8060175657272337,
-      "sampling/sampling_logp_difference/mean": 0.01994446888566017,
+      "step_time": 1390.6516450636088
+    },
+    {
+      "clip_ratio/high_max": 0.005659139377530664,
+      "clip_ratio/high_mean": 0.0027562346367631108,
+      "clip_ratio/low_mean": 0.0021051473886473104,
+      "clip_ratio/low_min": 0.00043640066287480294,
+      "clip_ratio/region_mean": 0.0048613820574246345,
+      "completion_length": 3013.03359375,
+      "completions/clipped_ratio": 0.015625,
+      "completions/max_length": 12004.6,
+      "completions/max_terminated_length": 11320.4,
+      "completions/mean_length": 3013.03359375,
+      "completions/mean_terminated_length": 2888.912255859375,
+      "completions/min_length": 414.2,
+      "completions/min_terminated_length": 414.2,
+      "epoch": 0.1282051282051282,
+      "frac_reward_zero_std": 0.034375,
+      "grad_norm": 0.11701524257659912,
+      "kl": 6.595802799868397,
+      "learning_rate": 1e-06,
+      "loss": 0.0687,
+      "num_tokens": 21000328.0,
+      "reward": -0.17579184994101524,
+      "reward_std": 0.4495803236961365,
       "step": 40,
-      "step_time": 554.565231207572
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.00416666679084301,
-      "completions/max_length": 1859.8,
-      "completions/max_terminated_length": 1757.3,
-      "completions/mean_length": 513.3979309082031,
-      "completions/mean_terminated_length": 499.66423950195315,
-      "completions/min_length": 117.0,
-      "completions/min_terminated_length": 117.0,
-      "entropy": 0.17271990440785884,
-      "epoch": 0.060024009603841535,
-      "frac_reward_zero_std": 0.06666666865348816,
-      "grad_norm": 0.09258268028497696,
-      "learning_rate": 2.9341317365269463e-06,
-      "loss": 0.0015,
-      "num_tokens": 5370018.0,
-      "reward": -0.3178896278142929,
-      "reward_std": 0.1635244082659483,
-      "rewards/grpo_reward_function/mean": -0.3178896352648735,
-      "rewards/grpo_reward_function/std": 0.45689679607748984,
-      "sampling/importance_sampling_ratio/max": 2.195555794239044,
-      "sampling/importance_sampling_ratio/mean": 0.3522403955459595,
-      "sampling/importance_sampling_ratio/min": 7.164277021729504e-05,
-      "sampling/sampling_logp_difference/max": 2.4301879167556764,
-      "sampling/sampling_logp_difference/mean": 0.02071673283353448,
+      "step_time": 1382.434469479695
+    },
+    {
+      "clip_ratio/high_max": 0.006088442541658879,
+      "clip_ratio/high_mean": 0.0028750557685270905,
+      "clip_ratio/low_mean": 0.0021623167864163405,
+      "clip_ratio/low_min": 0.0002990317210787907,
+      "clip_ratio/region_mean": 0.005037372524384409,
+      "completion_length": 2944.84765625,
+      "completions/clipped_ratio": 0.01640625,
+      "completions/max_length": 12398.4,
+      "completions/max_terminated_length": 10397.2,
+      "completions/mean_length": 2944.84765625,
+      "completions/mean_terminated_length": 2805.46142578125,
+      "completions/min_length": 505.4,
+      "completions/min_terminated_length": 505.4,
+      "epoch": 0.16025641025641027,
+      "frac_reward_zero_std": 0.015625,
+      "grad_norm": 8.11508560180664,
+      "kl": 0.21396316497121007,
+      "learning_rate": 1e-06,
+      "loss": 0.2042,
+      "num_tokens": 26241013.0,
+      "reward": -0.23499981164932252,
+      "reward_std": 0.4823799729347229,
       "step": 50,
-      "step_time": 548.9367319711484
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.00416666679084301,
-      "completions/max_length": 2202.6,
-      "completions/max_terminated_length": 1812.6,
-      "completions/mean_length": 506.99168395996094,
-      "completions/mean_terminated_length": 492.1666778564453,
-      "completions/min_length": 127.8,
-      "completions/min_terminated_length": 127.8,
-      "entropy": 0.160004629381001,
-      "epoch": 0.07202881152460984,
-      "frac_reward_zero_std": 0.0416666679084301,
-      "grad_norm": 0.05922295153141022,
-      "learning_rate": 3.5329341317365273e-06,
-      "loss": -0.0033,
-      "num_tokens": 6466162.0,
-      "reward": -0.34021527171134947,
-      "reward_std": 0.18639734461903573,
-      "rewards/grpo_reward_function/mean": -0.34021526128053664,
-      "rewards/grpo_reward_function/std": 0.5187867254018783,
-      "sampling/importance_sampling_ratio/max": 2.010967791080475,
-      "sampling/importance_sampling_ratio/mean": 0.30313637256622317,
-      "sampling/importance_sampling_ratio/min": 3.7146345167826667e-06,
-      "sampling/sampling_logp_difference/max": 2.6932833194732666,
-      "sampling/sampling_logp_difference/mean": 0.02041825857013464,
+      "step_time": 1388.5390573423356
+    },
+    {
+      "clip_ratio/high_max": 0.004905331420013681,
+      "clip_ratio/high_mean": 0.0026467385643627493,
+      "clip_ratio/low_mean": 0.0018470981682185083,
+      "clip_ratio/low_min": 0.0003405745592317544,
+      "clip_ratio/region_mean": 0.004493836738402024,
+      "completion_length": 3000.59453125,
+      "completions/clipped_ratio": 0.0125,
+      "completions/max_length": 12504.2,
+      "completions/max_terminated_length": 11336.2,
+      "completions/mean_length": 3000.59453125,
+      "completions/mean_terminated_length": 2900.00673828125,
+      "completions/min_length": 465.8,
+      "completions/min_terminated_length": 465.8,
+      "epoch": 0.19230769230769232,
+      "frac_reward_zero_std": 0.015625,
+      "grad_norm": 0.10289935767650604,
+      "kl": 0.04673767304047942,
+      "learning_rate": 1e-06,
+      "loss": 0.0029,
+      "num_tokens": 31588358.0,
+      "reward": -0.09221201464533806,
+      "reward_std": 0.4848791122436523,
       "step": 60,
-      "step_time": 530.4414538932033
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.016666666977107523,
-      "completions/max_length": 2579.7,
-      "completions/max_terminated_length": 2077.7,
-      "completions/mean_length": 557.7854309082031,
-      "completions/mean_terminated_length": 499.6593933105469,
-      "completions/min_length": 121.5,
-      "completions/min_terminated_length": 121.5,
-      "entropy": 0.16313071362674236,
-      "epoch": 0.08403361344537816,
-      "frac_reward_zero_std": 0.10000000223517418,
-      "grad_norm": 0.014785214327275753,
-      "learning_rate": 4.131736526946108e-06,
-      "loss": 0.0424,
-      "num_tokens": 7608683.0,
-      "reward": -0.33393135815858843,
-      "reward_std": 0.19106332510709761,
-      "rewards/grpo_reward_function/mean": -0.33393134772777555,
-      "rewards/grpo_reward_function/std": 0.5677398703992367,
-      "sampling/importance_sampling_ratio/max": 1.9730541229248046,
-      "sampling/importance_sampling_ratio/mean": 0.3427995890378952,
-      "sampling/importance_sampling_ratio/min": 1.0688633483368904e-05,
-      "sampling/sampling_logp_difference/max": 2.8619158267974854,
-      "sampling/sampling_logp_difference/mean": 0.0201931843534112,
+      "step_time": 1406.4417366620153
+    },
+    {
+      "clip_ratio/high_max": 0.005218895472353324,
+      "clip_ratio/high_mean": 0.0026741086272522806,
+      "clip_ratio/low_mean": 0.002022538718301803,
+      "clip_ratio/low_min": 0.00039923477415868545,
+      "clip_ratio/region_mean": 0.004696647363016382,
+      "completion_length": 2902.43515625,
+      "completions/clipped_ratio": 0.01171875,
+      "completions/max_length": 11472.4,
+      "completions/max_terminated_length": 10626.2,
+      "completions/mean_length": 2902.43515625,
+      "completions/mean_terminated_length": 2828.052734375,
+      "completions/min_length": 442.0,
+      "completions/min_terminated_length": 442.0,
+      "epoch": 0.22435897435897437,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.5660527944564819,
+      "kl": 0.07278029285371304,
+      "learning_rate": 1e-06,
+      "loss": 0.0031,
+      "num_tokens": 36777143.0,
+      "reward": -0.11244684159755707,
+      "reward_std": 0.4777476072311401,
       "step": 70,
-      "step_time": 541.6015901661478
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.01666666716337204,
-      "completions/max_length": 3292.7,
-      "completions/max_terminated_length": 2336.3,
-      "completions/mean_length": 605.0791870117188,
-      "completions/mean_terminated_length": 546.5141571044921,
-      "completions/min_length": 142.3,
-      "completions/min_terminated_length": 142.3,
-      "entropy": 0.16853776723146438,
-      "epoch": 0.09603841536614646,
-      "frac_reward_zero_std": 0.10000000298023223,
-      "grad_norm": 0.0669359341263771,
-      "learning_rate": 4.730538922155689e-06,
-      "loss": 0.003,
-      "num_tokens": 8693089.0,
-      "reward": -0.36407424658536913,
-      "reward_std": 0.15138040184974672,
-      "rewards/grpo_reward_function/mean": -0.364074233174324,
-      "rewards/grpo_reward_function/std": 0.4984104484319687,
-      "sampling/importance_sampling_ratio/max": 2.417657721042633,
-      "sampling/importance_sampling_ratio/mean": 0.3272848010063171,
-      "sampling/importance_sampling_ratio/min": 6.19425904005766e-05,
-      "sampling/sampling_logp_difference/max": 2.832179582118988,
-      "sampling/sampling_logp_difference/mean": 0.019849142245948314,
+      "step_time": 1481.242667537555
+    },
+    {
+      "clip_ratio/high_max": 0.0049126958358101545,
+      "clip_ratio/high_mean": 0.0026609906810335813,
+      "clip_ratio/low_mean": 0.0019596937811002134,
+      "clip_ratio/low_min": 0.0004173820205323864,
+      "clip_ratio/region_mean": 0.004620684438850731,
+      "completion_length": 2825.84609375,
+      "completions/clipped_ratio": 0.009375,
+      "completions/max_length": 11470.0,
+      "completions/max_terminated_length": 10055.2,
+      "completions/mean_length": 2825.84609375,
+      "completions/mean_terminated_length": 2761.252490234375,
+      "completions/min_length": 409.4,
+      "completions/min_terminated_length": 409.4,
+      "epoch": 0.2564102564102564,
+      "frac_reward_zero_std": 0.015625,
+      "grad_norm": 0.07384040206670761,
+      "kl": 0.0690781265962869,
+      "learning_rate": 1e-06,
+      "loss": 0.0036,
+      "num_tokens": 41906774.0,
+      "reward": -0.16573150753974913,
+      "reward_std": 0.4685970783233643,
       "step": 80,
-      "step_time": 559.4207322074101
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.018750000558793545,
-      "completions/max_length": 3217.2,
-      "completions/max_terminated_length": 2202.3,
-      "completions/mean_length": 652.0937744140625,
-      "completions/mean_terminated_length": 587.0378173828125,
-      "completions/min_length": 164.3,
-      "completions/min_terminated_length": 164.3,
-      "entropy": 0.16202539429068566,
-      "epoch": 0.10804321728691477,
-      "frac_reward_zero_std": 0.10000000223517418,
-      "grad_norm": 0.03904345631599426,
-      "learning_rate": 5.32934131736527e-06,
-      "loss": -0.0041,
-      "num_tokens": 9849890.0,
-      "reward": -0.3628114402294159,
-      "reward_std": 0.2544385172426701,
-      "rewards/grpo_reward_function/mean": -0.3628114327788353,
-      "rewards/grpo_reward_function/std": 0.6255016416311264,
-      "sampling/importance_sampling_ratio/max": 1.7914996325969696,
-      "sampling/importance_sampling_ratio/mean": 0.3064177379012108,
-      "sampling/importance_sampling_ratio/min": 1.5591655392199753e-05,
-      "sampling/sampling_logp_difference/max": 2.840235471725464,
-      "sampling/sampling_logp_difference/mean": 0.01814730800688267,
+      "step_time": 1543.2884240075946
+    },
+    {
+      "clip_ratio/high_max": 0.005226275697350502,
+      "clip_ratio/high_mean": 0.002715380094014108,
+      "clip_ratio/low_mean": 0.0019015513695194385,
+      "clip_ratio/low_min": 0.00043704208910639865,
+      "clip_ratio/region_mean": 0.004616931488271803,
+      "completion_length": 2924.34453125,
+      "completions/clipped_ratio": 0.0140625,
+      "completions/max_length": 12426.8,
+      "completions/max_terminated_length": 10779.2,
+      "completions/mean_length": 2924.34453125,
+      "completions/mean_terminated_length": 2818.486572265625,
+      "completions/min_length": 430.6,
+      "completions/min_terminated_length": 430.6,
+      "epoch": 0.28846153846153844,
+      "frac_reward_zero_std": 0.015625,
+      "grad_norm": 0.11525672674179077,
+      "kl": 0.08227849821560085,
+      "learning_rate": 1e-06,
+      "loss": 0.0038,
+      "num_tokens": 47119235.0,
+      "reward": -0.18828697204589845,
+      "reward_std": 0.48752092123031615,
       "step": 90,
-      "step_time": 555.9057860235683
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.010416666977107525,
-      "completions/max_length": 2567.4,
-      "completions/max_terminated_length": 2092.1,
-      "completions/mean_length": 548.7250213623047,
-      "completions/mean_terminated_length": 511.10838317871094,
-      "completions/min_length": 116.3,
-      "completions/min_terminated_length": 116.3,
-      "entropy": 0.16799122765660285,
-      "epoch": 0.12004801920768307,
-      "frac_reward_zero_std": 0.10000000223517418,
-      "grad_norm": 0.07849112898111343,
-      "learning_rate": 5.928143712574851e-06,
-      "loss": -0.0009,
-      "num_tokens": 10951862.0,
-      "reward": -0.37651871144771576,
-      "reward_std": 0.17827629819512367,
-      "rewards/grpo_reward_function/mean": -0.3765186980366707,
-      "rewards/grpo_reward_function/std": 0.4988637834787369,
-      "sampling/importance_sampling_ratio/max": 2.2177215456962585,
-      "sampling/importance_sampling_ratio/mean": 0.3616850808262825,
-      "sampling/importance_sampling_ratio/min": 0.0,
-      "sampling/sampling_logp_difference/max": 2.7038833022117617,
-      "sampling/sampling_logp_difference/mean": 0.019389390759170056,
+      "step_time": 1559.9560344189406
+    },
+    {
+      "clip_ratio/high_max": 0.005342797550838441,
+      "clip_ratio/high_mean": 0.0026746522344183177,
+      "clip_ratio/low_mean": 0.001785056365770288,
+      "clip_ratio/low_min": 0.00037644226977135987,
+      "clip_ratio/region_mean": 0.004459708638023585,
+      "completion_length": 2836.5890625,
+      "completions/clipped_ratio": 0.01484375,
+      "completions/max_length": 11882.2,
+      "completions/max_terminated_length": 10955.4,
+      "completions/mean_length": 2836.5890625,
+      "completions/mean_terminated_length": 2718.124755859375,
+      "completions/min_length": 441.6,
+      "completions/min_terminated_length": 441.6,
+      "epoch": 0.32051282051282054,
+      "frac_reward_zero_std": 0.01875,
+      "grad_norm": 0.06436898559331894,
+      "kl": 0.07589586917310953,
+      "learning_rate": 1e-06,
+      "loss": 0.0036,
+      "num_tokens": 52239397.0,
+      "reward": -0.13886644393205644,
+      "reward_std": 0.46603543162345884,
       "step": 100,
-      "step_time": 539.0174913492053
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.012500000186264515,
-      "completions/max_length": 2678.3,
-      "completions/max_terminated_length": 1888.8,
-      "completions/mean_length": 535.3271057128907,
-      "completions/mean_terminated_length": 491.43089904785154,
-      "completions/min_length": 127.5,
-      "completions/min_terminated_length": 127.5,
-      "entropy": 0.1744688918814063,
-      "epoch": 0.13205282112845138,
-      "frac_reward_zero_std": 0.0833333358168602,
-      "grad_norm": 0.043163955211639404,
-      "learning_rate": 6.526946107784432e-06,
-      "loss": -0.0173,
-      "num_tokens": 12060171.0,
-      "reward": -0.2666824638843536,
-      "reward_std": 0.12587157338857652,
-      "rewards/grpo_reward_function/mean": -0.26668245121836665,
-      "rewards/grpo_reward_function/std": 0.40698017328977587,
-      "sampling/importance_sampling_ratio/max": 1.899654006958008,
-      "sampling/importance_sampling_ratio/mean": 0.37773958817124365,
-      "sampling/importance_sampling_ratio/min": 3.46376573256979e-14,
-      "sampling/sampling_logp_difference/max": 2.272650396823883,
-      "sampling/sampling_logp_difference/mean": 0.01878545032814145,
+      "step_time": 1552.164419277385
+    },
+    {
+      "clip_ratio/high_max": 0.0049980103562120345,
+      "clip_ratio/high_mean": 0.002596232306677848,
+      "clip_ratio/low_mean": 0.0018050389306154102,
+      "clip_ratio/low_min": 0.00034353168994130103,
+      "clip_ratio/region_mean": 0.0044012712081894275,
+      "completion_length": 2951.7625,
+      "completions/clipped_ratio": 0.0171875,
+      "completions/max_length": 12211.6,
+      "completions/max_terminated_length": 10464.0,
+      "completions/mean_length": 2951.7625,
+      "completions/mean_terminated_length": 2824.860107421875,
+      "completions/min_length": 531.4,
+      "completions/min_terminated_length": 531.4,
+      "epoch": 0.3525641025641026,
+      "frac_reward_zero_std": 0.01875,
+      "grad_norm": 0.05486765503883362,
+      "kl": 0.07463542837649584,
+      "learning_rate": 1e-06,
+      "loss": 0.0035,
+      "num_tokens": 57540881.0,
+      "reward": -0.12071207985281944,
+      "reward_std": 0.4572619080543518,
       "step": 110,
-      "step_time": 554.1334780954755
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.01666666716337204,
-      "completions/max_length": 3077.6,
-      "completions/max_terminated_length": 1915.6,
-      "completions/mean_length": 565.0541870117188,
-      "completions/mean_terminated_length": 504.5126983642578,
-      "completions/min_length": 138.5,
-      "completions/min_terminated_length": 138.5,
-      "entropy": 0.17697170842438936,
-      "epoch": 0.14405762304921968,
-      "frac_reward_zero_std": 0.0833333358168602,
-      "grad_norm": 0.07391675561666489,
-      "learning_rate": 7.125748502994012e-06,
-      "loss": 0.042,
-      "num_tokens": 13168921.0,
-      "reward": -0.3262443482875824,
-      "reward_std": 0.20909521877765655,
-      "rewards/grpo_reward_function/mean": -0.3262443423271179,
-      "rewards/grpo_reward_function/std": 0.5003126785159111,
-      "sampling/importance_sampling_ratio/max": 2.301289737224579,
-      "sampling/importance_sampling_ratio/mean": 0.38998747766017916,
-      "sampling/importance_sampling_ratio/min": 7.411608444201079e-05,
-      "sampling/sampling_logp_difference/max": 2.7557363152503966,
-      "sampling/sampling_logp_difference/mean": 0.01885297931730747,
+      "step_time": 1562.7062059473246
+    },
+    {
+      "clip_ratio/high_max": 0.004836446885019541,
+      "clip_ratio/high_mean": 0.0024426573538221417,
+      "clip_ratio/low_mean": 0.001846739774919115,
+      "clip_ratio/low_min": 0.0004425366063514957,
+      "clip_ratio/region_mean": 0.004289397125830874,
+      "completion_length": 2979.4328125,
+      "completions/clipped_ratio": 0.0171875,
+      "completions/max_length": 12256.2,
+      "completions/max_terminated_length": 11061.8,
+      "completions/mean_length": 2979.4328125,
+      "completions/mean_terminated_length": 2837.741796875,
+      "completions/min_length": 425.0,
+      "completions/min_terminated_length": 425.0,
+      "epoch": 0.38461538461538464,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.06968267261981964,
+      "kl": 0.0659493409562856,
+      "learning_rate": 1e-06,
+      "loss": 0.0033,
+      "num_tokens": 62880435.0,
+      "reward": -0.143556846678257,
+      "reward_std": 0.46702405214309695,
       "step": 120,
-      "step_time": 549.9578140962869
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.00416666679084301,
-      "completions/max_length": 2290.1,
-      "completions/max_terminated_length": 1886.2,
-      "completions/mean_length": 499.789599609375,
-      "completions/mean_terminated_length": 484.77695617675784,
-      "completions/min_length": 127.0,
-      "completions/min_terminated_length": 127.0,
-      "entropy": 0.19473983831703662,
-      "epoch": 0.15606242496998798,
-      "frac_reward_zero_std": 0.03333333432674408,
-      "grad_norm": 0.06443756073713303,
-      "learning_rate": 7.724550898203594e-06,
-      "loss": -0.0203,
-      "num_tokens": 14212700.0,
-      "reward": -0.26288305670022966,
-      "reward_std": 0.1738448791205883,
-      "rewards/grpo_reward_function/mean": -0.26288305073976515,
-      "rewards/grpo_reward_function/std": 0.48268924951553344,
-      "sampling/importance_sampling_ratio/max": 2.1470563650131225,
-      "sampling/importance_sampling_ratio/mean": 0.35831653475761416,
-      "sampling/importance_sampling_ratio/min": 1.968140890369341e-05,
-      "sampling/sampling_logp_difference/max": 2.0650948524475097,
-      "sampling/sampling_logp_difference/mean": 0.01968124657869339,
+      "step_time": 1574.180371798575
+    },
+    {
+      "clip_ratio/high_max": 0.004874877282418311,
+      "clip_ratio/high_mean": 0.0025931915617547928,
+      "clip_ratio/low_mean": 0.0016370577533962205,
+      "clip_ratio/low_min": 0.00026831475042854436,
+      "clip_ratio/region_mean": 0.004230249358806759,
+      "completion_length": 3022.1375,
+      "completions/clipped_ratio": 0.0171875,
+      "completions/max_length": 12396.6,
+      "completions/max_terminated_length": 10747.4,
+      "completions/mean_length": 3022.1375,
+      "completions/mean_terminated_length": 2876.268359375,
+      "completions/min_length": 552.2,
+      "completions/min_terminated_length": 552.2,
+      "epoch": 0.4166666666666667,
+      "frac_reward_zero_std": 0.01875,
+      "grad_norm": 0.07230757176876068,
+      "kl": 0.06750464583747089,
+      "learning_rate": 1e-06,
+      "loss": 0.0031,
+      "num_tokens": 68253983.0,
+      "reward": -0.13109836876392364,
+      "reward_std": 0.4396223485469818,
       "step": 130,
-      "step_time": 535.0768479405903
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.01250000037252903,
-      "completions/max_length": 3015.9,
-      "completions/max_terminated_length": 1749.5,
-      "completions/mean_length": 492.37709045410156,
-      "completions/mean_terminated_length": 446.93010864257815,
-      "completions/min_length": 114.7,
-      "completions/min_terminated_length": 114.7,
-      "entropy": 0.21128173358738422,
-      "epoch": 0.16806722689075632,
-      "frac_reward_zero_std": 0.05833333432674408,
-      "grad_norm": 0.09337731450796127,
-      "learning_rate": 8.323353293413174e-06,
-      "loss": 0.0277,
-      "num_tokens": 15293077.0,
-      "reward": -0.26287811398506167,
-      "reward_std": 0.14675465896725653,
-      "rewards/grpo_reward_function/mean": -0.26287810802459716,
-      "rewards/grpo_reward_function/std": 0.37649901360273363,
-      "sampling/importance_sampling_ratio/max": 2.3814776659011843,
-      "sampling/importance_sampling_ratio/mean": 0.46577124297618866,
-      "sampling/importance_sampling_ratio/min": 1.3427511260960534e-06,
-      "sampling/sampling_logp_difference/max": 2.155888545513153,
-      "sampling/sampling_logp_difference/mean": 0.019245322328060865,
+      "step_time": 1543.8562961935997
+    },
+    {
+      "clip_ratio/high_max": 0.005032225453760475,
+      "clip_ratio/high_mean": 0.002709129359573126,
+      "clip_ratio/low_mean": 0.001721945361350663,
+      "clip_ratio/low_min": 0.00039209548631333744,
+      "clip_ratio/region_mean": 0.00443107471219264,
+      "completion_length": 2910.215625,
+      "completions/clipped_ratio": 0.0109375,
+      "completions/max_length": 12393.2,
+      "completions/max_terminated_length": 11423.0,
+      "completions/mean_length": 2910.215625,
+      "completions/mean_terminated_length": 2811.431982421875,
+      "completions/min_length": 439.4,
+      "completions/min_terminated_length": 439.4,
+      "epoch": 0.44871794871794873,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.07082553207874298,
+      "kl": 0.07416702015325427,
+      "learning_rate": 1e-06,
+      "loss": 0.0033,
+      "num_tokens": 73489251.0,
+      "reward": -0.18260307610034943,
+      "reward_std": 0.4691228806972504,
       "step": 140,
-      "step_time": 552.9976656335406
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 4.9971032422035935e-05,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 4.9971032422035935e-05,
-      "completions/clipped_ratio": 0.01041666679084301,
-      "completions/max_length": 2447.6,
-      "completions/max_terminated_length": 1785.5,
-      "completions/mean_length": 552.295849609375,
-      "completions/mean_terminated_length": 515.5673645019531,
-      "completions/min_length": 144.7,
-      "completions/min_terminated_length": 144.7,
-      "entropy": 0.2773955374956131,
-      "epoch": 0.18007202881152462,
-      "frac_reward_zero_std": 0.02500000074505806,
-      "grad_norm": 0.1007687970995903,
-      "learning_rate": 8.922155688622756e-06,
-      "loss": -0.0141,
-      "num_tokens": 16425531.0,
-      "reward": -0.26935882605612277,
-      "reward_std": 0.12829533144831656,
-      "rewards/grpo_reward_function/mean": -0.2693588202819228,
-      "rewards/grpo_reward_function/std": 0.33063299730420115,
-      "sampling/importance_sampling_ratio/max": 2.1382891178131103,
-      "sampling/importance_sampling_ratio/mean": 0.4445547193288803,
-      "sampling/importance_sampling_ratio/min": 7.412413807410812e-05,
-      "sampling/sampling_logp_difference/max": 1.7894923090934753,
-      "sampling/sampling_logp_difference/mean": 0.019091704115271568,
+      "step_time": 1559.5770216166973
+    },
+    {
+      "clip_ratio/high_max": 0.004947838565567508,
+      "clip_ratio/high_mean": 0.0025261392001993954,
+      "clip_ratio/low_mean": 0.001768623449606821,
+      "clip_ratio/low_min": 0.00031947743555065243,
+      "clip_ratio/region_mean": 0.0042947627021931115,
+      "completion_length": 3018.18359375,
+      "completions/clipped_ratio": 0.0140625,
+      "completions/max_length": 12244.0,
+      "completions/max_terminated_length": 11541.0,
+      "completions/mean_length": 3018.18359375,
+      "completions/mean_terminated_length": 2896.637255859375,
+      "completions/min_length": 440.2,
+      "completions/min_terminated_length": 440.2,
+      "epoch": 0.4807692307692308,
+      "frac_reward_zero_std": 0.009375,
+      "grad_norm": 0.12085209041833878,
+      "kl": 0.0732723297085613,
+      "learning_rate": 1e-06,
+      "loss": 0.0032,
+      "num_tokens": 78920458.0,
+      "reward": -0.0532445028424263,
+      "reward_std": 0.4656288862228394,
       "step": 150,
-      "step_time": 547.844954107888
-    },
-    {
-      "clip_ratio/high_max": 4.80769231216982e-05,
-      "clip_ratio/high_mean": 8.012820762814954e-06,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 8.012820762814954e-06,
-      "completions/clipped_ratio": 0.010416666977107525,
-      "completions/max_length": 2388.9,
-      "completions/max_terminated_length": 1982.4,
-      "completions/mean_length": 544.4896026611328,
-      "completions/mean_terminated_length": 507.44931945800784,
-      "completions/min_length": 122.8,
-      "completions/min_terminated_length": 122.8,
-      "entropy": 0.3212181769311428,
-      "epoch": 0.19207683073229292,
-      "frac_reward_zero_std": 0.06666666865348816,
-      "grad_norm": 0.06833557039499283,
-      "learning_rate": 9.520958083832336e-06,
-      "loss": 0.0134,
-      "num_tokens": 17564686.0,
-      "reward": -0.28017824441194533,
-      "reward_std": 0.20728585943579675,
-      "rewards/grpo_reward_function/mean": -0.28017824441194533,
-      "rewards/grpo_reward_function/std": 0.5502792000770569,
-      "sampling/importance_sampling_ratio/max": 2.1408735513687134,
-      "sampling/importance_sampling_ratio/mean": 0.5016659319400787,
-      "sampling/importance_sampling_ratio/min": 0.000778414961314411,
-      "sampling/sampling_logp_difference/max": 1.89249027967453,
-      "sampling/sampling_logp_difference/mean": 0.01902961954474449,
+      "step_time": 1578.2899328649044
+    },
+    {
+      "clip_ratio/high_max": 0.004828901478322223,
+      "clip_ratio/high_mean": 0.002561330530443229,
+      "clip_ratio/low_mean": 0.00175229228916578,
+      "clip_ratio/low_min": 0.00037519693905778694,
+      "clip_ratio/region_mean": 0.004313622799236328,
+      "completion_length": 2982.4328125,
+      "completions/clipped_ratio": 0.01875,
+      "completions/max_length": 12009.2,
+      "completions/max_terminated_length": 10664.8,
+      "completions/mean_length": 2982.4328125,
+      "completions/mean_terminated_length": 2828.51669921875,
+      "completions/min_length": 435.6,
+      "completions/min_terminated_length": 435.6,
+      "epoch": 0.5128205128205128,
+      "frac_reward_zero_std": 0.00625,
+      "grad_norm": 0.03947965055704117,
+      "kl": 0.07998953224159777,
+      "learning_rate": 1e-06,
+      "loss": 0.0058,
+      "num_tokens": 84203428.0,
+      "reward": -0.10095882713794709,
+      "reward_std": 0.5111252725124359,
       "step": 160,
-      "step_time": 536.2867619435303
+      "step_time": 1560.6413940612226
+    },
+    {
+      "clip_ratio/high_max": 0.0050891352235339585,
+      "clip_ratio/high_mean": 0.002622560871532187,
+      "clip_ratio/low_mean": 0.0017492718761786819,
+      "clip_ratio/low_min": 0.0002873761834052857,
+      "clip_ratio/region_mean": 0.004371832753531635,
+      "completion_length": 3032.325,
+      "completions/clipped_ratio": 0.01328125,
+      "completions/max_length": 11998.6,
+      "completions/max_terminated_length": 11143.4,
+      "completions/mean_length": 3032.325,
+      "completions/mean_terminated_length": 2927.192138671875,
+      "completions/min_length": 433.2,
+      "completions/min_terminated_length": 433.2,
+      "epoch": 0.5448717948717948,
+      "frac_reward_zero_std": 0.021875,
+      "grad_norm": 0.058390598744153976,
+      "kl": 0.07743198843672872,
+      "learning_rate": 1e-06,
+      "loss": 0.0048,
+      "num_tokens": 89623596.0,
+      "reward": -0.15904466956853866,
+      "reward_std": 0.4742555618286133,
+      "step": 170,
+      "step_time": 1577.4129925977438
+    },
+    {
+      "clip_ratio/high_max": 0.0044553753221407534,
+      "clip_ratio/high_mean": 0.0023813721840269864,
+      "clip_ratio/low_mean": 0.0017972777190152557,
+      "clip_ratio/low_min": 0.00036517053522402423,
+      "clip_ratio/region_mean": 0.004178649949608371,
+      "completion_length": 2987.81640625,
+      "completions/clipped_ratio": 0.01796875,
+      "completions/max_length": 11952.2,
+      "completions/max_terminated_length": 11009.6,
+      "completions/mean_length": 2987.81640625,
+      "completions/mean_terminated_length": 2860.55859375,
+      "completions/min_length": 475.6,
+      "completions/min_terminated_length": 475.6,
+      "epoch": 0.5769230769230769,
+      "frac_reward_zero_std": 0.028125,
+      "grad_norm": 0.04907602816820145,
+      "kl": 0.07805888415314258,
+      "learning_rate": 1e-06,
+      "loss": 0.0032,
+      "num_tokens": 94955621.0,
+      "reward": -0.11422853395342827,
+      "reward_std": 0.4412929594516754,
+      "step": 180,
+      "step_time": 1558.3106328308581
+    },
+    {
+      "clip_ratio/high_max": 0.005000349332112819,
+      "clip_ratio/high_mean": 0.0025746225466718895,
+      "clip_ratio/low_mean": 0.0017240287095773965,
+      "clip_ratio/low_min": 0.0003595072033931501,
+      "clip_ratio/region_mean": 0.004298651299905032,
+      "completion_length": 2952.6140625,
+      "completions/clipped_ratio": 0.01328125,
+      "completions/max_length": 12079.8,
+      "completions/max_terminated_length": 10907.2,
+      "completions/mean_length": 2952.6140625,
+      "completions/mean_terminated_length": 2858.857958984375,
+      "completions/min_length": 423.0,
+      "completions/min_terminated_length": 423.0,
+      "epoch": 0.6089743589743589,
+      "frac_reward_zero_std": 0.00625,
+      "grad_norm": 0.030720015987753868,
+      "kl": 0.0846875250339508,
+      "learning_rate": 1e-06,
+      "loss": 0.0033,
+      "num_tokens": 100229867.0,
+      "reward": -0.11310222297906876,
+      "reward_std": 0.4591243326663971,
+      "step": 190,
+      "step_time": 1537.4745403777808
+    },
+    {
+      "clip_ratio/high_max": 0.005211382452398538,
+      "clip_ratio/high_mean": 0.002718880938482471,
+      "clip_ratio/low_mean": 0.0016983163979602978,
+      "clip_ratio/low_min": 0.0002450207251968095,
+      "clip_ratio/region_mean": 0.004417197342263535,
+      "completion_length": 2891.078125,
+      "completions/clipped_ratio": 0.0140625,
+      "completions/max_length": 12289.4,
+      "completions/max_terminated_length": 11364.6,
+      "completions/mean_length": 2891.078125,
+      "completions/mean_terminated_length": 2776.784423828125,
+      "completions/min_length": 498.4,
+      "completions/min_terminated_length": 498.4,
+      "epoch": 0.6410256410256411,
+      "frac_reward_zero_std": 0.009375,
+      "grad_norm": 0.039781469851732254,
+      "kl": 0.09279958680272102,
+      "learning_rate": 1e-06,
+      "loss": 0.0036,
+      "num_tokens": 105422531.0,
+      "reward": -0.07257482595741749,
+      "reward_std": 0.48845980763435365,
+      "step": 200,
+      "step_time": 1575.8219865739345
+    },
+    {
+      "clip_ratio/high_max": 0.005096805078210309,
+      "clip_ratio/high_mean": 0.002603888895828277,
+      "clip_ratio/low_mean": 0.0017408629733836277,
+      "clip_ratio/low_min": 0.00032388986510341057,
+      "clip_ratio/region_mean": 0.004344751877943054,
+      "completion_length": 2946.13046875,
+      "completions/clipped_ratio": 0.0109375,
+      "completions/max_length": 12265.2,
+      "completions/max_terminated_length": 11185.6,
+      "completions/mean_length": 2946.13046875,
+      "completions/mean_terminated_length": 2858.392626953125,
+      "completions/min_length": 472.0,
+      "completions/min_terminated_length": 472.0,
+      "epoch": 0.6730769230769231,
+      "frac_reward_zero_std": 0.003125,
+      "grad_norm": 0.07236344367265701,
+      "kl": 0.10056616412475705,
+      "learning_rate": 1e-06,
+      "loss": 0.0042,
+      "num_tokens": 110705178.0,
+      "reward": -0.1346050776541233,
+      "reward_std": 0.5192839920520782,
+      "step": 210,
+      "step_time": 1570.4351487074048
+    },
+    {
+      "clip_ratio/high_max": 0.004975579242454842,
+      "clip_ratio/high_mean": 0.0025332159362733363,
+      "clip_ratio/low_mean": 0.0016626121301669627,
+      "clip_ratio/low_min": 0.0003221739400032675,
+      "clip_ratio/region_mean": 0.004195828136289492,
+      "completion_length": 3088.19765625,
+      "completions/clipped_ratio": 0.01953125,
+      "completions/max_length": 12473.2,
+      "completions/max_terminated_length": 10285.8,
+      "completions/mean_length": 3088.19765625,
+      "completions/mean_terminated_length": 2927.726904296875,
+      "completions/min_length": 497.0,
+      "completions/min_terminated_length": 497.0,
+      "epoch": 0.7051282051282052,
+      "frac_reward_zero_std": 0.00625,
+      "grad_norm": 0.03493595868349075,
+      "kl": 0.07714751800522208,
+      "learning_rate": 1e-06,
+      "loss": 0.0034,
+      "num_tokens": 116201787.0,
+      "reward": -0.10951823592185975,
+      "reward_std": 0.4896236062049866,
+      "step": 220,
+      "step_time": 1562.683198102936
+    },
+    {
+      "clip_ratio/high_max": 0.004581616725772619,
+      "clip_ratio/high_mean": 0.002467807347420603,
+      "clip_ratio/low_mean": 0.0017717583192279561,
+      "clip_ratio/low_min": 0.00032751168619142844,
+      "clip_ratio/region_mean": 0.004239565646275878,
+      "completion_length": 3070.24140625,
+      "completions/clipped_ratio": 0.0171875,
+      "completions/max_length": 12038.8,
+      "completions/max_terminated_length": 11155.2,
+      "completions/mean_length": 3070.24140625,
+      "completions/mean_terminated_length": 2950.84599609375,
+      "completions/min_length": 494.6,
+      "completions/min_terminated_length": 494.6,
+      "epoch": 0.7371794871794872,
+      "frac_reward_zero_std": 0.00625,
+      "grad_norm": 0.052283648401498795,
+      "kl": 0.09406707161106169,
+      "learning_rate": 1e-06,
+      "loss": 0.0034,
+      "num_tokens": 121654760.0,
+      "reward": -0.12395322620868683,
+      "reward_std": 0.5191160619258881,
+      "step": 230,
+      "step_time": 1571.9505534119903
+    },
+    {
+      "clip_ratio/high_max": 0.004726700356695801,
+      "clip_ratio/high_mean": 0.002542574534891173,
+      "clip_ratio/low_mean": 0.0017785578849725426,
+      "clip_ratio/low_min": 0.0004070574679644778,
+      "clip_ratio/region_mean": 0.004321132384939119,
+      "completion_length": 2982.69453125,
+      "completions/clipped_ratio": 0.021875,
+      "completions/max_length": 12443.8,
+      "completions/max_terminated_length": 11316.0,
+      "completions/mean_length": 2982.69453125,
+      "completions/mean_terminated_length": 2801.6517578125,
+      "completions/min_length": 499.8,
+      "completions/min_terminated_length": 499.8,
+      "epoch": 0.7692307692307693,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.043314285576343536,
+      "kl": 0.09030559090897441,
+      "learning_rate": 1e-06,
+      "loss": 0.0034,
+      "num_tokens": 126941533.0,
+      "reward": -0.12113026678562164,
+      "reward_std": 0.450360906124115,
+      "step": 240,
+      "step_time": 1579.2397240422665
+    },
+    {
+      "clip_ratio/high_max": 0.005093035561731085,
+      "clip_ratio/high_mean": 0.0025205660553183407,
+      "clip_ratio/low_mean": 0.0017172211373690516,
+      "clip_ratio/low_min": 0.0003752105563762598,
+      "clip_ratio/region_mean": 0.004237787169404328,
+      "completion_length": 3072.68984375,
+      "completions/clipped_ratio": 0.01796875,
+      "completions/max_length": 12340.4,
+      "completions/max_terminated_length": 10899.8,
+      "completions/mean_length": 3072.68984375,
+      "completions/mean_terminated_length": 2927.21953125,
+      "completions/min_length": 473.2,
+      "completions/min_terminated_length": 473.2,
+      "epoch": 0.8012820512820513,
+      "frac_reward_zero_std": 0.009375,
+      "grad_norm": 0.08550040423870087,
+      "kl": 0.09120887373574078,
+      "learning_rate": 1e-06,
+      "loss": 0.0036,
+      "num_tokens": 132397820.0,
+      "reward": -0.06451257765293121,
+      "reward_std": 0.44462894201278685,
+      "step": 250,
+      "step_time": 1581.8617346592248
+    },
+    {
+      "clip_ratio/high_max": 0.004506677895551548,
+      "clip_ratio/high_mean": 0.0024061154370428996,
+      "clip_ratio/low_mean": 0.0019920992257539184,
+      "clip_ratio/low_min": 0.0003064149113924941,
+      "clip_ratio/region_mean": 0.004398214694811031,
+      "completion_length": 2950.0640625,
+      "completions/clipped_ratio": 0.0078125,
+      "completions/max_length": 12021.2,
+      "completions/max_terminated_length": 11219.8,
+      "completions/mean_length": 2950.0640625,
+      "completions/mean_terminated_length": 2889.98876953125,
+      "completions/min_length": 454.2,
+      "completions/min_terminated_length": 454.2,
+      "epoch": 0.8333333333333334,
+      "frac_reward_zero_std": 0.009375,
+      "grad_norm": 0.06680677831172943,
+      "kl": 0.08958462765440345,
+      "learning_rate": 1e-06,
+      "loss": 0.0036,
+      "num_tokens": 137689982.0,
+      "reward": -0.06169652491807938,
+      "reward_std": 0.4440364301204681,
+      "step": 260,
+      "step_time": 1548.698206745088
+    },
+    {
+      "clip_ratio/high_max": 0.004509920469718054,
+      "clip_ratio/high_mean": 0.0023977969191037118,
+      "clip_ratio/low_mean": 0.001890558234299533,
+      "clip_ratio/low_min": 0.00027352118922863156,
+      "clip_ratio/region_mean": 0.004288355156313628,
+      "completion_length": 3148.22578125,
+      "completions/clipped_ratio": 0.009375,
+      "completions/max_length": 12400.4,
+      "completions/max_terminated_length": 11354.6,
+      "completions/mean_length": 3148.22578125,
+      "completions/mean_terminated_length": 3064.138818359375,
+      "completions/min_length": 646.2,
+      "completions/min_terminated_length": 646.2,
+      "epoch": 0.8653846153846154,
+      "frac_reward_zero_std": 0.003125,
+      "grad_norm": 0.03709687665104866,
+      "kl": 0.5683331843931227,
+      "learning_rate": 1e-06,
+      "loss": 0.0106,
+      "num_tokens": 143248091.0,
+      "reward": -0.012586714699864388,
+      "reward_std": 0.44408372044563293,
+      "step": 270,
+      "step_time": 1599.3815574798732
+    },
+    {
+      "clip_ratio/high_max": 0.005211462150327861,
+      "clip_ratio/high_mean": 0.0027846890938235448,
+      "clip_ratio/low_mean": 0.001632543487357907,
+      "clip_ratio/low_min": 0.00021985859493725,
+      "clip_ratio/region_mean": 0.004417232552077622,
+      "completion_length": 3046.190625,
+      "completions/clipped_ratio": 0.01640625,
+      "completions/max_length": 12181.6,
+      "completions/max_terminated_length": 11029.2,
+      "completions/mean_length": 3046.190625,
+      "completions/mean_terminated_length": 2912.7576171875,
+      "completions/min_length": 584.4,
+      "completions/min_terminated_length": 584.4,
+      "epoch": 0.8974358974358975,
+      "frac_reward_zero_std": 0.015625,
+      "grad_norm": 0.03959239274263382,
+      "kl": 0.07848452138714493,
+      "learning_rate": 1e-06,
+      "loss": 0.0029,
+      "num_tokens": 148676855.0,
+      "reward": -0.06608309932053089,
+      "reward_std": 0.46152743697166443,
+      "step": 280,
+      "step_time": 1586.691116861999
+    },
+    {
+      "clip_ratio/high_max": 0.0044549714657478034,
+      "clip_ratio/high_mean": 0.00228276529815048,
+      "clip_ratio/low_mean": 0.0018995442485902458,
+      "clip_ratio/low_min": 0.0003753020158910658,
+      "clip_ratio/region_mean": 0.004182309511816129,
+      "completion_length": 3087.03671875,
+      "completions/clipped_ratio": 0.01328125,
+      "completions/max_length": 12111.0,
+      "completions/max_terminated_length": 10906.0,
+      "completions/mean_length": 3087.03671875,
+      "completions/mean_terminated_length": 2989.638037109375,
+      "completions/min_length": 512.6,
+      "completions/min_terminated_length": 512.6,
+      "epoch": 0.9294871794871795,
+      "frac_reward_zero_std": 0.0125,
+      "grad_norm": 0.04725560173392296,
+      "kl": 0.08775982772931457,
+      "learning_rate": 1e-06,
+      "loss": 0.0034,
+      "num_tokens": 154130022.0,
+      "reward": -0.09921637326478958,
+      "reward_std": 0.439280241727829,
+      "step": 290,
+      "step_time": 1544.2581059247254
+    },
+    {
+      "clip_ratio/high_max": 0.0051462724048178645,
+      "clip_ratio/high_mean": 0.0026393055770313366,
+      "clip_ratio/low_mean": 0.0017790017009247094,
+      "clip_ratio/low_min": 0.00037529252331296447,
+      "clip_ratio/region_mean": 0.004418307275045663,
+      "completion_length": 2989.03828125,
+      "completions/clipped_ratio": 0.0109375,
+      "completions/max_length": 12238.6,
+      "completions/max_terminated_length": 11042.8,
+      "completions/mean_length": 2989.03828125,
+      "completions/mean_terminated_length": 2906.191015625,
+      "completions/min_length": 609.8,
+      "completions/min_terminated_length": 609.8,
+      "epoch": 0.9615384615384616,
+      "frac_reward_zero_std": 0.00625,
+      "grad_norm": 0.028459740802645683,
+      "kl": 0.08268881598487496,
+      "learning_rate": 1e-06,
+      "loss": 0.0033,
+      "num_tokens": 159413967.0,
+      "reward": -0.05725412368774414,
+      "reward_std": 0.41517805457115176,
+      "step": 300,
+      "step_time": 1574.2186388261616
+    },
+    {
+      "clip_ratio/high_max": 0.005078576249070465,
+      "clip_ratio/high_mean": 0.0026279066572897137,
+      "clip_ratio/low_mean": 0.0016655289655318484,
+      "clip_ratio/low_min": 0.00025660531136963984,
+      "clip_ratio/region_mean": 0.004293435596628115,
+      "completion_length": 3030.18125,
+      "completions/clipped_ratio": 0.01171875,
+      "completions/max_length": 11755.8,
+      "completions/max_terminated_length": 11019.2,
+      "completions/mean_length": 3030.18125,
+      "completions/mean_terminated_length": 2951.3966796875,
+      "completions/min_length": 498.8,
+      "completions/min_terminated_length": 498.8,
+      "epoch": 0.9935897435897436,
+      "frac_reward_zero_std": 0.015625,
+      "grad_norm": 0.046144235879182816,
+      "kl": 0.10964574567042291,
+      "learning_rate": 1e-06,
+      "loss": 0.0039,
+      "num_tokens": 164813083.0,
+      "reward": -0.07799501270055771,
+      "reward_std": 0.4621491074562073,
+      "step": 310,
+      "step_time": 1557.5355068217964
+    },
+    {
+      "clip_ratio/high_max": 0.0043965487508103255,
+      "clip_ratio/high_mean": 0.0022311770444503054,
+      "clip_ratio/low_mean": 0.001934563010581769,
+      "clip_ratio/low_min": 0.00036791299098695165,
+      "clip_ratio/region_mean": 0.00416574003174901,
+      "completion_length": 3090.36796875,
+      "completions/clipped_ratio": 0.0140625,
+      "completions/max_length": 12469.4,
+      "completions/max_terminated_length": 10506.6,
+      "completions/mean_length": 3090.36796875,
+      "completions/mean_terminated_length": 2961.30146484375,
+      "completions/min_length": 505.4,
+      "completions/min_terminated_length": 505.4,
+      "epoch": 1.0256410256410255,
+      "frac_reward_zero_std": 0.015625,
+      "grad_norm": 0.028119955211877823,
+      "kl": 0.0761645772960037,
+      "learning_rate": 1e-06,
+      "loss": 0.0031,
+      "num_tokens": 170277530.0,
+      "reward": 0.032285493612289426,
+      "reward_std": 0.477168345451355,
+      "step": 320,
+      "step_time": 1611.2154974307864
+    },
+    {
+      "clip_ratio/high_max": 0.004805623926222325,
+      "clip_ratio/high_mean": 0.002526776128797792,
+      "clip_ratio/low_mean": 0.0018500737234717235,
+      "clip_ratio/low_min": 0.0004184581914159935,
+      "clip_ratio/region_mean": 0.004376849811524153,
+      "completion_length": 3006.925,
+      "completions/clipped_ratio": 0.009375,
+      "completions/max_length": 12146.0,
+      "completions/max_terminated_length": 10995.2,
+      "completions/mean_length": 3006.925,
+      "completions/mean_terminated_length": 2928.357421875,
+      "completions/min_length": 498.0,
+      "completions/min_terminated_length": 498.0,
+      "epoch": 1.0576923076923077,
+      "frac_reward_zero_std": 0.009375,
+      "grad_norm": 0.031013675034046173,
+      "kl": 0.19318538415245712,
+      "learning_rate": 1e-06,
+      "loss": 0.0052,
+      "num_tokens": 175636454.0,
+      "reward": -6.786882877349854e-05,
+      "reward_std": 0.41322546601295473,
+      "step": 330,
+      "step_time": 1601.2282649662345
+    },
+    {
+      "clip_ratio/high_max": 0.004916327743558213,
+      "clip_ratio/high_mean": 0.0025603161833714695,
+      "clip_ratio/low_mean": 0.001742898570955731,
+      "clip_ratio/low_min": 0.00033117499042418786,
+      "clip_ratio/region_mean": 0.004303214780520648,
+      "completion_length": 2992.98671875,
+      "completions/clipped_ratio": 0.009375,
+      "completions/max_length": 12516.4,
+      "completions/max_terminated_length": 12072.0,
+      "completions/mean_length": 2992.98671875,
+      "completions/mean_terminated_length": 2912.883056640625,
+      "completions/min_length": 504.2,
+      "completions/min_terminated_length": 504.2,
+      "epoch": 1.0897435897435896,
+      "frac_reward_zero_std": 0.009375,
+      "grad_norm": 0.025729358196258545,
+      "kl": 0.08641395247541368,
+      "learning_rate": 1e-06,
+      "loss": 0.0034,
+      "num_tokens": 180997513.0,
+      "reward": 0.0007174957543611526,
+      "reward_std": 0.44232319593429564,
+      "step": 340,
+      "step_time": 1617.1940580930561
+    },
+    {
+      "clip_ratio/high_max": 0.005060376110486686,
+      "clip_ratio/high_mean": 0.0024434949416900055,
+      "clip_ratio/low_mean": 0.0018809563858667388,
+      "clip_ratio/low_min": 0.00038810843543615194,
+      "clip_ratio/region_mean": 0.004324451251886785,
+      "completion_length": 3072.35078125,
+      "completions/clipped_ratio": 0.01328125,
+      "completions/max_length": 12338.6,
+      "completions/max_terminated_length": 10769.8,
+      "completions/mean_length": 3072.35078125,
+      "completions/mean_terminated_length": 2963.105810546875,
+      "completions/min_length": 410.0,
+      "completions/min_terminated_length": 410.0,
+      "epoch": 1.1217948717948718,
+      "frac_reward_zero_std": 0.015625,
+      "grad_norm": 0.040223997086286545,
+      "kl": 1.0846694989595562,
+      "learning_rate": 1e-06,
+      "loss": 0.0157,
+      "num_tokens": 186367566.0,
+      "reward": -0.006166191399097442,
+      "reward_std": 0.40017282366752627,
+      "step": 350,
+      "step_time": 1591.564345838502
+    },
+    {
+      "clip_ratio/high_max": 0.005041535885538906,
+      "clip_ratio/high_mean": 0.002488660369999707,
+      "clip_ratio/low_mean": 0.001798816752852872,
+      "clip_ratio/low_min": 0.000395576364826411,
+      "clip_ratio/region_mean": 0.004287477117031813,
+      "completion_length": 2985.525,
+      "completions/clipped_ratio": 0.01171875,
+      "completions/max_length": 12207.2,
+      "completions/max_terminated_length": 10894.8,
+      "completions/mean_length": 2985.525,
+      "completions/mean_terminated_length": 2891.112939453125,
+      "completions/min_length": 525.8,
+      "completions/min_terminated_length": 525.8,
+      "epoch": 1.1538461538461537,
+      "frac_reward_zero_std": 0.009375,
+      "grad_norm": 0.016880759969353676,
+      "kl": 0.08941859514452517,
+      "learning_rate": 1e-06,
+      "loss": 0.0032,
+      "num_tokens": 191676006.0,
+      "reward": 0.014957365393638612,
+      "reward_std": 0.4513224720954895,
+      "step": 360,
+      "step_time": 1626.5114137839525
+    },
+    {
+      "clip_ratio/high_max": 0.005261168797733262,
+      "clip_ratio/high_mean": 0.0024944149190559983,
+      "clip_ratio/low_mean": 0.0016844567406224086,
+      "clip_ratio/low_min": 0.0003325651761770132,
+      "clip_ratio/region_mean": 0.004178871621843427,
+      "completion_length": 3179.51796875,
+      "completions/clipped_ratio": 0.0140625,
+      "completions/max_length": 12343.6,
+      "completions/max_terminated_length": 11213.6,
+      "completions/mean_length": 3179.51796875,
+      "completions/mean_terminated_length": 3066.6662109375,
+      "completions/min_length": 521.6,
+      "completions/min_terminated_length": 521.6,
+      "epoch": 1.185897435897436,
+      "frac_reward_zero_std": 0.003125,
+      "grad_norm": 0.02692381851375103,
+      "kl": 0.08805363420397043,
+      "learning_rate": 1e-06,
+      "loss": 0.0033,
+      "num_tokens": 197259477.0,
+      "reward": 0.022652272135019302,
+      "reward_std": 0.45602840185165405,
+      "step": 370,
+      "step_time": 1598.871085421741
+    },
+    {
+      "clip_ratio/high_max": 0.004819638578919694,
+      "clip_ratio/high_mean": 0.0024953169282525777,
+      "clip_ratio/low_mean": 0.0015949052845826372,
+      "clip_ratio/low_min": 0.0004186905334790936,
+      "clip_ratio/region_mean": 0.004090222186641767,
+      "completion_length": 3240.63515625,
+      "completions/clipped_ratio": 0.01640625,
+      "completions/max_length": 12522.4,
+      "completions/max_terminated_length": 11620.4,
+      "completions/mean_length": 3240.63515625,
+      "completions/mean_terminated_length": 3111.850390625,
+      "completions/min_length": 449.0,
+      "completions/min_terminated_length": 449.0,
+      "epoch": 1.217948717948718,
+      "frac_reward_zero_std": 0.01875,
+      "grad_norm": 0.0450025238096714,
+      "kl": 0.08879370880313217,
+      "learning_rate": 1e-06,
+      "loss": 0.0035,
+      "num_tokens": 202918994.0,
+      "reward": 0.016886448860168456,
+      "reward_std": 0.4428517997264862,
+      "step": 380,
+      "step_time": 1608.9123431723565
+    },
+    {
+      "clip_ratio/high_max": 0.004545018979115412,
+      "clip_ratio/high_mean": 0.00236421900044661,
+      "clip_ratio/low_mean": 0.0018956056766910479,
+      "clip_ratio/low_min": 0.0002857181410945486,
+      "clip_ratio/region_mean": 0.004259824682958424,
+      "completion_length": 3109.42265625,
+      "completions/clipped_ratio": 0.0171875,
+      "completions/max_length": 12422.0,
+      "completions/max_terminated_length": 11103.8,
+      "completions/mean_length": 3109.42265625,
+      "completions/mean_terminated_length": 2966.233984375,
+      "completions/min_length": 596.2,
+      "completions/min_terminated_length": 596.2,
+      "epoch": 1.25,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.022042548283934593,
+      "kl": 0.07720592515543104,
+      "learning_rate": 1e-06,
+      "loss": 0.0029,
+      "num_tokens": 208431635.0,
+      "reward": -0.03826795257627964,
+      "reward_std": 0.4387412369251251,
+      "step": 390,
+      "step_time": 1600.6475246451796
+    },
+    {
+      "clip_ratio/high_max": 0.004809655511053279,
+      "clip_ratio/high_mean": 0.0025483112316578628,
+      "clip_ratio/low_mean": 0.001705722790211439,
+      "clip_ratio/low_min": 0.0003331981468363665,
+      "clip_ratio/region_mean": 0.004254033981123939,
+      "completion_length": 3085.91484375,
+      "completions/clipped_ratio": 0.021875,
+      "completions/max_length": 12568.8,
+      "completions/max_terminated_length": 11250.8,
+      "completions/mean_length": 3085.91484375,
+      "completions/mean_terminated_length": 2922.940380859375,
+      "completions/min_length": 456.4,
+      "completions/min_terminated_length": 456.4,
+      "epoch": 1.282051282051282,
+      "frac_reward_zero_std": 0.003125,
+      "grad_norm": 0.022028187289834023,
+      "kl": 0.08367300066165625,
+      "learning_rate": 1e-06,
+      "loss": 0.003,
+      "num_tokens": 213906230.0,
+      "reward": -0.024722617119550705,
+      "reward_std": 0.4778853476047516,
+      "step": 400,
+      "step_time": 1613.2906013470142
+    },
+    {
+      "clip_ratio/high_max": 0.0047057366231456395,
+      "clip_ratio/high_mean": 0.002369255464873277,
+      "clip_ratio/low_mean": 0.0021425710641779006,
+      "clip_ratio/low_min": 0.0004687798318627756,
+      "clip_ratio/region_mean": 0.004511826549423859,
+      "completion_length": 2841.19375,
+      "completions/clipped_ratio": 0.00390625,
+      "completions/max_length": 11727.8,
+      "completions/max_terminated_length": 10734.2,
+      "completions/mean_length": 2841.19375,
+      "completions/mean_terminated_length": 2808.933056640625,
+      "completions/min_length": 479.0,
+      "completions/min_terminated_length": 479.0,
+      "epoch": 1.314102564102564,
+      "frac_reward_zero_std": 0.009375,
+      "grad_norm": 0.03915712609887123,
+      "kl": 0.09414773643948138,
+      "learning_rate": 1e-06,
+      "loss": 0.0034,
+      "num_tokens": 219005902.0,
+      "reward": 0.07219201363623143,
+      "reward_std": 0.4331868588924408,
+      "step": 410,
+      "step_time": 1596.245654945448
+    },
+    {
+      "clip_ratio/high_max": 0.005167479033116251,
+      "clip_ratio/high_mean": 0.0025030544755281882,
+      "clip_ratio/low_mean": 0.0017537950101541356,
+      "clip_ratio/low_min": 0.00045870429603382944,
+      "clip_ratio/region_mean": 0.004256849479861558,
+      "completion_length": 3214.5515625,
+      "completions/clipped_ratio": 0.0125,
+      "completions/max_length": 12340.2,
+      "completions/max_terminated_length": 11401.6,
+      "completions/mean_length": 3214.5515625,
+      "completions/mean_terminated_length": 3117.20849609375,
+      "completions/min_length": 586.4,
+      "completions/min_terminated_length": 586.4,
+      "epoch": 1.3461538461538463,
+      "frac_reward_zero_std": 0.003125,
+      "grad_norm": 0.03607157990336418,
+      "kl": 0.10746489656157791,
+      "learning_rate": 1e-06,
+      "loss": 0.0037,
+      "num_tokens": 224646816.0,
+      "reward": -0.009741688147187233,
+      "reward_std": 0.4510599136352539,
+      "step": 420,
+      "step_time": 1592.6302075430751
+    },
+    {
+      "clip_ratio/high_max": 0.005128868587780744,
+      "clip_ratio/high_mean": 0.002558647794649005,
+      "clip_ratio/low_mean": 0.0017414343266864308,
+      "clip_ratio/low_min": 0.0003404129995033145,
+      "clip_ratio/region_mean": 0.004300082137342542,
+      "completion_length": 3075.0328125,
+      "completions/clipped_ratio": 0.00859375,
+      "completions/max_length": 12154.2,
+      "completions/max_terminated_length": 11370.6,
+      "completions/mean_length": 3075.0328125,
+      "completions/mean_terminated_length": 3001.669970703125,
+      "completions/min_length": 430.4,
+      "completions/min_terminated_length": 430.4,
+      "epoch": 1.3782051282051282,
+      "frac_reward_zero_std": 0.009375,
+      "grad_norm": 0.031504735350608826,
+      "kl": 0.09796748813241721,
+      "learning_rate": 1e-06,
+      "loss": 0.0039,
+      "num_tokens": 230085374.0,
+      "reward": 0.018697647005319597,
+      "reward_std": 0.4792678475379944,
+      "step": 430,
+      "step_time": 1587.6049383770674
+    },
+    {
+      "clip_ratio/high_max": 0.004814739798894152,
+      "clip_ratio/high_mean": 0.002512058726279065,
+      "clip_ratio/low_mean": 0.001695432874839753,
+      "clip_ratio/low_min": 0.0004412926449731458,
+      "clip_ratio/region_mean": 0.00420749161276035,
+      "completion_length": 3140.99140625,
+      "completions/clipped_ratio": 0.01484375,
+      "completions/max_length": 12021.8,
+      "completions/max_terminated_length": 10888.4,
+      "completions/mean_length": 3140.99140625,
+      "completions/mean_terminated_length": 3044.864990234375,
+      "completions/min_length": 545.6,
+      "completions/min_terminated_length": 545.6,
+      "epoch": 1.4102564102564101,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.04127124324440956,
+      "kl": 0.09882787386886775,
+      "learning_rate": 1e-06,
+      "loss": 0.0037,
+      "num_tokens": 235589751.0,
+      "reward": 0.03319392055273056,
+      "reward_std": 0.4292157649993896,
+      "step": 440,
+      "step_time": 1557.1211522892118
     }
   ],
   "logging_steps": 10,
-  "max_steps": 833,
-  "num_input_tokens_seen": 18381921,
-  "num_train_epochs": 1,
-  "save_steps": 167,
+  "max_steps": 624,
+  "num_input_tokens_seen": 236723765,
+  "num_train_epochs": 2,
+  "save_steps": 63,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
@@ -540,7 +1172,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 2,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:437756794d1fd37b2071eda7302025349a9a8e2096d03eecc649e4dede814921
-size 9041
+oid sha256:0a583037941678db9e186c03957a03add71d67fdbeee2980aec32a31f9e985a6
+size 9233