Model save

README.md

@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/ryanzhang123-the-university-of-auckland/RLCR/runs/7uoij859) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/ryanzhang123-the-university-of-auckland/RLCR/runs/vgprtp13) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5cbb52a1207ba56f8b0e8fe6458e0e811a1a2e6ecb80ef1af9e299fd57178083
+oid sha256:fd417f8052f768e2bb60d6f379953d6c0e49f32e7a32a179c7feeb00525c423a
 size 3554214752

trainer_state.json

@@ -3,7 +3,7 @@
   "best_model_checkpoint": null,
   "epoch": 2.9790660225442833,
   "eval_steps": 50,
-  "global_step": 230,
+  "global_step": 57,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -14,188 +14,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.025,
-      "completions/max_length": 1310.4,
-      "completions/max_terminated_length": 1006.8,
-      "completions/mean_length": 270.43125,
-      "completions/mean_terminated_length": 237.94033203125,
-      "completions/min_length": 30.8,
-      "completions/min_terminated_length": 30.8,
-      "epoch": 0.0322061191626409,
-      "grad_norm": 0.035579483956098557,
-      "learning_rate": 1e-06,
-      "loss": -0.0011,
-      "num_tokens": 154629.0,
-      "reward": 0.415625,
-      "reward_std": 0.12983438968658448,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.83125,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 5
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.028125,
-      "completions/max_length": 1536.0,
-      "completions/max_terminated_length": 1284.8,
-      "completions/mean_length": 262.8453125,
-      "completions/mean_terminated_length": 225.93606567382812,
-      "completions/min_length": 27.6,
-      "completions/min_terminated_length": 27.6,
-      "epoch": 0.1288244766505636,
-      "grad_norm": 0.005246501415967941,
-      "learning_rate": 8.333333333333333e-07,
-      "loss": 0.0155,
-      "num_tokens": 800930.0,
-      "reward": 0.43046875,
-      "reward_std": 0.1471163362264633,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.8609375,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 10
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.009375,
-      "completions/max_length": 1399.0,
-      "completions/max_terminated_length": 1281.0,
-      "completions/mean_length": 232.3828125,
-      "completions/mean_terminated_length": 219.98731079101563,
-      "completions/min_length": 19.6,
-      "completions/min_terminated_length": 19.6,
-      "epoch": 0.1932367149758454,
-      "grad_norm": 0.33856287598609924,
-      "learning_rate": 1e-06,
-      "loss": 0.0134,
-      "num_tokens": 1427423.0,
-      "reward": 0.4484375,
-      "reward_std": 0.11558105051517487,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.896875,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 15
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.025,
-      "completions/max_length": 1536.0,
-      "completions/max_terminated_length": 1241.4,
-      "completions/mean_length": 230.415625,
-      "completions/mean_terminated_length": 196.781982421875,
-      "completions/min_length": 21.0,
-      "completions/min_terminated_length": 21.0,
-      "epoch": 0.2576489533011272,
-      "grad_norm": 0.00546436570584774,
-      "learning_rate": 1e-06,
-      "loss": 0.0153,
-      "num_tokens": 2065281.0,
-      "reward": 0.44921875,
-      "reward_std": 0.10368926376104355,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.8984375,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 20
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0125,
-      "completions/max_length": 1536.0,
-      "completions/max_terminated_length": 1338.2,
-      "completions/mean_length": 229.59375,
-      "completions/mean_terminated_length": 213.0333465576172,
-      "completions/min_length": 19.8,
-      "completions/min_terminated_length": 19.8,
-      "epoch": 0.322061191626409,
-      "grad_norm": 0.0021914097014814615,
-      "learning_rate": 1e-06,
-      "loss": 0.0134,
-      "num_tokens": 2737021.0,
-      "reward": 0.465625,
-      "reward_std": 0.08104550689458848,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.93125,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 25
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0125,
+      "completions/clipped_ratio": 0.016796875,
       "completions/max_length": 1536.0,
-      "completions/max_terminated_length": 1115.8,
-      "completions/mean_length": 204.8921875,
-      "completions/mean_terminated_length": 188.00881958007812,
-      "completions/min_length": 18.6,
-      "completions/min_terminated_length": 18.6,
-      "epoch": 0.3864734299516908,
-      "grad_norm": 0.31889012455940247,
-      "learning_rate": 1e-06,
-      "loss": 0.0083,
-      "num_tokens": 3360408.0,
-      "reward": 0.4734375,
-      "reward_std": 0.06902517676353455,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.946875,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 30
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0109375,
-      "completions/max_length": 1464.2,
-      "completions/max_terminated_length": 1223.4,
-      "completions/mean_length": 178.346875,
-      "completions/mean_terminated_length": 163.26475830078124,
+      "completions/max_terminated_length": 1448.0,
+      "completions/mean_length": 268.53125,
+      "completions/mean_terminated_length": 246.8666229248047,
       "completions/min_length": 19.0,
       "completions/min_terminated_length": 19.0,
-      "epoch": 0.45088566827697263,
-      "grad_norm": 0.0017821220681071281,
+      "epoch": 0.2576489533011272,
+      "grad_norm": 0.24433724582195282,
       "learning_rate": 1e-06,
-      "loss": 0.0029,
-      "num_tokens": 3953966.0,
-      "reward": 0.4875,
-      "reward_std": 0.03230288065969944,
-      "rewards/accuracy_reward": 0.0,
+      "loss": 0.054,
+      "num_tokens": 2605240.0,
+      "reward": 0.4578125,
+      "reward_std": 0.25091842710971834,
+      "rewards/accuracy_reward": 0.105859375,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.975,
+      "rewards/format_reward": 0.809765625,
       "rewards/mean_confidence_reward": 0.0,
-      "step": 35
+      "step": 5
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -203,26 +41,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0171875,
+      "completions/clipped_ratio": 0.01796875,
       "completions/max_length": 1536.0,
-      "completions/max_terminated_length": 1127.8,
-      "completions/mean_length": 181.365625,
-      "completions/mean_terminated_length": 157.82559204101562,
-      "completions/min_length": 17.4,
-      "completions/min_terminated_length": 17.4,
+      "completions/max_terminated_length": 1433.0,
+      "completions/mean_length": 250.86875,
+      "completions/mean_terminated_length": 227.32728271484376,
+      "completions/min_length": 17.6,
+      "completions/min_terminated_length": 17.6,
       "epoch": 0.5152979066022544,
-      "grad_norm": 0.0025463541969656944,
+      "grad_norm": 0.003483051899820566,
       "learning_rate": 1e-06,
-      "loss": 0.0087,
-      "num_tokens": 4593824.0,
-      "reward": 0.48046875,
-      "reward_std": 0.05219025611877441,
-      "rewards/accuracy_reward": 0.0,
+      "loss": 0.0464,
+      "num_tokens": 5188600.0,
+      "reward": 0.47890625,
+      "reward_std": 0.22365978062152864,
+      "rewards/accuracy_reward": 0.104296875,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9609375,
+      "rewards/format_reward": 0.853515625,
       "rewards/mean_confidence_reward": 0.0,
-      "step": 40
+      "step": 10
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -230,211 +68,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0140625,
+      "completions/clipped_ratio": 0.01796875,
       "completions/max_length": 1536.0,
-      "completions/max_terminated_length": 1005.2,
-      "completions/mean_length": 177.809375,
-      "completions/mean_terminated_length": 158.4675064086914,
-      "completions/min_length": 18.2,
-      "completions/min_terminated_length": 18.2,
-      "epoch": 0.5797101449275363,
-      "grad_norm": 0.0010303064482286572,
-      "learning_rate": 1e-06,
-      "loss": 0.0095,
-      "num_tokens": 5161622.0,
-      "reward": 0.48671875,
-      "reward_std": 0.037565046548843385,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9734375,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 45
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0140625,
-      "completions/max_length": 1499.8,
-      "completions/max_terminated_length": 1231.2,
-      "completions/mean_length": 184.93125,
-      "completions/mean_terminated_length": 165.76856384277343,
-      "completions/min_length": 19.0,
-      "completions/min_terminated_length": 19.0,
-      "epoch": 0.644122383252818,
-      "grad_norm": 0.00440658675506711,
-      "learning_rate": 1e-06,
-      "loss": 0.0058,
-      "num_tokens": 5771802.0,
-      "reward": 0.4828125,
-      "reward_std": 0.04029847476631403,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.965625,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 50
-    },
-    {
-      "epoch": 0.644122383252818,
-      "eval_completions/clipped_ratio": 0.009963768115942028,
-      "eval_completions/max_length": 571.5652173913044,
-      "eval_completions/max_terminated_length": 503.2463768115942,
-      "eval_completions/mean_length": 159.33152173913044,
-      "eval_completions/mean_terminated_length": 147.455357675967,
-      "eval_completions/min_length": 39.65942028985507,
-      "eval_completions/min_terminated_length": 39.65942028985507,
-      "eval_loss": 0.0,
-      "eval_num_tokens": 5771802.0,
-      "eval_reward": 0.4855072463768116,
-      "eval_reward_std": 0.03922215536020804,
-      "eval_rewards/accuracy_reward": 0.0,
-      "eval_rewards/brier_reward": 0.0,
-      "eval_rewards/confidence_one_or_zero": 0.0,
-      "eval_rewards/format_reward": 0.9710144927536232,
-      "eval_rewards/mean_confidence_reward": 0.0,
-      "eval_runtime": 458.7136,
-      "eval_samples_per_second": 0.301,
-      "eval_steps_per_second": 0.039,
-      "step": 50
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0109375,
-      "completions/max_length": 1460.2,
-      "completions/max_terminated_length": 1257.6,
-      "completions/mean_length": 168.40625,
-      "completions/mean_terminated_length": 153.32435913085936,
-      "completions/min_length": 18.4,
-      "completions/min_terminated_length": 18.4,
-      "epoch": 0.7085346215780999,
-      "grad_norm": 0.0,
-      "learning_rate": 1e-06,
-      "loss": 0.0042,
-      "num_tokens": 6394134.0,
-      "reward": 0.4875,
-      "reward_std": 0.03382910937070847,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.975,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 55
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 1440.2,
-      "completions/max_terminated_length": 1214.0,
-      "completions/mean_length": 172.475,
-      "completions/mean_terminated_length": 162.49291381835937,
-      "completions/min_length": 17.4,
-      "completions/min_terminated_length": 17.4,
+      "completions/max_terminated_length": 1412.0,
+      "completions/mean_length": 231.741015625,
+      "completions/mean_terminated_length": 207.91524353027344,
+      "completions/min_length": 19.2,
+      "completions/min_terminated_length": 19.2,
       "epoch": 0.7729468599033816,
-      "grad_norm": 0.2661661207675934,
+      "grad_norm": 0.002466069534420967,
       "learning_rate": 1e-06,
-      "loss": 0.0038,
-      "num_tokens": 6965774.0,
-      "reward": 0.4859375,
-      "reward_std": 0.03672229778021574,
-      "rewards/accuracy_reward": 0.0,
+      "loss": 0.0538,
+      "num_tokens": 7692025.0,
+      "reward": 0.521875,
+      "reward_std": 0.21285174489021302,
+      "rewards/accuracy_reward": 0.134375,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.971875,
+      "rewards/format_reward": 0.909375,
       "rewards/mean_confidence_reward": 0.0,
-      "step": 60
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0109375,
-      "completions/max_length": 1536.0,
-      "completions/max_terminated_length": 1079.2,
-      "completions/mean_length": 162.890625,
-      "completions/mean_terminated_length": 147.69927520751952,
-      "completions/min_length": 22.0,
-      "completions/min_terminated_length": 22.0,
-      "epoch": 0.8373590982286635,
-      "grad_norm": 0.0013718578265979886,
-      "learning_rate": 1e-06,
-      "loss": 0.0054,
-      "num_tokens": 7534152.0,
-      "reward": 0.48984375,
-      "reward_std": 0.027199982292950155,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9796875,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 65
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 1425.2,
-      "completions/max_terminated_length": 1097.4,
-      "completions/mean_length": 147.8484375,
-      "completions/mean_terminated_length": 136.90765686035155,
-      "completions/min_length": 21.8,
-      "completions/min_terminated_length": 21.8,
-      "epoch": 0.9017713365539453,
-      "grad_norm": 0.0,
-      "learning_rate": 1e-06,
-      "loss": 0.0032,
-      "num_tokens": 8069063.0,
-      "reward": 0.48984375,
-      "reward_std": 0.027199983410537244,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9796875,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 70
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0140625,
-      "completions/max_length": 1483.4,
-      "completions/max_terminated_length": 1025.8,
-      "completions/mean_length": 146.9515625,
-      "completions/mean_terminated_length": 127.07637023925781,
-      "completions/min_length": 22.6,
-      "completions/min_terminated_length": 22.6,
-      "epoch": 0.966183574879227,
-      "grad_norm": 0.0007732895319350064,
-      "learning_rate": 1e-06,
-      "loss": 0.006,
-      "num_tokens": 8632176.0,
-      "reward": 0.490625,
-      "reward_std": 0.026516503281891347,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.98125,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 75
+      "step": 15
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -442,107 +95,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 1351.0,
-      "completions/max_terminated_length": 1082.4,
-      "completions/mean_length": 152.284375,
-      "completions/mean_terminated_length": 141.77883911132812,
-      "completions/min_length": 19.4,
-      "completions/min_terminated_length": 19.4,
+      "completions/clipped_ratio": 0.01015625,
+      "completions/max_length": 1534.4,
+      "completions/max_terminated_length": 1478.6,
+      "completions/mean_length": 208.05107421875,
+      "completions/mean_terminated_length": 194.41161499023437,
+      "completions/min_length": 23.4,
+      "completions/min_terminated_length": 23.4,
       "epoch": 1.0515297906602254,
-      "grad_norm": 0.0026292074471712112,
+      "grad_norm": 0.0027691556606441736,
       "learning_rate": 1e-06,
-      "loss": 0.0069,
-      "num_tokens": 9244190.0,
-      "reward": 0.4875,
-      "reward_std": 0.03382910899817944,
-      "rewards/accuracy_reward": 0.0,
+      "loss": 0.049,
+      "num_tokens": 10130483.0,
+      "reward": 0.5451171875,
+      "reward_std": 0.19625933170318605,
+      "rewards/accuracy_reward": 0.146484375,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.975,
+      "rewards/format_reward": 0.94375,
       "rewards/mean_confidence_reward": 0.0,
-      "step": 80
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0125,
-      "completions/max_length": 1522.4,
-      "completions/max_terminated_length": 964.4,
-      "completions/mean_length": 137.1328125,
-      "completions/mean_terminated_length": 119.32202758789063,
-      "completions/min_length": 23.8,
-      "completions/min_terminated_length": 23.8,
-      "epoch": 1.1159420289855073,
-      "grad_norm": 0.0015776819782331586,
-      "learning_rate": 1e-06,
-      "loss": 0.0057,
-      "num_tokens": 9821539.0,
-      "reward": 0.48984375,
-      "reward_std": 0.027199982851743697,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9796875,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 85
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0125,
-      "completions/max_length": 1487.4,
-      "completions/max_terminated_length": 1213.4,
-      "completions/mean_length": 157.5234375,
-      "completions/mean_terminated_length": 139.98736267089845,
-      "completions/min_length": 18.6,
-      "completions/min_terminated_length": 18.6,
-      "epoch": 1.180354267310789,
-      "grad_norm": 0.0026901797391474247,
-      "learning_rate": 1e-06,
-      "loss": 0.0038,
-      "num_tokens": 10433074.0,
-      "reward": 0.4921875,
-      "reward_std": 0.022097086161375047,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.984375,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 90
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.003125,
-      "completions/max_length": 1289.4,
-      "completions/max_terminated_length": 1170.0,
-      "completions/mean_length": 145.9109375,
-      "completions/mean_terminated_length": 141.56537780761718,
-      "completions/min_length": 20.8,
-      "completions/min_terminated_length": 20.8,
-      "epoch": 1.2447665056360708,
-      "grad_norm": 0.0015041164588183165,
-      "learning_rate": 1e-06,
-      "loss": 0.0027,
-      "num_tokens": 11015073.0,
-      "reward": 0.49609375,
-      "reward_std": 0.011048543266952038,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9921875,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 95
+      "step": 20
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -550,130 +122,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0125,
-      "completions/max_length": 1430.8,
-      "completions/max_terminated_length": 1069.8,
-      "completions/mean_length": 147.3671875,
-      "completions/mean_terminated_length": 129.90637969970703,
-      "completions/min_length": 21.6,
-      "completions/min_terminated_length": 21.6,
-      "epoch": 1.3091787439613527,
-      "grad_norm": 0.0012177121825516224,
-      "learning_rate": 1e-06,
-      "loss": 0.0045,
-      "num_tokens": 11587028.0,
-      "reward": 0.48984375,
-      "reward_std": 0.02719998303800821,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9796875,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 100
-    },
-    {
+      "completions/clipped_ratio": 0.012109375,
+      "completions/max_length": 1536.0,
+      "completions/max_terminated_length": 1288.4,
+      "completions/mean_length": 198.357421875,
+      "completions/mean_terminated_length": 181.95052490234374,
+      "completions/min_length": 19.6,
+      "completions/min_terminated_length": 19.6,
       "epoch": 1.3091787439613527,
-      "eval_completions/clipped_ratio": 0.01177536231884058,
-      "eval_completions/max_length": 512.1449275362319,
-      "eval_completions/max_terminated_length": 432.1304347826087,
-      "eval_completions/mean_length": 143.50996376811594,
-      "eval_completions/mean_terminated_length": 128.20479691546896,
-      "eval_completions/min_length": 38.8768115942029,
-      "eval_completions/min_terminated_length": 38.8768115942029,
-      "eval_loss": 0.0,
-      "eval_num_tokens": 11587028.0,
-      "eval_reward": 0.4913949275362319,
-      "eval_reward_std": 0.022569278552048447,
-      "eval_rewards/accuracy_reward": 0.0,
-      "eval_rewards/brier_reward": 0.0,
-      "eval_rewards/confidence_one_or_zero": 0.0,
-      "eval_rewards/format_reward": 0.9827898550724637,
-      "eval_rewards/mean_confidence_reward": 0.0,
-      "eval_runtime": 402.0293,
-      "eval_samples_per_second": 0.343,
-      "eval_steps_per_second": 0.045,
-      "step": 100
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.003125,
-      "completions/max_length": 1192.2,
-      "completions/max_terminated_length": 1007.8,
-      "completions/mean_length": 130.6890625,
-      "completions/mean_terminated_length": 126.32133483886719,
-      "completions/min_length": 23.8,
-      "completions/min_terminated_length": 23.8,
-      "epoch": 1.3735909822866343,
-      "grad_norm": 0.0019685616716742516,
-      "learning_rate": 1e-06,
-      "loss": 0.0014,
-      "num_tokens": 12151989.0,
-      "reward": 0.496875,
-      "reward_std": 0.00883883461356163,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.99375,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 105
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 1458.8,
-      "completions/max_terminated_length": 1068.6,
-      "completions/mean_length": 123.403125,
-      "completions/mean_terminated_length": 112.2623046875,
-      "completions/min_length": 21.4,
-      "completions/min_terminated_length": 21.4,
-      "epoch": 1.4380032206119162,
-      "grad_norm": 0.001040562056005001,
-      "learning_rate": 1e-06,
-      "loss": 0.0039,
-      "num_tokens": 12721343.0,
-      "reward": 0.49453125,
-      "reward_std": 0.015467960573732852,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9890625,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 110
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.009375,
-      "completions/max_length": 1273.0,
-      "completions/max_terminated_length": 1066.8,
-      "completions/mean_length": 138.396875,
-      "completions/mean_terminated_length": 125.22679138183594,
-      "completions/min_length": 24.0,
-      "completions/min_terminated_length": 24.0,
-      "epoch": 1.502415458937198,
-      "grad_norm": 0.002089962363243103,
+      "grad_norm": 0.0031809473875910044,
       "learning_rate": 1e-06,
-      "loss": 0.0033,
-      "num_tokens": 13313469.0,
-      "reward": 0.49296875,
-      "reward_std": 0.019887377880513667,
-      "rewards/accuracy_reward": 0.0,
+      "loss": 0.0439,
+      "num_tokens": 12549638.0,
+      "reward": 0.556640625,
+      "reward_std": 0.19344157576560975,
+      "rewards/accuracy_reward": 0.1640625,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9859375,
+      "rewards/format_reward": 0.94921875,
       "rewards/mean_confidence_reward": 0.0,
-      "step": 115
+      "step": 25
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -681,211 +149,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.003125,
-      "completions/max_length": 1064.0,
-      "completions/max_terminated_length": 1004.6,
-      "completions/mean_length": 136.971875,
-      "completions/mean_terminated_length": 133.62301940917968,
-      "completions/min_length": 23.2,
-      "completions/min_terminated_length": 23.2,
+      "completions/clipped_ratio": 0.006640625,
+      "completions/max_length": 1536.0,
+      "completions/max_terminated_length": 1428.2,
+      "completions/mean_length": 164.6171875,
+      "completions/mean_terminated_length": 155.45379943847655,
+      "completions/min_length": 22.0,
+      "completions/min_terminated_length": 22.0,
       "epoch": 1.5668276972624797,
-      "grad_norm": 0.33233243227005005,
-      "learning_rate": 1e-06,
-      "loss": 0.0022,
-      "num_tokens": 13879963.0,
-      "reward": 0.49609375,
-      "reward_std": 0.011048543080687524,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9921875,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 120
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.009375,
-      "completions/max_length": 1188.4,
-      "completions/max_terminated_length": 888.6,
-      "completions/mean_length": 123.18125,
-      "completions/mean_terminated_length": 111.09695281982422,
-      "completions/min_length": 20.6,
-      "completions/min_terminated_length": 20.6,
-      "epoch": 1.6312399355877618,
-      "grad_norm": 0.0011026322608813643,
-      "learning_rate": 1e-06,
-      "loss": 0.0032,
-      "num_tokens": 14404831.0,
-      "reward": 0.4953125,
-      "reward_std": 0.013258251920342445,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.990625,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 125
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0046875,
-      "completions/max_length": 1117.8,
-      "completions/max_terminated_length": 1037.2,
-      "completions/mean_length": 122.975,
-      "completions/mean_terminated_length": 116.31961364746094,
-      "completions/min_length": 17.6,
-      "completions/min_terminated_length": 17.6,
-      "epoch": 1.6956521739130435,
-      "grad_norm": 0.0014723028289154172,
-      "learning_rate": 1e-06,
-      "loss": 0.0027,
-      "num_tokens": 14976751.0,
-      "reward": 0.49609375,
-      "reward_std": 0.011048543080687524,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9921875,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 130
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0046875,
-      "completions/max_length": 1286.2,
-      "completions/max_terminated_length": 984.2,
-      "completions/mean_length": 109.5984375,
-      "completions/mean_terminated_length": 102.81706085205079,
-      "completions/min_length": 20.2,
-      "completions/min_terminated_length": 20.2,
-      "epoch": 1.7600644122383253,
-      "grad_norm": 0.0021880464628338814,
-      "learning_rate": 1e-06,
-      "loss": 0.0011,
-      "num_tokens": 15543422.0,
-      "reward": 0.49609375,
-      "reward_std": 0.009522313810884952,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9921875,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 135
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0046875,
-      "completions/max_length": 1336.0,
-      "completions/max_terminated_length": 1114.0,
-      "completions/mean_length": 128.7359375,
-      "completions/mean_terminated_length": 122.13045043945313,
-      "completions/min_length": 19.2,
-      "completions/min_terminated_length": 19.2,
-      "epoch": 1.8244766505636072,
-      "grad_norm": 0.000525438750628382,
-      "learning_rate": 1e-06,
-      "loss": 0.0026,
-      "num_tokens": 16118933.0,
-      "reward": 0.496875,
-      "reward_std": 0.00883883461356163,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.99375,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 140
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.00625,
-      "completions/max_length": 1218.6,
-      "completions/max_terminated_length": 880.0,
-      "completions/mean_length": 127.0296875,
-      "completions/mean_terminated_length": 118.19228668212891,
-      "completions/min_length": 22.4,
-      "completions/min_terminated_length": 22.4,
-      "epoch": 1.8888888888888888,
-      "grad_norm": 0.0006711249006912112,
+      "grad_norm": 0.003916527610272169,
       "learning_rate": 1e-06,
-      "loss": 0.0032,
-      "num_tokens": 16660720.0,
-      "reward": 0.4953125,
-      "reward_std": 0.01325825173407793,
-      "rewards/accuracy_reward": 0.0,
+      "loss": 0.0237,
+      "num_tokens": 14889882.0,
+      "reward": 0.5884765625,
+      "reward_std": 0.19649214446544647,
+      "rewards/accuracy_reward": 0.203515625,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.990625,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 145
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.00625,
-      "completions/max_length": 1304.8,
-      "completions/max_terminated_length": 1041.2,
-      "completions/mean_length": 131.8859375,
-      "completions/mean_terminated_length": 123.68300018310546,
-      "completions/min_length": 16.8,
-      "completions/min_terminated_length": 16.8,
-      "epoch": 1.9533011272141707,
-      "grad_norm": 0.4140954911708832,
-      "learning_rate": 1e-06,
-      "loss": 0.0023,
-      "num_tokens": 17209407.0,
-      "reward": 0.49609375,
-      "reward_std": 0.011048543266952038,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9921875,
+      "rewards/format_reward": 0.9734375,
       "rewards/mean_confidence_reward": 0.0,
-      "step": 150
-    },
-    {
-      "epoch": 1.9533011272141707,
-      "eval_completions/clipped_ratio": 0.004528985507246377,
-      "eval_completions/max_length": 394.6231884057971,
-      "eval_completions/max_terminated_length": 367.69565217391306,
-      "eval_completions/mean_length": 114.42391304347827,
-      "eval_completions/mean_terminated_length": 110.32013475722161,
-      "eval_completions/min_length": 34.32608695652174,
-      "eval_completions/min_terminated_length": 34.32608695652174,
-      "eval_loss": 0.0,
-      "eval_num_tokens": 17209407.0,
-      "eval_reward": 0.49682971014492755,
-      "eval_reward_std": 0.008966933665932089,
-      "eval_rewards/accuracy_reward": 0.0,
-      "eval_rewards/brier_reward": 0.0,
-      "eval_rewards/confidence_one_or_zero": 0.0,
-      "eval_rewards/format_reward": 0.9936594202898551,
-      "eval_rewards/mean_confidence_reward": 0.0,
-      "eval_runtime": 318.6923,
-      "eval_samples_per_second": 0.433,
-      "eval_steps_per_second": 0.056,
-      "step": 150
+      "step": 30
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -893,26 +176,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0015625,
-      "completions/max_length": 1172.2,
-      "completions/max_terminated_length": 959.6,
-      "completions/mean_length": 110.8133331298828,
-      "completions/mean_terminated_length": 108.54268798828124,
+      "completions/clipped_ratio": 0.005078125,
+      "completions/max_length": 1519.6,
+      "completions/max_terminated_length": 1345.6,
+      "completions/mean_length": 146.447265625,
+      "completions/mean_terminated_length": 139.37295532226562,
       "completions/min_length": 20.0,
       "completions/min_terminated_length": 20.0,
-      "epoch": 2.0386473429951693,
-      "grad_norm": 0.002583252266049385,
+      "epoch": 1.8244766505636072,
+      "grad_norm": 0.003678182838484645,
       "learning_rate": 1e-06,
-      "loss": 0.0008,
-      "num_tokens": 17743744.0,
-      "reward": 0.4984375,
-      "reward_std": 0.004419417306780815,
-      "rewards/accuracy_reward": 0.0,
+      "loss": 0.0283,
+      "num_tokens": 17175619.0,
+      "reward": 0.60078125,
+      "reward_std": 0.1955224335193634,
+      "rewards/accuracy_reward": 0.216796875,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.996875,
+      "rewards/format_reward": 0.984765625,
       "rewards/mean_confidence_reward": 0.0,
-      "step": 155
+      "step": 35
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -920,26 +203,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0046875,
-      "completions/max_length": 1262.2,
-      "completions/max_terminated_length": 1153.0,
-      "completions/mean_length": 119.7703125,
-      "completions/mean_terminated_length": 113.08980102539063,
-      "completions/min_length": 20.0,
-      "completions/min_terminated_length": 20.0,
+      "completions/clipped_ratio": 0.00574111238532109,
+      "completions/max_length": 1488.6,
+      "completions/max_terminated_length": 1305.6,
+      "completions/mean_length": 141.31993713378907,
+      "completions/mean_terminated_length": 133.36893768310546,
+      "completions/min_length": 20.8,
+      "completions/min_terminated_length": 20.8,
       "epoch": 2.103059581320451,
-      "grad_norm": 0.5405756831169128,
+      "grad_norm": 0.31614598631858826,
       "learning_rate": 1e-06,
-      "loss": 0.0021,
-      "num_tokens": 18318485.0,
-      "reward": 0.49765625,
-      "reward_std": 0.006629125773906707,
-      "rewards/accuracy_reward": 0.0,
+      "loss": 0.028,
+      "num_tokens": 19445973.0,
+      "reward": 0.61953125,
+      "reward_std": 0.20828622579574585,
+      "rewards/accuracy_reward": 0.257421875,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9953125,
+      "rewards/format_reward": 0.981640625,
       "rewards/mean_confidence_reward": 0.0,
-      "step": 160
+      "step": 40
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -947,26 +230,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0046875,
-      "completions/max_length": 1343.2,
-      "completions/max_terminated_length": 1049.6,
-      "completions/mean_length": 122.5515625,
-      "completions/mean_terminated_length": 115.89584350585938,
-      "completions/min_length": 19.8,
-      "completions/min_terminated_length": 19.8,
-      "epoch": 2.1674718196457325,
-      "grad_norm": 0.0,
+      "completions/clipped_ratio": 0.006640625,
+      "completions/max_length": 1536.0,
+      "completions/max_terminated_length": 1335.6,
+      "completions/mean_length": 132.665234375,
+      "completions/mean_terminated_length": 123.2823257446289,
+      "completions/min_length": 21.0,
+      "completions/min_terminated_length": 21.0,
+      "epoch": 2.360708534621578,
+      "grad_norm": 0.0037020090967416763,
       "learning_rate": 1e-06,
-      "loss": 0.0024,
-      "num_tokens": 18834918.0,
-      "reward": 0.49609375,
-      "reward_std": 0.011048543266952038,
-      "rewards/accuracy_reward": 0.0,
+      "loss": 0.0276,
+      "num_tokens": 21705156.0,
+      "reward": 0.6427734375,
+      "reward_std": 0.21320482790470124,
+      "rewards/accuracy_reward": 0.298828125,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9921875,
+      "rewards/format_reward": 0.98671875,
       "rewards/mean_confidence_reward": 0.0,
-      "step": 165
+      "step": 45
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -975,318 +258,48 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0015625,
-      "completions/max_length": 1011.8,
-      "completions/max_terminated_length": 898.2,
-      "completions/mean_length": 108.3484375,
-      "completions/mean_terminated_length": 106.08817596435547,
-      "completions/min_length": 17.0,
-      "completions/min_terminated_length": 17.0,
-      "epoch": 2.2318840579710146,
-      "grad_norm": 0.0007173310150392354,
-      "learning_rate": 1e-06,
-      "loss": 0.0006,
-      "num_tokens": 19381893.0,
-      "reward": 0.49921875,
-      "reward_std": 0.0022097086533904076,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9984375,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 170
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.003125,
-      "completions/max_length": 1189.4,
-      "completions/max_terminated_length": 978.8,
-      "completions/mean_length": 119.48125,
-      "completions/mean_terminated_length": 115.07453308105468,
+      "completions/max_length": 1374.2,
+      "completions/max_terminated_length": 1175.2,
+      "completions/mean_length": 113.80234375,
+      "completions/mean_terminated_length": 111.57572784423829,
       "completions/min_length": 18.4,
       "completions/min_terminated_length": 18.4,
-      "epoch": 2.2962962962962963,
-      "grad_norm": 0.0,
-      "learning_rate": 1e-06,
-      "loss": 0.001,
-      "num_tokens": 19936521.0,
-      "reward": 0.49765625,
-      "reward_std": 0.0066291259601712225,
-      "rewards/accuracy_reward": 0.0,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9953125,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 175
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.00625,
-      "completions/max_length": 1237.4,
-      "completions/max_terminated_length": 973.4,
-      "completions/mean_length": 106.9171875,
-      "completions/mean_terminated_length": 97.92234344482422,
-      "completions/min_length": 18.6,
-      "completions/min_terminated_length": 18.6,
-      "epoch": 2.360708534621578,
-      "grad_norm": 0.0012030096258968115,
+      "epoch": 2.6183574879227054,
+      "grad_norm": 0.3358711898326874,
       "learning_rate": 1e-06,
-      "loss": 0.0034,
-      "num_tokens": 20497364.0,
-      "reward": 0.49296875,
-      "reward_std": 0.019887377507984638,
-      "rewards/accuracy_reward": 0.0,
+      "loss": 0.0166,
+      "num_tokens": 23885994.0,
+      "reward": 0.665625,
+      "reward_std": 0.2116748332977295,
+      "rewards/accuracy_reward": 0.338671875,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9859375,
+      "rewards/format_reward": 0.992578125,
       "rewards/mean_confidence_reward": 0.0,
-      "step": 180
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 1392.2,
-      "completions/max_terminated_length": 805.2,
-      "completions/mean_length": 115.64375,
-      "completions/mean_terminated_length": 104.5155517578125,
-      "completions/min_length": 18.2,
-      "completions/min_terminated_length": 18.2,
-      "epoch": 2.399355877616747,
-      "grad_norm": 0.011998126283288002,
-      "learning_rate": 1e-06,
-      "loss": 0.0081,
-      "num_tokens": 21064400.0,
-      "reward": 0.59453125,
-      "reward_std": 0.18795942664146423,
-      "rewards/accuracy_reward": 0.1984375,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.990625,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 185
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 868.6,
-      "completions/max_terminated_length": 868.6,
-      "completions/mean_length": 99.8234375,
-      "completions/mean_terminated_length": 99.8234375,
-      "completions/min_length": 16.8,
-      "completions/min_terminated_length": 16.8,
-      "epoch": 2.463768115942029,
-      "grad_norm": 0.01248540636152029,
-      "learning_rate": 1e-06,
-      "loss": 0.0108,
-      "num_tokens": 21602655.0,
-      "reward": 0.61171875,
-      "reward_std": 0.19059042632579803,
-      "rewards/accuracy_reward": 0.225,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9984375,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 190
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0015625,
-      "completions/max_length": 674.2,
-      "completions/max_terminated_length": 518.6,
-      "completions/mean_length": 41.5515625,
-      "completions/mean_terminated_length": 39.206139373779294,
-      "completions/min_length": 16.4,
-      "completions/min_terminated_length": 16.4,
-      "epoch": 2.528180354267311,
-      "grad_norm": 0.01337147131562233,
-      "learning_rate": 1e-06,
-      "loss": 0.0071,
-      "num_tokens": 22149424.0,
-      "reward": 0.65859375,
-      "reward_std": 0.2084730714559555,
-      "rewards/accuracy_reward": 0.31875,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9984375,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 195
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 65.8,
-      "completions/max_terminated_length": 65.8,
-      "completions/mean_length": 18.2390625,
-      "completions/mean_terminated_length": 18.2390625,
-      "completions/min_length": 16.0,
-      "completions/min_terminated_length": 16.0,
-      "epoch": 2.5925925925925926,
-      "grad_norm": 0.012995072640478611,
-      "learning_rate": 1e-06,
-      "loss": 0.0017,
-      "num_tokens": 22617817.0,
-      "reward": 0.71640625,
-      "reward_std": 0.20620474815368653,
-      "rewards/accuracy_reward": 0.4328125,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 1.0,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 200
+      "step": 50
     },
     {
-      "epoch": 2.5925925925925926,
-      "eval_completions/clipped_ratio": 0.0,
-      "eval_completions/max_length": 19.565217391304348,
-      "eval_completions/max_terminated_length": 19.565217391304348,
-      "eval_completions/mean_length": 17.383152173913043,
-      "eval_completions/mean_terminated_length": 17.383152173913043,
-      "eval_completions/min_length": 16.89855072463768,
-      "eval_completions/min_terminated_length": 16.89855072463768,
+      "epoch": 2.6183574879227054,
+      "eval_completions/clipped_ratio": 0.0036231884057971015,
+      "eval_completions/max_length": 385.3985507246377,
+      "eval_completions/max_terminated_length": 350.32608695652175,
+      "eval_completions/mean_length": 117.5498188405797,
+      "eval_completions/mean_terminated_length": 112.37914088152456,
+      "eval_completions/min_length": 38.92028985507246,
+      "eval_completions/min_terminated_length": 38.92028985507246,
       "eval_loss": 0.0,
-      "eval_num_tokens": 22617817.0,
-      "eval_reward": 0.7391304347826086,
-      "eval_reward_std": 0.1717898635112721,
-      "eval_rewards/accuracy_reward": 0.4791666666666667,
+      "eval_num_tokens": 23885994.0,
+      "eval_reward": 0.6829710144927537,
+      "eval_reward_std": 0.21439332471809525,
+      "eval_rewards/accuracy_reward": 0.3740942028985507,
       "eval_rewards/brier_reward": 0.0,
       "eval_rewards/confidence_one_or_zero": 0.0,
-      "eval_rewards/format_reward": 0.9990942028985508,
+      "eval_rewards/format_reward": 0.9918478260869565,
       "eval_rewards/mean_confidence_reward": 0.0,
-      "eval_runtime": 37.0255,
-      "eval_samples_per_second": 3.727,
-      "eval_steps_per_second": 0.486,
-      "step": 200
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 145.0,
-      "completions/max_terminated_length": 145.0,
-      "completions/mean_length": 18.1015625,
-      "completions/mean_terminated_length": 18.1015625,
-      "completions/min_length": 16.0,
-      "completions/min_terminated_length": 16.0,
-      "epoch": 2.6570048309178746,
-      "grad_norm": 0.006278031039983034,
-      "learning_rate": 1e-06,
-      "loss": 0.0015,
-      "num_tokens": 23134698.0,
-      "reward": 0.78046875,
-      "reward_std": 0.14939574152231216,
-      "rewards/accuracy_reward": 0.5625,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 0.9984375,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 205
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 20.2,
-      "completions/max_terminated_length": 20.2,
-      "completions/mean_length": 17.04375,
-      "completions/mean_terminated_length": 17.04375,
-      "completions/min_length": 16.2,
-      "completions/min_terminated_length": 16.2,
-      "epoch": 2.7214170692431563,
-      "grad_norm": 0.007409343961626291,
-      "learning_rate": 1e-06,
-      "loss": 0.0002,
-      "num_tokens": 23654262.0,
-      "reward": 0.72890625,
-      "reward_std": 0.049137798324227336,
-      "rewards/accuracy_reward": 0.4578125,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 1.0,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 210
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 19.2,
-      "completions/max_terminated_length": 19.2,
-      "completions/mean_length": 17.01875,
-      "completions/mean_terminated_length": 17.01875,
-      "completions/min_length": 17.0,
-      "completions/min_terminated_length": 17.0,
-      "epoch": 2.785829307568438,
-      "grad_norm": 0.0,
-      "learning_rate": 1e-06,
-      "loss": 0.0,
-      "num_tokens": 24120378.0,
-      "reward": 0.77421875,
-      "reward_std": 0.0022097086533904076,
-      "rewards/accuracy_reward": 0.5484375,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 1.0,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 215
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 18.2,
-      "completions/max_terminated_length": 18.2,
-      "completions/mean_length": 17.00625,
-      "completions/mean_terminated_length": 17.00625,
-      "completions/min_length": 16.6,
-      "completions/min_terminated_length": 16.6,
-      "epoch": 2.85024154589372,
-      "grad_norm": 0.003187602385878563,
-      "learning_rate": 1e-06,
-      "loss": 0.0,
-      "num_tokens": 24586918.0,
-      "reward": 0.77421875,
-      "reward_std": 0.0022097086533904076,
-      "rewards/accuracy_reward": 0.5484375,
-      "rewards/brier_reward": 0.0,
-      "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 1.0,
-      "rewards/mean_confidence_reward": 0.0,
-      "step": 220
+      "eval_runtime": 283.8399,
+      "eval_samples_per_second": 0.486,
+      "eval_steps_per_second": 0.063,
+      "step": 50
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1294,26 +307,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 17.0,
-      "completions/max_terminated_length": 17.0,
-      "completions/mean_length": 16.9984375,
-      "completions/mean_terminated_length": 16.9984375,
-      "completions/min_length": 16.8,
-      "completions/min_terminated_length": 16.8,
-      "epoch": 2.9146537842190017,
-      "grad_norm": 0.0,
+      "completions/clipped_ratio": 0.001953125,
+      "completions/max_length": 1499.4,
+      "completions/max_terminated_length": 1196.8,
+      "completions/mean_length": 109.181640625,
+      "completions/mean_terminated_length": 106.39108734130859,
+      "completions/min_length": 19.2,
+      "completions/min_terminated_length": 19.2,
+      "epoch": 2.8760064412238324,
+      "grad_norm": 0.004169007763266563,
       "learning_rate": 1e-06,
-      "loss": 0.0,
-      "num_tokens": 25043109.0,
-      "reward": 0.83046875,
-      "reward_std": 0.0022097086533904076,
-      "rewards/accuracy_reward": 0.6609375,
+      "loss": 0.0147,
+      "num_tokens": 26106195.0,
+      "reward": 0.680078125,
+      "reward_std": 0.2129088193178177,
+      "rewards/accuracy_reward": 0.3640625,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 1.0,
+      "rewards/format_reward": 0.99609375,
       "rewards/mean_confidence_reward": 0.0,
-      "step": 225
+      "step": 55
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1321,40 +334,33 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 17.6,
-      "completions/max_terminated_length": 17.6,
-      "completions/mean_length": 17.0046875,
-      "completions/mean_terminated_length": 17.0046875,
-      "completions/min_length": 17.0,
-      "completions/min_terminated_length": 17.0,
+      "completions/clipped_ratio": 0.005563718463302725,
+      "completions/max_length": 1536.0,
+      "completions/max_terminated_length": 1031.0,
+      "completions/mean_length": 108.30762481689453,
+      "completions/mean_terminated_length": 100.29183578491211,
+      "completions/min_length": 22.5,
+      "completions/min_terminated_length": 22.5,
       "epoch": 2.9790660225442833,
-      "grad_norm": 0.0,
-      "learning_rate": 1e-06,
-      "loss": 0.0,
-      "num_tokens": 25527624.0,
-      "reward": 0.76875,
-      "reward_std": 0.0,
-      "rewards/accuracy_reward": 0.5375,
+      "num_tokens": 26989330.0,
+      "reward": 0.6923828125,
+      "reward_std": 0.2056785449385643,
+      "rewards/accuracy_reward": 0.3896484375,
       "rewards/brier_reward": 0.0,
       "rewards/confidence_one_or_zero": 0.0,
-      "rewards/format_reward": 1.0,
+      "rewards/format_reward": 0.9951171875,
       "rewards/mean_confidence_reward": 0.0,
-      "step": 230
-    },
-    {
-      "epoch": 2.9790660225442833,
-      "step": 230,
+      "step": 57,
       "total_flos": 0.0,
-      "train_loss": 0.0006395614726225967,
-      "train_runtime": 1235.3467,
-      "train_samples_per_second": 3.016,
-      "train_steps_per_second": 0.187
+      "train_loss": 0.034499143952863257,
+      "train_runtime": 7332.8053,
+      "train_samples_per_second": 0.508,
+      "train_steps_per_second": 0.008
     }
   ],
   "logging_steps": 5,
-  "max_steps": 231,
-  "num_input_tokens_seen": 25527624,
+  "max_steps": 57,
+  "num_input_tokens_seen": 26989330,
   "num_train_epochs": 3,
   "save_steps": 60,
   "stateful_callbacks": {
@@ -1364,13 +370,13 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 2,
+  "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d621838afb400aedd4f5bc11f1106f7c3393295c516e1afd83b5de543331a2c3
+oid sha256:8fb871c75d4ce1a66d4015a05ff738fe7ff4eb2e301a47d588ffc368350e7357
 size 7992