| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 1.6666666666666665, |
| "eval_steps": 50, |
| "global_step": 250, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 143.3125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 185.0, |
| "completions/max_terminated_length": 185.0, |
| "completions/mean_length": 143.3125, |
| "completions/mean_terminated_length": 143.3125, |
| "completions/min_length": 126.5, |
| "completions/min_terminated_length": 126.5, |
| "epoch": 0.006666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.12731829285621643, |
| "kl": 0.0, |
| "learning_rate": 0.0, |
| "loss": 0.06993113458156586, |
| "num_tokens": 12045.0, |
| "reward": 0.28475117683410645, |
| "reward_std": 0.7594513893127441, |
| "rewards/reward_fn/mean": 0.28475117683410645, |
| "rewards/reward_fn/std": 0.7594513893127441, |
| "step": 1, |
| "step_time": 90.46665180695709 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 148.4375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 164.5, |
| "completions/max_terminated_length": 164.5, |
| "completions/mean_length": 148.4375, |
| "completions/mean_terminated_length": 148.4375, |
| "completions/min_length": 137.5, |
| "completions/min_terminated_length": 137.5, |
| "epoch": 0.013333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07862907648086548, |
| "kl": 0.0, |
| "learning_rate": 8.333333333333333e-07, |
| "loss": 0.039252012968063354, |
| "num_tokens": 24300.0, |
| "reward": -0.6684748530387878, |
| "reward_std": 0.34289104118943214, |
| "rewards/reward_fn/mean": -0.6684748530387878, |
| "rewards/reward_fn/std": 0.34289104118943214, |
| "step": 2, |
| "step_time": 62.789824239036534 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 204.75, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 247.5, |
| "completions/max_terminated_length": 247.5, |
| "completions/mean_length": 204.75, |
| "completions/mean_terminated_length": 204.75, |
| "completions/min_length": 137.0, |
| "completions/min_terminated_length": 137.0, |
| "epoch": 0.02, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.0595012903213501, |
| "kl": 0.0, |
| "learning_rate": 1.6666666666666667e-06, |
| "loss": -0.042902350425720215, |
| "num_tokens": 36984.0, |
| "reward": -1.3745315074920654, |
| "reward_std": 0.13413189351558685, |
| "rewards/reward_fn/mean": -1.3745315074920654, |
| "rewards/reward_fn/std": 0.13413189351558685, |
| "step": 3, |
| "step_time": 85.68428186204983 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 175.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 201.0, |
| "completions/max_terminated_length": 201.0, |
| "completions/mean_length": 175.0, |
| "completions/mean_terminated_length": 175.0, |
| "completions/min_length": 151.5, |
| "completions/min_terminated_length": 151.5, |
| "epoch": 0.02666666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.052226342260837555, |
| "kl": 0.0, |
| "learning_rate": 2.5e-06, |
| "loss": 0.017850903794169426, |
| "num_tokens": 48984.0, |
| "reward": -1.2789445519447327, |
| "reward_std": 0.42042604088783264, |
| "rewards/reward_fn/mean": -1.2789445519447327, |
| "rewards/reward_fn/std": 0.42042604088783264, |
| "step": 4, |
| "step_time": 71.62924331304384 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 230.75, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 359.5, |
| "completions/max_terminated_length": 359.5, |
| "completions/mean_length": 230.75, |
| "completions/mean_terminated_length": 230.75, |
| "completions/min_length": 179.0, |
| "completions/min_terminated_length": 179.0, |
| "epoch": 0.03333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07887183874845505, |
| "kl": 0.0, |
| "learning_rate": 3.3333333333333333e-06, |
| "loss": 0.003247857093811035, |
| "num_tokens": 61932.0, |
| "reward": -0.7773191928863525, |
| "reward_std": 0.9348528981208801, |
| "rewards/reward_fn/mean": -0.7773191928863525, |
| "rewards/reward_fn/std": 0.9348528981208801, |
| "step": 5, |
| "step_time": 111.19219641707605 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 223.8125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 301.5, |
| "completions/max_terminated_length": 301.5, |
| "completions/mean_length": 223.8125, |
| "completions/mean_terminated_length": 223.8125, |
| "completions/min_length": 172.5, |
| "completions/min_terminated_length": 172.5, |
| "epoch": 0.04, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08544809371232986, |
| "kl": 0.0, |
| "learning_rate": 4.166666666666667e-06, |
| "loss": -0.01605057343840599, |
| "num_tokens": 74937.0, |
| "reward": -0.011491045355796814, |
| "reward_std": 1.0746989250183105, |
| "rewards/reward_fn/mean": -0.011491045355796814, |
| "rewards/reward_fn/std": 1.0746989250183105, |
| "step": 6, |
| "step_time": 101.91355575795751 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 239.0625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 286.5, |
| "completions/max_terminated_length": 286.5, |
| "completions/mean_length": 239.0625, |
| "completions/mean_terminated_length": 239.0625, |
| "completions/min_length": 194.5, |
| "completions/min_terminated_length": 194.5, |
| "epoch": 0.04666666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.06637871265411377, |
| "kl": 0.0, |
| "learning_rate": 5e-06, |
| "loss": 0.022047851234674454, |
| "num_tokens": 88986.0, |
| "reward": -0.7737771570682526, |
| "reward_std": 0.08403173834085464, |
| "rewards/reward_fn/mean": -0.7737771570682526, |
| "rewards/reward_fn/std": 0.08403173834085464, |
| "step": 7, |
| "step_time": 94.80128556105774 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 160.4375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 192.0, |
| "completions/max_terminated_length": 192.0, |
| "completions/mean_length": 160.4375, |
| "completions/mean_terminated_length": 160.4375, |
| "completions/min_length": 123.5, |
| "completions/min_terminated_length": 123.5, |
| "epoch": 0.05333333333333334, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 5.833333333333334e-06, |
| "loss": 0.0, |
| "num_tokens": 101153.0, |
| "reward": -1.5, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": -1.5, |
| "rewards/reward_fn/std": 0.0, |
| "step": 8, |
| "step_time": 66.85633395699551 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 123.25, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 127.5, |
| "completions/max_terminated_length": 127.5, |
| "completions/mean_length": 123.25, |
| "completions/mean_terminated_length": 123.25, |
| "completions/min_length": 120.5, |
| "completions/min_terminated_length": 120.5, |
| "epoch": 0.06, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.08828099071979523, |
| "kl": 0.0, |
| "learning_rate": 6.666666666666667e-06, |
| "loss": 0.0025132223963737488, |
| "num_tokens": 112757.0, |
| "reward": -0.7912696972489357, |
| "reward_std": 0.12075159698724747, |
| "rewards/reward_fn/mean": -0.7912696972489357, |
| "rewards/reward_fn/std": 0.12075159698724747, |
| "step": 9, |
| "step_time": 50.840260804921854 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 162.625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 205.0, |
| "completions/max_terminated_length": 205.0, |
| "completions/mean_length": 162.625, |
| "completions/mean_terminated_length": 162.625, |
| "completions/min_length": 131.0, |
| "completions/min_terminated_length": 131.0, |
| "epoch": 0.06666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08045986294746399, |
| "kl": 0.0, |
| "learning_rate": 7.5e-06, |
| "loss": -0.03439512848854065, |
| "num_tokens": 124327.0, |
| "reward": -0.5544086694717407, |
| "reward_std": 0.5964015126228333, |
| "rewards/reward_fn/mean": -0.5544086694717407, |
| "rewards/reward_fn/std": 0.5964015126228333, |
| "step": 10, |
| "step_time": 69.93904624809511 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 157.5, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 193.0, |
| "completions/max_terminated_length": 193.0, |
| "completions/mean_length": 157.5, |
| "completions/mean_terminated_length": 157.5, |
| "completions/min_length": 141.0, |
| "completions/min_terminated_length": 141.0, |
| "epoch": 0.07333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.15679270029067993, |
| "kl": 0.0, |
| "learning_rate": 8.333333333333334e-06, |
| "loss": -0.03179311752319336, |
| "num_tokens": 136575.0, |
| "reward": 0.13369514793157578, |
| "reward_std": 0.6897610425949097, |
| "rewards/reward_fn/mean": 0.13369514793157578, |
| "rewards/reward_fn/std": 0.6897610425949097, |
| "step": 11, |
| "step_time": 68.66538558196044 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 250.4375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 342.0, |
| "completions/max_terminated_length": 342.0, |
| "completions/mean_length": 250.4375, |
| "completions/mean_terminated_length": 250.4375, |
| "completions/min_length": 167.5, |
| "completions/min_terminated_length": 167.5, |
| "epoch": 0.08, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1365279257297516, |
| "kl": 0.0, |
| "learning_rate": 9.166666666666666e-06, |
| "loss": 0.052371568977832794, |
| "num_tokens": 149926.0, |
| "reward": -0.7794331163167953, |
| "reward_std": 0.5107784122228622, |
| "rewards/reward_fn/mean": -0.7794331163167953, |
| "rewards/reward_fn/std": 0.5107784122228622, |
| "step": 12, |
| "step_time": 107.52361460705288 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 147.3125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 188.0, |
| "completions/max_terminated_length": 188.0, |
| "completions/mean_length": 147.3125, |
| "completions/mean_terminated_length": 147.3125, |
| "completions/min_length": 127.0, |
| "completions/min_terminated_length": 127.0, |
| "epoch": 0.08666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.14017197489738464, |
| "kl": 0.0, |
| "learning_rate": 1e-05, |
| "loss": -0.03942764177918434, |
| "num_tokens": 162051.0, |
| "reward": -0.24546438455581665, |
| "reward_std": 0.148831894621253, |
| "rewards/reward_fn/mean": -0.24546438455581665, |
| "rewards/reward_fn/std": 0.148831894621253, |
| "step": 13, |
| "step_time": 67.4536545749288 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 246.625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 294.5, |
| "completions/max_terminated_length": 294.5, |
| "completions/mean_length": 246.625, |
| "completions/mean_terminated_length": 246.625, |
| "completions/min_length": 181.5, |
| "completions/min_terminated_length": 181.5, |
| "epoch": 0.09333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.25423967838287354, |
| "kl": 0.0, |
| "learning_rate": 1.0833333333333334e-05, |
| "loss": 0.09142481535673141, |
| "num_tokens": 175253.0, |
| "reward": -1.1983885765075684, |
| "reward_std": 0.5847422480583191, |
| "rewards/reward_fn/mean": -1.1983885765075684, |
| "rewards/reward_fn/std": 0.5847422480583191, |
| "step": 14, |
| "step_time": 93.72659771697363 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 218.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 321.5, |
| "completions/max_terminated_length": 321.5, |
| "completions/mean_length": 218.9375, |
| "completions/mean_terminated_length": 218.9375, |
| "completions/min_length": 162.0, |
| "completions/min_terminated_length": 162.0, |
| "epoch": 0.1, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08925358206033707, |
| "kl": 0.0, |
| "learning_rate": 1.1666666666666668e-05, |
| "loss": 0.002535127103328705, |
| "num_tokens": 188108.0, |
| "reward": -0.9819782972335815, |
| "reward_std": 0.563330739736557, |
| "rewards/reward_fn/mean": -0.9819782972335815, |
| "rewards/reward_fn/std": 0.563330739736557, |
| "step": 15, |
| "step_time": 100.25624653795967 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 199.0625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 308.5, |
| "completions/max_terminated_length": 308.5, |
| "completions/mean_length": 199.0625, |
| "completions/mean_terminated_length": 199.0625, |
| "completions/min_length": 139.0, |
| "completions/min_terminated_length": 139.0, |
| "epoch": 0.10666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.11524740606546402, |
| "kl": 0.0, |
| "learning_rate": 1.25e-05, |
| "loss": 0.07866720855236053, |
| "num_tokens": 200701.0, |
| "reward": -0.6619112491607666, |
| "reward_std": 0.587931826710701, |
| "rewards/reward_fn/mean": -0.6619112491607666, |
| "rewards/reward_fn/std": 0.587931826710701, |
| "step": 16, |
| "step_time": 99.46528565394692 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 174.75, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 191.5, |
| "completions/max_terminated_length": 191.5, |
| "completions/mean_length": 174.75, |
| "completions/mean_terminated_length": 174.75, |
| "completions/min_length": 153.5, |
| "completions/min_terminated_length": 153.5, |
| "epoch": 0.11333333333333333, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.07091796398162842, |
| "kl": 0.0, |
| "learning_rate": 1.3333333333333333e-05, |
| "loss": -0.030646469444036484, |
| "num_tokens": 213113.0, |
| "reward": -1.163275420665741, |
| "reward_std": 0.4873338043689728, |
| "rewards/reward_fn/mean": -1.163275420665741, |
| "rewards/reward_fn/std": 0.4873338043689728, |
| "step": 17, |
| "step_time": 67.82858649804257 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 154.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 189.0, |
| "completions/max_terminated_length": 189.0, |
| "completions/mean_length": 154.6875, |
| "completions/mean_terminated_length": 154.6875, |
| "completions/min_length": 131.0, |
| "completions/min_terminated_length": 131.0, |
| "epoch": 0.12, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07412096858024597, |
| "kl": 0.0, |
| "learning_rate": 1.4166666666666668e-05, |
| "loss": 0.06621558219194412, |
| "num_tokens": 224596.0, |
| "reward": 0.09915217757225037, |
| "reward_std": 0.3922642320394516, |
| "rewards/reward_fn/mean": 0.09915217757225037, |
| "rewards/reward_fn/std": 0.3922642320394516, |
| "step": 18, |
| "step_time": 68.17481762805255 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 165.5625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 194.0, |
| "completions/max_terminated_length": 194.0, |
| "completions/mean_length": 165.5625, |
| "completions/mean_terminated_length": 165.5625, |
| "completions/min_length": 136.5, |
| "completions/min_terminated_length": 136.5, |
| "epoch": 0.12666666666666668, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0963885560631752, |
| "kl": 0.0, |
| "learning_rate": 1.5e-05, |
| "loss": 0.0464300662279129, |
| "num_tokens": 237013.0, |
| "reward": -0.12774711847305298, |
| "reward_std": 1.0569286346435547, |
| "rewards/reward_fn/mean": -0.12774711847305298, |
| "rewards/reward_fn/std": 1.0569286346435547, |
| "step": 19, |
| "step_time": 73.6154217319563 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 156.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 193.5, |
| "completions/max_terminated_length": 193.5, |
| "completions/mean_length": 156.9375, |
| "completions/mean_terminated_length": 156.9375, |
| "completions/min_length": 127.5, |
| "completions/min_terminated_length": 127.5, |
| "epoch": 0.13333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09283237904310226, |
| "kl": 0.0, |
| "learning_rate": 1.5833333333333333e-05, |
| "loss": 0.03124336153268814, |
| "num_tokens": 249164.0, |
| "reward": -0.7668359875679016, |
| "reward_std": 0.5728714317083359, |
| "rewards/reward_fn/mean": -0.7668359875679016, |
| "rewards/reward_fn/std": 0.5728714317083359, |
| "step": 20, |
| "step_time": 68.1898756630253 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 168.125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 229.5, |
| "completions/max_terminated_length": 229.5, |
| "completions/mean_length": 168.125, |
| "completions/mean_terminated_length": 168.125, |
| "completions/min_length": 120.5, |
| "completions/min_terminated_length": 120.5, |
| "epoch": 0.14, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.094845250248909, |
| "kl": 0.0, |
| "learning_rate": 1.6666666666666667e-05, |
| "loss": 0.09362012147903442, |
| "num_tokens": 260574.0, |
| "reward": -0.8211829662322998, |
| "reward_std": 0.7844448983669281, |
| "rewards/reward_fn/mean": -0.8211829662322998, |
| "rewards/reward_fn/std": 0.7844448983669281, |
| "step": 21, |
| "step_time": 76.81375163403573 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 181.0625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 223.5, |
| "completions/max_terminated_length": 223.5, |
| "completions/mean_length": 181.0625, |
| "completions/mean_terminated_length": 181.0625, |
| "completions/min_length": 140.0, |
| "completions/min_terminated_length": 140.0, |
| "epoch": 0.14666666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.054967641830444336, |
| "kl": 0.0, |
| "learning_rate": 1.75e-05, |
| "loss": 0.05134895443916321, |
| "num_tokens": 273087.0, |
| "reward": -1.3103257417678833, |
| "reward_std": 0.3521471619606018, |
| "rewards/reward_fn/mean": -1.3103257417678833, |
| "rewards/reward_fn/std": 0.3521471619606018, |
| "step": 22, |
| "step_time": 77.7118174440111 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 159.625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 234.5, |
| "completions/max_terminated_length": 234.5, |
| "completions/mean_length": 159.625, |
| "completions/mean_terminated_length": 159.625, |
| "completions/min_length": 123.0, |
| "completions/min_terminated_length": 123.0, |
| "epoch": 0.15333333333333332, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09953799843788147, |
| "kl": 0.0, |
| "learning_rate": 1.8333333333333333e-05, |
| "loss": -0.02984502911567688, |
| "num_tokens": 284569.0, |
| "reward": -0.8976435363292694, |
| "reward_std": 0.3961769565939903, |
| "rewards/reward_fn/mean": -0.8976435363292694, |
| "rewards/reward_fn/std": 0.3961769565939903, |
| "step": 23, |
| "step_time": 77.59863371500978 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 164.625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 178.5, |
| "completions/max_terminated_length": 178.5, |
| "completions/mean_length": 164.625, |
| "completions/mean_terminated_length": 164.625, |
| "completions/min_length": 138.0, |
| "completions/min_terminated_length": 138.0, |
| "epoch": 0.16, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08858790993690491, |
| "kl": 0.0, |
| "learning_rate": 1.9166666666666667e-05, |
| "loss": 0.045571208000183105, |
| "num_tokens": 296811.0, |
| "reward": -0.16475331783294678, |
| "reward_std": 0.3908658064901829, |
| "rewards/reward_fn/mean": -0.16475331783294678, |
| "rewards/reward_fn/std": 0.3908658064901829, |
| "step": 24, |
| "step_time": 64.24019336199854 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 176.25, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 216.5, |
| "completions/max_terminated_length": 216.5, |
| "completions/mean_length": 176.25, |
| "completions/mean_terminated_length": 176.25, |
| "completions/min_length": 150.5, |
| "completions/min_terminated_length": 150.5, |
| "epoch": 0.16666666666666666, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.07616627216339111, |
| "kl": 0.0, |
| "learning_rate": 2e-05, |
| "loss": -0.0033947303891181946, |
| "num_tokens": 309199.0, |
| "reward": -0.6221126019954681, |
| "reward_std": 0.17310830950737, |
| "rewards/reward_fn/mean": -0.6221126019954681, |
| "rewards/reward_fn/std": 0.17310830950737, |
| "step": 25, |
| "step_time": 73.73626439704094 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 246.625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 294.5, |
| "completions/max_terminated_length": 294.5, |
| "completions/mean_length": 246.625, |
| "completions/mean_terminated_length": 246.625, |
| "completions/min_length": 208.0, |
| "completions/min_terminated_length": 208.0, |
| "epoch": 0.17333333333333334, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.0425107404589653, |
| "kl": 0.0, |
| "learning_rate": 2.0833333333333336e-05, |
| "loss": -0.018586011603474617, |
| "num_tokens": 323297.0, |
| "reward": -1.3331425189971924, |
| "reward_std": 0.3089599609375, |
| "rewards/reward_fn/mean": -1.3331425189971924, |
| "rewards/reward_fn/std": 0.3089599609375, |
| "step": 26, |
| "step_time": 94.67684844596079 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 132.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 164.5, |
| "completions/max_terminated_length": 164.5, |
| "completions/mean_length": 132.6875, |
| "completions/mean_terminated_length": 132.6875, |
| "completions/min_length": 101.5, |
| "completions/min_terminated_length": 101.5, |
| "epoch": 0.18, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08376004546880722, |
| "kl": 0.0, |
| "learning_rate": 2.1666666666666667e-05, |
| "loss": -0.05854324996471405, |
| "num_tokens": 334436.0, |
| "reward": 0.26202554255723953, |
| "reward_std": 0.5281454622745514, |
| "rewards/reward_fn/mean": 0.26202554255723953, |
| "rewards/reward_fn/std": 0.5281454622745514, |
| "step": 27, |
| "step_time": 60.064499742002226 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 194.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 271.0, |
| "completions/max_terminated_length": 271.0, |
| "completions/mean_length": 194.6875, |
| "completions/mean_terminated_length": 194.6875, |
| "completions/min_length": 156.5, |
| "completions/min_terminated_length": 156.5, |
| "epoch": 0.18666666666666668, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1175316870212555, |
| "kl": 0.0, |
| "learning_rate": 2.25e-05, |
| "loss": 0.003547981381416321, |
| "num_tokens": 347655.0, |
| "reward": -0.8296540975570679, |
| "reward_std": 0.8124817311763763, |
| "rewards/reward_fn/mean": -0.8296540975570679, |
| "rewards/reward_fn/std": 0.8124817311763763, |
| "step": 28, |
| "step_time": 89.95816739893053 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 209.4375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 309.5, |
| "completions/max_terminated_length": 309.5, |
| "completions/mean_length": 209.4375, |
| "completions/mean_terminated_length": 209.4375, |
| "completions/min_length": 152.0, |
| "completions/min_terminated_length": 152.0, |
| "epoch": 0.19333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09398413449525833, |
| "kl": 0.0, |
| "learning_rate": 2.3333333333333336e-05, |
| "loss": -0.15494152903556824, |
| "num_tokens": 360318.0, |
| "reward": -0.8558410555124283, |
| "reward_std": 0.6401710212230682, |
| "rewards/reward_fn/mean": -0.8558410555124283, |
| "rewards/reward_fn/std": 0.6401710212230682, |
| "step": 29, |
| "step_time": 96.83519398601493 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 232.4375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 328.0, |
| "completions/max_terminated_length": 328.0, |
| "completions/mean_length": 232.4375, |
| "completions/mean_terminated_length": 232.4375, |
| "completions/min_length": 141.0, |
| "completions/min_terminated_length": 141.0, |
| "epoch": 0.2, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.061997007578611374, |
| "kl": 0.0, |
| "learning_rate": 2.4166666666666667e-05, |
| "loss": 0.11603285372257233, |
| "num_tokens": 373245.0, |
| "reward": -0.9066768288612366, |
| "reward_std": 0.8537713587284088, |
| "rewards/reward_fn/mean": -0.9066768288612366, |
| "rewards/reward_fn/std": 0.8537713587284088, |
| "step": 30, |
| "step_time": 105.47558436403051 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 157.4375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 213.0, |
| "completions/max_terminated_length": 213.0, |
| "completions/mean_length": 157.4375, |
| "completions/mean_terminated_length": 157.4375, |
| "completions/min_length": 123.0, |
| "completions/min_terminated_length": 123.0, |
| "epoch": 0.20666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07278493046760559, |
| "kl": 0.0, |
| "learning_rate": 2.5e-05, |
| "loss": 0.14571788907051086, |
| "num_tokens": 384564.0, |
| "reward": -0.9546848833560944, |
| "reward_std": 0.41054398447158746, |
| "rewards/reward_fn/mean": -0.9546848833560944, |
| "rewards/reward_fn/std": 0.41054398447158746, |
| "step": 31, |
| "step_time": 72.25721991300816 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 260.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 334.5, |
| "completions/max_terminated_length": 334.5, |
| "completions/mean_length": 260.0, |
| "completions/mean_terminated_length": 260.0, |
| "completions/min_length": 188.5, |
| "completions/min_terminated_length": 188.5, |
| "epoch": 0.21333333333333335, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.056718725711107254, |
| "kl": 0.0, |
| "learning_rate": 2.5833333333333336e-05, |
| "loss": 0.042716674506664276, |
| "num_tokens": 398956.0, |
| "reward": -1.2975031733512878, |
| "reward_std": 0.3875386118888855, |
| "rewards/reward_fn/mean": -1.2975031733512878, |
| "rewards/reward_fn/std": 0.3875386118888855, |
| "step": 32, |
| "step_time": 108.24371674098074 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 199.25, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 241.0, |
| "completions/max_terminated_length": 241.0, |
| "completions/mean_length": 199.25, |
| "completions/mean_terminated_length": 199.25, |
| "completions/min_length": 159.5, |
| "completions/min_terminated_length": 159.5, |
| "epoch": 0.22, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08926832675933838, |
| "kl": 0.0, |
| "learning_rate": 2.6666666666666667e-05, |
| "loss": -0.039713893085718155, |
| "num_tokens": 412240.0, |
| "reward": -0.13461190462112427, |
| "reward_std": 0.5121839940547943, |
| "rewards/reward_fn/mean": -0.13461190462112427, |
| "rewards/reward_fn/std": 0.5121839940547943, |
| "step": 33, |
| "step_time": 82.1538975770236 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 274.1875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 426.0, |
| "completions/max_terminated_length": 426.0, |
| "completions/mean_length": 274.1875, |
| "completions/mean_terminated_length": 274.1875, |
| "completions/min_length": 153.0, |
| "completions/min_terminated_length": 153.0, |
| "epoch": 0.22666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09963853657245636, |
| "kl": 0.0, |
| "learning_rate": 2.7500000000000004e-05, |
| "loss": -0.15619568526744843, |
| "num_tokens": 426331.0, |
| "reward": -1.0308716893196106, |
| "reward_std": 0.6284461915493011, |
| "rewards/reward_fn/mean": -1.0308716893196106, |
| "rewards/reward_fn/std": 0.6284461915493011, |
| "step": 34, |
| "step_time": 127.3287339269882 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 204.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 252.0, |
| "completions/max_terminated_length": 252.0, |
| "completions/mean_length": 204.6875, |
| "completions/mean_terminated_length": 204.6875, |
| "completions/min_length": 156.0, |
| "completions/min_terminated_length": 156.0, |
| "epoch": 0.23333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.15367895364761353, |
| "kl": 0.0, |
| "learning_rate": 2.8333333333333335e-05, |
| "loss": -0.03458409756422043, |
| "num_tokens": 438846.0, |
| "reward": -0.30794087052345276, |
| "reward_std": 0.7503761947154999, |
| "rewards/reward_fn/mean": -0.30794087052345276, |
| "rewards/reward_fn/std": 0.7503761947154999, |
| "step": 35, |
| "step_time": 85.05355873703957 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 223.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 324.5, |
| "completions/max_terminated_length": 324.5, |
| "completions/mean_length": 223.9375, |
| "completions/mean_terminated_length": 223.9375, |
| "completions/min_length": 141.0, |
| "completions/min_terminated_length": 141.0, |
| "epoch": 0.24, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.2046246975660324, |
| "kl": 0.0, |
| "learning_rate": 2.916666666666667e-05, |
| "loss": -0.07736283540725708, |
| "num_tokens": 451773.0, |
| "reward": -0.29854077845811844, |
| "reward_std": 0.44150011567398906, |
| "rewards/reward_fn/mean": -0.29854077845811844, |
| "rewards/reward_fn/std": 0.44150011567398906, |
| "step": 36, |
| "step_time": 101.35352180700283 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 145.8125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 173.0, |
| "completions/max_terminated_length": 173.0, |
| "completions/mean_length": 145.8125, |
| "completions/mean_terminated_length": 145.8125, |
| "completions/min_length": 125.0, |
| "completions/min_terminated_length": 125.0, |
| "epoch": 0.24666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.11604579538106918, |
| "kl": 0.0, |
| "learning_rate": 3e-05, |
| "loss": 0.09416678547859192, |
| "num_tokens": 463706.0, |
| "reward": 0.04717719554901123, |
| "reward_std": 0.4922184348106384, |
| "rewards/reward_fn/mean": 0.04717719554901123, |
| "rewards/reward_fn/std": 0.4922184348106384, |
| "step": 37, |
| "step_time": 62.971165234048385 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 208.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 234.0, |
| "completions/max_terminated_length": 234.0, |
| "completions/mean_length": 208.6875, |
| "completions/mean_terminated_length": 208.6875, |
| "completions/min_length": 196.0, |
| "completions/min_terminated_length": 196.0, |
| "epoch": 0.25333333333333335, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1814187616109848, |
| "kl": 0.0, |
| "learning_rate": 3.0833333333333335e-05, |
| "loss": -0.002286073751747608, |
| "num_tokens": 477237.0, |
| "reward": 0.18147915601730347, |
| "reward_std": 0.40245659090578556, |
| "rewards/reward_fn/mean": 0.18147915601730347, |
| "rewards/reward_fn/std": 0.40245659090578556, |
| "step": 38, |
| "step_time": 81.6735560669913 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 232.0625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 341.5, |
| "completions/max_terminated_length": 341.5, |
| "completions/mean_length": 232.0625, |
| "completions/mean_terminated_length": 232.0625, |
| "completions/min_length": 177.5, |
| "completions/min_terminated_length": 177.5, |
| "epoch": 0.26, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08577155321836472, |
| "kl": 0.0, |
| "learning_rate": 3.1666666666666666e-05, |
| "loss": -0.02454685978591442, |
| "num_tokens": 490294.0, |
| "reward": -0.25618940591812134, |
| "reward_std": 0.6517162024974823, |
| "rewards/reward_fn/mean": -0.25618940591812134, |
| "rewards/reward_fn/std": 0.6517162024974823, |
| "step": 39, |
| "step_time": 110.83180778095266 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 256.5625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 327.0, |
| "completions/max_terminated_length": 327.0, |
| "completions/mean_length": 256.5625, |
| "completions/mean_terminated_length": 256.5625, |
| "completions/min_length": 188.5, |
| "completions/min_terminated_length": 188.5, |
| "epoch": 0.26666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09231025725603104, |
| "kl": 0.0, |
| "learning_rate": 3.2500000000000004e-05, |
| "loss": -0.07835742831230164, |
| "num_tokens": 503831.0, |
| "reward": -0.2845441624522209, |
| "reward_std": 0.41623825021088123, |
| "rewards/reward_fn/mean": -0.2845441624522209, |
| "rewards/reward_fn/std": 0.41623825021088123, |
| "step": 40, |
| "step_time": 105.11443146498641 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 184.375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 216.5, |
| "completions/max_terminated_length": 216.5, |
| "completions/mean_length": 184.375, |
| "completions/mean_terminated_length": 184.375, |
| "completions/min_length": 157.5, |
| "completions/min_terminated_length": 157.5, |
| "epoch": 0.2733333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.11624035984277725, |
| "kl": 0.0, |
| "learning_rate": 3.3333333333333335e-05, |
| "loss": -0.07812672853469849, |
| "num_tokens": 516661.0, |
| "reward": -0.5956845507025719, |
| "reward_std": 0.27048898907378316, |
| "rewards/reward_fn/mean": -0.5956845507025719, |
| "rewards/reward_fn/std": 0.27048898907378316, |
| "step": 41, |
| "step_time": 75.3128279329976 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 149.5625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 174.0, |
| "completions/max_terminated_length": 174.0, |
| "completions/mean_length": 149.5625, |
| "completions/mean_terminated_length": 149.5625, |
| "completions/min_length": 135.5, |
| "completions/min_terminated_length": 135.5, |
| "epoch": 0.28, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.11803104728460312, |
| "kl": 0.0, |
| "learning_rate": 3.4166666666666666e-05, |
| "loss": -0.08490889519453049, |
| "num_tokens": 528838.0, |
| "reward": -0.35317955911159515, |
| "reward_std": 0.17408898659050465, |
| "rewards/reward_fn/mean": -0.35317955911159515, |
| "rewards/reward_fn/std": 0.17408898659050465, |
| "step": 42, |
| "step_time": 63.69105087395292 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 157.875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 197.5, |
| "completions/max_terminated_length": 197.5, |
| "completions/mean_length": 157.875, |
| "completions/mean_terminated_length": 157.875, |
| "completions/min_length": 121.0, |
| "completions/min_terminated_length": 121.0, |
| "epoch": 0.2866666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07110287994146347, |
| "kl": 0.0, |
| "learning_rate": 3.5e-05, |
| "loss": 0.016056224703788757, |
| "num_tokens": 540356.0, |
| "reward": 0.11271905899047852, |
| "reward_std": 0.010076586622744799, |
| "rewards/reward_fn/mean": 0.11271905899047852, |
| "rewards/reward_fn/std": 0.010076586622744799, |
| "step": 43, |
| "step_time": 68.61112404498272 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 184.8125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 235.5, |
| "completions/max_terminated_length": 235.5, |
| "completions/mean_length": 184.8125, |
| "completions/mean_terminated_length": 184.8125, |
| "completions/min_length": 147.0, |
| "completions/min_terminated_length": 147.0, |
| "epoch": 0.29333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09673087298870087, |
| "kl": 0.0, |
| "learning_rate": 3.5833333333333335e-05, |
| "loss": -0.034359708428382874, |
| "num_tokens": 553137.0, |
| "reward": -0.38676056265830994, |
| "reward_std": 0.5605057073407806, |
| "rewards/reward_fn/mean": -0.38676056265830994, |
| "rewards/reward_fn/std": 0.5605057073407806, |
| "step": 44, |
| "step_time": 79.45262654602993 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 160.4375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 173.5, |
| "completions/max_terminated_length": 173.5, |
| "completions/mean_length": 160.4375, |
| "completions/mean_terminated_length": 160.4375, |
| "completions/min_length": 145.0, |
| "completions/min_terminated_length": 145.0, |
| "epoch": 0.3, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07127348333597183, |
| "kl": 0.0, |
| "learning_rate": 3.6666666666666666e-05, |
| "loss": -0.017228223383426666, |
| "num_tokens": 565552.0, |
| "reward": -0.00951889157295227, |
| "reward_std": 0.1373694152571261, |
| "rewards/reward_fn/mean": -0.00951889157295227, |
| "rewards/reward_fn/std": 0.1373694152571261, |
| "step": 45, |
| "step_time": 64.07329139101785 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 294.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 378.0, |
| "completions/max_terminated_length": 378.0, |
| "completions/mean_length": 294.9375, |
| "completions/mean_terminated_length": 294.9375, |
| "completions/min_length": 209.5, |
| "completions/min_terminated_length": 209.5, |
| "epoch": 0.30666666666666664, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09467363357543945, |
| "kl": 0.0, |
| "learning_rate": 3.7500000000000003e-05, |
| "loss": -0.03529605269432068, |
| "num_tokens": 579951.0, |
| "reward": -1.2961561679840088, |
| "reward_std": 0.26750241219997406, |
| "rewards/reward_fn/mean": -1.2961561679840088, |
| "rewards/reward_fn/std": 0.26750241219997406, |
| "step": 46, |
| "step_time": 116.57304848608328 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 199.3125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 254.5, |
| "completions/max_terminated_length": 254.5, |
| "completions/mean_length": 199.3125, |
| "completions/mean_terminated_length": 199.3125, |
| "completions/min_length": 150.0, |
| "completions/min_terminated_length": 150.0, |
| "epoch": 0.31333333333333335, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.14669747650623322, |
| "kl": 0.0, |
| "learning_rate": 3.8333333333333334e-05, |
| "loss": -0.10157700628042221, |
| "num_tokens": 593324.0, |
| "reward": -0.15056359767913818, |
| "reward_std": 0.18598411977291107, |
| "rewards/reward_fn/mean": -0.15056359767913818, |
| "rewards/reward_fn/std": 0.18598411977291107, |
| "step": 47, |
| "step_time": 89.92860590590863 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 234.8125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 290.0, |
| "completions/max_terminated_length": 290.0, |
| "completions/mean_length": 234.8125, |
| "completions/mean_terminated_length": 234.8125, |
| "completions/min_length": 210.5, |
| "completions/min_terminated_length": 210.5, |
| "epoch": 0.32, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08877667784690857, |
| "kl": 0.0, |
| "learning_rate": 3.9166666666666665e-05, |
| "loss": -0.04996691644191742, |
| "num_tokens": 606257.0, |
| "reward": -0.5346458461135626, |
| "reward_std": 0.1979452222585678, |
| "rewards/reward_fn/mean": -0.5346458461135626, |
| "rewards/reward_fn/std": 0.1979452222585678, |
| "step": 48, |
| "step_time": 92.41859935398679 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 118.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 142.5, |
| "completions/max_terminated_length": 142.5, |
| "completions/mean_length": 118.0, |
| "completions/mean_terminated_length": 118.0, |
| "completions/min_length": 92.0, |
| "completions/min_terminated_length": 92.0, |
| "epoch": 0.32666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1030936986207962, |
| "kl": 0.0, |
| "learning_rate": 4e-05, |
| "loss": 0.03239426016807556, |
| "num_tokens": 616705.0, |
| "reward": 0.04530230909585953, |
| "reward_std": 0.5476353615522385, |
| "rewards/reward_fn/mean": 0.04530230909585953, |
| "rewards/reward_fn/std": 0.5476353615522385, |
| "step": 49, |
| "step_time": 53.05935197096551 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 174.5625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 219.5, |
| "completions/max_terminated_length": 219.5, |
| "completions/mean_length": 174.5625, |
| "completions/mean_terminated_length": 174.5625, |
| "completions/min_length": 134.5, |
| "completions/min_terminated_length": 134.5, |
| "epoch": 0.3333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.18021085858345032, |
| "kl": 0.0, |
| "learning_rate": 4.0833333333333334e-05, |
| "loss": -0.07679768651723862, |
| "num_tokens": 628386.0, |
| "reward": -0.2784375250339508, |
| "reward_std": 0.6172725409269333, |
| "rewards/reward_fn/mean": -0.2784375250339508, |
| "rewards/reward_fn/std": 0.6172725409269333, |
| "step": 50, |
| "step_time": 73.86384335003095 |
| }, |
| { |
| "epoch": 0.3333333333333333, |
| "eval_clip_ratio/high_max": 0.0, |
| "eval_clip_ratio/high_mean": 0.0, |
| "eval_clip_ratio/low_mean": 0.0, |
| "eval_clip_ratio/low_min": 0.0, |
| "eval_clip_ratio/region_mean": 0.0, |
| "eval_completion_length": 244.61125, |
| "eval_completions/clipped_ratio": 0.0, |
| "eval_completions/max_length": 305.91, |
| "eval_completions/max_terminated_length": 305.91, |
| "eval_completions/mean_length": 244.61125, |
| "eval_completions/mean_terminated_length": 244.61125, |
| "eval_completions/min_length": 190.08, |
| "eval_completions/min_terminated_length": 190.08, |
| "eval_frac_reward_zero_std": 0.01, |
| "eval_kl": 0.0, |
| "eval_loss": 0.002150977263227105, |
| "eval_num_tokens": 628386.0, |
| "eval_reward": -0.2439578665420413, |
| "eval_reward_std": 0.40503982904250735, |
| "eval_rewards/reward_fn/mean": -0.2439578665420413, |
| "eval_rewards/reward_fn/std": 0.40503982904250735, |
| "eval_runtime": 4225.7071, |
| "eval_samples_per_second": 0.024, |
| "eval_steps_per_second": 0.003, |
| "step": 50 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 165.0625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 200.0, |
| "completions/max_terminated_length": 200.0, |
| "completions/mean_length": 165.0625, |
| "completions/mean_terminated_length": 165.0625, |
| "completions/min_length": 138.0, |
| "completions/min_terminated_length": 138.0, |
| "epoch": 0.34, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07993482053279877, |
| "kl": 0.0, |
| "learning_rate": 4.166666666666667e-05, |
| "loss": 0.020043745636940002, |
| "num_tokens": 639827.0, |
| "reward": -0.4589511938393116, |
| "reward_std": 0.24016011551429983, |
| "rewards/reward_fn/mean": -0.4589511938393116, |
| "rewards/reward_fn/std": 0.24016011551429983, |
| "step": 51, |
| "step_time": 69.2877980620251 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 168.5, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 213.5, |
| "completions/max_terminated_length": 213.5, |
| "completions/mean_length": 168.5, |
| "completions/mean_terminated_length": 168.5, |
| "completions/min_length": 148.5, |
| "completions/min_terminated_length": 148.5, |
| "epoch": 0.3466666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.10122694820165634, |
| "kl": 0.0, |
| "learning_rate": 4.25e-05, |
| "loss": 0.010254371911287308, |
| "num_tokens": 652363.0, |
| "reward": -0.5353640392422676, |
| "reward_std": 0.12437161646721506, |
| "rewards/reward_fn/mean": -0.5353640392422676, |
| "rewards/reward_fn/std": 0.12437161646721506, |
| "step": 52, |
| "step_time": 73.87324213504326 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 222.625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 288.5, |
| "completions/max_terminated_length": 288.5, |
| "completions/mean_length": 222.625, |
| "completions/mean_terminated_length": 222.625, |
| "completions/min_length": 191.5, |
| "completions/min_terminated_length": 191.5, |
| "epoch": 0.35333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0843462198972702, |
| "kl": 0.0, |
| "learning_rate": 4.3333333333333334e-05, |
| "loss": -0.03871044144034386, |
| "num_tokens": 665613.0, |
| "reward": 0.12957476451992989, |
| "reward_std": 0.21294647455215454, |
| "rewards/reward_fn/mean": 0.12957476451992989, |
| "rewards/reward_fn/std": 0.21294647455215454, |
| "step": 53, |
| "step_time": 92.9070069710142 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 193.0625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 228.0, |
| "completions/max_terminated_length": 228.0, |
| "completions/mean_length": 193.0625, |
| "completions/mean_terminated_length": 193.0625, |
| "completions/min_length": 162.5, |
| "completions/min_terminated_length": 162.5, |
| "epoch": 0.36, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08981351554393768, |
| "kl": 0.0, |
| "learning_rate": 4.4166666666666665e-05, |
| "loss": -0.005176428705453873, |
| "num_tokens": 678446.0, |
| "reward": -0.22327297925949097, |
| "reward_std": 0.20772529065288836, |
| "rewards/reward_fn/mean": -0.22327297925949097, |
| "rewards/reward_fn/std": 0.20772529065288836, |
| "step": 54, |
| "step_time": 78.26076654792996 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 239.125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 293.0, |
| "completions/max_terminated_length": 293.0, |
| "completions/mean_length": 239.125, |
| "completions/mean_terminated_length": 239.125, |
| "completions/min_length": 195.0, |
| "completions/min_terminated_length": 195.0, |
| "epoch": 0.36666666666666664, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06711370497941971, |
| "kl": 0.0, |
| "learning_rate": 4.5e-05, |
| "loss": 0.04008489102125168, |
| "num_tokens": 692144.0, |
| "reward": -0.32132431864738464, |
| "reward_std": 0.12893771713788738, |
| "rewards/reward_fn/mean": -0.32132431864738464, |
| "rewards/reward_fn/std": 0.12893771713788738, |
| "step": 55, |
| "step_time": 95.0031545261154 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 277.5, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 323.0, |
| "completions/max_terminated_length": 323.0, |
| "completions/mean_length": 277.5, |
| "completions/mean_terminated_length": 277.5, |
| "completions/min_length": 231.0, |
| "completions/min_terminated_length": 231.0, |
| "epoch": 0.37333333333333335, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.074734166264534, |
| "kl": 0.0, |
| "learning_rate": 4.5833333333333334e-05, |
| "loss": -0.027020789682865143, |
| "num_tokens": 705904.0, |
| "reward": 0.2992004454135895, |
| "reward_std": 0.5213650465011597, |
| "rewards/reward_fn/mean": 0.2992004454135895, |
| "rewards/reward_fn/std": 0.5213650465011597, |
| "step": 56, |
| "step_time": 102.65910005901242 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 209.3125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 243.5, |
| "completions/max_terminated_length": 243.5, |
| "completions/mean_length": 209.3125, |
| "completions/mean_terminated_length": 209.3125, |
| "completions/min_length": 199.0, |
| "completions/min_terminated_length": 199.0, |
| "epoch": 0.38, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.04772329702973366, |
| "kl": 0.0, |
| "learning_rate": 4.666666666666667e-05, |
| "loss": 0.01125314086675644, |
| "num_tokens": 718213.0, |
| "reward": -0.6336008310317993, |
| "reward_std": 0.2387970182016943, |
| "rewards/reward_fn/mean": -0.6336008310317993, |
| "rewards/reward_fn/std": 0.2387970182016943, |
| "step": 57, |
| "step_time": 83.00291228800779 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 262.4375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 343.5, |
| "completions/max_terminated_length": 343.5, |
| "completions/mean_length": 262.4375, |
| "completions/mean_terminated_length": 262.4375, |
| "completions/min_length": 208.5, |
| "completions/min_terminated_length": 208.5, |
| "epoch": 0.38666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.10154283046722412, |
| "kl": 0.0, |
| "learning_rate": 4.75e-05, |
| "loss": -0.06698265671730042, |
| "num_tokens": 731612.0, |
| "reward": 0.13350587338209152, |
| "reward_std": 0.46326927840709686, |
| "rewards/reward_fn/mean": 0.13350587338209152, |
| "rewards/reward_fn/std": 0.46326927840709686, |
| "step": 58, |
| "step_time": 106.23267988796579 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 283.875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 305.0, |
| "completions/max_terminated_length": 305.0, |
| "completions/mean_length": 283.875, |
| "completions/mean_terminated_length": 283.875, |
| "completions/min_length": 223.0, |
| "completions/min_terminated_length": 223.0, |
| "epoch": 0.3933333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08558438718318939, |
| "kl": 0.0, |
| "learning_rate": 4.8333333333333334e-05, |
| "loss": -0.006462842226028442, |
| "num_tokens": 746274.0, |
| "reward": 0.035797879099845886, |
| "reward_std": 0.6359168291091919, |
| "rewards/reward_fn/mean": 0.035797879099845886, |
| "rewards/reward_fn/std": 0.6359168291091919, |
| "step": 59, |
| "step_time": 103.01650287100347 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 220.125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 254.5, |
| "completions/max_terminated_length": 254.5, |
| "completions/mean_length": 220.125, |
| "completions/mean_terminated_length": 220.125, |
| "completions/min_length": 186.5, |
| "completions/min_terminated_length": 186.5, |
| "epoch": 0.4, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1676492542028427, |
| "kl": 0.0, |
| "learning_rate": 4.9166666666666665e-05, |
| "loss": -0.056491315364837646, |
| "num_tokens": 758820.0, |
| "reward": -0.6773550808429718, |
| "reward_std": 0.28672073036432266, |
| "rewards/reward_fn/mean": -0.6773550808429718, |
| "rewards/reward_fn/std": 0.28672073036432266, |
| "step": 60, |
| "step_time": 85.76503377000336 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 203.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 217.0, |
| "completions/max_terminated_length": 217.0, |
| "completions/mean_length": 203.9375, |
| "completions/mean_terminated_length": 203.9375, |
| "completions/min_length": 196.0, |
| "completions/min_terminated_length": 196.0, |
| "epoch": 0.4066666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07807639241218567, |
| "kl": 0.0, |
| "learning_rate": 5e-05, |
| "loss": -0.006218082271516323, |
| "num_tokens": 771731.0, |
| "reward": 0.7148753702640533, |
| "reward_std": 0.07848419807851315, |
| "rewards/reward_fn/mean": 0.7148753702640533, |
| "rewards/reward_fn/std": 0.07848419807851315, |
| "step": 61, |
| "step_time": 74.89335807203315 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 299.75, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 403.5, |
| "completions/max_terminated_length": 403.5, |
| "completions/mean_length": 299.75, |
| "completions/mean_terminated_length": 299.75, |
| "completions/min_length": 223.5, |
| "completions/min_terminated_length": 223.5, |
| "epoch": 0.41333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07576266676187515, |
| "kl": 0.0, |
| "learning_rate": 4.999957692144361e-05, |
| "loss": 0.09580255299806595, |
| "num_tokens": 786671.0, |
| "reward": -0.053020179271698, |
| "reward_std": 0.4070226475596428, |
| "rewards/reward_fn/mean": -0.053020179271698, |
| "rewards/reward_fn/std": 0.4070226475596428, |
| "step": 62, |
| "step_time": 123.00635565299308 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 251.375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 385.5, |
| "completions/max_terminated_length": 385.5, |
| "completions/mean_length": 251.375, |
| "completions/mean_terminated_length": 251.375, |
| "completions/min_length": 200.0, |
| "completions/min_terminated_length": 200.0, |
| "epoch": 0.42, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1709807962179184, |
| "kl": 0.0, |
| "learning_rate": 4.999830770009406e-05, |
| "loss": 0.0437316857278347, |
| "num_tokens": 799901.0, |
| "reward": 0.33461277186870575, |
| "reward_std": 0.4237028732895851, |
| "rewards/reward_fn/mean": 0.33461277186870575, |
| "rewards/reward_fn/std": 0.4237028732895851, |
| "step": 63, |
| "step_time": 118.21145736804465 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 196.1875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 230.5, |
| "completions/max_terminated_length": 230.5, |
| "completions/mean_length": 196.1875, |
| "completions/mean_terminated_length": 196.1875, |
| "completions/min_length": 168.0, |
| "completions/min_terminated_length": 168.0, |
| "epoch": 0.4266666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09982649981975555, |
| "kl": 0.0, |
| "learning_rate": 4.9996192378909786e-05, |
| "loss": 0.007593698799610138, |
| "num_tokens": 812552.0, |
| "reward": 0.2559966593980789, |
| "reward_std": 0.6412993967533112, |
| "rewards/reward_fn/mean": 0.2559966593980789, |
| "rewards/reward_fn/std": 0.6412993967533112, |
| "step": 64, |
| "step_time": 77.57821400102694 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 267.375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 325.0, |
| "completions/max_terminated_length": 325.0, |
| "completions/mean_length": 267.375, |
| "completions/mean_terminated_length": 267.375, |
| "completions/min_length": 220.5, |
| "completions/min_terminated_length": 220.5, |
| "epoch": 0.43333333333333335, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08924577385187149, |
| "kl": 0.0, |
| "learning_rate": 4.9993231029486544e-05, |
| "loss": 0.0375928059220314, |
| "num_tokens": 826230.0, |
| "reward": -0.2883334718644619, |
| "reward_std": 0.28712435346096754, |
| "rewards/reward_fn/mean": -0.2883334718644619, |
| "rewards/reward_fn/std": 0.28712435346096754, |
| "step": 65, |
| "step_time": 104.71041284193052 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 218.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 283.5, |
| "completions/max_terminated_length": 283.5, |
| "completions/mean_length": 218.6875, |
| "completions/mean_terminated_length": 218.6875, |
| "completions/min_length": 153.5, |
| "completions/min_terminated_length": 153.5, |
| "epoch": 0.44, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.11544258147478104, |
| "kl": 0.0, |
| "learning_rate": 4.998942375205502e-05, |
| "loss": -0.07060755789279938, |
| "num_tokens": 839281.0, |
| "reward": 0.40776287019252777, |
| "reward_std": 0.41822005808353424, |
| "rewards/reward_fn/mean": 0.40776287019252777, |
| "rewards/reward_fn/std": 0.41822005808353424, |
| "step": 66, |
| "step_time": 91.1544282680843 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 173.8125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 195.5, |
| "completions/max_terminated_length": 195.5, |
| "completions/mean_length": 173.8125, |
| "completions/mean_terminated_length": 173.8125, |
| "completions/min_length": 144.5, |
| "completions/min_terminated_length": 144.5, |
| "epoch": 0.44666666666666666, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.1583082377910614, |
| "kl": 0.0, |
| "learning_rate": 4.99847706754774e-05, |
| "loss": 0.0034518204629421234, |
| "num_tokens": 850894.0, |
| "reward": 0.48864591121673584, |
| "reward_std": 0.6118558049201965, |
| "rewards/reward_fn/mean": 0.48864591121673584, |
| "rewards/reward_fn/std": 0.6118558049201965, |
| "step": 67, |
| "step_time": 69.27147658000467 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 223.625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 254.0, |
| "completions/max_terminated_length": 254.0, |
| "completions/mean_length": 223.625, |
| "completions/mean_terminated_length": 223.625, |
| "completions/min_length": 167.5, |
| "completions/min_terminated_length": 167.5, |
| "epoch": 0.4533333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09846646338701248, |
| "kl": 0.0, |
| "learning_rate": 4.997927195724303e-05, |
| "loss": -0.030570141971111298, |
| "num_tokens": 863432.0, |
| "reward": 0.6720858812332153, |
| "reward_std": 0.18690402715583332, |
| "rewards/reward_fn/mean": 0.6720858812332153, |
| "rewards/reward_fn/std": 0.18690402715583332, |
| "step": 68, |
| "step_time": 84.2234050030238 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 239.75, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 257.0, |
| "completions/max_terminated_length": 257.0, |
| "completions/mean_length": 239.75, |
| "completions/mean_terminated_length": 239.75, |
| "completions/min_length": 217.0, |
| "completions/min_terminated_length": 217.0, |
| "epoch": 0.46, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07295793294906616, |
| "kl": 0.0, |
| "learning_rate": 4.997292778346312e-05, |
| "loss": 0.015293777920305729, |
| "num_tokens": 877908.0, |
| "reward": 0.4418184161186218, |
| "reward_std": 0.2608553245663643, |
| "rewards/reward_fn/mean": 0.4418184161186218, |
| "rewards/reward_fn/std": 0.2608553245663643, |
| "step": 69, |
| "step_time": 87.94977311202092 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 284.3125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 331.5, |
| "completions/max_terminated_length": 331.5, |
| "completions/mean_length": 284.3125, |
| "completions/mean_terminated_length": 284.3125, |
| "completions/min_length": 226.5, |
| "completions/min_terminated_length": 226.5, |
| "epoch": 0.4666666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.05379607528448105, |
| "kl": 0.0, |
| "learning_rate": 4.996573836886435e-05, |
| "loss": -0.02144354209303856, |
| "num_tokens": 891793.0, |
| "reward": 0.2061479091644287, |
| "reward_std": 0.3776038587093353, |
| "rewards/reward_fn/mean": 0.2061479091644287, |
| "rewards/reward_fn/std": 0.3776038587093353, |
| "step": 70, |
| "step_time": 116.79733415995724 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 225.5625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 283.5, |
| "completions/max_terminated_length": 283.5, |
| "completions/mean_length": 225.5625, |
| "completions/mean_terminated_length": 225.5625, |
| "completions/min_length": 189.0, |
| "completions/min_terminated_length": 189.0, |
| "epoch": 0.47333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.11158322542905807, |
| "kl": 0.0, |
| "learning_rate": 4.995770395678171e-05, |
| "loss": -0.021105844527482986, |
| "num_tokens": 905042.0, |
| "reward": 0.7146752774715424, |
| "reward_std": 0.3213859871029854, |
| "rewards/reward_fn/mean": 0.7146752774715424, |
| "rewards/reward_fn/std": 0.3213859871029854, |
| "step": 71, |
| "step_time": 92.01113651989726 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 178.3125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 215.5, |
| "completions/max_terminated_length": 215.5, |
| "completions/mean_length": 178.3125, |
| "completions/mean_terminated_length": 178.3125, |
| "completions/min_length": 144.0, |
| "completions/min_terminated_length": 144.0, |
| "epoch": 0.48, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1308518946170807, |
| "kl": 0.0, |
| "learning_rate": 4.9948824819150185e-05, |
| "loss": -0.017831377685070038, |
| "num_tokens": 916575.0, |
| "reward": 0.8005787134170532, |
| "reward_std": 0.09497696161270142, |
| "rewards/reward_fn/mean": 0.8005787134170532, |
| "rewards/reward_fn/std": 0.09497696161270142, |
| "step": 72, |
| "step_time": 72.7636261759908 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 224.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 285.0, |
| "completions/max_terminated_length": 285.0, |
| "completions/mean_length": 224.6875, |
| "completions/mean_terminated_length": 224.6875, |
| "completions/min_length": 178.0, |
| "completions/min_terminated_length": 178.0, |
| "epoch": 0.4866666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.10509830713272095, |
| "kl": 0.0, |
| "learning_rate": 4.993910125649561e-05, |
| "loss": 0.07416598498821259, |
| "num_tokens": 929778.0, |
| "reward": 0.6564783751964569, |
| "reward_std": 0.26796439323516097, |
| "rewards/reward_fn/mean": 0.6564783751964569, |
| "rewards/reward_fn/std": 0.26796439323516097, |
| "step": 73, |
| "step_time": 92.07556184596615 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 273.5625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 349.0, |
| "completions/max_terminated_length": 349.0, |
| "completions/mean_length": 273.5625, |
| "completions/mean_terminated_length": 273.5625, |
| "completions/min_length": 215.0, |
| "completions/min_terminated_length": 215.0, |
| "epoch": 0.49333333333333335, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09755323827266693, |
| "kl": 0.0, |
| "learning_rate": 4.992853359792444e-05, |
| "loss": -0.037562236189842224, |
| "num_tokens": 943499.0, |
| "reward": -0.27786024287343025, |
| "reward_std": 0.3537736553698778, |
| "rewards/reward_fn/mean": -0.27786024287343025, |
| "rewards/reward_fn/std": 0.3537736553698778, |
| "step": 74, |
| "step_time": 115.20405016100267 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 260.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 344.5, |
| "completions/max_terminated_length": 344.5, |
| "completions/mean_length": 260.9375, |
| "completions/mean_terminated_length": 260.9375, |
| "completions/min_length": 185.0, |
| "completions/min_terminated_length": 185.0, |
| "epoch": 0.5, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03571086376905441, |
| "kl": 0.0, |
| "learning_rate": 4.9917122201112656e-05, |
| "loss": -0.016071610152721405, |
| "num_tokens": 956394.0, |
| "reward": 0.9999693334102631, |
| "reward_std": 5.47565687156748e-05, |
| "rewards/reward_fn/mean": 0.9999693334102631, |
| "rewards/reward_fn/std": 5.47565687156748e-05, |
| "step": 75, |
| "step_time": 105.76170146500226 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 271.5625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 356.0, |
| "completions/max_terminated_length": 356.0, |
| "completions/mean_length": 271.5625, |
| "completions/mean_terminated_length": 271.5625, |
| "completions/min_length": 198.0, |
| "completions/min_terminated_length": 198.0, |
| "epoch": 0.5066666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0005527990870177746, |
| "kl": 0.0, |
| "learning_rate": 4.990486745229364e-05, |
| "loss": 0.00025011063553392887, |
| "num_tokens": 969475.0, |
| "reward": 0.9999998211860657, |
| "reward_std": 5.813290879785882e-07, |
| "rewards/reward_fn/mean": 0.9999998211860657, |
| "rewards/reward_fn/std": 5.813290879785882e-07, |
| "step": 76, |
| "step_time": 108.75159504503245 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 227.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 282.5, |
| "completions/max_terminated_length": 282.5, |
| "completions/mean_length": 227.6875, |
| "completions/mean_terminated_length": 227.6875, |
| "completions/min_length": 200.0, |
| "completions/min_terminated_length": 200.0, |
| "epoch": 0.5133333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.10346979647874832, |
| "kl": 0.0, |
| "learning_rate": 4.989176976624511e-05, |
| "loss": 0.009114399552345276, |
| "num_tokens": 982878.0, |
| "reward": 0.5748505890369415, |
| "reward_std": 0.61591437458992, |
| "rewards/reward_fn/mean": 0.5748505890369415, |
| "rewards/reward_fn/std": 0.61591437458992, |
| "step": 77, |
| "step_time": 96.31299933302216 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 307.4375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 414.0, |
| "completions/max_terminated_length": 414.0, |
| "completions/mean_length": 307.4375, |
| "completions/mean_terminated_length": 307.4375, |
| "completions/min_length": 244.0, |
| "completions/min_terminated_length": 244.0, |
| "epoch": 0.52, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.11586609482765198, |
| "kl": 0.0, |
| "learning_rate": 4.987782958627508e-05, |
| "loss": -0.00447622686624527, |
| "num_tokens": 996965.0, |
| "reward": 0.17336545884609222, |
| "reward_std": 0.4828975200653076, |
| "rewards/reward_fn/mean": 0.17336545884609222, |
| "rewards/reward_fn/std": 0.4828975200653076, |
| "step": 78, |
| "step_time": 142.4235202560085 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 314.1875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 414.5, |
| "completions/max_terminated_length": 414.5, |
| "completions/mean_length": 314.1875, |
| "completions/mean_terminated_length": 314.1875, |
| "completions/min_length": 221.0, |
| "completions/min_terminated_length": 221.0, |
| "epoch": 0.5266666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.10377015173435211, |
| "kl": 0.0, |
| "learning_rate": 4.9863047384206835e-05, |
| "loss": 0.06035196781158447, |
| "num_tokens": 1011376.0, |
| "reward": -0.3835201971232891, |
| "reward_std": 0.330098245292902, |
| "rewards/reward_fn/mean": -0.3835201971232891, |
| "rewards/reward_fn/std": 0.330098245292902, |
| "step": 79, |
| "step_time": 134.57437156600645 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 253.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 302.5, |
| "completions/max_terminated_length": 302.5, |
| "completions/mean_length": 253.6875, |
| "completions/mean_terminated_length": 253.6875, |
| "completions/min_length": 224.5, |
| "completions/min_terminated_length": 224.5, |
| "epoch": 0.5333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08293555676937103, |
| "kl": 0.0, |
| "learning_rate": 4.9847423660363e-05, |
| "loss": -0.019359871745109558, |
| "num_tokens": 1024363.0, |
| "reward": 0.9290256202220917, |
| "reward_std": 0.049609191715717316, |
| "rewards/reward_fn/mean": 0.9290256202220917, |
| "rewards/reward_fn/std": 0.049609191715717316, |
| "step": 80, |
| "step_time": 103.19731909304392 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 292.4375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 326.5, |
| "completions/max_terminated_length": 326.5, |
| "completions/mean_length": 292.4375, |
| "completions/mean_terminated_length": 292.4375, |
| "completions/min_length": 253.0, |
| "completions/min_terminated_length": 253.0, |
| "epoch": 0.54, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1021459549665451, |
| "kl": 0.0, |
| "learning_rate": 4.983095894354858e-05, |
| "loss": 0.010597247630357742, |
| "num_tokens": 1038818.0, |
| "reward": 0.6705855131149292, |
| "reward_std": 0.5288838520646095, |
| "rewards/reward_fn/mean": 0.6705855131149292, |
| "rewards/reward_fn/std": 0.5288838520646095, |
| "step": 81, |
| "step_time": 104.1593450899818 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 186.1875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 214.5, |
| "completions/max_terminated_length": 214.5, |
| "completions/mean_length": 186.1875, |
| "completions/mean_terminated_length": 186.1875, |
| "completions/min_length": 158.5, |
| "completions/min_terminated_length": 158.5, |
| "epoch": 0.5466666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.13033370673656464, |
| "kl": 0.0, |
| "learning_rate": 4.9813653791033057e-05, |
| "loss": -0.013745991513133049, |
| "num_tokens": 1051325.0, |
| "reward": 0.4789735674858093, |
| "reward_std": 0.6973891258239746, |
| "rewards/reward_fn/mean": 0.4789735674858093, |
| "rewards/reward_fn/std": 0.6973891258239746, |
| "step": 82, |
| "step_time": 73.42914309195476 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 350.8125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 428.5, |
| "completions/max_terminated_length": 428.5, |
| "completions/mean_length": 350.8125, |
| "completions/mean_terminated_length": 350.8125, |
| "completions/min_length": 274.5, |
| "completions/min_terminated_length": 274.5, |
| "epoch": 0.5533333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08932679146528244, |
| "kl": 0.0, |
| "learning_rate": 4.979550878853154e-05, |
| "loss": -0.06357850879430771, |
| "num_tokens": 1066626.0, |
| "reward": -0.35760005563497543, |
| "reward_std": 0.4595603495836258, |
| "rewards/reward_fn/mean": -0.35760005563497543, |
| "rewards/reward_fn/std": 0.4595603495836258, |
| "step": 83, |
| "step_time": 131.997143570974 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 254.4375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 320.5, |
| "completions/max_terminated_length": 320.5, |
| "completions/mean_length": 254.4375, |
| "completions/mean_terminated_length": 254.4375, |
| "completions/min_length": 214.5, |
| "completions/min_terminated_length": 214.5, |
| "epoch": 0.56, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06409084051847458, |
| "kl": 0.0, |
| "learning_rate": 4.9776524550184965e-05, |
| "loss": 0.025061018764972687, |
| "num_tokens": 1080409.0, |
| "reward": 0.7382991313934326, |
| "reward_std": 0.05440911108235014, |
| "rewards/reward_fn/mean": 0.7382991313934326, |
| "rewards/reward_fn/std": 0.05440911108235014, |
| "step": 84, |
| "step_time": 101.2649952210486 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 250.75, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 267.5, |
| "completions/max_terminated_length": 267.5, |
| "completions/mean_length": 250.75, |
| "completions/mean_terminated_length": 250.75, |
| "completions/min_length": 225.0, |
| "completions/min_terminated_length": 225.0, |
| "epoch": 0.5666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07058465480804443, |
| "kl": 0.0, |
| "learning_rate": 4.975670171853926e-05, |
| "loss": -0.009824715554714203, |
| "num_tokens": 1093269.0, |
| "reward": 0.5432418249547482, |
| "reward_std": 0.011591001180931926, |
| "rewards/reward_fn/mean": 0.5432418249547482, |
| "rewards/reward_fn/std": 0.011591001180931926, |
| "step": 85, |
| "step_time": 94.21445810300065 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 319.75, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 418.0, |
| "completions/max_terminated_length": 418.0, |
| "completions/mean_length": 319.75, |
| "completions/mean_terminated_length": 319.75, |
| "completions/min_length": 272.0, |
| "completions/min_terminated_length": 272.0, |
| "epoch": 0.5733333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0850214958190918, |
| "kl": 0.0, |
| "learning_rate": 4.973604096452361e-05, |
| "loss": -0.04513334110379219, |
| "num_tokens": 1107625.0, |
| "reward": 0.30771853029727936, |
| "reward_std": 0.30676984786987305, |
| "rewards/reward_fn/mean": 0.30771853029727936, |
| "rewards/reward_fn/std": 0.30676984786987305, |
| "step": 86, |
| "step_time": 128.4876309260726 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 376.5, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 635.0, |
| "completions/max_terminated_length": 635.0, |
| "completions/mean_length": 376.5, |
| "completions/mean_terminated_length": 376.5, |
| "completions/min_length": 265.5, |
| "completions/min_terminated_length": 265.5, |
| "epoch": 0.58, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0707458034157753, |
| "kl": 0.0, |
| "learning_rate": 4.971454298742779e-05, |
| "loss": -0.15971024334430695, |
| "num_tokens": 1123865.0, |
| "reward": 0.49016936123371124, |
| "reward_std": 0.2892051860690117, |
| "rewards/reward_fn/mean": 0.49016936123371124, |
| "rewards/reward_fn/std": 0.2892051860690117, |
| "step": 87, |
| "step_time": 187.60785773402313 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 306.5, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 394.5, |
| "completions/max_terminated_length": 394.5, |
| "completions/mean_length": 306.5, |
| "completions/mean_terminated_length": 306.5, |
| "completions/min_length": 237.5, |
| "completions/min_terminated_length": 237.5, |
| "epoch": 0.5866666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05612671747803688, |
| "kl": 0.0, |
| "learning_rate": 4.9692208514878444e-05, |
| "loss": -0.011051323264837265, |
| "num_tokens": 1138457.0, |
| "reward": 0.6979325413703918, |
| "reward_std": 0.3039002874547805, |
| "rewards/reward_fn/mean": 0.6979325413703918, |
| "rewards/reward_fn/std": 0.3039002874547805, |
| "step": 88, |
| "step_time": 120.77513207792072 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 200.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 221.0, |
| "completions/max_terminated_length": 221.0, |
| "completions/mean_length": 200.9375, |
| "completions/mean_terminated_length": 200.9375, |
| "completions/min_length": 185.0, |
| "completions/min_terminated_length": 185.0, |
| "epoch": 0.5933333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08993715792894363, |
| "kl": 0.0, |
| "learning_rate": 4.966903830281449e-05, |
| "loss": 0.003101535141468048, |
| "num_tokens": 1150488.0, |
| "reward": 0.5542193427681923, |
| "reward_std": 0.0270778898228059, |
| "rewards/reward_fn/mean": 0.5542193427681923, |
| "rewards/reward_fn/std": 0.0270778898228059, |
| "step": 89, |
| "step_time": 82.04292890999932 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 266.5625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 295.0, |
| "completions/max_terminated_length": 295.0, |
| "completions/mean_length": 266.5625, |
| "completions/mean_terminated_length": 266.5625, |
| "completions/min_length": 239.5, |
| "completions/min_terminated_length": 239.5, |
| "epoch": 0.6, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08966866135597229, |
| "kl": 0.0, |
| "learning_rate": 4.9645033135461494e-05, |
| "loss": 0.005549095571041107, |
| "num_tokens": 1164649.0, |
| "reward": 0.4357367791235447, |
| "reward_std": 0.11432263441383839, |
| "rewards/reward_fn/mean": 0.4357367791235447, |
| "rewards/reward_fn/std": 0.11432263441383839, |
| "step": 90, |
| "step_time": 99.49214323301567 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 214.875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 273.0, |
| "completions/max_terminated_length": 273.0, |
| "completions/mean_length": 214.875, |
| "completions/mean_terminated_length": 214.875, |
| "completions/min_length": 170.0, |
| "completions/min_terminated_length": 170.0, |
| "epoch": 0.6066666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06210985407233238, |
| "kl": 0.0, |
| "learning_rate": 4.962019382530521e-05, |
| "loss": -0.010337941348552704, |
| "num_tokens": 1177647.0, |
| "reward": 0.7368387281894684, |
| "reward_std": 0.1752328469083153, |
| "rewards/reward_fn/mean": 0.7368387281894684, |
| "rewards/reward_fn/std": 0.1752328469083153, |
| "step": 91, |
| "step_time": 88.74471832800191 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 428.375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 519.5, |
| "completions/max_terminated_length": 519.5, |
| "completions/mean_length": 428.375, |
| "completions/mean_terminated_length": 428.375, |
| "completions/min_length": 329.0, |
| "completions/min_terminated_length": 329.0, |
| "epoch": 0.6133333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07177510857582092, |
| "kl": 0.0, |
| "learning_rate": 4.9594521213063974e-05, |
| "loss": -0.009097330272197723, |
| "num_tokens": 1194149.0, |
| "reward": -0.3315518796443939, |
| "reward_std": 0.46374107152223587, |
| "rewards/reward_fn/mean": -0.3315518796443939, |
| "rewards/reward_fn/std": 0.46374107152223587, |
| "step": 92, |
| "step_time": 173.7315434149932 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 365.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 496.0, |
| "completions/max_terminated_length": 496.0, |
| "completions/mean_length": 365.0, |
| "completions/mean_terminated_length": 365.0, |
| "completions/min_length": 246.5, |
| "completions/min_terminated_length": 246.5, |
| "epoch": 0.62, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.04886671528220177, |
| "kl": 0.0, |
| "learning_rate": 4.9568016167660334e-05, |
| "loss": 0.025912761688232422, |
| "num_tokens": 1209413.0, |
| "reward": 0.5116970045492053, |
| "reward_std": 0.09442839026451111, |
| "rewards/reward_fn/mean": 0.5116970045492053, |
| "rewards/reward_fn/std": 0.09442839026451111, |
| "step": 93, |
| "step_time": 159.12018209404778 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 443.25, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 612.0, |
| "completions/max_terminated_length": 612.0, |
| "completions/mean_length": 443.25, |
| "completions/mean_terminated_length": 443.25, |
| "completions/min_length": 336.0, |
| "completions/min_terminated_length": 336.0, |
| "epoch": 0.6266666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.05622043460607529, |
| "kl": 0.0, |
| "learning_rate": 4.9540679586191605e-05, |
| "loss": 0.03646225109696388, |
| "num_tokens": 1225689.0, |
| "reward": 0.4065200388431549, |
| "reward_std": 0.29066115617752075, |
| "rewards/reward_fn/mean": 0.4065200388431549, |
| "rewards/reward_fn/std": 0.29066115617752075, |
| "step": 94, |
| "step_time": 187.76930392504437 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 393.0625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 557.5, |
| "completions/max_terminated_length": 557.5, |
| "completions/mean_length": 393.0625, |
| "completions/mean_terminated_length": 393.0625, |
| "completions/min_length": 282.0, |
| "completions/min_terminated_length": 282.0, |
| "epoch": 0.6333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06194169446825981, |
| "kl": 0.0, |
| "learning_rate": 4.951251239389948e-05, |
| "loss": 0.003901831805706024, |
| "num_tokens": 1242106.0, |
| "reward": 0.32503822073340416, |
| "reward_std": 0.19283149391412735, |
| "rewards/reward_fn/mean": 0.32503822073340416, |
| "rewards/reward_fn/std": 0.19283149391412735, |
| "step": 95, |
| "step_time": 165.4877885619644 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 311.375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 373.5, |
| "completions/max_terminated_length": 373.5, |
| "completions/mean_length": 311.375, |
| "completions/mean_terminated_length": 311.375, |
| "completions/min_length": 266.0, |
| "completions/min_terminated_length": 266.0, |
| "epoch": 0.64, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.045090898871421814, |
| "kl": 0.0, |
| "learning_rate": 4.948351554413879e-05, |
| "loss": 0.05610096454620361, |
| "num_tokens": 1256464.0, |
| "reward": 0.4024457186460495, |
| "reward_std": 0.4338012635707855, |
| "rewards/reward_fn/mean": 0.4024457186460495, |
| "rewards/reward_fn/std": 0.4338012635707855, |
| "step": 96, |
| "step_time": 123.16074389498681 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 214.875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 219.0, |
| "completions/max_terminated_length": 219.0, |
| "completions/mean_length": 214.875, |
| "completions/mean_terminated_length": 214.875, |
| "completions/min_length": 210.0, |
| "completions/min_terminated_length": 210.0, |
| "epoch": 0.6466666666666666, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.03964964672923088, |
| "kl": 0.0, |
| "learning_rate": 4.9453690018345144e-05, |
| "loss": -0.0001890324056148529, |
| "num_tokens": 1268902.0, |
| "reward": 0.5716236382722855, |
| "reward_std": 0.016463955864310265, |
| "rewards/reward_fn/mean": 0.5716236382722855, |
| "rewards/reward_fn/std": 0.016463955864310265, |
| "step": 97, |
| "step_time": 92.26312756206607 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 461.8125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 757.0, |
| "completions/max_terminated_length": 757.0, |
| "completions/mean_length": 461.8125, |
| "completions/mean_terminated_length": 461.8125, |
| "completions/min_length": 303.5, |
| "completions/min_terminated_length": 303.5, |
| "epoch": 0.6533333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0490785576403141, |
| "kl": 0.0, |
| "learning_rate": 4.942303682600178e-05, |
| "loss": 0.010549291968345642, |
| "num_tokens": 1285595.0, |
| "reward": 0.47012635320425034, |
| "reward_std": 0.2943352471457814, |
| "rewards/reward_fn/mean": 0.47012635320425034, |
| "rewards/reward_fn/std": 0.2943352471457814, |
| "step": 98, |
| "step_time": 214.64041374594672 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 282.1875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 372.0, |
| "completions/max_terminated_length": 372.0, |
| "completions/mean_length": 282.1875, |
| "completions/mean_terminated_length": 282.1875, |
| "completions/min_length": 213.0, |
| "completions/min_terminated_length": 213.0, |
| "epoch": 0.66, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1215004101395607, |
| "kl": 0.0, |
| "learning_rate": 4.939155700460536e-05, |
| "loss": -0.11692754924297333, |
| "num_tokens": 1300774.0, |
| "reward": -1.0974531769752502, |
| "reward_std": 0.8694423735141754, |
| "rewards/reward_fn/mean": -1.0974531769752502, |
| "rewards/reward_fn/std": 0.8694423735141754, |
| "step": 99, |
| "step_time": 115.8598381759366 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 378.3125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 457.0, |
| "completions/max_terminated_length": 457.0, |
| "completions/mean_length": 378.3125, |
| "completions/mean_terminated_length": 378.3125, |
| "completions/min_length": 312.0, |
| "completions/min_terminated_length": 312.0, |
| "epoch": 0.6666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06170595437288284, |
| "kl": 0.0, |
| "learning_rate": 4.9359251619630886e-05, |
| "loss": -0.032986074686050415, |
| "num_tokens": 1316947.0, |
| "reward": 0.24835945293307304, |
| "reward_std": 0.1980152428150177, |
| "rewards/reward_fn/mean": 0.24835945293307304, |
| "rewards/reward_fn/std": 0.1980152428150177, |
| "step": 100, |
| "step_time": 138.67001789598726 |
| }, |
| { |
| "epoch": 0.6666666666666666, |
| "eval_clip_ratio/high_max": 0.0, |
| "eval_clip_ratio/high_mean": 0.0, |
| "eval_clip_ratio/low_mean": 0.0, |
| "eval_clip_ratio/low_min": 0.0, |
| "eval_clip_ratio/region_mean": 0.0, |
| "eval_completion_length": 348.90625, |
| "eval_completions/clipped_ratio": 0.0, |
| "eval_completions/max_length": 474.82, |
| "eval_completions/max_terminated_length": 474.82, |
| "eval_completions/mean_length": 348.90625, |
| "eval_completions/mean_terminated_length": 348.90625, |
| "eval_completions/min_length": 266.68, |
| "eval_completions/min_terminated_length": 266.68, |
| "eval_frac_reward_zero_std": 0.1, |
| "eval_kl": 0.0, |
| "eval_loss": 0.008204526267945766, |
| "eval_num_tokens": 1316947.0, |
| "eval_reward": -0.090777420476079, |
| "eval_reward_std": 0.40388403258689026, |
| "eval_rewards/reward_fn/mean": -0.090777420476079, |
| "eval_rewards/reward_fn/std": 0.40388403258689026, |
| "eval_runtime": 6565.5982, |
| "eval_samples_per_second": 0.015, |
| "eval_steps_per_second": 0.002, |
| "step": 100 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 212.875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 219.0, |
| "completions/max_terminated_length": 219.0, |
| "completions/mean_length": 212.875, |
| "completions/mean_terminated_length": 212.875, |
| "completions/min_length": 208.5, |
| "completions/min_terminated_length": 208.5, |
| "epoch": 0.6733333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09589043259620667, |
| "kl": 0.0, |
| "learning_rate": 4.9326121764495596e-05, |
| "loss": 0.0010354742407798767, |
| "num_tokens": 1329313.0, |
| "reward": 0.8882656097412109, |
| "reward_std": 0.027551015838980675, |
| "rewards/reward_fn/mean": 0.8882656097412109, |
| "rewards/reward_fn/std": 0.027551015838980675, |
| "step": 101, |
| "step_time": 113.81590729398886 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 255.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 292.5, |
| "completions/max_terminated_length": 292.5, |
| "completions/mean_length": 255.0, |
| "completions/mean_terminated_length": 255.0, |
| "completions/min_length": 206.5, |
| "completions/min_terminated_length": 206.5, |
| "epoch": 0.68, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06758790463209152, |
| "kl": 0.0, |
| "learning_rate": 4.9292168560522014e-05, |
| "loss": 0.05638433247804642, |
| "num_tokens": 1343929.0, |
| "reward": 0.5735078454017639, |
| "reward_std": 0.19756827503442764, |
| "rewards/reward_fn/mean": 0.5735078454017639, |
| "rewards/reward_fn/std": 0.19756827503442764, |
| "step": 102, |
| "step_time": 97.67106665606843 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 476.125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 726.0, |
| "completions/max_terminated_length": 726.0, |
| "completions/mean_length": 476.125, |
| "completions/mean_terminated_length": 476.125, |
| "completions/min_length": 348.5, |
| "completions/min_terminated_length": 348.5, |
| "epoch": 0.6866666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06600646674633026, |
| "kl": 0.0, |
| "learning_rate": 4.925739315689991e-05, |
| "loss": -0.1070128008723259, |
| "num_tokens": 1361355.0, |
| "reward": 0.0030741989612579346, |
| "reward_std": 0.6040342599153519, |
| "rewards/reward_fn/mean": 0.0030741989612579346, |
| "rewards/reward_fn/std": 0.6040342599153519, |
| "step": 103, |
| "step_time": 218.79046111099888 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 210.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 213.5, |
| "completions/max_terminated_length": 213.5, |
| "completions/mean_length": 210.0, |
| "completions/mean_terminated_length": 210.0, |
| "completions/min_length": 208.0, |
| "completions/min_terminated_length": 208.0, |
| "epoch": 0.6933333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05047248676419258, |
| "kl": 0.0, |
| "learning_rate": 4.9221796730647516e-05, |
| "loss": -0.00010214000940322876, |
| "num_tokens": 1373667.0, |
| "reward": 0.07565045356750488, |
| "reward_std": 0.0007886506391514558, |
| "rewards/reward_fn/mean": 0.07565045356750488, |
| "rewards/reward_fn/std": 0.0007886506391514558, |
| "step": 104, |
| "step_time": 87.27632812299998 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 287.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 349.0, |
| "completions/max_terminated_length": 349.0, |
| "completions/mean_length": 287.6875, |
| "completions/mean_terminated_length": 287.6875, |
| "completions/min_length": 242.0, |
| "completions/min_terminated_length": 242.0, |
| "epoch": 0.7, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03672612085938454, |
| "kl": 0.0, |
| "learning_rate": 4.9185380486571595e-05, |
| "loss": 0.0014976412057876587, |
| "num_tokens": 1387110.0, |
| "reward": 0.5444922223687172, |
| "reward_std": 7.728397611117543e-05, |
| "rewards/reward_fn/mean": 0.5444922223687172, |
| "rewards/reward_fn/std": 7.728397611117543e-05, |
| "step": 105, |
| "step_time": 115.72042374301236 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 293.3125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 323.0, |
| "completions/max_terminated_length": 323.0, |
| "completions/mean_length": 293.3125, |
| "completions/mean_terminated_length": 293.3125, |
| "completions/min_length": 250.5, |
| "completions/min_terminated_length": 250.5, |
| "epoch": 0.7066666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.04970452934503555, |
| "kl": 0.0, |
| "learning_rate": 4.914814565722671e-05, |
| "loss": -0.0025767460465431213, |
| "num_tokens": 1401147.0, |
| "reward": 0.08888586703687906, |
| "reward_std": 0.3375508785247803, |
| "rewards/reward_fn/mean": 0.08888586703687906, |
| "rewards/reward_fn/std": 0.3375508785247803, |
| "step": 106, |
| "step_time": 110.42762503004633 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 519.25, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 949.5, |
| "completions/max_terminated_length": 949.5, |
| "completions/mean_length": 519.25, |
| "completions/mean_terminated_length": 519.25, |
| "completions/min_length": 347.5, |
| "completions/min_terminated_length": 347.5, |
| "epoch": 0.7133333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07587578892707825, |
| "kl": 0.0, |
| "learning_rate": 4.9110093502873476e-05, |
| "loss": 0.04362190514802933, |
| "num_tokens": 1419135.0, |
| "reward": 0.2313607558608055, |
| "reward_std": 0.5956478118896484, |
| "rewards/reward_fn/mean": 0.2313607558608055, |
| "rewards/reward_fn/std": 0.5956478118896484, |
| "step": 107, |
| "step_time": 312.2387495030416 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 300.5, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 342.0, |
| "completions/max_terminated_length": 342.0, |
| "completions/mean_length": 300.5, |
| "completions/mean_terminated_length": 300.5, |
| "completions/min_length": 251.0, |
| "completions/min_terminated_length": 251.0, |
| "epoch": 0.72, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.06615033745765686, |
| "kl": 0.0, |
| "learning_rate": 4.907122531143594e-05, |
| "loss": 0.01366664282977581, |
| "num_tokens": 1433295.0, |
| "reward": 0.5283045321702957, |
| "reward_std": 0.2961108386516571, |
| "rewards/reward_fn/mean": 0.5283045321702957, |
| "rewards/reward_fn/std": 0.2961108386516571, |
| "step": 108, |
| "step_time": 115.16167661710642 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 302.25, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 352.5, |
| "completions/max_terminated_length": 352.5, |
| "completions/mean_length": 302.25, |
| "completions/mean_terminated_length": 302.25, |
| "completions/min_length": 264.0, |
| "completions/min_terminated_length": 264.0, |
| "epoch": 0.7266666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0835169330239296, |
| "kl": 0.0, |
| "learning_rate": 4.9031542398457974e-05, |
| "loss": -0.005957708694040775, |
| "num_tokens": 1448819.0, |
| "reward": 0.5500120520591736, |
| "reward_std": 0.288537934422493, |
| "rewards/reward_fn/mean": 0.5500120520591736, |
| "rewards/reward_fn/std": 0.288537934422493, |
| "step": 109, |
| "step_time": 111.85748529294506 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 300.75, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 343.5, |
| "completions/max_terminated_length": 343.5, |
| "completions/mean_length": 300.75, |
| "completions/mean_terminated_length": 300.75, |
| "completions/min_length": 224.0, |
| "completions/min_terminated_length": 224.0, |
| "epoch": 0.7333333333333333, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.05528109520673752, |
| "kl": 0.0, |
| "learning_rate": 4.8991046107058735e-05, |
| "loss": 0.003592744469642639, |
| "num_tokens": 1462735.0, |
| "reward": 0.4155409634113312, |
| "reward_std": 0.1777665913105011, |
| "rewards/reward_fn/mean": 0.4155409634113312, |
| "rewards/reward_fn/std": 0.1777665913105011, |
| "step": 110, |
| "step_time": 106.56734898797004 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 240.875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 265.5, |
| "completions/max_terminated_length": 265.5, |
| "completions/mean_length": 240.875, |
| "completions/mean_terminated_length": 240.875, |
| "completions/min_length": 199.0, |
| "completions/min_terminated_length": 199.0, |
| "epoch": 0.74, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.0353812538087368, |
| "kl": 0.0, |
| "learning_rate": 4.894973780788722e-05, |
| "loss": -0.011618122458457947, |
| "num_tokens": 1476493.0, |
| "reward": 0.7792612016201019, |
| "reward_std": 0.10726096481084824, |
| "rewards/reward_fn/mean": 0.7792612016201019, |
| "rewards/reward_fn/std": 0.10726096481084824, |
| "step": 111, |
| "step_time": 96.58656353410333 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 507.1875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 806.0, |
| "completions/max_terminated_length": 806.0, |
| "completions/mean_length": 507.1875, |
| "completions/mean_terminated_length": 507.1875, |
| "completions/min_length": 375.5, |
| "completions/min_terminated_length": 375.5, |
| "epoch": 0.7466666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.034782811999320984, |
| "kl": 0.0, |
| "learning_rate": 4.890761889907589e-05, |
| "loss": 0.031690001487731934, |
| "num_tokens": 1493936.0, |
| "reward": 0.5777778774499893, |
| "reward_std": 0.11972552537918091, |
| "rewards/reward_fn/mean": 0.5777778774499893, |
| "rewards/reward_fn/std": 0.11972552537918091, |
| "step": 112, |
| "step_time": 232.74652250896906 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 298.875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 504.5, |
| "completions/max_terminated_length": 504.5, |
| "completions/mean_length": 298.875, |
| "completions/mean_terminated_length": 298.875, |
| "completions/min_length": 206.5, |
| "completions/min_terminated_length": 206.5, |
| "epoch": 0.7533333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.048440419137477875, |
| "kl": 0.0, |
| "learning_rate": 4.88646908061933e-05, |
| "loss": -0.025000344961881638, |
| "num_tokens": 1507358.0, |
| "reward": 0.9986199140548706, |
| "reward_std": 0.003903499803300292, |
| "rewards/reward_fn/mean": 0.9986199140548706, |
| "rewards/reward_fn/std": 0.003903499803300292, |
| "step": 113, |
| "step_time": 145.51719051605323 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 286.4375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 330.5, |
| "completions/max_terminated_length": 330.5, |
| "completions/mean_length": 286.4375, |
| "completions/mean_terminated_length": 286.4375, |
| "completions/min_length": 260.0, |
| "completions/min_terminated_length": 260.0, |
| "epoch": 0.76, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.05592720955610275, |
| "kl": 0.0, |
| "learning_rate": 4.8820954982195905e-05, |
| "loss": 0.0003649294376373291, |
| "num_tokens": 1521589.0, |
| "reward": 0.6258035600185394, |
| "reward_std": 0.35387808084487915, |
| "rewards/reward_fn/mean": 0.6258035600185394, |
| "rewards/reward_fn/std": 0.35387808084487915, |
| "step": 114, |
| "step_time": 104.28781160601648 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 300.3125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 358.0, |
| "completions/max_terminated_length": 358.0, |
| "completions/mean_length": 300.3125, |
| "completions/mean_terminated_length": 300.3125, |
| "completions/min_length": 258.5, |
| "completions/min_terminated_length": 258.5, |
| "epoch": 0.7666666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 8.306245581479743e-05, |
| "kl": 0.0, |
| "learning_rate": 4.877641290737884e-05, |
| "loss": 6.103819760028273e-05, |
| "num_tokens": 1535322.0, |
| "reward": 0.9999999701976776, |
| "reward_std": 1.0566774477638319e-07, |
| "rewards/reward_fn/mean": 0.9999999701976776, |
| "rewards/reward_fn/std": 1.0566774477638319e-07, |
| "step": 115, |
| "step_time": 119.61036076099845 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 458.4375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 778.0, |
| "completions/max_terminated_length": 778.0, |
| "completions/mean_length": 458.4375, |
| "completions/mean_terminated_length": 458.4375, |
| "completions/min_length": 252.0, |
| "completions/min_terminated_length": 252.0, |
| "epoch": 0.7733333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0683111697435379, |
| "kl": 0.0, |
| "learning_rate": 4.873106608932585e-05, |
| "loss": -0.1494099348783493, |
| "num_tokens": 1551873.0, |
| "reward": 0.7072757333517075, |
| "reward_std": 0.1626912999496426, |
| "rewards/reward_fn/mean": 0.7072757333517075, |
| "rewards/reward_fn/std": 0.1626912999496426, |
| "step": 116, |
| "step_time": 219.83419971400872 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 214.375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 234.0, |
| "completions/max_terminated_length": 234.0, |
| "completions/mean_length": 214.375, |
| "completions/mean_terminated_length": 214.375, |
| "completions/min_length": 187.5, |
| "completions/min_terminated_length": 187.5, |
| "epoch": 0.78, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.04911733418703079, |
| "kl": 0.0, |
| "learning_rate": 4.868491606285823e-05, |
| "loss": -0.002008102834224701, |
| "num_tokens": 1564847.0, |
| "reward": 0.851028710603714, |
| "reward_std": 0.11776234209537506, |
| "rewards/reward_fn/mean": 0.851028710603714, |
| "rewards/reward_fn/std": 0.11776234209537506, |
| "step": 117, |
| "step_time": 79.10500066704117 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 394.1875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 526.5, |
| "completions/max_terminated_length": 526.5, |
| "completions/mean_length": 394.1875, |
| "completions/mean_terminated_length": 394.1875, |
| "completions/min_length": 273.5, |
| "completions/min_terminated_length": 273.5, |
| "epoch": 0.7866666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06439276784658432, |
| "kl": 0.0, |
| "learning_rate": 4.8637964389982926e-05, |
| "loss": 0.05288882926106453, |
| "num_tokens": 1581274.0, |
| "reward": 0.5939511358737946, |
| "reward_std": 0.6298229396343231, |
| "rewards/reward_fn/mean": 0.5939511358737946, |
| "rewards/reward_fn/std": 0.6298229396343231, |
| "step": 118, |
| "step_time": 160.8602781650261 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 294.25, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 362.5, |
| "completions/max_terminated_length": 362.5, |
| "completions/mean_length": 294.25, |
| "completions/mean_terminated_length": 294.25, |
| "completions/min_length": 249.0, |
| "completions/min_terminated_length": 249.0, |
| "epoch": 0.7933333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06369972974061966, |
| "kl": 0.0, |
| "learning_rate": 4.859021265983959e-05, |
| "loss": 0.04611533135175705, |
| "num_tokens": 1596198.0, |
| "reward": 0.07208789885044098, |
| "reward_std": 0.21891965717077255, |
| "rewards/reward_fn/mean": 0.07208789885044098, |
| "rewards/reward_fn/std": 0.21891965717077255, |
| "step": 119, |
| "step_time": 126.26092710503144 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 341.875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 400.0, |
| "completions/max_terminated_length": 400.0, |
| "completions/mean_length": 341.875, |
| "completions/mean_terminated_length": 341.875, |
| "completions/min_length": 308.0, |
| "completions/min_terminated_length": 308.0, |
| "epoch": 0.8, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.028934789821505547, |
| "kl": 0.0, |
| "learning_rate": 4.854166248864689e-05, |
| "loss": 0.028521619737148285, |
| "num_tokens": 1611036.0, |
| "reward": 0.5731896162033081, |
| "reward_std": 0.009335569106042385, |
| "rewards/reward_fn/mean": 0.5731896162033081, |
| "rewards/reward_fn/std": 0.009335569106042385, |
| "step": 120, |
| "step_time": 131.31865840405226 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 355.5, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 470.0, |
| "completions/max_terminated_length": 470.0, |
| "completions/mean_length": 355.5, |
| "completions/mean_terminated_length": 355.5, |
| "completions/min_length": 300.0, |
| "completions/min_terminated_length": 300.0, |
| "epoch": 0.8066666666666666, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.04000210389494896, |
| "kl": 0.0, |
| "learning_rate": 4.849231551964771e-05, |
| "loss": 0.014162637293338776, |
| "num_tokens": 1626324.0, |
| "reward": 0.8219311833381653, |
| "reward_std": 0.0823940634727478, |
| "rewards/reward_fn/mean": 0.8219311833381653, |
| "rewards/reward_fn/std": 0.0823940634727478, |
| "step": 121, |
| "step_time": 140.04646876995685 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 400.3125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 576.0, |
| "completions/max_terminated_length": 576.0, |
| "completions/mean_length": 400.3125, |
| "completions/mean_terminated_length": 400.3125, |
| "completions/min_length": 317.5, |
| "completions/min_terminated_length": 317.5, |
| "epoch": 0.8133333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07007366418838501, |
| "kl": 0.0, |
| "learning_rate": 4.844217342305363e-05, |
| "loss": -0.025220118463039398, |
| "num_tokens": 1642993.0, |
| "reward": 0.027626875787973404, |
| "reward_std": 0.2794133871793747, |
| "rewards/reward_fn/mean": 0.027626875787973404, |
| "rewards/reward_fn/std": 0.2794133871793747, |
| "step": 122, |
| "step_time": 171.93886475102045 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 379.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 593.5, |
| "completions/max_terminated_length": 593.5, |
| "completions/mean_length": 379.0, |
| "completions/mean_terminated_length": 379.0, |
| "completions/min_length": 239.5, |
| "completions/min_terminated_length": 239.5, |
| "epoch": 0.82, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.20304864645004272, |
| "kl": 0.0, |
| "learning_rate": 4.839123789598829e-05, |
| "loss": 0.12271372228860855, |
| "num_tokens": 1659169.0, |
| "reward": 0.0422142893075943, |
| "reward_std": 0.42568448930978775, |
| "rewards/reward_fn/mean": 0.0422142893075943, |
| "rewards/reward_fn/std": 0.42568448930978775, |
| "step": 123, |
| "step_time": 200.5066597990226 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 406.0625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 700.0, |
| "completions/max_terminated_length": 700.0, |
| "completions/mean_length": 406.0625, |
| "completions/mean_terminated_length": 406.0625, |
| "completions/min_length": 299.0, |
| "completions/min_terminated_length": 299.0, |
| "epoch": 0.8266666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.041422288864851, |
| "kl": 0.0, |
| "learning_rate": 4.8339510662430046e-05, |
| "loss": -0.0771825909614563, |
| "num_tokens": 1675098.0, |
| "reward": 0.5340617299079895, |
| "reward_std": 0.03479880467057228, |
| "rewards/reward_fn/mean": 0.5340617299079895, |
| "rewards/reward_fn/std": 0.03479880467057228, |
| "step": 124, |
| "step_time": 213.72491782298312 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 272.1875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 315.0, |
| "completions/max_terminated_length": 315.0, |
| "completions/mean_length": 272.1875, |
| "completions/mean_terminated_length": 272.1875, |
| "completions/min_length": 203.5, |
| "completions/min_terminated_length": 203.5, |
| "epoch": 0.8333333333333334, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.05468154326081276, |
| "kl": 0.0, |
| "learning_rate": 4.828699347315356e-05, |
| "loss": -0.055062174797058105, |
| "num_tokens": 1689133.0, |
| "reward": 0.8563609719276428, |
| "reward_std": 0.12412404268980026, |
| "rewards/reward_fn/mean": 0.8563609719276428, |
| "rewards/reward_fn/std": 0.12412404268980026, |
| "step": 125, |
| "step_time": 99.89914110297104 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 208.125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 210.0, |
| "completions/max_terminated_length": 210.0, |
| "completions/mean_length": 208.125, |
| "completions/mean_terminated_length": 208.125, |
| "completions/min_length": 200.5, |
| "completions/min_terminated_length": 200.5, |
| "epoch": 0.84, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.062068283557891846, |
| "kl": 0.0, |
| "learning_rate": 4.823368810567056e-05, |
| "loss": -0.013048075139522552, |
| "num_tokens": 1701455.0, |
| "reward": 0.9390365481376648, |
| "reward_std": 0.028855346143245697, |
| "rewards/reward_fn/mean": 0.9390365481376648, |
| "rewards/reward_fn/std": 0.028855346143245697, |
| "step": 126, |
| "step_time": 89.4614521729527 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 239.5625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 247.5, |
| "completions/max_terminated_length": 247.5, |
| "completions/mean_length": 239.5625, |
| "completions/mean_terminated_length": 239.5625, |
| "completions/min_length": 201.5, |
| "completions/min_terminated_length": 201.5, |
| "epoch": 0.8466666666666667, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.817959636416969e-05, |
| "loss": 0.0, |
| "num_tokens": 1714064.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 1.0, |
| "rewards/reward_fn/std": 0.0, |
| "step": 127, |
| "step_time": 89.98527943901718 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 299.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 407.0, |
| "completions/max_terminated_length": 407.0, |
| "completions/mean_length": 299.6875, |
| "completions/mean_terminated_length": 299.6875, |
| "completions/min_length": 232.5, |
| "completions/min_terminated_length": 232.5, |
| "epoch": 0.8533333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05948776379227638, |
| "kl": 0.0, |
| "learning_rate": 4.81247200794554e-05, |
| "loss": 0.01649428904056549, |
| "num_tokens": 1728195.0, |
| "reward": 0.0764324888586998, |
| "reward_std": 0.2988954664178891, |
| "rewards/reward_fn/mean": 0.0764324888586998, |
| "rewards/reward_fn/std": 0.2988954664178891, |
| "step": 128, |
| "step_time": 226.33545375603717 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 476.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 904.0, |
| "completions/max_terminated_length": 904.0, |
| "completions/mean_length": 476.0, |
| "completions/mean_terminated_length": 476.0, |
| "completions/min_length": 264.0, |
| "completions/min_terminated_length": 264.0, |
| "epoch": 0.86, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.10832423716783524, |
| "kl": 0.0, |
| "learning_rate": 4.806906110888606e-05, |
| "loss": -0.07909408956766129, |
| "num_tokens": 1745547.0, |
| "reward": -0.04347759485244751, |
| "reward_std": 0.7136133313179016, |
| "rewards/reward_fn/mean": -0.04347759485244751, |
| "rewards/reward_fn/std": 0.7136133313179016, |
| "step": 129, |
| "step_time": 249.15961863403209 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 195.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 222.5, |
| "completions/max_terminated_length": 222.5, |
| "completions/mean_length": 195.0, |
| "completions/mean_terminated_length": 195.0, |
| "completions/min_length": 162.0, |
| "completions/min_terminated_length": 162.0, |
| "epoch": 0.8666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07652340829372406, |
| "kl": 0.0, |
| "learning_rate": 4.8012621336311016e-05, |
| "loss": -0.023258736357092857, |
| "num_tokens": 1759219.0, |
| "reward": 0.5377570688724518, |
| "reward_std": 0.21886634081602097, |
| "rewards/reward_fn/mean": 0.5377570688724518, |
| "rewards/reward_fn/std": 0.21886634081602097, |
| "step": 130, |
| "step_time": 77.46455274592154 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 870.0625, |
| "completions/clipped_ratio": 0.1875, |
| "completions/max_length": 1466.0, |
| "completions/max_terminated_length": 1397.0, |
| "completions/mean_length": 870.0625, |
| "completions/mean_terminated_length": 654.8875122070312, |
| "completions/min_length": 300.0, |
| "completions/min_terminated_length": 300.0, |
| "epoch": 0.8733333333333333, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.01964535377919674, |
| "kl": 0.0, |
| "learning_rate": 4.7955402672006854e-05, |
| "loss": 0.19607774913311005, |
| "num_tokens": 1782388.0, |
| "reward": 0.2218644618988037, |
| "reward_std": 0.6362914443016052, |
| "rewards/reward_fn/mean": 0.2218644618988037, |
| "rewards/reward_fn/std": 0.6362914443016052, |
| "step": 131, |
| "step_time": 409.15760393301025 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 352.25, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 441.0, |
| "completions/max_terminated_length": 441.0, |
| "completions/mean_length": 352.25, |
| "completions/mean_terminated_length": 352.25, |
| "completions/min_length": 299.0, |
| "completions/min_terminated_length": 299.0, |
| "epoch": 0.88, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.04634862020611763, |
| "kl": 0.0, |
| "learning_rate": 4.789740705261278e-05, |
| "loss": 0.0035696476697921753, |
| "num_tokens": 1797352.0, |
| "reward": 0.5494393706321716, |
| "reward_std": 0.2591811418533325, |
| "rewards/reward_fn/mean": 0.5494393706321716, |
| "rewards/reward_fn/std": 0.2591811418533325, |
| "step": 132, |
| "step_time": 164.5742819889565 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 352.0625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 600.0, |
| "completions/max_terminated_length": 600.0, |
| "completions/mean_length": 352.0625, |
| "completions/mean_terminated_length": 352.0625, |
| "completions/min_length": 254.0, |
| "completions/min_terminated_length": 254.0, |
| "epoch": 0.8866666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.04685305058956146, |
| "kl": 0.0, |
| "learning_rate": 4.783863644106502e-05, |
| "loss": -0.13369393348693848, |
| "num_tokens": 1812129.0, |
| "reward": 0.6904256045818329, |
| "reward_std": 0.1820424646139145, |
| "rewards/reward_fn/mean": 0.6904256045818329, |
| "rewards/reward_fn/std": 0.1820424646139145, |
| "step": 133, |
| "step_time": 185.4885982349515 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 467.3125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 832.5, |
| "completions/max_terminated_length": 832.5, |
| "completions/mean_length": 467.3125, |
| "completions/mean_terminated_length": 467.3125, |
| "completions/min_length": 346.0, |
| "completions/min_terminated_length": 346.0, |
| "epoch": 0.8933333333333333, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.05197684094309807, |
| "kl": 0.0, |
| "learning_rate": 4.777909282653042e-05, |
| "loss": 0.02681293897330761, |
| "num_tokens": 1828782.0, |
| "reward": 0.6091441139578819, |
| "reward_std": 0.21364624798297882, |
| "rewards/reward_fn/mean": 0.6091441139578819, |
| "rewards/reward_fn/std": 0.21364624798297882, |
| "step": 134, |
| "step_time": 412.91265689296415 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 368.375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 580.5, |
| "completions/max_terminated_length": 580.5, |
| "completions/mean_length": 368.375, |
| "completions/mean_terminated_length": 368.375, |
| "completions/min_length": 212.0, |
| "completions/min_terminated_length": 212.0, |
| "epoch": 0.9, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1014743223786354, |
| "kl": 0.0, |
| "learning_rate": 4.771877822433911e-05, |
| "loss": 0.07772844284772873, |
| "num_tokens": 1844388.0, |
| "reward": 0.7372333705425262, |
| "reward_std": 0.02323738712834711, |
| "rewards/reward_fn/mean": 0.7372333705425262, |
| "rewards/reward_fn/std": 0.02323738712834711, |
| "step": 135, |
| "step_time": 168.2943005109555 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 364.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 605.0, |
| "completions/max_terminated_length": 605.0, |
| "completions/mean_length": 364.6875, |
| "completions/mean_terminated_length": 364.6875, |
| "completions/min_length": 276.0, |
| "completions/min_terminated_length": 276.0, |
| "epoch": 0.9066666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.14220644533634186, |
| "kl": 0.0, |
| "learning_rate": 4.765769467591625e-05, |
| "loss": 0.2252202183008194, |
| "num_tokens": 1858895.0, |
| "reward": 0.6875, |
| "reward_std": 0.883883535861969, |
| "rewards/reward_fn/mean": 0.6875, |
| "rewards/reward_fn/std": 0.883883535861969, |
| "step": 136, |
| "step_time": 171.65731632098323 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 199.625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 243.0, |
| "completions/max_terminated_length": 243.0, |
| "completions/mean_length": 199.625, |
| "completions/mean_terminated_length": 199.625, |
| "completions/min_length": 181.0, |
| "completions/min_terminated_length": 181.0, |
| "epoch": 0.9133333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05413385108113289, |
| "kl": 0.0, |
| "learning_rate": 4.759584424871302e-05, |
| "loss": -0.02340317703783512, |
| "num_tokens": 1872657.0, |
| "reward": 0.6003958731889725, |
| "reward_std": 0.10894697532057762, |
| "rewards/reward_fn/mean": 0.6003958731889725, |
| "rewards/reward_fn/std": 0.10894697532057762, |
| "step": 137, |
| "step_time": 82.92960694391513 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 580.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 983.5, |
| "completions/max_terminated_length": 983.5, |
| "completions/mean_length": 580.9375, |
| "completions/mean_terminated_length": 580.9375, |
| "completions/min_length": 338.5, |
| "completions/min_terminated_length": 338.5, |
| "epoch": 0.92, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.038850050419569016, |
| "kl": 0.0, |
| "learning_rate": 4.7533229036136553e-05, |
| "loss": -0.07479565590620041, |
| "num_tokens": 1892224.0, |
| "reward": 0.30399832874536514, |
| "reward_std": 0.08702933220774867, |
| "rewards/reward_fn/mean": 0.30399832874536514, |
| "rewards/reward_fn/std": 0.08702933220774867, |
| "step": 138, |
| "step_time": 276.3797710710205 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 335.75, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 382.0, |
| "completions/max_terminated_length": 382.0, |
| "completions/mean_length": 335.75, |
| "completions/mean_terminated_length": 335.75, |
| "completions/min_length": 276.0, |
| "completions/min_terminated_length": 276.0, |
| "epoch": 0.9266666666666666, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.7469851157479177e-05, |
| "loss": 0.0, |
| "num_tokens": 1906212.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 1.0, |
| "rewards/reward_fn/std": 0.0, |
| "step": 139, |
| "step_time": 115.95142895297613 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 247.1875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 251.5, |
| "completions/max_terminated_length": 251.5, |
| "completions/mean_length": 247.1875, |
| "completions/mean_terminated_length": 247.1875, |
| "completions/min_length": 233.5, |
| "completions/min_terminated_length": 233.5, |
| "epoch": 0.9333333333333333, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.011438372544944286, |
| "kl": 0.0, |
| "learning_rate": 4.740571275784659e-05, |
| "loss": -3.4224241971969604e-05, |
| "num_tokens": 1918951.0, |
| "reward": 0.5426880717277527, |
| "reward_std": 1.0973985808959696e-05, |
| "rewards/reward_fn/mean": 0.5426880717277527, |
| "rewards/reward_fn/std": 1.0973985808959696e-05, |
| "step": 140, |
| "step_time": 95.01890190399718 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 521.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 757.0, |
| "completions/max_terminated_length": 757.0, |
| "completions/mean_length": 521.0, |
| "completions/mean_terminated_length": 521.0, |
| "completions/min_length": 299.5, |
| "completions/min_terminated_length": 299.5, |
| "epoch": 0.94, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.024509411305189133, |
| "kl": 0.0, |
| "learning_rate": 4.734081600808531e-05, |
| "loss": -0.09396696835756302, |
| "num_tokens": 1936599.0, |
| "reward": 0.41570543497800827, |
| "reward_std": 0.3383743853873966, |
| "rewards/reward_fn/mean": 0.41570543497800827, |
| "rewards/reward_fn/std": 0.3383743853873966, |
| "step": 141, |
| "step_time": 234.9987866890151 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 233.5, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 250.5, |
| "completions/max_terminated_length": 250.5, |
| "completions/mean_length": 233.5, |
| "completions/mean_terminated_length": 233.5, |
| "completions/min_length": 229.0, |
| "completions/min_terminated_length": 229.0, |
| "epoch": 0.9466666666666667, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.72751631047092e-05, |
| "loss": 0.0, |
| "num_tokens": 1949183.0, |
| "reward": 0.5619802176952362, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.5619802176952362, |
| "rewards/reward_fn/std": 0.0, |
| "step": 142, |
| "step_time": 90.94075334299123 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 423.0625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 761.5, |
| "completions/max_terminated_length": 761.5, |
| "completions/mean_length": 423.0625, |
| "completions/mean_terminated_length": 423.0625, |
| "completions/min_length": 286.0, |
| "completions/min_terminated_length": 286.0, |
| "epoch": 0.9533333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1250963658094406, |
| "kl": 0.0, |
| "learning_rate": 4.7208756269825104e-05, |
| "loss": 0.12352190911769867, |
| "num_tokens": 1966080.0, |
| "reward": 0.34782883524894714, |
| "reward_std": 0.6882529407739639, |
| "rewards/reward_fn/mean": 0.34782883524894714, |
| "rewards/reward_fn/std": 0.6882529407739639, |
| "step": 143, |
| "step_time": 226.21472980699036 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 208.125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 216.5, |
| "completions/max_terminated_length": 216.5, |
| "completions/mean_length": 208.125, |
| "completions/mean_terminated_length": 208.125, |
| "completions/min_length": 194.5, |
| "completions/min_terminated_length": 194.5, |
| "epoch": 0.96, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.04892691969871521, |
| "kl": 0.0, |
| "learning_rate": 4.714159775105765e-05, |
| "loss": -0.016497205942869186, |
| "num_tokens": 1979202.0, |
| "reward": 0.7357056438922882, |
| "reward_std": 0.09705093502998352, |
| "rewards/reward_fn/mean": 0.7357056438922882, |
| "rewards/reward_fn/std": 0.09705093502998352, |
| "step": 144, |
| "step_time": 84.11677284108009 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 203.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 216.5, |
| "completions/max_terminated_length": 216.5, |
| "completions/mean_length": 203.6875, |
| "completions/mean_terminated_length": 203.6875, |
| "completions/min_length": 196.0, |
| "completions/min_terminated_length": 196.0, |
| "epoch": 0.9666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05953475832939148, |
| "kl": 0.0, |
| "learning_rate": 4.707368982147318e-05, |
| "loss": -0.014895366504788399, |
| "num_tokens": 1992269.0, |
| "reward": 0.324895441532135, |
| "reward_std": 0.049588216134793583, |
| "rewards/reward_fn/mean": 0.324895441532135, |
| "rewards/reward_fn/std": 0.049588216134793583, |
| "step": 145, |
| "step_time": 84.07973394100554 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 257.5, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 295.0, |
| "completions/max_terminated_length": 295.0, |
| "completions/mean_length": 257.5, |
| "completions/mean_terminated_length": 257.5, |
| "completions/min_length": 235.0, |
| "completions/min_terminated_length": 235.0, |
| "epoch": 0.9733333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06280596554279327, |
| "kl": 0.0, |
| "learning_rate": 4.700503477950278e-05, |
| "loss": 0.003337457776069641, |
| "num_tokens": 2007037.0, |
| "reward": 0.49471792578697205, |
| "reward_std": 0.21104929596185684, |
| "rewards/reward_fn/mean": 0.49471792578697205, |
| "rewards/reward_fn/std": 0.21104929596185684, |
| "step": 146, |
| "step_time": 96.53027527104132 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 498.8125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 731.0, |
| "completions/max_terminated_length": 731.0, |
| "completions/mean_length": 498.8125, |
| "completions/mean_terminated_length": 498.8125, |
| "completions/min_length": 334.0, |
| "completions/min_terminated_length": 334.0, |
| "epoch": 0.98, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.034298237413167953, |
| "kl": 0.0, |
| "learning_rate": 4.693563494886455e-05, |
| "loss": -0.06357244402170181, |
| "num_tokens": 2024218.0, |
| "reward": 0.9669809341430664, |
| "reward_std": 0.04393957555294037, |
| "rewards/reward_fn/mean": 0.9669809341430664, |
| "rewards/reward_fn/std": 0.04393957555294037, |
| "step": 147, |
| "step_time": 206.35053608001908 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 312.4375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 378.5, |
| "completions/max_terminated_length": 378.5, |
| "completions/mean_length": 312.4375, |
| "completions/mean_terminated_length": 312.4375, |
| "completions/min_length": 245.0, |
| "completions/min_terminated_length": 245.0, |
| "epoch": 0.9866666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 5.0839618779718876e-05, |
| "kl": 0.0, |
| "learning_rate": 4.6865492678484895e-05, |
| "loss": 0.00010071517317555845, |
| "num_tokens": 2038105.0, |
| "reward": 0.5656213238835335, |
| "reward_std": 4.5056875563886933e-08, |
| "rewards/reward_fn/mean": 0.5656213238835335, |
| "rewards/reward_fn/std": 4.5056875563886933e-08, |
| "step": 148, |
| "step_time": 124.25263723096577 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 567.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 914.5, |
| "completions/max_terminated_length": 914.5, |
| "completions/mean_length": 567.9375, |
| "completions/mean_terminated_length": 567.9375, |
| "completions/min_length": 354.5, |
| "completions/min_terminated_length": 354.5, |
| "epoch": 0.9933333333333333, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.040333859622478485, |
| "kl": 0.0, |
| "learning_rate": 4.679461034241906e-05, |
| "loss": -0.033196814358234406, |
| "num_tokens": 2056408.0, |
| "reward": 0.7618521153926849, |
| "reward_std": 0.45696911215782166, |
| "rewards/reward_fn/mean": 0.7618521153926849, |
| "rewards/reward_fn/std": 0.45696911215782166, |
| "step": 149, |
| "step_time": 253.0286384250503 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 252.1875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 302.5, |
| "completions/max_terminated_length": 302.5, |
| "completions/mean_length": 252.1875, |
| "completions/mean_terminated_length": 252.1875, |
| "completions/min_length": 223.0, |
| "completions/min_terminated_length": 223.0, |
| "epoch": 1.0, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06843557208776474, |
| "kl": 0.0, |
| "learning_rate": 4.672299033977076e-05, |
| "loss": 0.05354718863964081, |
| "num_tokens": 2071083.0, |
| "reward": 0.6244544088840485, |
| "reward_std": 0.09683120995759964, |
| "rewards/reward_fn/mean": 0.6244544088840485, |
| "rewards/reward_fn/std": 0.09683120995759964, |
| "step": 150, |
| "step_time": 98.54597930196906 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 329.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 382.5, |
| "completions/max_terminated_length": 382.5, |
| "completions/mean_length": 329.9375, |
| "completions/mean_terminated_length": 329.9375, |
| "completions/min_length": 274.5, |
| "completions/min_terminated_length": 274.5, |
| "epoch": 1.0066666666666666, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.665063509461097e-05, |
| "loss": 0.0, |
| "num_tokens": 2085242.0, |
| "reward": 0.9492384493350983, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.9492384493350983, |
| "rewards/reward_fn/std": 0.0, |
| "step": 151, |
| "step_time": 125.676316802972 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 773.125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1563.5, |
| "completions/max_terminated_length": 1563.5, |
| "completions/mean_length": 773.125, |
| "completions/mean_terminated_length": 773.125, |
| "completions/min_length": 421.5, |
| "completions/min_terminated_length": 421.5, |
| "epoch": 1.0133333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.026009313762187958, |
| "kl": 0.0, |
| "learning_rate": 4.657754705589591e-05, |
| "loss": -0.1481044441461563, |
| "num_tokens": 2107260.0, |
| "reward": 0.8665153086185455, |
| "reward_std": 0.24713269983112696, |
| "rewards/reward_fn/mean": 0.8665153086185455, |
| "rewards/reward_fn/std": 0.24713269983112696, |
| "step": 152, |
| "step_time": 419.2542784839752 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 453.875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 555.0, |
| "completions/max_terminated_length": 555.0, |
| "completions/mean_length": 453.875, |
| "completions/mean_terminated_length": 453.875, |
| "completions/min_length": 310.5, |
| "completions/min_terminated_length": 310.5, |
| "epoch": 1.02, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.00012790317123290151, |
| "kl": 0.0, |
| "learning_rate": 4.650372869738414e-05, |
| "loss": -0.00028114268206991255, |
| "num_tokens": 2123154.0, |
| "reward": 0.9999999403953552, |
| "reward_std": 1.2744008870413381e-07, |
| "rewards/reward_fn/mean": 0.9999999403953552, |
| "rewards/reward_fn/std": 1.2744008870413381e-07, |
| "step": 153, |
| "step_time": 161.3001378289773 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 255.3125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 278.0, |
| "completions/max_terminated_length": 278.0, |
| "completions/mean_length": 255.3125, |
| "completions/mean_terminated_length": 255.3125, |
| "completions/min_length": 220.5, |
| "completions/min_terminated_length": 220.5, |
| "epoch": 1.0266666666666666, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.0790843740105629, |
| "kl": 0.0, |
| "learning_rate": 4.642918251755281e-05, |
| "loss": 0.010872147977352142, |
| "num_tokens": 2136751.0, |
| "reward": 0.695651650428772, |
| "reward_std": 0.1743757575750351, |
| "rewards/reward_fn/mean": 0.695651650428772, |
| "rewards/reward_fn/std": 0.1743757575750351, |
| "step": 154, |
| "step_time": 90.34915228991304 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 333.5, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 378.0, |
| "completions/max_terminated_length": 378.0, |
| "completions/mean_length": 333.5, |
| "completions/mean_terminated_length": 333.5, |
| "completions/min_length": 280.0, |
| "completions/min_terminated_length": 280.0, |
| "epoch": 1.0333333333333334, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.6353911039513145e-05, |
| "loss": 0.0, |
| "num_tokens": 2150983.0, |
| "reward": 0.9483389258384705, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.9483389258384705, |
| "rewards/reward_fn/std": 0.0, |
| "step": 155, |
| "step_time": 128.63163546600845 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 602.25, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1006.5, |
| "completions/max_terminated_length": 1006.5, |
| "completions/mean_length": 602.25, |
| "completions/mean_terminated_length": 602.25, |
| "completions/min_length": 456.0, |
| "completions/min_terminated_length": 456.0, |
| "epoch": 1.04, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05929885059595108, |
| "kl": 0.0, |
| "learning_rate": 4.627791681092499e-05, |
| "loss": -0.05129897594451904, |
| "num_tokens": 2170827.0, |
| "reward": 0.5733692049980164, |
| "reward_std": 0.2317853346467018, |
| "rewards/reward_fn/mean": 0.5733692049980164, |
| "rewards/reward_fn/std": 0.2317853346467018, |
| "step": 156, |
| "step_time": 285.2149426349206 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 537.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 688.0, |
| "completions/max_terminated_length": 688.0, |
| "completions/mean_length": 537.0, |
| "completions/mean_terminated_length": 537.0, |
| "completions/min_length": 419.5, |
| "completions/min_terminated_length": 419.5, |
| "epoch": 1.0466666666666666, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.036035988479852676, |
| "kl": 0.0, |
| "learning_rate": 4.620120240391065e-05, |
| "loss": -0.06904249638319016, |
| "num_tokens": 2188515.0, |
| "reward": 0.9940311908721924, |
| "reward_std": 0.01009331364184618, |
| "rewards/reward_fn/mean": 0.9940311908721924, |
| "rewards/reward_fn/std": 0.01009331364184618, |
| "step": 157, |
| "step_time": 206.95500113192247 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 209.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 210.0, |
| "completions/max_terminated_length": 210.0, |
| "completions/mean_length": 209.6875, |
| "completions/mean_terminated_length": 209.6875, |
| "completions/min_length": 209.0, |
| "completions/min_terminated_length": 209.0, |
| "epoch": 1.0533333333333332, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.612377041496776e-05, |
| "loss": 0.0, |
| "num_tokens": 2200886.0, |
| "reward": 0.5638016313314438, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.5638016313314438, |
| "rewards/reward_fn/std": 0.0, |
| "step": 158, |
| "step_time": 91.06828080205014 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 209.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 210.0, |
| "completions/max_terminated_length": 210.0, |
| "completions/mean_length": 209.9375, |
| "completions/mean_terminated_length": 209.9375, |
| "completions/min_length": 209.5, |
| "completions/min_terminated_length": 209.5, |
| "epoch": 1.06, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.00034379371209070086, |
| "kl": 0.0, |
| "learning_rate": 4.604562346488144e-05, |
| "loss": -3.690889570862055e-05, |
| "num_tokens": 2213205.0, |
| "reward": 0.5148106962442398, |
| "reward_std": 4.6602090719716216e-07, |
| "rewards/reward_fn/mean": 0.5148106962442398, |
| "rewards/reward_fn/std": 4.6602090719716216e-07, |
| "step": 159, |
| "step_time": 91.57397655490786 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 654.3125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 905.5, |
| "completions/max_terminated_length": 905.5, |
| "completions/mean_length": 654.3125, |
| "completions/mean_terminated_length": 654.3125, |
| "completions/min_length": 387.0, |
| "completions/min_terminated_length": 387.0, |
| "epoch": 1.0666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02906476892530918, |
| "kl": 0.0, |
| "learning_rate": 4.5966764198635606e-05, |
| "loss": 0.06294281035661697, |
| "num_tokens": 2233018.0, |
| "reward": 0.006258882582187653, |
| "reward_std": 0.15943723261341347, |
| "rewards/reward_fn/mean": 0.006258882582187653, |
| "rewards/reward_fn/std": 0.15943723261341347, |
| "step": 160, |
| "step_time": 322.56884266098496 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 337.1875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 500.5, |
| "completions/max_terminated_length": 500.5, |
| "completions/mean_length": 337.1875, |
| "completions/mean_terminated_length": 337.1875, |
| "completions/min_length": 276.0, |
| "completions/min_terminated_length": 276.0, |
| "epoch": 1.0733333333333333, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.588719528532342e-05, |
| "loss": 0.0, |
| "num_tokens": 2247301.0, |
| "reward": 0.5473424270749092, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.5473424270749092, |
| "rewards/reward_fn/std": 0.0, |
| "step": 161, |
| "step_time": 156.99171748693334 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 703.9375, |
| "completions/clipped_ratio": 0.0625, |
| "completions/max_length": 1155.5, |
| "completions/max_terminated_length": 913.0, |
| "completions/mean_length": 703.9375, |
| "completions/mean_terminated_length": 643.0357666015625, |
| "completions/min_length": 487.0, |
| "completions/min_terminated_length": 487.0, |
| "epoch": 1.08, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.066600002348423, |
| "kl": 0.0, |
| "learning_rate": 4.580691941805695e-05, |
| "loss": 0.07555639743804932, |
| "num_tokens": 2268652.0, |
| "reward": 0.5355610847473145, |
| "reward_std": 0.733004629611969, |
| "rewards/reward_fn/mean": 0.5355610847473145, |
| "rewards/reward_fn/std": 0.733004629611969, |
| "step": 162, |
| "step_time": 318.9592123919865 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 916.625, |
| "completions/clipped_ratio": 0.125, |
| "completions/max_length": 1129.0, |
| "completions/max_terminated_length": 904.0, |
| "completions/mean_length": 916.625, |
| "completions/mean_terminated_length": 845.875, |
| "completions/min_length": 699.5, |
| "completions/min_terminated_length": 699.5, |
| "epoch": 1.0866666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.015455679967999458, |
| "kl": 0.0, |
| "learning_rate": 4.572593931387604e-05, |
| "loss": 0.0705941915512085, |
| "num_tokens": 2292702.0, |
| "reward": -0.2573578804731369, |
| "reward_std": 0.42069763968612506, |
| "rewards/reward_fn/mean": -0.2573578804731369, |
| "rewards/reward_fn/std": 0.42069763968612506, |
| "step": 163, |
| "step_time": 332.97736496001016 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 331.5625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 382.5, |
| "completions/max_terminated_length": 382.5, |
| "completions/mean_length": 331.5625, |
| "completions/mean_terminated_length": 331.5625, |
| "completions/min_length": 259.5, |
| "completions/min_terminated_length": 259.5, |
| "epoch": 1.0933333333333333, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.5644257713656356e-05, |
| "loss": 0.0, |
| "num_tokens": 2306919.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 1.0, |
| "rewards/reward_fn/std": 0.0, |
| "step": 164, |
| "step_time": 126.35907765099546 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 495.4375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 933.0, |
| "completions/max_terminated_length": 933.0, |
| "completions/mean_length": 495.4375, |
| "completions/mean_terminated_length": 495.4375, |
| "completions/min_length": 323.5, |
| "completions/min_terminated_length": 323.5, |
| "epoch": 1.1, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05776069313287735, |
| "kl": 0.0, |
| "learning_rate": 4.556187738201656e-05, |
| "loss": 0.006418749690055847, |
| "num_tokens": 2324894.0, |
| "reward": 0.7401622533798218, |
| "reward_std": 0.410004697740078, |
| "rewards/reward_fn/mean": 0.7401622533798218, |
| "rewards/reward_fn/std": 0.410004697740078, |
| "step": 165, |
| "step_time": 318.11655589198926 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 368.5, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 424.0, |
| "completions/max_terminated_length": 424.0, |
| "completions/mean_length": 368.5, |
| "completions/mean_terminated_length": 368.5, |
| "completions/min_length": 297.5, |
| "completions/min_terminated_length": 297.5, |
| "epoch": 1.1066666666666667, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.54788011072248e-05, |
| "loss": 0.0, |
| "num_tokens": 2339502.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 1.0, |
| "rewards/reward_fn/std": 0.0, |
| "step": 166, |
| "step_time": 129.41105996404076 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 209.8125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 210.0, |
| "completions/max_terminated_length": 210.0, |
| "completions/mean_length": 209.8125, |
| "completions/mean_terminated_length": 209.8125, |
| "completions/min_length": 209.0, |
| "completions/min_terminated_length": 209.0, |
| "epoch": 1.1133333333333333, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.539503170110431e-05, |
| "loss": 0.0, |
| "num_tokens": 2351923.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 1.0, |
| "rewards/reward_fn/std": 0.0, |
| "step": 167, |
| "step_time": 96.18989022210008 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 608.0625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 873.5, |
| "completions/max_terminated_length": 873.5, |
| "completions/mean_length": 608.0625, |
| "completions/mean_terminated_length": 608.0625, |
| "completions/min_length": 387.0, |
| "completions/min_terminated_length": 387.0, |
| "epoch": 1.12, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.0253182090818882, |
| "kl": 0.0, |
| "learning_rate": 4.531057199893824e-05, |
| "loss": -0.0755884125828743, |
| "num_tokens": 2371084.0, |
| "reward": 0.6556649804115295, |
| "reward_std": 0.1775762438774109, |
| "rewards/reward_fn/mean": 0.6556649804115295, |
| "rewards/reward_fn/std": 0.1775762438774109, |
| "step": 168, |
| "step_time": 257.9385768309585 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 351.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 391.5, |
| "completions/max_terminated_length": 391.5, |
| "completions/mean_length": 351.6875, |
| "completions/mean_terminated_length": 351.6875, |
| "completions/min_length": 294.0, |
| "completions/min_terminated_length": 294.0, |
| "epoch": 1.1266666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.05709303170442581, |
| "kl": 0.0, |
| "learning_rate": 4.522542485937369e-05, |
| "loss": -0.023601766675710678, |
| "num_tokens": 2386415.0, |
| "reward": 0.6813934743404388, |
| "reward_std": 0.054284658282995224, |
| "rewards/reward_fn/mean": 0.6813934743404388, |
| "rewards/reward_fn/std": 0.054284658282995224, |
| "step": 169, |
| "step_time": 120.74896475899732 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 673.8125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 923.5, |
| "completions/max_terminated_length": 923.5, |
| "completions/mean_length": 673.8125, |
| "completions/mean_terminated_length": 673.8125, |
| "completions/min_length": 403.5, |
| "completions/min_terminated_length": 403.5, |
| "epoch": 1.1333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.08081483095884323, |
| "kl": 0.0, |
| "learning_rate": 4.5139593164324986e-05, |
| "loss": -0.06292446702718735, |
| "num_tokens": 2407308.0, |
| "reward": 0.5982428789138794, |
| "reward_std": 0.30377645045518875, |
| "rewards/reward_fn/mean": 0.5982428789138794, |
| "rewards/reward_fn/std": 0.30377645045518875, |
| "step": 170, |
| "step_time": 268.2763004680164 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 233.4375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 234.0, |
| "completions/max_terminated_length": 234.0, |
| "completions/mean_length": 233.4375, |
| "completions/mean_terminated_length": 233.4375, |
| "completions/min_length": 232.5, |
| "completions/min_terminated_length": 232.5, |
| "epoch": 1.1400000000000001, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.50530798188761e-05, |
| "loss": 0.0, |
| "num_tokens": 2419827.0, |
| "reward": 0.9509938955307007, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.9509938955307007, |
| "rewards/reward_fn/std": 0.0, |
| "step": 171, |
| "step_time": 91.46864466898842 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 209.8125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 210.0, |
| "completions/max_terminated_length": 210.0, |
| "completions/mean_length": 209.8125, |
| "completions/mean_terminated_length": 209.8125, |
| "completions/min_length": 209.5, |
| "completions/min_terminated_length": 209.5, |
| "epoch": 1.1466666666666667, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.496588775118232e-05, |
| "loss": 0.0, |
| "num_tokens": 2432200.0, |
| "reward": 0.8966733813285828, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.8966733813285828, |
| "rewards/reward_fn/std": 0.0, |
| "step": 172, |
| "step_time": 91.63760911300778 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 316.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 319.0, |
| "completions/max_terminated_length": 319.0, |
| "completions/mean_length": 316.0, |
| "completions/mean_terminated_length": 316.0, |
| "completions/min_length": 297.5, |
| "completions/min_terminated_length": 297.5, |
| "epoch": 1.1533333333333333, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.036608047783374786, |
| "kl": 0.0, |
| "learning_rate": 4.48780199123712e-05, |
| "loss": 0.00680706650018692, |
| "num_tokens": 2446848.0, |
| "reward": 0.619898796081543, |
| "reward_std": 0.09930147230625153, |
| "rewards/reward_fn/mean": 0.619898796081543, |
| "rewards/reward_fn/std": 0.09930147230625153, |
| "step": 173, |
| "step_time": 102.21163495199289 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 724.875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1117.0, |
| "completions/max_terminated_length": 1117.0, |
| "completions/mean_length": 724.875, |
| "completions/mean_terminated_length": 724.875, |
| "completions/min_length": 393.0, |
| "completions/min_terminated_length": 393.0, |
| "epoch": 1.16, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.03367030993103981, |
| "kl": 0.0, |
| "learning_rate": 4.478947927644258e-05, |
| "loss": 0.06719475984573364, |
| "num_tokens": 2467862.0, |
| "reward": 0.277934730052948, |
| "reward_std": 0.3036574125289917, |
| "rewards/reward_fn/mean": 0.277934730052948, |
| "rewards/reward_fn/std": 0.3036574125289917, |
| "step": 174, |
| "step_time": 317.1425336949178 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 339.25, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 379.0, |
| "completions/max_terminated_length": 379.0, |
| "completions/mean_length": 339.25, |
| "completions/mean_terminated_length": 339.25, |
| "completions/min_length": 318.0, |
| "completions/min_terminated_length": 318.0, |
| "epoch": 1.1666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07057405263185501, |
| "kl": 0.0, |
| "learning_rate": 4.4700268840168045e-05, |
| "loss": -0.013459604233503342, |
| "num_tokens": 2483850.0, |
| "reward": 0.4681895226240158, |
| "reward_std": 0.23409754782915115, |
| "rewards/reward_fn/mean": 0.4681895226240158, |
| "rewards/reward_fn/std": 0.23409754782915115, |
| "step": 175, |
| "step_time": 119.56902152101975 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 435.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 508.5, |
| "completions/max_terminated_length": 508.5, |
| "completions/mean_length": 435.9375, |
| "completions/mean_terminated_length": 435.9375, |
| "completions/min_length": 390.5, |
| "completions/min_terminated_length": 390.5, |
| "epoch": 1.1733333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07291749119758606, |
| "kl": 0.0, |
| "learning_rate": 4.4610391622989396e-05, |
| "loss": 0.01635035127401352, |
| "num_tokens": 2501393.0, |
| "reward": 0.4043854922056198, |
| "reward_std": 0.7844874858856201, |
| "rewards/reward_fn/mean": 0.4043854922056198, |
| "rewards/reward_fn/std": 0.7844874858856201, |
| "step": 176, |
| "step_time": 152.86159872601274 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 385.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 490.5, |
| "completions/max_terminated_length": 490.5, |
| "completions/mean_length": 385.9375, |
| "completions/mean_terminated_length": 385.9375, |
| "completions/min_length": 370.0, |
| "completions/min_terminated_length": 370.0, |
| "epoch": 1.18, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.4519850666916484e-05, |
| "loss": 0.0, |
| "num_tokens": 2516200.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 1.0, |
| "rewards/reward_fn/std": 0.0, |
| "step": 177, |
| "step_time": 144.09567704610527 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 482.0625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 489.0, |
| "completions/max_terminated_length": 489.0, |
| "completions/mean_length": 482.0625, |
| "completions/mean_terminated_length": 482.0625, |
| "completions/min_length": 479.5, |
| "completions/min_terminated_length": 479.5, |
| "epoch": 1.1866666666666668, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.054842259734869, |
| "kl": 0.0, |
| "learning_rate": 4.442864903642428e-05, |
| "loss": 0.004520110785961151, |
| "num_tokens": 2534569.0, |
| "reward": 0.48581717908382416, |
| "reward_std": 0.5444052815437317, |
| "rewards/reward_fn/mean": 0.48581717908382416, |
| "rewards/reward_fn/std": 0.5444052815437317, |
| "step": 178, |
| "step_time": 148.80164092604537 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 209.5625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 210.0, |
| "completions/max_terminated_length": 210.0, |
| "completions/mean_length": 209.5625, |
| "completions/mean_terminated_length": 209.5625, |
| "completions/min_length": 209.0, |
| "completions/min_terminated_length": 209.0, |
| "epoch": 1.1933333333333334, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.43367898183491e-05, |
| "loss": 0.0, |
| "num_tokens": 2546914.0, |
| "reward": 0.5261223241686821, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.5261223241686821, |
| "rewards/reward_fn/std": 0.0, |
| "step": 179, |
| "step_time": 92.55844796006568 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 509.875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 643.5, |
| "completions/max_terminated_length": 643.5, |
| "completions/mean_length": 509.875, |
| "completions/mean_terminated_length": 509.875, |
| "completions/min_length": 426.0, |
| "completions/min_terminated_length": 426.0, |
| "epoch": 1.2, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.4244276121784195e-05, |
| "loss": 0.0, |
| "num_tokens": 2563760.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 1.0, |
| "rewards/reward_fn/std": 0.0, |
| "step": 180, |
| "step_time": 184.21834510291228 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 372.75, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 387.5, |
| "completions/max_terminated_length": 387.5, |
| "completions/mean_length": 372.75, |
| "completions/mean_terminated_length": 372.75, |
| "completions/min_length": 369.0, |
| "completions/min_terminated_length": 369.0, |
| "epoch": 1.2066666666666666, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.415111107797445e-05, |
| "loss": 0.0, |
| "num_tokens": 2578380.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 1.0, |
| "rewards/reward_fn/std": 0.0, |
| "step": 181, |
| "step_time": 119.07426050008507 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 380.75, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 490.5, |
| "completions/max_terminated_length": 490.5, |
| "completions/mean_length": 380.75, |
| "completions/mean_terminated_length": 380.75, |
| "completions/min_length": 284.0, |
| "completions/min_terminated_length": 284.0, |
| "epoch": 1.2133333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.10535330325365067, |
| "kl": 0.0, |
| "learning_rate": 4.405729784021046e-05, |
| "loss": -0.029865777119994164, |
| "num_tokens": 2593112.0, |
| "reward": 0.625, |
| "reward_std": 0.816463440656662, |
| "rewards/reward_fn/mean": 0.625, |
| "rewards/reward_fn/std": 0.816463440656662, |
| "step": 182, |
| "step_time": 144.44632358610397 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 866.625, |
| "completions/clipped_ratio": 0.125, |
| "completions/max_length": 1213.0, |
| "completions/max_terminated_length": 1204.0, |
| "completions/mean_length": 866.625, |
| "completions/mean_terminated_length": 758.5416870117188, |
| "completions/min_length": 523.0, |
| "completions/min_terminated_length": 523.0, |
| "epoch": 1.22, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03957837074995041, |
| "kl": 0.0, |
| "learning_rate": 4.396283958372173e-05, |
| "loss": 0.08437013626098633, |
| "num_tokens": 2617154.0, |
| "reward": -0.035997092723846436, |
| "reward_std": 0.7678898870944977, |
| "rewards/reward_fn/mean": -0.035997092723846436, |
| "rewards/reward_fn/std": 0.7678898870944977, |
| "step": 183, |
| "step_time": 338.4485408520559 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 358.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 429.5, |
| "completions/max_terminated_length": 429.5, |
| "completions/mean_length": 358.9375, |
| "completions/mean_terminated_length": 358.9375, |
| "completions/min_length": 311.5, |
| "completions/min_terminated_length": 311.5, |
| "epoch": 1.2266666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0457036979496479, |
| "kl": 0.0, |
| "learning_rate": 4.386773950556931e-05, |
| "loss": 0.04372432082891464, |
| "num_tokens": 2633545.0, |
| "reward": 0.32335518300533295, |
| "reward_std": 0.10355385020375252, |
| "rewards/reward_fn/mean": 0.32335518300533295, |
| "rewards/reward_fn/std": 0.10355385020375252, |
| "step": 184, |
| "step_time": 132.418771808967 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 559.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 729.5, |
| "completions/max_terminated_length": 729.5, |
| "completions/mean_length": 559.0, |
| "completions/mean_terminated_length": 559.0, |
| "completions/min_length": 411.0, |
| "completions/min_terminated_length": 411.0, |
| "epoch": 1.2333333333333334, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.04097546264529228, |
| "kl": 0.0, |
| "learning_rate": 4.377200082453749e-05, |
| "loss": 0.010574601590633392, |
| "num_tokens": 2651889.0, |
| "reward": 0.8245929479598999, |
| "reward_std": 0.12071260064840317, |
| "rewards/reward_fn/mean": 0.8245929479598999, |
| "rewards/reward_fn/std": 0.12071260064840317, |
| "step": 185, |
| "step_time": 216.6957085240283 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 261.625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 275.0, |
| "completions/max_terminated_length": 275.0, |
| "completions/mean_length": 261.625, |
| "completions/mean_terminated_length": 261.625, |
| "completions/min_length": 253.5, |
| "completions/min_terminated_length": 253.5, |
| "epoch": 1.24, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05110285058617592, |
| "kl": 0.0, |
| "learning_rate": 4.36756267810249e-05, |
| "loss": -0.0024216175079345703, |
| "num_tokens": 2665851.0, |
| "reward": 0.3784840330481529, |
| "reward_std": 0.10511010140180588, |
| "rewards/reward_fn/mean": 0.3784840330481529, |
| "rewards/reward_fn/std": 0.10511010140180588, |
| "step": 186, |
| "step_time": 103.37580211204477 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 316.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 372.0, |
| "completions/max_terminated_length": 372.0, |
| "completions/mean_length": 316.9375, |
| "completions/mean_terminated_length": 316.9375, |
| "completions/min_length": 262.0, |
| "completions/min_terminated_length": 262.0, |
| "epoch": 1.2466666666666666, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.357862063693486e-05, |
| "loss": 0.0, |
| "num_tokens": 2679610.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 1.0, |
| "rewards/reward_fn/std": 0.0, |
| "step": 187, |
| "step_time": 113.27226198796416 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 361.875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 427.5, |
| "completions/max_terminated_length": 427.5, |
| "completions/mean_length": 361.875, |
| "completions/mean_terminated_length": 361.875, |
| "completions/min_length": 325.0, |
| "completions/min_terminated_length": 325.0, |
| "epoch": 1.2533333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0703757181763649, |
| "kl": 0.0, |
| "learning_rate": 4.34809856755649e-05, |
| "loss": 0.01859419420361519, |
| "num_tokens": 2694040.0, |
| "reward": 0.9870609939098358, |
| "reward_std": 0.036594912206055596, |
| "rewards/reward_fn/mean": 0.9870609939098358, |
| "rewards/reward_fn/std": 0.036594912206055596, |
| "step": 188, |
| "step_time": 128.19551112595946 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 361.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 449.5, |
| "completions/max_terminated_length": 449.5, |
| "completions/mean_length": 361.9375, |
| "completions/mean_terminated_length": 361.9375, |
| "completions/min_length": 294.5, |
| "completions/min_terminated_length": 294.5, |
| "epoch": 1.26, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.06226472556591034, |
| "kl": 0.0, |
| "learning_rate": 4.3382725201495723e-05, |
| "loss": 0.0047224462032318115, |
| "num_tokens": 2709535.0, |
| "reward": 0.6252674013376236, |
| "reward_std": 0.1955031156539917, |
| "rewards/reward_fn/mean": 0.6252674013376236, |
| "rewards/reward_fn/std": 0.1955031156539917, |
| "step": 189, |
| "step_time": 135.10929551004665 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 761.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1120.5, |
| "completions/max_terminated_length": 1120.5, |
| "completions/mean_length": 761.6875, |
| "completions/mean_terminated_length": 761.6875, |
| "completions/min_length": 444.0, |
| "completions/min_terminated_length": 444.0, |
| "epoch": 1.2666666666666666, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.0425034761428833, |
| "kl": 0.0, |
| "learning_rate": 4.3283842540479264e-05, |
| "loss": -0.004420645534992218, |
| "num_tokens": 2731914.0, |
| "reward": 0.17015497386455536, |
| "reward_std": 0.03678224980831146, |
| "rewards/reward_fn/mean": 0.17015497386455536, |
| "rewards/reward_fn/std": 0.03678224980831146, |
| "step": 190, |
| "step_time": 313.89839813805884 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 301.0625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 371.0, |
| "completions/max_terminated_length": 371.0, |
| "completions/mean_length": 301.0625, |
| "completions/mean_terminated_length": 301.0625, |
| "completions/min_length": 253.5, |
| "completions/min_terminated_length": 253.5, |
| "epoch": 1.2733333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09773189574480057, |
| "kl": 0.0, |
| "learning_rate": 4.318434103932622e-05, |
| "loss": 0.08142915368080139, |
| "num_tokens": 2747275.0, |
| "reward": 0.6540311872959137, |
| "reward_std": 0.19406485185027122, |
| "rewards/reward_fn/mean": 0.6540311872959137, |
| "rewards/reward_fn/std": 0.19406485185027122, |
| "step": 191, |
| "step_time": 116.78903921408346 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 482.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 730.0, |
| "completions/max_terminated_length": 730.0, |
| "completions/mean_length": 482.0, |
| "completions/mean_terminated_length": 482.0, |
| "completions/min_length": 296.5, |
| "completions/min_terminated_length": 296.5, |
| "epoch": 1.28, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.028390681371092796, |
| "kl": 0.0, |
| "learning_rate": 4.30842240657927e-05, |
| "loss": 0.003453485667705536, |
| "num_tokens": 2764083.0, |
| "reward": 0.8803534507751465, |
| "reward_std": 0.3384115397930145, |
| "rewards/reward_fn/mean": 0.8803534507751465, |
| "rewards/reward_fn/std": 0.3384115397930145, |
| "step": 192, |
| "step_time": 211.86879105505068 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 365.5625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 472.0, |
| "completions/max_terminated_length": 472.0, |
| "completions/mean_length": 365.5625, |
| "completions/mean_terminated_length": 365.5625, |
| "completions/min_length": 302.0, |
| "completions/min_terminated_length": 302.0, |
| "epoch": 1.2866666666666666, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.2983495008466276e-05, |
| "loss": 0.0, |
| "num_tokens": 2778700.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 1.0, |
| "rewards/reward_fn/std": 0.0, |
| "step": 193, |
| "step_time": 138.9105662080692 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 225.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 236.5, |
| "completions/max_terminated_length": 236.5, |
| "completions/mean_length": 225.6875, |
| "completions/mean_terminated_length": 225.6875, |
| "completions/min_length": 214.5, |
| "completions/min_terminated_length": 214.5, |
| "epoch": 1.2933333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.5753271579742432, |
| "kl": 0.0, |
| "learning_rate": 4.288215727665129e-05, |
| "loss": -0.010599890723824501, |
| "num_tokens": 2792055.0, |
| "reward": 0.49704940617084503, |
| "reward_std": 0.031217220712960625, |
| "rewards/reward_fn/mean": 0.49704940617084503, |
| "rewards/reward_fn/std": 0.031217220712960625, |
| "step": 194, |
| "step_time": 89.62762422900414 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 389.1875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 607.0, |
| "completions/max_terminated_length": 607.0, |
| "completions/mean_length": 389.1875, |
| "completions/mean_terminated_length": 389.1875, |
| "completions/min_length": 259.5, |
| "completions/min_terminated_length": 259.5, |
| "epoch": 1.3, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.05310267210006714, |
| "kl": 0.0, |
| "learning_rate": 4.278021430025343e-05, |
| "loss": -0.12552142143249512, |
| "num_tokens": 2807642.0, |
| "reward": 0.3906529098749161, |
| "reward_std": 0.24650515615940094, |
| "rewards/reward_fn/mean": 0.3906529098749161, |
| "rewards/reward_fn/std": 0.24650515615940094, |
| "step": 195, |
| "step_time": 185.09018681605812 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 385.3125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 501.5, |
| "completions/max_terminated_length": 501.5, |
| "completions/mean_length": 385.3125, |
| "completions/mean_terminated_length": 385.3125, |
| "completions/min_length": 340.0, |
| "completions/min_terminated_length": 340.0, |
| "epoch": 1.3066666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.027251476421952248, |
| "kl": 0.0, |
| "learning_rate": 4.267766952966369e-05, |
| "loss": -0.006735360249876976, |
| "num_tokens": 2823903.0, |
| "reward": 0.9465951919555664, |
| "reward_std": 0.04546740645120906, |
| "rewards/reward_fn/mean": 0.9465951919555664, |
| "rewards/reward_fn/std": 0.04546740645120906, |
| "step": 196, |
| "step_time": 148.99407557304949 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 627.375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 968.0, |
| "completions/max_terminated_length": 968.0, |
| "completions/mean_length": 627.375, |
| "completions/mean_terminated_length": 627.375, |
| "completions/min_length": 425.0, |
| "completions/min_terminated_length": 425.0, |
| "epoch": 1.3133333333333335, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06101449579000473, |
| "kl": 0.0, |
| "learning_rate": 4.257452643564155e-05, |
| "loss": -0.060674093663692474, |
| "num_tokens": 2843725.0, |
| "reward": 0.4497956559062004, |
| "reward_std": 0.6334844715893269, |
| "rewards/reward_fn/mean": 0.4497956559062004, |
| "rewards/reward_fn/std": 0.6334844715893269, |
| "step": 197, |
| "step_time": 271.5556913619512 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 466.8125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 702.5, |
| "completions/max_terminated_length": 702.5, |
| "completions/mean_length": 466.8125, |
| "completions/mean_terminated_length": 466.8125, |
| "completions/min_length": 330.0, |
| "completions/min_terminated_length": 330.0, |
| "epoch": 1.32, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.05101463943719864, |
| "kl": 0.0, |
| "learning_rate": 4.24707885091975e-05, |
| "loss": -0.07410165667533875, |
| "num_tokens": 2860386.0, |
| "reward": 0.7855673432350159, |
| "reward_std": 0.14362023770809174, |
| "rewards/reward_fn/mean": 0.7855673432350159, |
| "rewards/reward_fn/std": 0.14362023770809174, |
| "step": 198, |
| "step_time": 198.9184314039885 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 272.125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 312.0, |
| "completions/max_terminated_length": 312.0, |
| "completions/mean_length": 272.125, |
| "completions/mean_terminated_length": 272.125, |
| "completions/min_length": 241.5, |
| "completions/min_terminated_length": 241.5, |
| "epoch": 1.3266666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.03716262802481651, |
| "kl": 0.0, |
| "learning_rate": 4.2366459261474933e-05, |
| "loss": -0.00589386373758316, |
| "num_tokens": 2874348.0, |
| "reward": 0.9446012079715729, |
| "reward_std": 0.022661298513412476, |
| "rewards/reward_fn/mean": 0.9446012079715729, |
| "rewards/reward_fn/std": 0.022661298513412476, |
| "step": 199, |
| "step_time": 99.15163314703386 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 247.1875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 276.5, |
| "completions/max_terminated_length": 276.5, |
| "completions/mean_length": 247.1875, |
| "completions/mean_terminated_length": 247.1875, |
| "completions/min_length": 226.5, |
| "completions/min_terminated_length": 226.5, |
| "epoch": 1.3333333333333333, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.05213377997279167, |
| "kl": 0.0, |
| "learning_rate": 4.226154222363124e-05, |
| "loss": 0.055981483310461044, |
| "num_tokens": 2888167.0, |
| "reward": 0.822132408618927, |
| "reward_std": 0.1446933150291443, |
| "rewards/reward_fn/mean": 0.822132408618927, |
| "rewards/reward_fn/std": 0.1446933150291443, |
| "step": 200, |
| "step_time": 99.35902613506187 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 394.25, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 477.5, |
| "completions/max_terminated_length": 477.5, |
| "completions/mean_length": 394.25, |
| "completions/mean_terminated_length": 394.25, |
| "completions/min_length": 305.5, |
| "completions/min_terminated_length": 305.5, |
| "epoch": 1.34, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07755188643932343, |
| "kl": 0.0, |
| "learning_rate": 4.215604094671835e-05, |
| "loss": -0.05246041715145111, |
| "num_tokens": 2904579.0, |
| "reward": 0.5525769889354706, |
| "reward_std": 0.588111624121666, |
| "rewards/reward_fn/mean": 0.5525769889354706, |
| "rewards/reward_fn/std": 0.588111624121666, |
| "step": 201, |
| "step_time": 144.34619156294502 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 401.75, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 550.5, |
| "completions/max_terminated_length": 550.5, |
| "completions/mean_length": 401.75, |
| "completions/mean_terminated_length": 401.75, |
| "completions/min_length": 251.5, |
| "completions/min_terminated_length": 251.5, |
| "epoch": 1.3466666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.044257309287786484, |
| "kl": 0.0, |
| "learning_rate": 4.2049959001562464e-05, |
| "loss": -0.0892098918557167, |
| "num_tokens": 2920423.0, |
| "reward": 0.39934873580932617, |
| "reward_std": 0.12950681149959564, |
| "rewards/reward_fn/mean": 0.39934873580932617, |
| "rewards/reward_fn/std": 0.12950681149959564, |
| "step": 202, |
| "step_time": 169.23466102196835 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 267.875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 288.0, |
| "completions/max_terminated_length": 288.0, |
| "completions/mean_length": 267.875, |
| "completions/mean_terminated_length": 267.875, |
| "completions/min_length": 256.0, |
| "completions/min_terminated_length": 256.0, |
| "epoch": 1.3533333333333333, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.09341531991958618, |
| "kl": 0.0, |
| "learning_rate": 4.194329997864331e-05, |
| "loss": 1.2942723515152466e-07, |
| "num_tokens": 2933605.0, |
| "reward": 0.5648120641708374, |
| "reward_std": 0.011593026109039783, |
| "rewards/reward_fn/mean": 0.5648120641708374, |
| "rewards/reward_fn/std": 0.011593026109039783, |
| "step": 203, |
| "step_time": 98.69075074000284 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 421.125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 471.5, |
| "completions/max_terminated_length": 471.5, |
| "completions/mean_length": 421.125, |
| "completions/mean_terminated_length": 421.125, |
| "completions/min_length": 351.5, |
| "completions/min_terminated_length": 351.5, |
| "epoch": 1.3599999999999999, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06170891597867012, |
| "kl": 0.0, |
| "learning_rate": 4.183606748797251e-05, |
| "loss": -0.03707977384328842, |
| "num_tokens": 2950479.0, |
| "reward": 0.4111311808228493, |
| "reward_std": 0.44924869388341904, |
| "rewards/reward_fn/mean": 0.4111311808228493, |
| "rewards/reward_fn/std": 0.44924869388341904, |
| "step": 204, |
| "step_time": 148.0194004449877 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 263.0625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 278.5, |
| "completions/max_terminated_length": 278.5, |
| "completions/mean_length": 263.0625, |
| "completions/mean_terminated_length": 263.0625, |
| "completions/min_length": 222.5, |
| "completions/min_terminated_length": 222.5, |
| "epoch": 1.3666666666666667, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.172826515897146e-05, |
| "loss": 0.0, |
| "num_tokens": 2963288.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 1.0, |
| "rewards/reward_fn/std": 0.0, |
| "step": 205, |
| "step_time": 89.47718564799288 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 498.125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 709.5, |
| "completions/max_terminated_length": 709.5, |
| "completions/mean_length": 498.125, |
| "completions/mean_terminated_length": 498.125, |
| "completions/min_length": 416.5, |
| "completions/min_terminated_length": 416.5, |
| "epoch": 1.3733333333333333, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.03950470685958862, |
| "kl": 0.0, |
| "learning_rate": 4.1619896640348445e-05, |
| "loss": 0.005682090297341347, |
| "num_tokens": 2980506.0, |
| "reward": 0.5262927748262882, |
| "reward_std": 0.1411045640707016, |
| "rewards/reward_fn/mean": 0.5262927748262882, |
| "rewards/reward_fn/std": 0.1411045640707016, |
| "step": 206, |
| "step_time": 207.79324868600816 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 489.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 656.0, |
| "completions/max_terminated_length": 656.0, |
| "completions/mean_length": 489.6875, |
| "completions/mean_terminated_length": 489.6875, |
| "completions/min_length": 413.0, |
| "completions/min_terminated_length": 413.0, |
| "epoch": 1.38, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.1510965599975196e-05, |
| "loss": 0.0, |
| "num_tokens": 2997557.0, |
| "reward": 0.8645179867744446, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.8645179867744446, |
| "rewards/reward_fn/std": 0.0, |
| "step": 207, |
| "step_time": 187.55295675288653 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 528.0625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 661.0, |
| "completions/max_terminated_length": 661.0, |
| "completions/mean_length": 528.0625, |
| "completions/mean_terminated_length": 528.0625, |
| "completions/min_length": 395.5, |
| "completions/min_terminated_length": 395.5, |
| "epoch": 1.3866666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.07827558368444443, |
| "kl": 0.0, |
| "learning_rate": 4.140147572476268e-05, |
| "loss": 0.026687294244766235, |
| "num_tokens": 3015694.0, |
| "reward": 0.7765228152275085, |
| "reward_std": 0.4247249960899353, |
| "rewards/reward_fn/mean": 0.7765228152275085, |
| "rewards/reward_fn/std": 0.4247249960899353, |
| "step": 208, |
| "step_time": 202.78064414602704 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 440.0625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 606.5, |
| "completions/max_terminated_length": 606.5, |
| "completions/mean_length": 440.0625, |
| "completions/mean_terminated_length": 440.0625, |
| "completions/min_length": 328.5, |
| "completions/min_terminated_length": 328.5, |
| "epoch": 1.3933333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0835847333073616, |
| "kl": 0.0, |
| "learning_rate": 4.129143072053638e-05, |
| "loss": -0.08566160500049591, |
| "num_tokens": 3032919.0, |
| "reward": 0.6553326100111008, |
| "reward_std": 0.08002846711315215, |
| "rewards/reward_fn/mean": 0.6553326100111008, |
| "rewards/reward_fn/std": 0.08002846711315215, |
| "step": 209, |
| "step_time": 179.81165817193687 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 528.5, |
| "completions/clipped_ratio": 0.0625, |
| "completions/max_length": 1155.5, |
| "completions/max_terminated_length": 654.5, |
| "completions/mean_length": 528.5, |
| "completions/mean_terminated_length": 441.4464416503906, |
| "completions/min_length": 273.5, |
| "completions/min_terminated_length": 273.5, |
| "epoch": 1.4, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.044825032353401184, |
| "kl": 0.0, |
| "learning_rate": 4.118083431191081e-05, |
| "loss": 0.21945233643054962, |
| "num_tokens": 3051503.0, |
| "reward": 0.45718520134687424, |
| "reward_std": 0.586285712197423, |
| "rewards/reward_fn/mean": 0.45718520134687424, |
| "rewards/reward_fn/std": 0.586285712197423, |
| "step": 210, |
| "step_time": 320.2217711000703 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 457.5625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 808.0, |
| "completions/max_terminated_length": 808.0, |
| "completions/mean_length": 457.5625, |
| "completions/mean_terminated_length": 457.5625, |
| "completions/min_length": 335.0, |
| "completions/min_terminated_length": 335.0, |
| "epoch": 1.4066666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.050018809735774994, |
| "kl": 0.0, |
| "learning_rate": 4.1069690242163484e-05, |
| "loss": -0.009994201362133026, |
| "num_tokens": 3068272.0, |
| "reward": 0.3910246044397354, |
| "reward_std": 0.43388164043426514, |
| "rewards/reward_fn/mean": 0.3910246044397354, |
| "rewards/reward_fn/std": 0.43388164043426514, |
| "step": 211, |
| "step_time": 242.25502372306073 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 519.625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 772.0, |
| "completions/max_terminated_length": 772.0, |
| "completions/mean_length": 519.625, |
| "completions/mean_terminated_length": 519.625, |
| "completions/min_length": 304.0, |
| "completions/min_terminated_length": 304.0, |
| "epoch": 1.4133333333333333, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.03964097052812576, |
| "kl": 0.0, |
| "learning_rate": 4.095800227310821e-05, |
| "loss": 0.08257991820573807, |
| "num_tokens": 3085970.0, |
| "reward": 0.43319354951381683, |
| "reward_std": 0.19981928169727325, |
| "rewards/reward_fn/mean": 0.43319354951381683, |
| "rewards/reward_fn/std": 0.19981928169727325, |
| "step": 212, |
| "step_time": 231.8354788030265 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 319.5625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 383.0, |
| "completions/max_terminated_length": 383.0, |
| "completions/mean_length": 319.5625, |
| "completions/mean_terminated_length": 319.5625, |
| "completions/min_length": 282.5, |
| "completions/min_terminated_length": 282.5, |
| "epoch": 1.42, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.03575386852025986, |
| "kl": 0.0, |
| "learning_rate": 4.0845774184967754e-05, |
| "loss": 0.021894656121730804, |
| "num_tokens": 3100747.0, |
| "reward": 0.959542840719223, |
| "reward_std": 0.023604698479175568, |
| "rewards/reward_fn/mean": 0.959542840719223, |
| "rewards/reward_fn/std": 0.023604698479175568, |
| "step": 213, |
| "step_time": 117.80390733794775 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 409.8125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 700.0, |
| "completions/max_terminated_length": 700.0, |
| "completions/mean_length": 409.8125, |
| "completions/mean_terminated_length": 409.8125, |
| "completions/min_length": 325.0, |
| "completions/min_terminated_length": 325.0, |
| "epoch": 1.4266666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.06581282615661621, |
| "kl": 0.0, |
| "learning_rate": 4.073300977624594e-05, |
| "loss": 0.1886315941810608, |
| "num_tokens": 3116696.0, |
| "reward": 0.896901398897171, |
| "reward_std": 0.061386555433273315, |
| "rewards/reward_fn/mean": 0.896901398897171, |
| "rewards/reward_fn/std": 0.061386555433273315, |
| "step": 214, |
| "step_time": 209.5079359910451 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 539.1875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 758.0, |
| "completions/max_terminated_length": 758.0, |
| "completions/mean_length": 539.1875, |
| "completions/mean_terminated_length": 539.1875, |
| "completions/min_length": 383.5, |
| "completions/min_terminated_length": 383.5, |
| "epoch": 1.4333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07676459103822708, |
| "kl": 0.0, |
| "learning_rate": 4.0619712863599e-05, |
| "loss": -0.09422039985656738, |
| "num_tokens": 3135171.0, |
| "reward": 0.6873325705528259, |
| "reward_std": 0.2853637635707855, |
| "rewards/reward_fn/mean": 0.6873325705528259, |
| "rewards/reward_fn/std": 0.2853637635707855, |
| "step": 215, |
| "step_time": 221.83494500699453 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 254.4375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 276.0, |
| "completions/max_terminated_length": 276.0, |
| "completions/mean_length": 254.4375, |
| "completions/mean_terminated_length": 254.4375, |
| "completions/min_length": 227.5, |
| "completions/min_terminated_length": 227.5, |
| "epoch": 1.44, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.05058872817065e-05, |
| "loss": 0.0, |
| "num_tokens": 3148138.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 1.0, |
| "rewards/reward_fn/std": 0.0, |
| "step": 216, |
| "step_time": 97.23418708896497 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 318.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 415.0, |
| "completions/max_terminated_length": 415.0, |
| "completions/mean_length": 318.6875, |
| "completions/mean_terminated_length": 318.6875, |
| "completions/min_length": 273.5, |
| "completions/min_terminated_length": 273.5, |
| "epoch": 1.4466666666666668, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.0035884876269847155, |
| "kl": 0.0, |
| "learning_rate": 4.039153688314145e-05, |
| "loss": -0.0018375769723206758, |
| "num_tokens": 3162613.0, |
| "reward": 0.9999989867210388, |
| "reward_std": 2.858738071154221e-06, |
| "rewards/reward_fn/mean": 0.9999989867210388, |
| "rewards/reward_fn/std": 2.858738071154221e-06, |
| "step": 217, |
| "step_time": 136.8080933529418 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 209.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 210.0, |
| "completions/max_terminated_length": 210.0, |
| "completions/mean_length": 209.9375, |
| "completions/mean_terminated_length": 209.9375, |
| "completions/min_length": 209.5, |
| "completions/min_terminated_length": 209.5, |
| "epoch": 1.4533333333333334, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.0276665538239996e-05, |
| "loss": 0.0, |
| "num_tokens": 3174996.0, |
| "reward": 0.5221271961927414, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.5221271961927414, |
| "rewards/reward_fn/std": 0.0, |
| "step": 218, |
| "step_time": 88.73153007001383 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 429.1875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 592.0, |
| "completions/max_terminated_length": 592.0, |
| "completions/mean_length": 429.1875, |
| "completions/mean_terminated_length": 429.1875, |
| "completions/min_length": 334.5, |
| "completions/min_terminated_length": 334.5, |
| "epoch": 1.46, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.10545268654823303, |
| "kl": 0.0, |
| "learning_rate": 4.0161277134970345e-05, |
| "loss": -0.02452065981924534, |
| "num_tokens": 3191567.0, |
| "reward": 0.622944176197052, |
| "reward_std": 0.289985753595829, |
| "rewards/reward_fn/mean": 0.622944176197052, |
| "rewards/reward_fn/std": 0.289985753595829, |
| "step": 219, |
| "step_time": 182.1568425160949 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 219.75, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 247.0, |
| "completions/max_terminated_length": 247.0, |
| "completions/mean_length": 219.75, |
| "completions/mean_terminated_length": 219.75, |
| "completions/min_length": 194.5, |
| "completions/min_terminated_length": 194.5, |
| "epoch": 1.4666666666666668, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 4.0045375578801214e-05, |
| "loss": 0.0, |
| "num_tokens": 3203891.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 1.0, |
| "rewards/reward_fn/std": 0.0, |
| "step": 220, |
| "step_time": 90.31312085100217 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 332.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 417.5, |
| "completions/max_terminated_length": 417.5, |
| "completions/mean_length": 332.0, |
| "completions/mean_terminated_length": 332.0, |
| "completions/min_length": 258.5, |
| "completions/min_terminated_length": 258.5, |
| "epoch": 1.4733333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06864330917596817, |
| "kl": 0.0, |
| "learning_rate": 3.9928964792569655e-05, |
| "loss": -0.04521460831165314, |
| "num_tokens": 3219307.0, |
| "reward": 0.750563770532608, |
| "reward_std": 0.2518393062055111, |
| "rewards/reward_fn/mean": 0.750563770532608, |
| "rewards/reward_fn/std": 0.2518393062055111, |
| "step": 221, |
| "step_time": 133.98197106306907 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 258.25, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 282.5, |
| "completions/max_terminated_length": 282.5, |
| "completions/mean_length": 258.25, |
| "completions/mean_terminated_length": 258.25, |
| "completions/min_length": 232.5, |
| "completions/min_terminated_length": 232.5, |
| "epoch": 1.48, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.0495823435485363, |
| "kl": 0.0, |
| "learning_rate": 3.981204871634827e-05, |
| "loss": -0.015496471896767616, |
| "num_tokens": 3233079.0, |
| "reward": 0.7546902000904083, |
| "reward_std": 0.003863011719658971, |
| "rewards/reward_fn/mean": 0.7546902000904083, |
| "rewards/reward_fn/std": 0.003863011719658971, |
| "step": 222, |
| "step_time": 92.20567861292511 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 201.625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 229.5, |
| "completions/max_terminated_length": 229.5, |
| "completions/mean_length": 201.625, |
| "completions/mean_terminated_length": 201.625, |
| "completions/min_length": 190.5, |
| "completions/min_terminated_length": 190.5, |
| "epoch": 1.4866666666666668, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 6.255746848182753e-05, |
| "kl": 0.0, |
| "learning_rate": 3.969463130731183e-05, |
| "loss": 0.0001317363348789513, |
| "num_tokens": 3245049.0, |
| "reward": 0.5408785976469517, |
| "reward_std": 4.5056875563886933e-08, |
| "rewards/reward_fn/mean": 0.5408785976469517, |
| "rewards/reward_fn/std": 4.5056875563886933e-08, |
| "step": 223, |
| "step_time": 83.72366849990794 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 282.5625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 307.5, |
| "completions/max_terminated_length": 307.5, |
| "completions/mean_length": 282.5625, |
| "completions/mean_terminated_length": 282.5625, |
| "completions/min_length": 254.5, |
| "completions/min_terminated_length": 254.5, |
| "epoch": 1.4933333333333334, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.057080987840890884, |
| "kl": 0.0, |
| "learning_rate": 3.957671653960337e-05, |
| "loss": -0.015905410051345825, |
| "num_tokens": 3259186.0, |
| "reward": 0.6227080449461937, |
| "reward_std": 0.15797464549541473, |
| "rewards/reward_fn/mean": 0.6227080449461937, |
| "rewards/reward_fn/std": 0.15797464549541473, |
| "step": 224, |
| "step_time": 98.58942143298918 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 407.75, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 556.0, |
| "completions/max_terminated_length": 556.0, |
| "completions/mean_length": 407.75, |
| "completions/mean_terminated_length": 407.75, |
| "completions/min_length": 326.0, |
| "completions/min_terminated_length": 326.0, |
| "epoch": 1.5, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0834810733795166, |
| "kl": 0.0, |
| "learning_rate": 3.945830840419966e-05, |
| "loss": -0.08507833629846573, |
| "num_tokens": 3275478.0, |
| "reward": 0.7662567794322968, |
| "reward_std": 0.24252163991332054, |
| "rewards/reward_fn/mean": 0.7662567794322968, |
| "rewards/reward_fn/std": 0.24252163991332054, |
| "step": 225, |
| "step_time": 167.97602311102673 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 223.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 236.5, |
| "completions/max_terminated_length": 236.5, |
| "completions/mean_length": 223.9375, |
| "completions/mean_terminated_length": 223.9375, |
| "completions/min_length": 216.0, |
| "completions/min_terminated_length": 216.0, |
| "epoch": 1.5066666666666668, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.05319474637508392, |
| "kl": 0.0, |
| "learning_rate": 3.933941090877615e-05, |
| "loss": 0.030159292742609978, |
| "num_tokens": 3288837.0, |
| "reward": 0.47812579572200775, |
| "reward_std": 0.04895319417119026, |
| "rewards/reward_fn/mean": 0.47812579572200775, |
| "rewards/reward_fn/std": 0.04895319417119026, |
| "step": 226, |
| "step_time": 88.83022219099803 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 271.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 291.5, |
| "completions/max_terminated_length": 291.5, |
| "completions/mean_length": 271.9375, |
| "completions/mean_terminated_length": 271.9375, |
| "completions/min_length": 261.5, |
| "completions/min_terminated_length": 261.5, |
| "epoch": 1.5133333333333332, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.0350649356842041, |
| "kl": 0.0, |
| "learning_rate": 3.9220028077571295e-05, |
| "loss": 0.001047968864440918, |
| "num_tokens": 3302796.0, |
| "reward": 0.9795733690261841, |
| "reward_std": 0.007507155183702707, |
| "rewards/reward_fn/mean": 0.9795733690261841, |
| "rewards/reward_fn/std": 0.007507155183702707, |
| "step": 227, |
| "step_time": 94.09043034503702 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 216.375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 216.5, |
| "completions/max_terminated_length": 216.5, |
| "completions/mean_length": 216.375, |
| "completions/mean_terminated_length": 216.375, |
| "completions/min_length": 215.5, |
| "completions/min_terminated_length": 215.5, |
| "epoch": 1.52, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.0425180047750473, |
| "kl": 0.0, |
| "learning_rate": 3.910016395125037e-05, |
| "loss": -0.00019904226064682007, |
| "num_tokens": 3315978.0, |
| "reward": 0.9242193102836609, |
| "reward_std": 0.008535998873412609, |
| "rewards/reward_fn/mean": 0.9242193102836609, |
| "rewards/reward_fn/std": 0.008535998873412609, |
| "step": 228, |
| "step_time": 83.15773937094491 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 209.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 210.0, |
| "completions/max_terminated_length": 210.0, |
| "completions/mean_length": 209.9375, |
| "completions/mean_terminated_length": 209.9375, |
| "completions/min_length": 209.5, |
| "completions/min_terminated_length": 209.5, |
| "epoch": 1.5266666666666666, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 3.897982258676867e-05, |
| "loss": 0.0, |
| "num_tokens": 3328353.0, |
| "reward": 0.09640464186668396, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.09640464186668396, |
| "rewards/reward_fn/std": 0.0, |
| "step": 229, |
| "step_time": 92.9228468539659 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 271.125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 288.0, |
| "completions/max_terminated_length": 288.0, |
| "completions/mean_length": 271.125, |
| "completions/mean_terminated_length": 271.125, |
| "completions/min_length": 254.0, |
| "completions/min_terminated_length": 254.0, |
| "epoch": 1.5333333333333332, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 3.885900805723429e-05, |
| "loss": 0.0, |
| "num_tokens": 3341515.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 1.0, |
| "rewards/reward_fn/std": 0.0, |
| "step": 230, |
| "step_time": 103.87323606194695 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 271.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 289.0, |
| "completions/max_terminated_length": 289.0, |
| "completions/mean_length": 271.0, |
| "completions/mean_terminated_length": 271.0, |
| "completions/min_length": 256.0, |
| "completions/min_terminated_length": 256.0, |
| "epoch": 1.54, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 3.873772445177015e-05, |
| "loss": 0.0, |
| "num_tokens": 3354707.0, |
| "reward": 0.5801214650273323, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.5801214650273323, |
| "rewards/reward_fn/std": 0.0, |
| "step": 231, |
| "step_time": 104.76834018190857 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 351.875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 449.5, |
| "completions/max_terminated_length": 449.5, |
| "completions/mean_length": 351.875, |
| "completions/mean_terminated_length": 351.875, |
| "completions/min_length": 292.0, |
| "completions/min_terminated_length": 292.0, |
| "epoch": 1.5466666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0679924488067627, |
| "kl": 0.0, |
| "learning_rate": 3.861597587537568e-05, |
| "loss": -0.0012257262133061886, |
| "num_tokens": 3370505.0, |
| "reward": 0.7388543784618378, |
| "reward_std": 0.04050896943635962, |
| "rewards/reward_fn/mean": 0.7388543784618378, |
| "rewards/reward_fn/std": 0.04050896943635962, |
| "step": 232, |
| "step_time": 135.8323201989988 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 450.5, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 612.0, |
| "completions/max_terminated_length": 612.0, |
| "completions/mean_length": 450.5, |
| "completions/mean_terminated_length": 450.5, |
| "completions/min_length": 369.0, |
| "completions/min_terminated_length": 369.0, |
| "epoch": 1.5533333333333332, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 3.8493766448787825e-05, |
| "loss": 0.0, |
| "num_tokens": 3386985.0, |
| "reward": 0.40595003962516785, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.40595003962516785, |
| "rewards/reward_fn/std": 0.0, |
| "step": 233, |
| "step_time": 180.66226357588312 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 221.4375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 236.5, |
| "completions/max_terminated_length": 236.5, |
| "completions/mean_length": 221.4375, |
| "completions/mean_terminated_length": 221.4375, |
| "completions/min_length": 216.5, |
| "completions/min_terminated_length": 216.5, |
| "epoch": 1.56, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.06356774270534515, |
| "kl": 0.0, |
| "learning_rate": 3.837110030834161e-05, |
| "loss": -0.02530668117105961, |
| "num_tokens": 3400344.0, |
| "reward": 0.8683105707168579, |
| "reward_std": 0.07456143200397491, |
| "rewards/reward_fn/mean": 0.8683105707168579, |
| "rewards/reward_fn/std": 0.07456143200397491, |
| "step": 234, |
| "step_time": 91.35750043700682 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 277.3125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 343.0, |
| "completions/max_terminated_length": 343.0, |
| "completions/mean_length": 277.3125, |
| "completions/mean_terminated_length": 277.3125, |
| "completions/min_length": 254.0, |
| "completions/min_terminated_length": 254.0, |
| "epoch": 1.5666666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 6.303178815869614e-05, |
| "kl": 0.0, |
| "learning_rate": 3.824798160583012e-05, |
| "loss": 0.00016298360424116254, |
| "num_tokens": 3413677.0, |
| "reward": 0.9469609558582306, |
| "reward_std": 5.6321095343037086e-08, |
| "rewards/reward_fn/mean": 0.9469609558582306, |
| "rewards/reward_fn/std": 5.6321095343037086e-08, |
| "step": 235, |
| "step_time": 117.30590885004494 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 306.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 367.0, |
| "completions/max_terminated_length": 367.0, |
| "completions/mean_length": 306.9375, |
| "completions/mean_terminated_length": 306.9375, |
| "completions/min_length": 274.0, |
| "completions/min_terminated_length": 274.0, |
| "epoch": 1.5733333333333333, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 3.8124414508364e-05, |
| "loss": 0.0, |
| "num_tokens": 3427252.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 1.0, |
| "rewards/reward_fn/std": 0.0, |
| "step": 236, |
| "step_time": 111.6909038599697 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 441.625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 532.5, |
| "completions/max_terminated_length": 532.5, |
| "completions/mean_length": 441.625, |
| "completions/mean_terminated_length": 441.625, |
| "completions/min_length": 355.0, |
| "completions/min_terminated_length": 355.0, |
| "epoch": 1.58, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06842601299285889, |
| "kl": 0.0, |
| "learning_rate": 3.8000403198230387e-05, |
| "loss": 0.025734104216098785, |
| "num_tokens": 3444510.0, |
| "reward": 0.4380612373352051, |
| "reward_std": 0.2177828741259873, |
| "rewards/reward_fn/mean": 0.4380612373352051, |
| "rewards/reward_fn/std": 0.2177828741259873, |
| "step": 237, |
| "step_time": 162.99026800697902 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 426.8125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 500.5, |
| "completions/max_terminated_length": 500.5, |
| "completions/mean_length": 426.8125, |
| "completions/mean_terminated_length": 426.8125, |
| "completions/min_length": 361.5, |
| "completions/min_terminated_length": 361.5, |
| "epoch": 1.5866666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.05216877907514572, |
| "kl": 0.0, |
| "learning_rate": 3.787595187275136e-05, |
| "loss": -0.022159036248922348, |
| "num_tokens": 3460707.0, |
| "reward": 0.17322568595409393, |
| "reward_std": 0.2979126274585724, |
| "rewards/reward_fn/mean": 0.17322568595409393, |
| "rewards/reward_fn/std": 0.2979126274585724, |
| "step": 238, |
| "step_time": 164.71848585800035 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 293.75, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 312.0, |
| "completions/max_terminated_length": 312.0, |
| "completions/mean_length": 293.75, |
| "completions/mean_terminated_length": 293.75, |
| "completions/min_length": 249.5, |
| "completions/min_terminated_length": 249.5, |
| "epoch": 1.5933333333333333, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 3.775106474414188e-05, |
| "loss": 0.0, |
| "num_tokens": 3474063.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 1.0, |
| "rewards/reward_fn/std": 0.0, |
| "step": 239, |
| "step_time": 98.01331254001707 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 210.0625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 210.5, |
| "completions/max_terminated_length": 210.5, |
| "completions/mean_length": 210.0625, |
| "completions/mean_terminated_length": 210.0625, |
| "completions/min_length": 210.0, |
| "completions/min_terminated_length": 210.0, |
| "epoch": 1.6, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 3.762574603936725e-05, |
| "loss": 0.0, |
| "num_tokens": 3486408.0, |
| "reward": 0.8980219066143036, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.8980219066143036, |
| "rewards/reward_fn/std": 0.0, |
| "step": 240, |
| "step_time": 90.27049679402262 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 335.625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 411.0, |
| "completions/max_terminated_length": 411.0, |
| "completions/mean_length": 335.625, |
| "completions/mean_terminated_length": 335.625, |
| "completions/min_length": 298.5, |
| "completions/min_terminated_length": 298.5, |
| "epoch": 1.6066666666666667, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 3.7500000000000003e-05, |
| "loss": 0.0, |
| "num_tokens": 3501146.0, |
| "reward": 0.5656213238835335, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.5656213238835335, |
| "rewards/reward_fn/std": 0.0, |
| "step": 241, |
| "step_time": 160.73021261492977 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 295.5625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 351.5, |
| "completions/max_terminated_length": 351.5, |
| "completions/mean_length": 295.5625, |
| "completions/mean_terminated_length": 295.5625, |
| "completions/min_length": 244.0, |
| "completions/min_terminated_length": 244.0, |
| "epoch": 1.6133333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.11344259977340698, |
| "kl": 0.0, |
| "learning_rate": 3.7373830882076354e-05, |
| "loss": 0.02902945876121521, |
| "num_tokens": 3516555.0, |
| "reward": 0.7674919366836548, |
| "reward_std": 0.018040144525002688, |
| "rewards/reward_fn/mean": 0.7674919366836548, |
| "rewards/reward_fn/std": 0.018040144525002688, |
| "step": 242, |
| "step_time": 111.73577347001992 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 294.3125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 334.5, |
| "completions/max_terminated_length": 334.5, |
| "completions/mean_length": 294.3125, |
| "completions/mean_terminated_length": 294.3125, |
| "completions/min_length": 248.0, |
| "completions/min_terminated_length": 248.0, |
| "epoch": 1.62, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 3.7247242955952175e-05, |
| "loss": 0.0, |
| "num_tokens": 3529936.0, |
| "reward": 1.0, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 1.0, |
| "rewards/reward_fn/std": 0.0, |
| "step": 243, |
| "step_time": 103.81889916706132 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 209.9375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 210.0, |
| "completions/max_terminated_length": 210.0, |
| "completions/mean_length": 209.9375, |
| "completions/mean_terminated_length": 209.9375, |
| "completions/min_length": 209.5, |
| "completions/min_terminated_length": 209.5, |
| "epoch": 1.6266666666666667, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 3.712024050615843e-05, |
| "loss": 0.0, |
| "num_tokens": 3542279.0, |
| "reward": 0.5284403860569, |
| "reward_std": 0.0, |
| "rewards/reward_fn/mean": 0.5284403860569, |
| "rewards/reward_fn/std": 0.0, |
| "step": 244, |
| "step_time": 96.00647811003728 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 313.0625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 373.5, |
| "completions/max_terminated_length": 373.5, |
| "completions/mean_length": 313.0625, |
| "completions/mean_terminated_length": 313.0625, |
| "completions/min_length": 264.0, |
| "completions/min_terminated_length": 264.0, |
| "epoch": 1.6333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07335827499628067, |
| "kl": 0.0, |
| "learning_rate": 3.699282783125616e-05, |
| "loss": 0.0014421269297599792, |
| "num_tokens": 3557960.0, |
| "reward": 0.7668299674987793, |
| "reward_std": 0.09549267427064478, |
| "rewards/reward_fn/mean": 0.7668299674987793, |
| "rewards/reward_fn/std": 0.09549267427064478, |
| "step": 245, |
| "step_time": 117.32995172601659 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 453.6875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 613.0, |
| "completions/max_terminated_length": 613.0, |
| "completions/mean_length": 453.6875, |
| "completions/mean_terminated_length": 453.6875, |
| "completions/min_length": 325.5, |
| "completions/min_terminated_length": 325.5, |
| "epoch": 1.6400000000000001, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.032244011759757996, |
| "kl": 0.0, |
| "learning_rate": 3.686500924369101e-05, |
| "loss": 0.050328221172094345, |
| "num_tokens": 3574691.0, |
| "reward": 0.12802473455667496, |
| "reward_std": 0.13411740958690643, |
| "rewards/reward_fn/mean": 0.12802473455667496, |
| "rewards/reward_fn/std": 0.13411740958690643, |
| "step": 246, |
| "step_time": 189.75963115796912 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 282.375, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 330.0, |
| "completions/max_terminated_length": 330.0, |
| "completions/mean_length": 282.375, |
| "completions/mean_terminated_length": 282.375, |
| "completions/min_length": 240.5, |
| "completions/min_terminated_length": 240.5, |
| "epoch": 1.6466666666666665, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.00042099610436707735, |
| "kl": 0.0, |
| "learning_rate": 3.673678906964727e-05, |
| "loss": 5.045448779128492e-05, |
| "num_tokens": 3587825.0, |
| "reward": 0.9999999105930328, |
| "reward_std": 2.634479017160629e-07, |
| "rewards/reward_fn/mean": 0.9999999105930328, |
| "rewards/reward_fn/std": 2.634479017160629e-07, |
| "step": 247, |
| "step_time": 101.9842315640999 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 390.75, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 486.0, |
| "completions/max_terminated_length": 486.0, |
| "completions/mean_length": 390.75, |
| "completions/mean_terminated_length": 390.75, |
| "completions/min_length": 329.5, |
| "completions/min_terminated_length": 329.5, |
| "epoch": 1.6533333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0787544846534729, |
| "kl": 0.0, |
| "learning_rate": 3.660817164890143e-05, |
| "loss": 0.014489062130451202, |
| "num_tokens": 3604285.0, |
| "reward": 0.8284005224704742, |
| "reward_std": 0.33756453602109104, |
| "rewards/reward_fn/mean": 0.8284005224704742, |
| "rewards/reward_fn/std": 0.33756453602109104, |
| "step": 248, |
| "step_time": 145.34176448197104 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 327.625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 385.0, |
| "completions/max_terminated_length": 385.0, |
| "completions/mean_length": 327.625, |
| "completions/mean_terminated_length": 327.625, |
| "completions/min_length": 293.0, |
| "completions/min_terminated_length": 293.0, |
| "epoch": 1.6600000000000001, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.05011061951518059, |
| "kl": 0.0, |
| "learning_rate": 3.6479161334675296e-05, |
| "loss": -0.005787290632724762, |
| "num_tokens": 3619271.0, |
| "reward": 0.9531709551811218, |
| "reward_std": 0.004021522589027882, |
| "rewards/reward_fn/mean": 0.9531709551811218, |
| "rewards/reward_fn/std": 0.004021522589027882, |
| "step": 249, |
| "step_time": 118.20399923500372 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 359.5, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 442.5, |
| "completions/max_terminated_length": 442.5, |
| "completions/mean_length": 359.5, |
| "completions/mean_terminated_length": 359.5, |
| "completions/min_length": 268.0, |
| "completions/min_terminated_length": 268.0, |
| "epoch": 1.6666666666666665, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.05501338839530945, |
| "kl": 0.0, |
| "learning_rate": 3.634976249348867e-05, |
| "loss": -0.006874836981296539, |
| "num_tokens": 3635063.0, |
| "reward": 0.9404093027114868, |
| "reward_std": 0.16401344537734985, |
| "rewards/reward_fn/mean": 0.9404093027114868, |
| "rewards/reward_fn/std": 0.16401344537734985, |
| "step": 250, |
| "step_time": 154.596884167986 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 600, |
| "num_input_tokens_seen": 3635063, |
| "num_train_epochs": 4, |
| "save_steps": 50, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 4, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
