{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.6666666666666665, "eval_steps": 50, "global_step": 250, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 143.3125, "completions/clipped_ratio": 0.0, "completions/max_length": 185.0, "completions/max_terminated_length": 185.0, "completions/mean_length": 143.3125, "completions/mean_terminated_length": 143.3125, "completions/min_length": 126.5, "completions/min_terminated_length": 126.5, "epoch": 0.006666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.12731829285621643, "kl": 0.0, "learning_rate": 0.0, "loss": 0.06993113458156586, "num_tokens": 12045.0, "reward": 0.28475117683410645, "reward_std": 0.7594513893127441, "rewards/reward_fn/mean": 0.28475117683410645, "rewards/reward_fn/std": 0.7594513893127441, "step": 1, "step_time": 90.46665180695709 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 148.4375, "completions/clipped_ratio": 0.0, "completions/max_length": 164.5, "completions/max_terminated_length": 164.5, "completions/mean_length": 148.4375, "completions/mean_terminated_length": 148.4375, "completions/min_length": 137.5, "completions/min_terminated_length": 137.5, "epoch": 0.013333333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 0.07862907648086548, "kl": 0.0, "learning_rate": 8.333333333333333e-07, "loss": 0.039252012968063354, "num_tokens": 24300.0, "reward": -0.6684748530387878, "reward_std": 0.34289104118943214, "rewards/reward_fn/mean": -0.6684748530387878, "rewards/reward_fn/std": 0.34289104118943214, "step": 2, "step_time": 62.789824239036534 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 204.75, "completions/clipped_ratio": 0.0, "completions/max_length": 247.5, "completions/max_terminated_length": 247.5, "completions/mean_length": 204.75, "completions/mean_terminated_length": 204.75, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "epoch": 0.02, "frac_reward_zero_std": 0.5, "grad_norm": 0.0595012903213501, "kl": 0.0, "learning_rate": 1.6666666666666667e-06, "loss": -0.042902350425720215, "num_tokens": 36984.0, "reward": -1.3745315074920654, "reward_std": 0.13413189351558685, "rewards/reward_fn/mean": -1.3745315074920654, "rewards/reward_fn/std": 0.13413189351558685, "step": 3, "step_time": 85.68428186204983 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 175.0, "completions/clipped_ratio": 0.0, "completions/max_length": 201.0, "completions/max_terminated_length": 201.0, "completions/mean_length": 175.0, "completions/mean_terminated_length": 175.0, "completions/min_length": 151.5, "completions/min_terminated_length": 151.5, "epoch": 0.02666666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 0.052226342260837555, "kl": 0.0, "learning_rate": 2.5e-06, "loss": 0.017850903794169426, "num_tokens": 48984.0, "reward": -1.2789445519447327, "reward_std": 0.42042604088783264, "rewards/reward_fn/mean": -1.2789445519447327, "rewards/reward_fn/std": 0.42042604088783264, "step": 4, "step_time": 71.62924331304384 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 230.75, "completions/clipped_ratio": 0.0, "completions/max_length": 359.5, "completions/max_terminated_length": 359.5, "completions/mean_length": 230.75, "completions/mean_terminated_length": 230.75, "completions/min_length": 179.0, "completions/min_terminated_length": 179.0, "epoch": 0.03333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.07887183874845505, "kl": 0.0, "learning_rate": 3.3333333333333333e-06, "loss": 0.003247857093811035, "num_tokens": 61932.0, "reward": -0.7773191928863525, "reward_std": 0.9348528981208801, "rewards/reward_fn/mean": -0.7773191928863525, "rewards/reward_fn/std": 0.9348528981208801, "step": 5, "step_time": 111.19219641707605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 223.8125, "completions/clipped_ratio": 0.0, "completions/max_length": 301.5, "completions/max_terminated_length": 301.5, "completions/mean_length": 223.8125, "completions/mean_terminated_length": 223.8125, "completions/min_length": 172.5, "completions/min_terminated_length": 172.5, "epoch": 0.04, "frac_reward_zero_std": 0.0, "grad_norm": 0.08544809371232986, "kl": 0.0, "learning_rate": 4.166666666666667e-06, "loss": -0.01605057343840599, "num_tokens": 74937.0, "reward": -0.011491045355796814, "reward_std": 1.0746989250183105, "rewards/reward_fn/mean": -0.011491045355796814, "rewards/reward_fn/std": 1.0746989250183105, "step": 6, "step_time": 101.91355575795751 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 239.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 286.5, "completions/max_terminated_length": 286.5, "completions/mean_length": 239.0625, "completions/mean_terminated_length": 239.0625, "completions/min_length": 194.5, "completions/min_terminated_length": 194.5, "epoch": 0.04666666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 0.06637871265411377, "kl": 0.0, "learning_rate": 5e-06, "loss": 0.022047851234674454, "num_tokens": 88986.0, "reward": -0.7737771570682526, "reward_std": 0.08403173834085464, "rewards/reward_fn/mean": -0.7737771570682526, "rewards/reward_fn/std": 0.08403173834085464, "step": 7, "step_time": 94.80128556105774 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 160.4375, "completions/clipped_ratio": 0.0, "completions/max_length": 192.0, "completions/max_terminated_length": 192.0, "completions/mean_length": 160.4375, "completions/mean_terminated_length": 160.4375, "completions/min_length": 123.5, "completions/min_terminated_length": 123.5, "epoch": 0.05333333333333334, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 5.833333333333334e-06, "loss": 0.0, "num_tokens": 101153.0, "reward": -1.5, "reward_std": 0.0, "rewards/reward_fn/mean": -1.5, "rewards/reward_fn/std": 0.0, "step": 8, "step_time": 66.85633395699551 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 123.25, "completions/clipped_ratio": 0.0, "completions/max_length": 127.5, "completions/max_terminated_length": 127.5, "completions/mean_length": 123.25, "completions/mean_terminated_length": 123.25, "completions/min_length": 120.5, "completions/min_terminated_length": 120.5, "epoch": 0.06, "frac_reward_zero_std": 0.5, "grad_norm": 0.08828099071979523, "kl": 0.0, "learning_rate": 6.666666666666667e-06, "loss": 0.0025132223963737488, "num_tokens": 112757.0, "reward": -0.7912696972489357, "reward_std": 0.12075159698724747, "rewards/reward_fn/mean": -0.7912696972489357, "rewards/reward_fn/std": 0.12075159698724747, "step": 9, "step_time": 50.840260804921854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 162.625, "completions/clipped_ratio": 0.0, "completions/max_length": 205.0, "completions/max_terminated_length": 205.0, "completions/mean_length": 162.625, "completions/mean_terminated_length": 162.625, "completions/min_length": 131.0, "completions/min_terminated_length": 131.0, "epoch": 0.06666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.08045986294746399, "kl": 0.0, "learning_rate": 7.5e-06, "loss": -0.03439512848854065, "num_tokens": 124327.0, "reward": -0.5544086694717407, "reward_std": 0.5964015126228333, "rewards/reward_fn/mean": -0.5544086694717407, "rewards/reward_fn/std": 0.5964015126228333, "step": 10, "step_time": 69.93904624809511 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 157.5, "completions/clipped_ratio": 0.0, "completions/max_length": 193.0, "completions/max_terminated_length": 193.0, "completions/mean_length": 157.5, "completions/mean_terminated_length": 157.5, "completions/min_length": 141.0, "completions/min_terminated_length": 141.0, "epoch": 0.07333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.15679270029067993, "kl": 0.0, "learning_rate": 8.333333333333334e-06, "loss": -0.03179311752319336, "num_tokens": 136575.0, "reward": 0.13369514793157578, "reward_std": 0.6897610425949097, "rewards/reward_fn/mean": 0.13369514793157578, "rewards/reward_fn/std": 0.6897610425949097, "step": 11, "step_time": 68.66538558196044 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 250.4375, "completions/clipped_ratio": 0.0, "completions/max_length": 342.0, "completions/max_terminated_length": 342.0, "completions/mean_length": 250.4375, "completions/mean_terminated_length": 250.4375, "completions/min_length": 167.5, "completions/min_terminated_length": 167.5, "epoch": 0.08, "frac_reward_zero_std": 0.0, "grad_norm": 0.1365279257297516, "kl": 0.0, "learning_rate": 9.166666666666666e-06, "loss": 0.052371568977832794, "num_tokens": 149926.0, "reward": -0.7794331163167953, "reward_std": 0.5107784122228622, "rewards/reward_fn/mean": -0.7794331163167953, "rewards/reward_fn/std": 0.5107784122228622, "step": 12, "step_time": 107.52361460705288 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 147.3125, "completions/clipped_ratio": 0.0, "completions/max_length": 188.0, "completions/max_terminated_length": 188.0, "completions/mean_length": 147.3125, "completions/mean_terminated_length": 147.3125, "completions/min_length": 127.0, "completions/min_terminated_length": 127.0, "epoch": 0.08666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.14017197489738464, "kl": 0.0, "learning_rate": 1e-05, "loss": -0.03942764177918434, "num_tokens": 162051.0, "reward": -0.24546438455581665, "reward_std": 0.148831894621253, "rewards/reward_fn/mean": -0.24546438455581665, "rewards/reward_fn/std": 0.148831894621253, "step": 13, "step_time": 67.4536545749288 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 246.625, "completions/clipped_ratio": 0.0, "completions/max_length": 294.5, "completions/max_terminated_length": 294.5, "completions/mean_length": 246.625, "completions/mean_terminated_length": 246.625, "completions/min_length": 181.5, "completions/min_terminated_length": 181.5, "epoch": 0.09333333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 0.25423967838287354, "kl": 0.0, "learning_rate": 1.0833333333333334e-05, "loss": 0.09142481535673141, "num_tokens": 175253.0, "reward": -1.1983885765075684, "reward_std": 0.5847422480583191, "rewards/reward_fn/mean": -1.1983885765075684, "rewards/reward_fn/std": 0.5847422480583191, "step": 14, "step_time": 93.72659771697363 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 218.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 321.5, "completions/max_terminated_length": 321.5, "completions/mean_length": 218.9375, "completions/mean_terminated_length": 218.9375, "completions/min_length": 162.0, "completions/min_terminated_length": 162.0, "epoch": 0.1, "frac_reward_zero_std": 0.0, "grad_norm": 0.08925358206033707, "kl": 0.0, "learning_rate": 1.1666666666666668e-05, "loss": 0.002535127103328705, "num_tokens": 188108.0, "reward": -0.9819782972335815, "reward_std": 0.563330739736557, "rewards/reward_fn/mean": -0.9819782972335815, "rewards/reward_fn/std": 0.563330739736557, "step": 15, "step_time": 100.25624653795967 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 199.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 308.5, "completions/max_terminated_length": 308.5, "completions/mean_length": 199.0625, "completions/mean_terminated_length": 199.0625, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "epoch": 0.10666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.11524740606546402, "kl": 0.0, "learning_rate": 1.25e-05, "loss": 0.07866720855236053, "num_tokens": 200701.0, "reward": -0.6619112491607666, "reward_std": 0.587931826710701, "rewards/reward_fn/mean": -0.6619112491607666, "rewards/reward_fn/std": 0.587931826710701, "step": 16, "step_time": 99.46528565394692 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 174.75, "completions/clipped_ratio": 0.0, "completions/max_length": 191.5, "completions/max_terminated_length": 191.5, "completions/mean_length": 174.75, "completions/mean_terminated_length": 174.75, "completions/min_length": 153.5, "completions/min_terminated_length": 153.5, "epoch": 0.11333333333333333, "frac_reward_zero_std": 0.5, "grad_norm": 0.07091796398162842, "kl": 0.0, "learning_rate": 1.3333333333333333e-05, "loss": -0.030646469444036484, "num_tokens": 213113.0, "reward": -1.163275420665741, "reward_std": 0.4873338043689728, "rewards/reward_fn/mean": -1.163275420665741, "rewards/reward_fn/std": 0.4873338043689728, "step": 17, "step_time": 67.82858649804257 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 154.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 189.0, "completions/max_terminated_length": 189.0, "completions/mean_length": 154.6875, "completions/mean_terminated_length": 154.6875, "completions/min_length": 131.0, "completions/min_terminated_length": 131.0, "epoch": 0.12, "frac_reward_zero_std": 0.0, "grad_norm": 0.07412096858024597, "kl": 0.0, "learning_rate": 1.4166666666666668e-05, "loss": 0.06621558219194412, "num_tokens": 224596.0, "reward": 0.09915217757225037, "reward_std": 0.3922642320394516, "rewards/reward_fn/mean": 0.09915217757225037, "rewards/reward_fn/std": 0.3922642320394516, "step": 18, "step_time": 68.17481762805255 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 165.5625, "completions/clipped_ratio": 0.0, "completions/max_length": 194.0, "completions/max_terminated_length": 194.0, "completions/mean_length": 165.5625, "completions/mean_terminated_length": 165.5625, "completions/min_length": 136.5, "completions/min_terminated_length": 136.5, "epoch": 0.12666666666666668, "frac_reward_zero_std": 0.0, "grad_norm": 0.0963885560631752, "kl": 0.0, "learning_rate": 1.5e-05, "loss": 0.0464300662279129, "num_tokens": 237013.0, "reward": -0.12774711847305298, "reward_std": 1.0569286346435547, "rewards/reward_fn/mean": -0.12774711847305298, "rewards/reward_fn/std": 1.0569286346435547, "step": 19, "step_time": 73.6154217319563 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 156.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 193.5, "completions/max_terminated_length": 193.5, "completions/mean_length": 156.9375, "completions/mean_terminated_length": 156.9375, "completions/min_length": 127.5, "completions/min_terminated_length": 127.5, "epoch": 0.13333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.09283237904310226, "kl": 0.0, "learning_rate": 1.5833333333333333e-05, "loss": 0.03124336153268814, "num_tokens": 249164.0, "reward": -0.7668359875679016, "reward_std": 0.5728714317083359, "rewards/reward_fn/mean": -0.7668359875679016, "rewards/reward_fn/std": 0.5728714317083359, "step": 20, "step_time": 68.1898756630253 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 168.125, "completions/clipped_ratio": 0.0, "completions/max_length": 229.5, "completions/max_terminated_length": 229.5, "completions/mean_length": 168.125, "completions/mean_terminated_length": 168.125, "completions/min_length": 120.5, "completions/min_terminated_length": 120.5, "epoch": 0.14, "frac_reward_zero_std": 0.0, "grad_norm": 0.094845250248909, "kl": 0.0, "learning_rate": 1.6666666666666667e-05, "loss": 0.09362012147903442, "num_tokens": 260574.0, "reward": -0.8211829662322998, "reward_std": 0.7844448983669281, "rewards/reward_fn/mean": -0.8211829662322998, "rewards/reward_fn/std": 0.7844448983669281, "step": 21, "step_time": 76.81375163403573 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 181.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 223.5, "completions/max_terminated_length": 223.5, "completions/mean_length": 181.0625, "completions/mean_terminated_length": 181.0625, "completions/min_length": 140.0, "completions/min_terminated_length": 140.0, "epoch": 0.14666666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 0.054967641830444336, "kl": 0.0, "learning_rate": 1.75e-05, "loss": 0.05134895443916321, "num_tokens": 273087.0, "reward": -1.3103257417678833, "reward_std": 0.3521471619606018, "rewards/reward_fn/mean": -1.3103257417678833, "rewards/reward_fn/std": 0.3521471619606018, "step": 22, "step_time": 77.7118174440111 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 159.625, "completions/clipped_ratio": 0.0, "completions/max_length": 234.5, "completions/max_terminated_length": 234.5, "completions/mean_length": 159.625, "completions/mean_terminated_length": 159.625, "completions/min_length": 123.0, "completions/min_terminated_length": 123.0, "epoch": 0.15333333333333332, "frac_reward_zero_std": 0.0, "grad_norm": 0.09953799843788147, "kl": 0.0, "learning_rate": 1.8333333333333333e-05, "loss": -0.02984502911567688, "num_tokens": 284569.0, "reward": -0.8976435363292694, "reward_std": 0.3961769565939903, "rewards/reward_fn/mean": -0.8976435363292694, "rewards/reward_fn/std": 0.3961769565939903, "step": 23, "step_time": 77.59863371500978 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 164.625, "completions/clipped_ratio": 0.0, "completions/max_length": 178.5, "completions/max_terminated_length": 178.5, "completions/mean_length": 164.625, "completions/mean_terminated_length": 164.625, "completions/min_length": 138.0, "completions/min_terminated_length": 138.0, "epoch": 0.16, "frac_reward_zero_std": 0.0, "grad_norm": 0.08858790993690491, "kl": 0.0, "learning_rate": 1.9166666666666667e-05, "loss": 0.045571208000183105, "num_tokens": 296811.0, "reward": -0.16475331783294678, "reward_std": 0.3908658064901829, "rewards/reward_fn/mean": -0.16475331783294678, "rewards/reward_fn/std": 0.3908658064901829, "step": 24, "step_time": 64.24019336199854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 176.25, "completions/clipped_ratio": 0.0, "completions/max_length": 216.5, "completions/max_terminated_length": 216.5, "completions/mean_length": 176.25, "completions/mean_terminated_length": 176.25, "completions/min_length": 150.5, "completions/min_terminated_length": 150.5, "epoch": 0.16666666666666666, "frac_reward_zero_std": 0.5, "grad_norm": 0.07616627216339111, "kl": 0.0, "learning_rate": 2e-05, "loss": -0.0033947303891181946, "num_tokens": 309199.0, "reward": -0.6221126019954681, "reward_std": 0.17310830950737, "rewards/reward_fn/mean": -0.6221126019954681, "rewards/reward_fn/std": 0.17310830950737, "step": 25, "step_time": 73.73626439704094 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 246.625, "completions/clipped_ratio": 0.0, "completions/max_length": 294.5, "completions/max_terminated_length": 294.5, "completions/mean_length": 246.625, "completions/mean_terminated_length": 246.625, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "epoch": 0.17333333333333334, "frac_reward_zero_std": 0.5, "grad_norm": 0.0425107404589653, "kl": 0.0, "learning_rate": 2.0833333333333336e-05, "loss": -0.018586011603474617, "num_tokens": 323297.0, "reward": -1.3331425189971924, "reward_std": 0.3089599609375, "rewards/reward_fn/mean": -1.3331425189971924, "rewards/reward_fn/std": 0.3089599609375, "step": 26, "step_time": 94.67684844596079 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 132.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 164.5, "completions/max_terminated_length": 164.5, "completions/mean_length": 132.6875, "completions/mean_terminated_length": 132.6875, "completions/min_length": 101.5, "completions/min_terminated_length": 101.5, "epoch": 0.18, "frac_reward_zero_std": 0.0, "grad_norm": 0.08376004546880722, "kl": 0.0, "learning_rate": 2.1666666666666667e-05, "loss": -0.05854324996471405, "num_tokens": 334436.0, "reward": 0.26202554255723953, "reward_std": 0.5281454622745514, "rewards/reward_fn/mean": 0.26202554255723953, "rewards/reward_fn/std": 0.5281454622745514, "step": 27, "step_time": 60.064499742002226 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 194.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 271.0, "completions/max_terminated_length": 271.0, "completions/mean_length": 194.6875, "completions/mean_terminated_length": 194.6875, "completions/min_length": 156.5, "completions/min_terminated_length": 156.5, "epoch": 0.18666666666666668, "frac_reward_zero_std": 0.0, "grad_norm": 0.1175316870212555, "kl": 0.0, "learning_rate": 2.25e-05, "loss": 0.003547981381416321, "num_tokens": 347655.0, "reward": -0.8296540975570679, "reward_std": 0.8124817311763763, "rewards/reward_fn/mean": -0.8296540975570679, "rewards/reward_fn/std": 0.8124817311763763, "step": 28, "step_time": 89.95816739893053 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 209.4375, "completions/clipped_ratio": 0.0, "completions/max_length": 309.5, "completions/max_terminated_length": 309.5, "completions/mean_length": 209.4375, "completions/mean_terminated_length": 209.4375, "completions/min_length": 152.0, "completions/min_terminated_length": 152.0, "epoch": 0.19333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.09398413449525833, "kl": 0.0, "learning_rate": 2.3333333333333336e-05, "loss": -0.15494152903556824, "num_tokens": 360318.0, "reward": -0.8558410555124283, "reward_std": 0.6401710212230682, "rewards/reward_fn/mean": -0.8558410555124283, "rewards/reward_fn/std": 0.6401710212230682, "step": 29, "step_time": 96.83519398601493 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 232.4375, "completions/clipped_ratio": 0.0, "completions/max_length": 328.0, "completions/max_terminated_length": 328.0, "completions/mean_length": 232.4375, "completions/mean_terminated_length": 232.4375, "completions/min_length": 141.0, "completions/min_terminated_length": 141.0, "epoch": 0.2, "frac_reward_zero_std": 0.0, "grad_norm": 0.061997007578611374, "kl": 0.0, "learning_rate": 2.4166666666666667e-05, "loss": 0.11603285372257233, "num_tokens": 373245.0, "reward": -0.9066768288612366, "reward_std": 0.8537713587284088, "rewards/reward_fn/mean": -0.9066768288612366, "rewards/reward_fn/std": 0.8537713587284088, "step": 30, "step_time": 105.47558436403051 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 157.4375, "completions/clipped_ratio": 0.0, "completions/max_length": 213.0, "completions/max_terminated_length": 213.0, "completions/mean_length": 157.4375, "completions/mean_terminated_length": 157.4375, "completions/min_length": 123.0, "completions/min_terminated_length": 123.0, "epoch": 0.20666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.07278493046760559, "kl": 0.0, "learning_rate": 2.5e-05, "loss": 0.14571788907051086, "num_tokens": 384564.0, "reward": -0.9546848833560944, "reward_std": 0.41054398447158746, "rewards/reward_fn/mean": -0.9546848833560944, "rewards/reward_fn/std": 0.41054398447158746, "step": 31, "step_time": 72.25721991300816 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 260.0, "completions/clipped_ratio": 0.0, "completions/max_length": 334.5, "completions/max_terminated_length": 334.5, "completions/mean_length": 260.0, "completions/mean_terminated_length": 260.0, "completions/min_length": 188.5, "completions/min_terminated_length": 188.5, "epoch": 0.21333333333333335, "frac_reward_zero_std": 0.5, "grad_norm": 0.056718725711107254, "kl": 0.0, "learning_rate": 2.5833333333333336e-05, "loss": 0.042716674506664276, "num_tokens": 398956.0, "reward": -1.2975031733512878, "reward_std": 0.3875386118888855, "rewards/reward_fn/mean": -1.2975031733512878, "rewards/reward_fn/std": 0.3875386118888855, "step": 32, "step_time": 108.24371674098074 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 199.25, "completions/clipped_ratio": 0.0, "completions/max_length": 241.0, "completions/max_terminated_length": 241.0, "completions/mean_length": 199.25, "completions/mean_terminated_length": 199.25, "completions/min_length": 159.5, "completions/min_terminated_length": 159.5, "epoch": 0.22, "frac_reward_zero_std": 0.0, "grad_norm": 0.08926832675933838, "kl": 0.0, "learning_rate": 2.6666666666666667e-05, "loss": -0.039713893085718155, "num_tokens": 412240.0, "reward": -0.13461190462112427, "reward_std": 0.5121839940547943, "rewards/reward_fn/mean": -0.13461190462112427, "rewards/reward_fn/std": 0.5121839940547943, "step": 33, "step_time": 82.1538975770236 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 274.1875, "completions/clipped_ratio": 0.0, "completions/max_length": 426.0, "completions/max_terminated_length": 426.0, "completions/mean_length": 274.1875, "completions/mean_terminated_length": 274.1875, "completions/min_length": 153.0, "completions/min_terminated_length": 153.0, "epoch": 0.22666666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 0.09963853657245636, "kl": 0.0, "learning_rate": 2.7500000000000004e-05, "loss": -0.15619568526744843, "num_tokens": 426331.0, "reward": -1.0308716893196106, "reward_std": 0.6284461915493011, "rewards/reward_fn/mean": -1.0308716893196106, "rewards/reward_fn/std": 0.6284461915493011, "step": 34, "step_time": 127.3287339269882 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 204.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 252.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 204.6875, "completions/mean_terminated_length": 204.6875, "completions/min_length": 156.0, "completions/min_terminated_length": 156.0, "epoch": 0.23333333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 0.15367895364761353, "kl": 0.0, "learning_rate": 2.8333333333333335e-05, "loss": -0.03458409756422043, "num_tokens": 438846.0, "reward": -0.30794087052345276, "reward_std": 0.7503761947154999, "rewards/reward_fn/mean": -0.30794087052345276, "rewards/reward_fn/std": 0.7503761947154999, "step": 35, "step_time": 85.05355873703957 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 223.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 324.5, "completions/max_terminated_length": 324.5, "completions/mean_length": 223.9375, "completions/mean_terminated_length": 223.9375, "completions/min_length": 141.0, "completions/min_terminated_length": 141.0, "epoch": 0.24, "frac_reward_zero_std": 0.0, "grad_norm": 0.2046246975660324, "kl": 0.0, "learning_rate": 2.916666666666667e-05, "loss": -0.07736283540725708, "num_tokens": 451773.0, "reward": -0.29854077845811844, "reward_std": 0.44150011567398906, "rewards/reward_fn/mean": -0.29854077845811844, "rewards/reward_fn/std": 0.44150011567398906, "step": 36, "step_time": 101.35352180700283 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 145.8125, "completions/clipped_ratio": 0.0, "completions/max_length": 173.0, "completions/max_terminated_length": 173.0, "completions/mean_length": 145.8125, "completions/mean_terminated_length": 145.8125, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "epoch": 0.24666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.11604579538106918, "kl": 0.0, "learning_rate": 3e-05, "loss": 0.09416678547859192, "num_tokens": 463706.0, "reward": 0.04717719554901123, "reward_std": 0.4922184348106384, "rewards/reward_fn/mean": 0.04717719554901123, "rewards/reward_fn/std": 0.4922184348106384, "step": 37, "step_time": 62.971165234048385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 208.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 234.0, "completions/max_terminated_length": 234.0, "completions/mean_length": 208.6875, "completions/mean_terminated_length": 208.6875, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "epoch": 0.25333333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 0.1814187616109848, "kl": 0.0, "learning_rate": 3.0833333333333335e-05, "loss": -0.002286073751747608, "num_tokens": 477237.0, "reward": 0.18147915601730347, "reward_std": 0.40245659090578556, "rewards/reward_fn/mean": 0.18147915601730347, "rewards/reward_fn/std": 0.40245659090578556, "step": 38, "step_time": 81.6735560669913 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 232.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 341.5, "completions/max_terminated_length": 341.5, "completions/mean_length": 232.0625, "completions/mean_terminated_length": 232.0625, "completions/min_length": 177.5, "completions/min_terminated_length": 177.5, "epoch": 0.26, "frac_reward_zero_std": 0.0, "grad_norm": 0.08577155321836472, "kl": 0.0, "learning_rate": 3.1666666666666666e-05, "loss": -0.02454685978591442, "num_tokens": 490294.0, "reward": -0.25618940591812134, "reward_std": 0.6517162024974823, "rewards/reward_fn/mean": -0.25618940591812134, "rewards/reward_fn/std": 0.6517162024974823, "step": 39, "step_time": 110.83180778095266 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 256.5625, "completions/clipped_ratio": 0.0, "completions/max_length": 327.0, "completions/max_terminated_length": 327.0, "completions/mean_length": 256.5625, "completions/mean_terminated_length": 256.5625, "completions/min_length": 188.5, "completions/min_terminated_length": 188.5, "epoch": 0.26666666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 0.09231025725603104, "kl": 0.0, "learning_rate": 3.2500000000000004e-05, "loss": -0.07835742831230164, "num_tokens": 503831.0, "reward": -0.2845441624522209, "reward_std": 0.41623825021088123, "rewards/reward_fn/mean": -0.2845441624522209, "rewards/reward_fn/std": 0.41623825021088123, "step": 40, "step_time": 105.11443146498641 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 184.375, "completions/clipped_ratio": 0.0, "completions/max_length": 216.5, "completions/max_terminated_length": 216.5, "completions/mean_length": 184.375, "completions/mean_terminated_length": 184.375, "completions/min_length": 157.5, "completions/min_terminated_length": 157.5, "epoch": 0.2733333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.11624035984277725, "kl": 0.0, "learning_rate": 3.3333333333333335e-05, "loss": -0.07812672853469849, "num_tokens": 516661.0, "reward": -0.5956845507025719, "reward_std": 0.27048898907378316, "rewards/reward_fn/mean": -0.5956845507025719, "rewards/reward_fn/std": 0.27048898907378316, "step": 41, "step_time": 75.3128279329976 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 149.5625, "completions/clipped_ratio": 0.0, "completions/max_length": 174.0, "completions/max_terminated_length": 174.0, "completions/mean_length": 149.5625, "completions/mean_terminated_length": 149.5625, "completions/min_length": 135.5, "completions/min_terminated_length": 135.5, "epoch": 0.28, "frac_reward_zero_std": 0.0, "grad_norm": 0.11803104728460312, "kl": 0.0, "learning_rate": 3.4166666666666666e-05, "loss": -0.08490889519453049, "num_tokens": 528838.0, "reward": -0.35317955911159515, "reward_std": 0.17408898659050465, "rewards/reward_fn/mean": -0.35317955911159515, "rewards/reward_fn/std": 0.17408898659050465, "step": 42, "step_time": 63.69105087395292 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 157.875, "completions/clipped_ratio": 0.0, "completions/max_length": 197.5, "completions/max_terminated_length": 197.5, "completions/mean_length": 157.875, "completions/mean_terminated_length": 157.875, "completions/min_length": 121.0, "completions/min_terminated_length": 121.0, "epoch": 0.2866666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.07110287994146347, "kl": 0.0, "learning_rate": 3.5e-05, "loss": 0.016056224703788757, "num_tokens": 540356.0, "reward": 0.11271905899047852, "reward_std": 0.010076586622744799, "rewards/reward_fn/mean": 0.11271905899047852, "rewards/reward_fn/std": 0.010076586622744799, "step": 43, "step_time": 68.61112404498272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 184.8125, "completions/clipped_ratio": 0.0, "completions/max_length": 235.5, "completions/max_terminated_length": 235.5, "completions/mean_length": 184.8125, "completions/mean_terminated_length": 184.8125, "completions/min_length": 147.0, "completions/min_terminated_length": 147.0, "epoch": 0.29333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.09673087298870087, "kl": 0.0, "learning_rate": 3.5833333333333335e-05, "loss": -0.034359708428382874, "num_tokens": 553137.0, "reward": -0.38676056265830994, "reward_std": 0.5605057073407806, "rewards/reward_fn/mean": -0.38676056265830994, "rewards/reward_fn/std": 0.5605057073407806, "step": 44, "step_time": 79.45262654602993 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 160.4375, "completions/clipped_ratio": 0.0, "completions/max_length": 173.5, "completions/max_terminated_length": 173.5, "completions/mean_length": 160.4375, "completions/mean_terminated_length": 160.4375, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "epoch": 0.3, "frac_reward_zero_std": 0.0, "grad_norm": 0.07127348333597183, "kl": 0.0, "learning_rate": 3.6666666666666666e-05, "loss": -0.017228223383426666, "num_tokens": 565552.0, "reward": -0.00951889157295227, "reward_std": 0.1373694152571261, "rewards/reward_fn/mean": -0.00951889157295227, "rewards/reward_fn/std": 0.1373694152571261, "step": 45, "step_time": 64.07329139101785 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 294.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 378.0, "completions/max_terminated_length": 378.0, "completions/mean_length": 294.9375, "completions/mean_terminated_length": 294.9375, "completions/min_length": 209.5, "completions/min_terminated_length": 209.5, "epoch": 0.30666666666666664, "frac_reward_zero_std": 0.0, "grad_norm": 0.09467363357543945, "kl": 0.0, "learning_rate": 3.7500000000000003e-05, "loss": -0.03529605269432068, "num_tokens": 579951.0, "reward": -1.2961561679840088, "reward_std": 0.26750241219997406, "rewards/reward_fn/mean": -1.2961561679840088, "rewards/reward_fn/std": 0.26750241219997406, "step": 46, "step_time": 116.57304848608328 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 199.3125, "completions/clipped_ratio": 0.0, "completions/max_length": 254.5, "completions/max_terminated_length": 254.5, "completions/mean_length": 199.3125, "completions/mean_terminated_length": 199.3125, "completions/min_length": 150.0, "completions/min_terminated_length": 150.0, "epoch": 0.31333333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 0.14669747650623322, "kl": 0.0, "learning_rate": 3.8333333333333334e-05, "loss": -0.10157700628042221, "num_tokens": 593324.0, "reward": -0.15056359767913818, "reward_std": 0.18598411977291107, "rewards/reward_fn/mean": -0.15056359767913818, "rewards/reward_fn/std": 0.18598411977291107, "step": 47, "step_time": 89.92860590590863 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 234.8125, "completions/clipped_ratio": 0.0, "completions/max_length": 290.0, "completions/max_terminated_length": 290.0, "completions/mean_length": 234.8125, "completions/mean_terminated_length": 234.8125, "completions/min_length": 210.5, "completions/min_terminated_length": 210.5, "epoch": 0.32, "frac_reward_zero_std": 0.0, "grad_norm": 0.08877667784690857, "kl": 0.0, "learning_rate": 3.9166666666666665e-05, "loss": -0.04996691644191742, "num_tokens": 606257.0, "reward": -0.5346458461135626, "reward_std": 0.1979452222585678, "rewards/reward_fn/mean": -0.5346458461135626, "rewards/reward_fn/std": 0.1979452222585678, "step": 48, "step_time": 92.41859935398679 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 118.0, "completions/clipped_ratio": 0.0, "completions/max_length": 142.5, "completions/max_terminated_length": 142.5, "completions/mean_length": 118.0, "completions/mean_terminated_length": 118.0, "completions/min_length": 92.0, "completions/min_terminated_length": 92.0, "epoch": 0.32666666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 0.1030936986207962, "kl": 0.0, "learning_rate": 4e-05, "loss": 0.03239426016807556, "num_tokens": 616705.0, "reward": 0.04530230909585953, "reward_std": 0.5476353615522385, "rewards/reward_fn/mean": 0.04530230909585953, "rewards/reward_fn/std": 0.5476353615522385, "step": 49, "step_time": 53.05935197096551 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 174.5625, "completions/clipped_ratio": 0.0, "completions/max_length": 219.5, "completions/max_terminated_length": 219.5, "completions/mean_length": 174.5625, "completions/mean_terminated_length": 174.5625, "completions/min_length": 134.5, "completions/min_terminated_length": 134.5, "epoch": 0.3333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.18021085858345032, "kl": 0.0, "learning_rate": 4.0833333333333334e-05, "loss": -0.07679768651723862, "num_tokens": 628386.0, "reward": -0.2784375250339508, "reward_std": 0.6172725409269333, "rewards/reward_fn/mean": -0.2784375250339508, "rewards/reward_fn/std": 0.6172725409269333, "step": 50, "step_time": 73.86384335003095 }, { "epoch": 0.3333333333333333, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completion_length": 244.61125, "eval_completions/clipped_ratio": 0.0, "eval_completions/max_length": 305.91, "eval_completions/max_terminated_length": 305.91, "eval_completions/mean_length": 244.61125, "eval_completions/mean_terminated_length": 244.61125, "eval_completions/min_length": 190.08, "eval_completions/min_terminated_length": 190.08, "eval_frac_reward_zero_std": 0.01, "eval_kl": 0.0, "eval_loss": 0.002150977263227105, "eval_num_tokens": 628386.0, "eval_reward": -0.2439578665420413, "eval_reward_std": 0.40503982904250735, "eval_rewards/reward_fn/mean": -0.2439578665420413, "eval_rewards/reward_fn/std": 0.40503982904250735, "eval_runtime": 4225.7071, "eval_samples_per_second": 0.024, "eval_steps_per_second": 0.003, "step": 50 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 165.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 200.0, "completions/max_terminated_length": 200.0, "completions/mean_length": 165.0625, "completions/mean_terminated_length": 165.0625, "completions/min_length": 138.0, "completions/min_terminated_length": 138.0, "epoch": 0.34, "frac_reward_zero_std": 0.0, "grad_norm": 0.07993482053279877, "kl": 0.0, "learning_rate": 4.166666666666667e-05, "loss": 0.020043745636940002, "num_tokens": 639827.0, "reward": -0.4589511938393116, "reward_std": 0.24016011551429983, "rewards/reward_fn/mean": -0.4589511938393116, "rewards/reward_fn/std": 0.24016011551429983, "step": 51, "step_time": 69.2877980620251 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 168.5, "completions/clipped_ratio": 0.0, "completions/max_length": 213.5, "completions/max_terminated_length": 213.5, "completions/mean_length": 168.5, "completions/mean_terminated_length": 168.5, "completions/min_length": 148.5, "completions/min_terminated_length": 148.5, "epoch": 0.3466666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.10122694820165634, "kl": 0.0, "learning_rate": 4.25e-05, "loss": 0.010254371911287308, "num_tokens": 652363.0, "reward": -0.5353640392422676, "reward_std": 0.12437161646721506, "rewards/reward_fn/mean": -0.5353640392422676, "rewards/reward_fn/std": 0.12437161646721506, "step": 52, "step_time": 73.87324213504326 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 222.625, "completions/clipped_ratio": 0.0, "completions/max_length": 288.5, "completions/max_terminated_length": 288.5, "completions/mean_length": 222.625, "completions/mean_terminated_length": 222.625, "completions/min_length": 191.5, "completions/min_terminated_length": 191.5, "epoch": 0.35333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.0843462198972702, "kl": 0.0, "learning_rate": 4.3333333333333334e-05, "loss": -0.03871044144034386, "num_tokens": 665613.0, "reward": 0.12957476451992989, "reward_std": 0.21294647455215454, "rewards/reward_fn/mean": 0.12957476451992989, "rewards/reward_fn/std": 0.21294647455215454, "step": 53, "step_time": 92.9070069710142 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 193.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 228.0, "completions/max_terminated_length": 228.0, "completions/mean_length": 193.0625, "completions/mean_terminated_length": 193.0625, "completions/min_length": 162.5, "completions/min_terminated_length": 162.5, "epoch": 0.36, "frac_reward_zero_std": 0.0, "grad_norm": 0.08981351554393768, "kl": 0.0, "learning_rate": 4.4166666666666665e-05, "loss": -0.005176428705453873, "num_tokens": 678446.0, "reward": -0.22327297925949097, "reward_std": 0.20772529065288836, "rewards/reward_fn/mean": -0.22327297925949097, "rewards/reward_fn/std": 0.20772529065288836, "step": 54, "step_time": 78.26076654792996 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 239.125, "completions/clipped_ratio": 0.0, "completions/max_length": 293.0, "completions/max_terminated_length": 293.0, "completions/mean_length": 239.125, "completions/mean_terminated_length": 239.125, "completions/min_length": 195.0, "completions/min_terminated_length": 195.0, "epoch": 0.36666666666666664, "frac_reward_zero_std": 0.0, "grad_norm": 0.06711370497941971, "kl": 0.0, "learning_rate": 4.5e-05, "loss": 0.04008489102125168, "num_tokens": 692144.0, "reward": -0.32132431864738464, "reward_std": 0.12893771713788738, "rewards/reward_fn/mean": -0.32132431864738464, "rewards/reward_fn/std": 0.12893771713788738, "step": 55, "step_time": 95.0031545261154 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 277.5, "completions/clipped_ratio": 0.0, "completions/max_length": 323.0, "completions/max_terminated_length": 323.0, "completions/mean_length": 277.5, "completions/mean_terminated_length": 277.5, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "epoch": 0.37333333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 0.074734166264534, "kl": 0.0, "learning_rate": 4.5833333333333334e-05, "loss": -0.027020789682865143, "num_tokens": 705904.0, "reward": 0.2992004454135895, "reward_std": 0.5213650465011597, "rewards/reward_fn/mean": 0.2992004454135895, "rewards/reward_fn/std": 0.5213650465011597, "step": 56, "step_time": 102.65910005901242 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 209.3125, "completions/clipped_ratio": 0.0, "completions/max_length": 243.5, "completions/max_terminated_length": 243.5, "completions/mean_length": 209.3125, "completions/mean_terminated_length": 209.3125, "completions/min_length": 199.0, "completions/min_terminated_length": 199.0, "epoch": 0.38, "frac_reward_zero_std": 0.0, "grad_norm": 0.04772329702973366, "kl": 0.0, "learning_rate": 4.666666666666667e-05, "loss": 0.01125314086675644, "num_tokens": 718213.0, "reward": -0.6336008310317993, "reward_std": 0.2387970182016943, "rewards/reward_fn/mean": -0.6336008310317993, "rewards/reward_fn/std": 0.2387970182016943, "step": 57, "step_time": 83.00291228800779 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 262.4375, "completions/clipped_ratio": 0.0, "completions/max_length": 343.5, "completions/max_terminated_length": 343.5, "completions/mean_length": 262.4375, "completions/mean_terminated_length": 262.4375, "completions/min_length": 208.5, "completions/min_terminated_length": 208.5, "epoch": 0.38666666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 0.10154283046722412, "kl": 0.0, "learning_rate": 4.75e-05, "loss": -0.06698265671730042, "num_tokens": 731612.0, "reward": 0.13350587338209152, "reward_std": 0.46326927840709686, "rewards/reward_fn/mean": 0.13350587338209152, "rewards/reward_fn/std": 0.46326927840709686, "step": 58, "step_time": 106.23267988796579 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 283.875, "completions/clipped_ratio": 0.0, "completions/max_length": 305.0, "completions/max_terminated_length": 305.0, "completions/mean_length": 283.875, "completions/mean_terminated_length": 283.875, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "epoch": 0.3933333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.08558438718318939, "kl": 0.0, "learning_rate": 4.8333333333333334e-05, "loss": -0.006462842226028442, "num_tokens": 746274.0, "reward": 0.035797879099845886, "reward_std": 0.6359168291091919, "rewards/reward_fn/mean": 0.035797879099845886, "rewards/reward_fn/std": 0.6359168291091919, "step": 59, "step_time": 103.01650287100347 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 220.125, "completions/clipped_ratio": 0.0, "completions/max_length": 254.5, "completions/max_terminated_length": 254.5, "completions/mean_length": 220.125, "completions/mean_terminated_length": 220.125, "completions/min_length": 186.5, "completions/min_terminated_length": 186.5, "epoch": 0.4, "frac_reward_zero_std": 0.0, "grad_norm": 0.1676492542028427, "kl": 0.0, "learning_rate": 4.9166666666666665e-05, "loss": -0.056491315364837646, "num_tokens": 758820.0, "reward": -0.6773550808429718, "reward_std": 0.28672073036432266, "rewards/reward_fn/mean": -0.6773550808429718, "rewards/reward_fn/std": 0.28672073036432266, "step": 60, "step_time": 85.76503377000336 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 203.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 217.0, "completions/max_terminated_length": 217.0, "completions/mean_length": 203.9375, "completions/mean_terminated_length": 203.9375, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "epoch": 0.4066666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.07807639241218567, "kl": 0.0, "learning_rate": 5e-05, "loss": -0.006218082271516323, "num_tokens": 771731.0, "reward": 0.7148753702640533, "reward_std": 0.07848419807851315, "rewards/reward_fn/mean": 0.7148753702640533, "rewards/reward_fn/std": 0.07848419807851315, "step": 61, "step_time": 74.89335807203315 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 299.75, "completions/clipped_ratio": 0.0, "completions/max_length": 403.5, "completions/max_terminated_length": 403.5, "completions/mean_length": 299.75, "completions/mean_terminated_length": 299.75, "completions/min_length": 223.5, "completions/min_terminated_length": 223.5, "epoch": 0.41333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.07576266676187515, "kl": 0.0, "learning_rate": 4.999957692144361e-05, "loss": 0.09580255299806595, "num_tokens": 786671.0, "reward": -0.053020179271698, "reward_std": 0.4070226475596428, "rewards/reward_fn/mean": -0.053020179271698, "rewards/reward_fn/std": 0.4070226475596428, "step": 62, "step_time": 123.00635565299308 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 251.375, "completions/clipped_ratio": 0.0, "completions/max_length": 385.5, "completions/max_terminated_length": 385.5, "completions/mean_length": 251.375, "completions/mean_terminated_length": 251.375, "completions/min_length": 200.0, "completions/min_terminated_length": 200.0, "epoch": 0.42, "frac_reward_zero_std": 0.0, "grad_norm": 0.1709807962179184, "kl": 0.0, "learning_rate": 4.999830770009406e-05, "loss": 0.0437316857278347, "num_tokens": 799901.0, "reward": 0.33461277186870575, "reward_std": 0.4237028732895851, "rewards/reward_fn/mean": 0.33461277186870575, "rewards/reward_fn/std": 0.4237028732895851, "step": 63, "step_time": 118.21145736804465 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 196.1875, "completions/clipped_ratio": 0.0, "completions/max_length": 230.5, "completions/max_terminated_length": 230.5, "completions/mean_length": 196.1875, "completions/mean_terminated_length": 196.1875, "completions/min_length": 168.0, "completions/min_terminated_length": 168.0, "epoch": 0.4266666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.09982649981975555, "kl": 0.0, "learning_rate": 4.9996192378909786e-05, "loss": 0.007593698799610138, "num_tokens": 812552.0, "reward": 0.2559966593980789, "reward_std": 0.6412993967533112, "rewards/reward_fn/mean": 0.2559966593980789, "rewards/reward_fn/std": 0.6412993967533112, "step": 64, "step_time": 77.57821400102694 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 267.375, "completions/clipped_ratio": 0.0, "completions/max_length": 325.0, "completions/max_terminated_length": 325.0, "completions/mean_length": 267.375, "completions/mean_terminated_length": 267.375, "completions/min_length": 220.5, "completions/min_terminated_length": 220.5, "epoch": 0.43333333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 0.08924577385187149, "kl": 0.0, "learning_rate": 4.9993231029486544e-05, "loss": 0.0375928059220314, "num_tokens": 826230.0, "reward": -0.2883334718644619, "reward_std": 0.28712435346096754, "rewards/reward_fn/mean": -0.2883334718644619, "rewards/reward_fn/std": 0.28712435346096754, "step": 65, "step_time": 104.71041284193052 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 218.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 283.5, "completions/max_terminated_length": 283.5, "completions/mean_length": 218.6875, "completions/mean_terminated_length": 218.6875, "completions/min_length": 153.5, "completions/min_terminated_length": 153.5, "epoch": 0.44, "frac_reward_zero_std": 0.0, "grad_norm": 0.11544258147478104, "kl": 0.0, "learning_rate": 4.998942375205502e-05, "loss": -0.07060755789279938, "num_tokens": 839281.0, "reward": 0.40776287019252777, "reward_std": 0.41822005808353424, "rewards/reward_fn/mean": 0.40776287019252777, "rewards/reward_fn/std": 0.41822005808353424, "step": 66, "step_time": 91.1544282680843 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 173.8125, "completions/clipped_ratio": 0.0, "completions/max_length": 195.5, "completions/max_terminated_length": 195.5, "completions/mean_length": 173.8125, "completions/mean_terminated_length": 173.8125, "completions/min_length": 144.5, "completions/min_terminated_length": 144.5, "epoch": 0.44666666666666666, "frac_reward_zero_std": 0.5, "grad_norm": 0.1583082377910614, "kl": 0.0, "learning_rate": 4.99847706754774e-05, "loss": 0.0034518204629421234, "num_tokens": 850894.0, "reward": 0.48864591121673584, "reward_std": 0.6118558049201965, "rewards/reward_fn/mean": 0.48864591121673584, "rewards/reward_fn/std": 0.6118558049201965, "step": 67, "step_time": 69.27147658000467 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 223.625, "completions/clipped_ratio": 0.0, "completions/max_length": 254.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 223.625, "completions/mean_terminated_length": 223.625, "completions/min_length": 167.5, "completions/min_terminated_length": 167.5, "epoch": 0.4533333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.09846646338701248, "kl": 0.0, "learning_rate": 4.997927195724303e-05, "loss": -0.030570141971111298, "num_tokens": 863432.0, "reward": 0.6720858812332153, "reward_std": 0.18690402715583332, "rewards/reward_fn/mean": 0.6720858812332153, "rewards/reward_fn/std": 0.18690402715583332, "step": 68, "step_time": 84.2234050030238 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 239.75, "completions/clipped_ratio": 0.0, "completions/max_length": 257.0, "completions/max_terminated_length": 257.0, "completions/mean_length": 239.75, "completions/mean_terminated_length": 239.75, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "epoch": 0.46, "frac_reward_zero_std": 0.0, "grad_norm": 0.07295793294906616, "kl": 0.0, "learning_rate": 4.997292778346312e-05, "loss": 0.015293777920305729, "num_tokens": 877908.0, "reward": 0.4418184161186218, "reward_std": 0.2608553245663643, "rewards/reward_fn/mean": 0.4418184161186218, "rewards/reward_fn/std": 0.2608553245663643, "step": 69, "step_time": 87.94977311202092 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 284.3125, "completions/clipped_ratio": 0.0, "completions/max_length": 331.5, "completions/max_terminated_length": 331.5, "completions/mean_length": 284.3125, "completions/mean_terminated_length": 284.3125, "completions/min_length": 226.5, "completions/min_terminated_length": 226.5, "epoch": 0.4666666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 0.05379607528448105, "kl": 0.0, "learning_rate": 4.996573836886435e-05, "loss": -0.02144354209303856, "num_tokens": 891793.0, "reward": 0.2061479091644287, "reward_std": 0.3776038587093353, "rewards/reward_fn/mean": 0.2061479091644287, "rewards/reward_fn/std": 0.3776038587093353, "step": 70, "step_time": 116.79733415995724 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 225.5625, "completions/clipped_ratio": 0.0, "completions/max_length": 283.5, "completions/max_terminated_length": 283.5, "completions/mean_length": 225.5625, "completions/mean_terminated_length": 225.5625, "completions/min_length": 189.0, "completions/min_terminated_length": 189.0, "epoch": 0.47333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.11158322542905807, "kl": 0.0, "learning_rate": 4.995770395678171e-05, "loss": -0.021105844527482986, "num_tokens": 905042.0, "reward": 0.7146752774715424, "reward_std": 0.3213859871029854, "rewards/reward_fn/mean": 0.7146752774715424, "rewards/reward_fn/std": 0.3213859871029854, "step": 71, "step_time": 92.01113651989726 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 178.3125, "completions/clipped_ratio": 0.0, "completions/max_length": 215.5, "completions/max_terminated_length": 215.5, "completions/mean_length": 178.3125, "completions/mean_terminated_length": 178.3125, "completions/min_length": 144.0, "completions/min_terminated_length": 144.0, "epoch": 0.48, "frac_reward_zero_std": 0.0, "grad_norm": 0.1308518946170807, "kl": 0.0, "learning_rate": 4.9948824819150185e-05, "loss": -0.017831377685070038, "num_tokens": 916575.0, "reward": 0.8005787134170532, "reward_std": 0.09497696161270142, "rewards/reward_fn/mean": 0.8005787134170532, "rewards/reward_fn/std": 0.09497696161270142, "step": 72, "step_time": 72.7636261759908 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 224.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 285.0, "completions/max_terminated_length": 285.0, "completions/mean_length": 224.6875, "completions/mean_terminated_length": 224.6875, "completions/min_length": 178.0, "completions/min_terminated_length": 178.0, "epoch": 0.4866666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.10509830713272095, "kl": 0.0, "learning_rate": 4.993910125649561e-05, "loss": 0.07416598498821259, "num_tokens": 929778.0, "reward": 0.6564783751964569, "reward_std": 0.26796439323516097, "rewards/reward_fn/mean": 0.6564783751964569, "rewards/reward_fn/std": 0.26796439323516097, "step": 73, "step_time": 92.07556184596615 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 273.5625, "completions/clipped_ratio": 0.0, "completions/max_length": 349.0, "completions/max_terminated_length": 349.0, "completions/mean_length": 273.5625, "completions/mean_terminated_length": 273.5625, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "epoch": 0.49333333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 0.09755323827266693, "kl": 0.0, "learning_rate": 4.992853359792444e-05, "loss": -0.037562236189842224, "num_tokens": 943499.0, "reward": -0.27786024287343025, "reward_std": 0.3537736553698778, "rewards/reward_fn/mean": -0.27786024287343025, "rewards/reward_fn/std": 0.3537736553698778, "step": 74, "step_time": 115.20405016100267 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 260.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 344.5, "completions/max_terminated_length": 344.5, "completions/mean_length": 260.9375, "completions/mean_terminated_length": 260.9375, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "epoch": 0.5, "frac_reward_zero_std": 0.0, "grad_norm": 0.03571086376905441, "kl": 0.0, "learning_rate": 4.9917122201112656e-05, "loss": -0.016071610152721405, "num_tokens": 956394.0, "reward": 0.9999693334102631, "reward_std": 5.47565687156748e-05, "rewards/reward_fn/mean": 0.9999693334102631, "rewards/reward_fn/std": 5.47565687156748e-05, "step": 75, "step_time": 105.76170146500226 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 271.5625, "completions/clipped_ratio": 0.0, "completions/max_length": 356.0, "completions/max_terminated_length": 356.0, "completions/mean_length": 271.5625, "completions/mean_terminated_length": 271.5625, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "epoch": 0.5066666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.0005527990870177746, "kl": 0.0, "learning_rate": 4.990486745229364e-05, "loss": 0.00025011063553392887, "num_tokens": 969475.0, "reward": 0.9999998211860657, "reward_std": 5.813290879785882e-07, "rewards/reward_fn/mean": 0.9999998211860657, "rewards/reward_fn/std": 5.813290879785882e-07, "step": 76, "step_time": 108.75159504503245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 227.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 282.5, "completions/max_terminated_length": 282.5, "completions/mean_length": 227.6875, "completions/mean_terminated_length": 227.6875, "completions/min_length": 200.0, "completions/min_terminated_length": 200.0, "epoch": 0.5133333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.10346979647874832, "kl": 0.0, "learning_rate": 4.989176976624511e-05, "loss": 0.009114399552345276, "num_tokens": 982878.0, "reward": 0.5748505890369415, "reward_std": 0.61591437458992, "rewards/reward_fn/mean": 0.5748505890369415, "rewards/reward_fn/std": 0.61591437458992, "step": 77, "step_time": 96.31299933302216 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 307.4375, "completions/clipped_ratio": 0.0, "completions/max_length": 414.0, "completions/max_terminated_length": 414.0, "completions/mean_length": 307.4375, "completions/mean_terminated_length": 307.4375, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "epoch": 0.52, "frac_reward_zero_std": 0.0, "grad_norm": 0.11586609482765198, "kl": 0.0, "learning_rate": 4.987782958627508e-05, "loss": -0.00447622686624527, "num_tokens": 996965.0, "reward": 0.17336545884609222, "reward_std": 0.4828975200653076, "rewards/reward_fn/mean": 0.17336545884609222, "rewards/reward_fn/std": 0.4828975200653076, "step": 78, "step_time": 142.4235202560085 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 314.1875, "completions/clipped_ratio": 0.0, "completions/max_length": 414.5, "completions/max_terminated_length": 414.5, "completions/mean_length": 314.1875, "completions/mean_terminated_length": 314.1875, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "epoch": 0.5266666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 0.10377015173435211, "kl": 0.0, "learning_rate": 4.9863047384206835e-05, "loss": 0.06035196781158447, "num_tokens": 1011376.0, "reward": -0.3835201971232891, "reward_std": 0.330098245292902, "rewards/reward_fn/mean": -0.3835201971232891, "rewards/reward_fn/std": 0.330098245292902, "step": 79, "step_time": 134.57437156600645 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 253.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 302.5, "completions/max_terminated_length": 302.5, "completions/mean_length": 253.6875, "completions/mean_terminated_length": 253.6875, "completions/min_length": 224.5, "completions/min_terminated_length": 224.5, "epoch": 0.5333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.08293555676937103, "kl": 0.0, "learning_rate": 4.9847423660363e-05, "loss": -0.019359871745109558, "num_tokens": 1024363.0, "reward": 0.9290256202220917, "reward_std": 0.049609191715717316, "rewards/reward_fn/mean": 0.9290256202220917, "rewards/reward_fn/std": 0.049609191715717316, "step": 80, "step_time": 103.19731909304392 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 292.4375, "completions/clipped_ratio": 0.0, "completions/max_length": 326.5, "completions/max_terminated_length": 326.5, "completions/mean_length": 292.4375, "completions/mean_terminated_length": 292.4375, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "epoch": 0.54, "frac_reward_zero_std": 0.0, "grad_norm": 0.1021459549665451, "kl": 0.0, "learning_rate": 4.983095894354858e-05, "loss": 0.010597247630357742, "num_tokens": 1038818.0, "reward": 0.6705855131149292, "reward_std": 0.5288838520646095, "rewards/reward_fn/mean": 0.6705855131149292, "rewards/reward_fn/std": 0.5288838520646095, "step": 81, "step_time": 104.1593450899818 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 186.1875, "completions/clipped_ratio": 0.0, "completions/max_length": 214.5, "completions/max_terminated_length": 214.5, "completions/mean_length": 186.1875, "completions/mean_terminated_length": 186.1875, "completions/min_length": 158.5, "completions/min_terminated_length": 158.5, "epoch": 0.5466666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 0.13033370673656464, "kl": 0.0, "learning_rate": 4.9813653791033057e-05, "loss": -0.013745991513133049, "num_tokens": 1051325.0, "reward": 0.4789735674858093, "reward_std": 0.6973891258239746, "rewards/reward_fn/mean": 0.4789735674858093, "rewards/reward_fn/std": 0.6973891258239746, "step": 82, "step_time": 73.42914309195476 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 350.8125, "completions/clipped_ratio": 0.0, "completions/max_length": 428.5, "completions/max_terminated_length": 428.5, "completions/mean_length": 350.8125, "completions/mean_terminated_length": 350.8125, "completions/min_length": 274.5, "completions/min_terminated_length": 274.5, "epoch": 0.5533333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.08932679146528244, "kl": 0.0, "learning_rate": 4.979550878853154e-05, "loss": -0.06357850879430771, "num_tokens": 1066626.0, "reward": -0.35760005563497543, "reward_std": 0.4595603495836258, "rewards/reward_fn/mean": -0.35760005563497543, "rewards/reward_fn/std": 0.4595603495836258, "step": 83, "step_time": 131.997143570974 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 254.4375, "completions/clipped_ratio": 0.0, "completions/max_length": 320.5, "completions/max_terminated_length": 320.5, "completions/mean_length": 254.4375, "completions/mean_terminated_length": 254.4375, "completions/min_length": 214.5, "completions/min_terminated_length": 214.5, "epoch": 0.56, "frac_reward_zero_std": 0.0, "grad_norm": 0.06409084051847458, "kl": 0.0, "learning_rate": 4.9776524550184965e-05, "loss": 0.025061018764972687, "num_tokens": 1080409.0, "reward": 0.7382991313934326, "reward_std": 0.05440911108235014, "rewards/reward_fn/mean": 0.7382991313934326, "rewards/reward_fn/std": 0.05440911108235014, "step": 84, "step_time": 101.2649952210486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 250.75, "completions/clipped_ratio": 0.0, "completions/max_length": 267.5, "completions/max_terminated_length": 267.5, "completions/mean_length": 250.75, "completions/mean_terminated_length": 250.75, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "epoch": 0.5666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.07058465480804443, "kl": 0.0, "learning_rate": 4.975670171853926e-05, "loss": -0.009824715554714203, "num_tokens": 1093269.0, "reward": 0.5432418249547482, "reward_std": 0.011591001180931926, "rewards/reward_fn/mean": 0.5432418249547482, "rewards/reward_fn/std": 0.011591001180931926, "step": 85, "step_time": 94.21445810300065 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 319.75, "completions/clipped_ratio": 0.0, "completions/max_length": 418.0, "completions/max_terminated_length": 418.0, "completions/mean_length": 319.75, "completions/mean_terminated_length": 319.75, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "epoch": 0.5733333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 0.0850214958190918, "kl": 0.0, "learning_rate": 4.973604096452361e-05, "loss": -0.04513334110379219, "num_tokens": 1107625.0, "reward": 0.30771853029727936, "reward_std": 0.30676984786987305, "rewards/reward_fn/mean": 0.30771853029727936, "rewards/reward_fn/std": 0.30676984786987305, "step": 86, "step_time": 128.4876309260726 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 376.5, "completions/clipped_ratio": 0.0, "completions/max_length": 635.0, "completions/max_terminated_length": 635.0, "completions/mean_length": 376.5, "completions/mean_terminated_length": 376.5, "completions/min_length": 265.5, "completions/min_terminated_length": 265.5, "epoch": 0.58, "frac_reward_zero_std": 0.0, "grad_norm": 0.0707458034157753, "kl": 0.0, "learning_rate": 4.971454298742779e-05, "loss": -0.15971024334430695, "num_tokens": 1123865.0, "reward": 0.49016936123371124, "reward_std": 0.2892051860690117, "rewards/reward_fn/mean": 0.49016936123371124, "rewards/reward_fn/std": 0.2892051860690117, "step": 87, "step_time": 187.60785773402313 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 306.5, "completions/clipped_ratio": 0.0, "completions/max_length": 394.5, "completions/max_terminated_length": 394.5, "completions/mean_length": 306.5, "completions/mean_terminated_length": 306.5, "completions/min_length": 237.5, "completions/min_terminated_length": 237.5, "epoch": 0.5866666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.05612671747803688, "kl": 0.0, "learning_rate": 4.9692208514878444e-05, "loss": -0.011051323264837265, "num_tokens": 1138457.0, "reward": 0.6979325413703918, "reward_std": 0.3039002874547805, "rewards/reward_fn/mean": 0.6979325413703918, "rewards/reward_fn/std": 0.3039002874547805, "step": 88, "step_time": 120.77513207792072 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 200.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 221.0, "completions/max_terminated_length": 221.0, "completions/mean_length": 200.9375, "completions/mean_terminated_length": 200.9375, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "epoch": 0.5933333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 0.08993715792894363, "kl": 0.0, "learning_rate": 4.966903830281449e-05, "loss": 0.003101535141468048, "num_tokens": 1150488.0, "reward": 0.5542193427681923, "reward_std": 0.0270778898228059, "rewards/reward_fn/mean": 0.5542193427681923, "rewards/reward_fn/std": 0.0270778898228059, "step": 89, "step_time": 82.04292890999932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 266.5625, "completions/clipped_ratio": 0.0, "completions/max_length": 295.0, "completions/max_terminated_length": 295.0, "completions/mean_length": 266.5625, "completions/mean_terminated_length": 266.5625, "completions/min_length": 239.5, "completions/min_terminated_length": 239.5, "epoch": 0.6, "frac_reward_zero_std": 0.0, "grad_norm": 0.08966866135597229, "kl": 0.0, "learning_rate": 4.9645033135461494e-05, "loss": 0.005549095571041107, "num_tokens": 1164649.0, "reward": 0.4357367791235447, "reward_std": 0.11432263441383839, "rewards/reward_fn/mean": 0.4357367791235447, "rewards/reward_fn/std": 0.11432263441383839, "step": 90, "step_time": 99.49214323301567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 214.875, "completions/clipped_ratio": 0.0, "completions/max_length": 273.0, "completions/max_terminated_length": 273.0, "completions/mean_length": 214.875, "completions/mean_terminated_length": 214.875, "completions/min_length": 170.0, "completions/min_terminated_length": 170.0, "epoch": 0.6066666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.06210985407233238, "kl": 0.0, "learning_rate": 4.962019382530521e-05, "loss": -0.010337941348552704, "num_tokens": 1177647.0, "reward": 0.7368387281894684, "reward_std": 0.1752328469083153, "rewards/reward_fn/mean": 0.7368387281894684, "rewards/reward_fn/std": 0.1752328469083153, "step": 91, "step_time": 88.74471832800191 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 428.375, "completions/clipped_ratio": 0.0, "completions/max_length": 519.5, "completions/max_terminated_length": 519.5, "completions/mean_length": 428.375, "completions/mean_terminated_length": 428.375, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "epoch": 0.6133333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.07177510857582092, "kl": 0.0, "learning_rate": 4.9594521213063974e-05, "loss": -0.009097330272197723, "num_tokens": 1194149.0, "reward": -0.3315518796443939, "reward_std": 0.46374107152223587, "rewards/reward_fn/mean": -0.3315518796443939, "rewards/reward_fn/std": 0.46374107152223587, "step": 92, "step_time": 173.7315434149932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 365.0, "completions/clipped_ratio": 0.0, "completions/max_length": 496.0, "completions/max_terminated_length": 496.0, "completions/mean_length": 365.0, "completions/mean_terminated_length": 365.0, "completions/min_length": 246.5, "completions/min_terminated_length": 246.5, "epoch": 0.62, "frac_reward_zero_std": 0.5, "grad_norm": 0.04886671528220177, "kl": 0.0, "learning_rate": 4.9568016167660334e-05, "loss": 0.025912761688232422, "num_tokens": 1209413.0, "reward": 0.5116970045492053, "reward_std": 0.09442839026451111, "rewards/reward_fn/mean": 0.5116970045492053, "rewards/reward_fn/std": 0.09442839026451111, "step": 93, "step_time": 159.12018209404778 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 443.25, "completions/clipped_ratio": 0.0, "completions/max_length": 612.0, "completions/max_terminated_length": 612.0, "completions/mean_length": 443.25, "completions/mean_terminated_length": 443.25, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "epoch": 0.6266666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 0.05622043460607529, "kl": 0.0, "learning_rate": 4.9540679586191605e-05, "loss": 0.03646225109696388, "num_tokens": 1225689.0, "reward": 0.4065200388431549, "reward_std": 0.29066115617752075, "rewards/reward_fn/mean": 0.4065200388431549, "rewards/reward_fn/std": 0.29066115617752075, "step": 94, "step_time": 187.76930392504437 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 393.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 557.5, "completions/max_terminated_length": 557.5, "completions/mean_length": 393.0625, "completions/mean_terminated_length": 393.0625, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "epoch": 0.6333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.06194169446825981, "kl": 0.0, "learning_rate": 4.951251239389948e-05, "loss": 0.003901831805706024, "num_tokens": 1242106.0, "reward": 0.32503822073340416, "reward_std": 0.19283149391412735, "rewards/reward_fn/mean": 0.32503822073340416, "rewards/reward_fn/std": 0.19283149391412735, "step": 95, "step_time": 165.4877885619644 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 311.375, "completions/clipped_ratio": 0.0, "completions/max_length": 373.5, "completions/max_terminated_length": 373.5, "completions/mean_length": 311.375, "completions/mean_terminated_length": 311.375, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "epoch": 0.64, "frac_reward_zero_std": 0.5, "grad_norm": 0.045090898871421814, "kl": 0.0, "learning_rate": 4.948351554413879e-05, "loss": 0.05610096454620361, "num_tokens": 1256464.0, "reward": 0.4024457186460495, "reward_std": 0.4338012635707855, "rewards/reward_fn/mean": 0.4024457186460495, "rewards/reward_fn/std": 0.4338012635707855, "step": 96, "step_time": 123.16074389498681 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 214.875, "completions/clipped_ratio": 0.0, "completions/max_length": 219.0, "completions/max_terminated_length": 219.0, "completions/mean_length": 214.875, "completions/mean_terminated_length": 214.875, "completions/min_length": 210.0, "completions/min_terminated_length": 210.0, "epoch": 0.6466666666666666, "frac_reward_zero_std": 0.5, "grad_norm": 0.03964964672923088, "kl": 0.0, "learning_rate": 4.9453690018345144e-05, "loss": -0.0001890324056148529, "num_tokens": 1268902.0, "reward": 0.5716236382722855, "reward_std": 0.016463955864310265, "rewards/reward_fn/mean": 0.5716236382722855, "rewards/reward_fn/std": 0.016463955864310265, "step": 97, "step_time": 92.26312756206607 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 461.8125, "completions/clipped_ratio": 0.0, "completions/max_length": 757.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 461.8125, "completions/mean_terminated_length": 461.8125, "completions/min_length": 303.5, "completions/min_terminated_length": 303.5, "epoch": 0.6533333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.0490785576403141, "kl": 0.0, "learning_rate": 4.942303682600178e-05, "loss": 0.010549291968345642, "num_tokens": 1285595.0, "reward": 0.47012635320425034, "reward_std": 0.2943352471457814, "rewards/reward_fn/mean": 0.47012635320425034, "rewards/reward_fn/std": 0.2943352471457814, "step": 98, "step_time": 214.64041374594672 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 282.1875, "completions/clipped_ratio": 0.0, "completions/max_length": 372.0, "completions/max_terminated_length": 372.0, "completions/mean_length": 282.1875, "completions/mean_terminated_length": 282.1875, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "epoch": 0.66, "frac_reward_zero_std": 0.0, "grad_norm": 0.1215004101395607, "kl": 0.0, "learning_rate": 4.939155700460536e-05, "loss": -0.11692754924297333, "num_tokens": 1300774.0, "reward": -1.0974531769752502, "reward_std": 0.8694423735141754, "rewards/reward_fn/mean": -1.0974531769752502, "rewards/reward_fn/std": 0.8694423735141754, "step": 99, "step_time": 115.8598381759366 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 378.3125, "completions/clipped_ratio": 0.0, "completions/max_length": 457.0, "completions/max_terminated_length": 457.0, "completions/mean_length": 378.3125, "completions/mean_terminated_length": 378.3125, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "epoch": 0.6666666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 0.06170595437288284, "kl": 0.0, "learning_rate": 4.9359251619630886e-05, "loss": -0.032986074686050415, "num_tokens": 1316947.0, "reward": 0.24835945293307304, "reward_std": 0.1980152428150177, "rewards/reward_fn/mean": 0.24835945293307304, "rewards/reward_fn/std": 0.1980152428150177, "step": 100, "step_time": 138.67001789598726 }, { "epoch": 0.6666666666666666, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completion_length": 348.90625, "eval_completions/clipped_ratio": 0.0, "eval_completions/max_length": 474.82, "eval_completions/max_terminated_length": 474.82, "eval_completions/mean_length": 348.90625, "eval_completions/mean_terminated_length": 348.90625, "eval_completions/min_length": 266.68, "eval_completions/min_terminated_length": 266.68, "eval_frac_reward_zero_std": 0.1, "eval_kl": 0.0, "eval_loss": 0.008204526267945766, "eval_num_tokens": 1316947.0, "eval_reward": -0.090777420476079, "eval_reward_std": 0.40388403258689026, "eval_rewards/reward_fn/mean": -0.090777420476079, "eval_rewards/reward_fn/std": 0.40388403258689026, "eval_runtime": 6565.5982, "eval_samples_per_second": 0.015, "eval_steps_per_second": 0.002, "step": 100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 212.875, "completions/clipped_ratio": 0.0, "completions/max_length": 219.0, "completions/max_terminated_length": 219.0, "completions/mean_length": 212.875, "completions/mean_terminated_length": 212.875, "completions/min_length": 208.5, "completions/min_terminated_length": 208.5, "epoch": 0.6733333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.09589043259620667, "kl": 0.0, "learning_rate": 4.9326121764495596e-05, "loss": 0.0010354742407798767, "num_tokens": 1329313.0, "reward": 0.8882656097412109, "reward_std": 0.027551015838980675, "rewards/reward_fn/mean": 0.8882656097412109, "rewards/reward_fn/std": 0.027551015838980675, "step": 101, "step_time": 113.81590729398886 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 255.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.5, "completions/max_terminated_length": 292.5, "completions/mean_length": 255.0, "completions/mean_terminated_length": 255.0, "completions/min_length": 206.5, "completions/min_terminated_length": 206.5, "epoch": 0.68, "frac_reward_zero_std": 0.0, "grad_norm": 0.06758790463209152, "kl": 0.0, "learning_rate": 4.9292168560522014e-05, "loss": 0.05638433247804642, "num_tokens": 1343929.0, "reward": 0.5735078454017639, "reward_std": 0.19756827503442764, "rewards/reward_fn/mean": 0.5735078454017639, "rewards/reward_fn/std": 0.19756827503442764, "step": 102, "step_time": 97.67106665606843 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 476.125, "completions/clipped_ratio": 0.0, "completions/max_length": 726.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 476.125, "completions/mean_terminated_length": 476.125, "completions/min_length": 348.5, "completions/min_terminated_length": 348.5, "epoch": 0.6866666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 0.06600646674633026, "kl": 0.0, "learning_rate": 4.925739315689991e-05, "loss": -0.1070128008723259, "num_tokens": 1361355.0, "reward": 0.0030741989612579346, "reward_std": 0.6040342599153519, "rewards/reward_fn/mean": 0.0030741989612579346, "rewards/reward_fn/std": 0.6040342599153519, "step": 103, "step_time": 218.79046111099888 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 210.0, "completions/clipped_ratio": 0.0, "completions/max_length": 213.5, "completions/max_terminated_length": 213.5, "completions/mean_length": 210.0, "completions/mean_terminated_length": 210.0, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "epoch": 0.6933333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 0.05047248676419258, "kl": 0.0, "learning_rate": 4.9221796730647516e-05, "loss": -0.00010214000940322876, "num_tokens": 1373667.0, "reward": 0.07565045356750488, "reward_std": 0.0007886506391514558, "rewards/reward_fn/mean": 0.07565045356750488, "rewards/reward_fn/std": 0.0007886506391514558, "step": 104, "step_time": 87.27632812299998 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 287.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 349.0, "completions/max_terminated_length": 349.0, "completions/mean_length": 287.6875, "completions/mean_terminated_length": 287.6875, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "epoch": 0.7, "frac_reward_zero_std": 0.0, "grad_norm": 0.03672612085938454, "kl": 0.0, "learning_rate": 4.9185380486571595e-05, "loss": 0.0014976412057876587, "num_tokens": 1387110.0, "reward": 0.5444922223687172, "reward_std": 7.728397611117543e-05, "rewards/reward_fn/mean": 0.5444922223687172, "rewards/reward_fn/std": 7.728397611117543e-05, "step": 105, "step_time": 115.72042374301236 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 293.3125, "completions/clipped_ratio": 0.0, "completions/max_length": 323.0, "completions/max_terminated_length": 323.0, "completions/mean_length": 293.3125, "completions/mean_terminated_length": 293.3125, "completions/min_length": 250.5, "completions/min_terminated_length": 250.5, "epoch": 0.7066666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 0.04970452934503555, "kl": 0.0, "learning_rate": 4.914814565722671e-05, "loss": -0.0025767460465431213, "num_tokens": 1401147.0, "reward": 0.08888586703687906, "reward_std": 0.3375508785247803, "rewards/reward_fn/mean": 0.08888586703687906, "rewards/reward_fn/std": 0.3375508785247803, "step": 106, "step_time": 110.42762503004633 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 519.25, "completions/clipped_ratio": 0.0, "completions/max_length": 949.5, "completions/max_terminated_length": 949.5, "completions/mean_length": 519.25, "completions/mean_terminated_length": 519.25, "completions/min_length": 347.5, "completions/min_terminated_length": 347.5, "epoch": 0.7133333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 0.07587578892707825, "kl": 0.0, "learning_rate": 4.9110093502873476e-05, "loss": 0.04362190514802933, "num_tokens": 1419135.0, "reward": 0.2313607558608055, "reward_std": 0.5956478118896484, "rewards/reward_fn/mean": 0.2313607558608055, "rewards/reward_fn/std": 0.5956478118896484, "step": 107, "step_time": 312.2387495030416 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 300.5, "completions/clipped_ratio": 0.0, "completions/max_length": 342.0, "completions/max_terminated_length": 342.0, "completions/mean_length": 300.5, "completions/mean_terminated_length": 300.5, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "epoch": 0.72, "frac_reward_zero_std": 0.5, "grad_norm": 0.06615033745765686, "kl": 0.0, "learning_rate": 4.907122531143594e-05, "loss": 0.01366664282977581, "num_tokens": 1433295.0, "reward": 0.5283045321702957, "reward_std": 0.2961108386516571, "rewards/reward_fn/mean": 0.5283045321702957, "rewards/reward_fn/std": 0.2961108386516571, "step": 108, "step_time": 115.16167661710642 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 302.25, "completions/clipped_ratio": 0.0, "completions/max_length": 352.5, "completions/max_terminated_length": 352.5, "completions/mean_length": 302.25, "completions/mean_terminated_length": 302.25, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "epoch": 0.7266666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.0835169330239296, "kl": 0.0, "learning_rate": 4.9031542398457974e-05, "loss": -0.005957708694040775, "num_tokens": 1448819.0, "reward": 0.5500120520591736, "reward_std": 0.288537934422493, "rewards/reward_fn/mean": 0.5500120520591736, "rewards/reward_fn/std": 0.288537934422493, "step": 109, "step_time": 111.85748529294506 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 300.75, "completions/clipped_ratio": 0.0, "completions/max_length": 343.5, "completions/max_terminated_length": 343.5, "completions/mean_length": 300.75, "completions/mean_terminated_length": 300.75, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "epoch": 0.7333333333333333, "frac_reward_zero_std": 0.5, "grad_norm": 0.05528109520673752, "kl": 0.0, "learning_rate": 4.8991046107058735e-05, "loss": 0.003592744469642639, "num_tokens": 1462735.0, "reward": 0.4155409634113312, "reward_std": 0.1777665913105011, "rewards/reward_fn/mean": 0.4155409634113312, "rewards/reward_fn/std": 0.1777665913105011, "step": 110, "step_time": 106.56734898797004 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 240.875, "completions/clipped_ratio": 0.0, "completions/max_length": 265.5, "completions/max_terminated_length": 265.5, "completions/mean_length": 240.875, "completions/mean_terminated_length": 240.875, "completions/min_length": 199.0, "completions/min_terminated_length": 199.0, "epoch": 0.74, "frac_reward_zero_std": 0.5, "grad_norm": 0.0353812538087368, "kl": 0.0, "learning_rate": 4.894973780788722e-05, "loss": -0.011618122458457947, "num_tokens": 1476493.0, "reward": 0.7792612016201019, "reward_std": 0.10726096481084824, "rewards/reward_fn/mean": 0.7792612016201019, "rewards/reward_fn/std": 0.10726096481084824, "step": 111, "step_time": 96.58656353410333 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 507.1875, "completions/clipped_ratio": 0.0, "completions/max_length": 806.0, "completions/max_terminated_length": 806.0, "completions/mean_length": 507.1875, "completions/mean_terminated_length": 507.1875, "completions/min_length": 375.5, "completions/min_terminated_length": 375.5, "epoch": 0.7466666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 0.034782811999320984, "kl": 0.0, "learning_rate": 4.890761889907589e-05, "loss": 0.031690001487731934, "num_tokens": 1493936.0, "reward": 0.5777778774499893, "reward_std": 0.11972552537918091, "rewards/reward_fn/mean": 0.5777778774499893, "rewards/reward_fn/std": 0.11972552537918091, "step": 112, "step_time": 232.74652250896906 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 298.875, "completions/clipped_ratio": 0.0, "completions/max_length": 504.5, "completions/max_terminated_length": 504.5, "completions/mean_length": 298.875, "completions/mean_terminated_length": 298.875, "completions/min_length": 206.5, "completions/min_terminated_length": 206.5, "epoch": 0.7533333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.048440419137477875, "kl": 0.0, "learning_rate": 4.88646908061933e-05, "loss": -0.025000344961881638, "num_tokens": 1507358.0, "reward": 0.9986199140548706, "reward_std": 0.003903499803300292, "rewards/reward_fn/mean": 0.9986199140548706, "rewards/reward_fn/std": 0.003903499803300292, "step": 113, "step_time": 145.51719051605323 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 286.4375, "completions/clipped_ratio": 0.0, "completions/max_length": 330.5, "completions/max_terminated_length": 330.5, "completions/mean_length": 286.4375, "completions/mean_terminated_length": 286.4375, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "epoch": 0.76, "frac_reward_zero_std": 0.5, "grad_norm": 0.05592720955610275, "kl": 0.0, "learning_rate": 4.8820954982195905e-05, "loss": 0.0003649294376373291, "num_tokens": 1521589.0, "reward": 0.6258035600185394, "reward_std": 0.35387808084487915, "rewards/reward_fn/mean": 0.6258035600185394, "rewards/reward_fn/std": 0.35387808084487915, "step": 114, "step_time": 104.28781160601648 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 300.3125, "completions/clipped_ratio": 0.0, "completions/max_length": 358.0, "completions/max_terminated_length": 358.0, "completions/mean_length": 300.3125, "completions/mean_terminated_length": 300.3125, "completions/min_length": 258.5, "completions/min_terminated_length": 258.5, "epoch": 0.7666666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 8.306245581479743e-05, "kl": 0.0, "learning_rate": 4.877641290737884e-05, "loss": 6.103819760028273e-05, "num_tokens": 1535322.0, "reward": 0.9999999701976776, "reward_std": 1.0566774477638319e-07, "rewards/reward_fn/mean": 0.9999999701976776, "rewards/reward_fn/std": 1.0566774477638319e-07, "step": 115, "step_time": 119.61036076099845 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 458.4375, "completions/clipped_ratio": 0.0, "completions/max_length": 778.0, "completions/max_terminated_length": 778.0, "completions/mean_length": 458.4375, "completions/mean_terminated_length": 458.4375, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "epoch": 0.7733333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.0683111697435379, "kl": 0.0, "learning_rate": 4.873106608932585e-05, "loss": -0.1494099348783493, "num_tokens": 1551873.0, "reward": 0.7072757333517075, "reward_std": 0.1626912999496426, "rewards/reward_fn/mean": 0.7072757333517075, "rewards/reward_fn/std": 0.1626912999496426, "step": 116, "step_time": 219.83419971400872 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 214.375, "completions/clipped_ratio": 0.0, "completions/max_length": 234.0, "completions/max_terminated_length": 234.0, "completions/mean_length": 214.375, "completions/mean_terminated_length": 214.375, "completions/min_length": 187.5, "completions/min_terminated_length": 187.5, "epoch": 0.78, "frac_reward_zero_std": 0.5, "grad_norm": 0.04911733418703079, "kl": 0.0, "learning_rate": 4.868491606285823e-05, "loss": -0.002008102834224701, "num_tokens": 1564847.0, "reward": 0.851028710603714, "reward_std": 0.11776234209537506, "rewards/reward_fn/mean": 0.851028710603714, "rewards/reward_fn/std": 0.11776234209537506, "step": 117, "step_time": 79.10500066704117 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 394.1875, "completions/clipped_ratio": 0.0, "completions/max_length": 526.5, "completions/max_terminated_length": 526.5, "completions/mean_length": 394.1875, "completions/mean_terminated_length": 394.1875, "completions/min_length": 273.5, "completions/min_terminated_length": 273.5, "epoch": 0.7866666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 0.06439276784658432, "kl": 0.0, "learning_rate": 4.8637964389982926e-05, "loss": 0.05288882926106453, "num_tokens": 1581274.0, "reward": 0.5939511358737946, "reward_std": 0.6298229396343231, "rewards/reward_fn/mean": 0.5939511358737946, "rewards/reward_fn/std": 0.6298229396343231, "step": 118, "step_time": 160.8602781650261 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 294.25, "completions/clipped_ratio": 0.0, "completions/max_length": 362.5, "completions/max_terminated_length": 362.5, "completions/mean_length": 294.25, "completions/mean_terminated_length": 294.25, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "epoch": 0.7933333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.06369972974061966, "kl": 0.0, "learning_rate": 4.859021265983959e-05, "loss": 0.04611533135175705, "num_tokens": 1596198.0, "reward": 0.07208789885044098, "reward_std": 0.21891965717077255, "rewards/reward_fn/mean": 0.07208789885044098, "rewards/reward_fn/std": 0.21891965717077255, "step": 119, "step_time": 126.26092710503144 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 341.875, "completions/clipped_ratio": 0.0, "completions/max_length": 400.0, "completions/max_terminated_length": 400.0, "completions/mean_length": 341.875, "completions/mean_terminated_length": 341.875, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "epoch": 0.8, "frac_reward_zero_std": 0.5, "grad_norm": 0.028934789821505547, "kl": 0.0, "learning_rate": 4.854166248864689e-05, "loss": 0.028521619737148285, "num_tokens": 1611036.0, "reward": 0.5731896162033081, "reward_std": 0.009335569106042385, "rewards/reward_fn/mean": 0.5731896162033081, "rewards/reward_fn/std": 0.009335569106042385, "step": 120, "step_time": 131.31865840405226 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 355.5, "completions/clipped_ratio": 0.0, "completions/max_length": 470.0, "completions/max_terminated_length": 470.0, "completions/mean_length": 355.5, "completions/mean_terminated_length": 355.5, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "epoch": 0.8066666666666666, "frac_reward_zero_std": 0.5, "grad_norm": 0.04000210389494896, "kl": 0.0, "learning_rate": 4.849231551964771e-05, "loss": 0.014162637293338776, "num_tokens": 1626324.0, "reward": 0.8219311833381653, "reward_std": 0.0823940634727478, "rewards/reward_fn/mean": 0.8219311833381653, "rewards/reward_fn/std": 0.0823940634727478, "step": 121, "step_time": 140.04646876995685 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 400.3125, "completions/clipped_ratio": 0.0, "completions/max_length": 576.0, "completions/max_terminated_length": 576.0, "completions/mean_length": 400.3125, "completions/mean_terminated_length": 400.3125, "completions/min_length": 317.5, "completions/min_terminated_length": 317.5, "epoch": 0.8133333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 0.07007366418838501, "kl": 0.0, "learning_rate": 4.844217342305363e-05, "loss": -0.025220118463039398, "num_tokens": 1642993.0, "reward": 0.027626875787973404, "reward_std": 0.2794133871793747, "rewards/reward_fn/mean": 0.027626875787973404, "rewards/reward_fn/std": 0.2794133871793747, "step": 122, "step_time": 171.93886475102045 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 379.0, "completions/clipped_ratio": 0.0, "completions/max_length": 593.5, "completions/max_terminated_length": 593.5, "completions/mean_length": 379.0, "completions/mean_terminated_length": 379.0, "completions/min_length": 239.5, "completions/min_terminated_length": 239.5, "epoch": 0.82, "frac_reward_zero_std": 0.0, "grad_norm": 0.20304864645004272, "kl": 0.0, "learning_rate": 4.839123789598829e-05, "loss": 0.12271372228860855, "num_tokens": 1659169.0, "reward": 0.0422142893075943, "reward_std": 0.42568448930978775, "rewards/reward_fn/mean": 0.0422142893075943, "rewards/reward_fn/std": 0.42568448930978775, "step": 123, "step_time": 200.5066597990226 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 406.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 700.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 406.0625, "completions/mean_terminated_length": 406.0625, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "epoch": 0.8266666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 0.041422288864851, "kl": 0.0, "learning_rate": 4.8339510662430046e-05, "loss": -0.0771825909614563, "num_tokens": 1675098.0, "reward": 0.5340617299079895, "reward_std": 0.03479880467057228, "rewards/reward_fn/mean": 0.5340617299079895, "rewards/reward_fn/std": 0.03479880467057228, "step": 124, "step_time": 213.72491782298312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 272.1875, "completions/clipped_ratio": 0.0, "completions/max_length": 315.0, "completions/max_terminated_length": 315.0, "completions/mean_length": 272.1875, "completions/mean_terminated_length": 272.1875, "completions/min_length": 203.5, "completions/min_terminated_length": 203.5, "epoch": 0.8333333333333334, "frac_reward_zero_std": 0.5, "grad_norm": 0.05468154326081276, "kl": 0.0, "learning_rate": 4.828699347315356e-05, "loss": -0.055062174797058105, "num_tokens": 1689133.0, "reward": 0.8563609719276428, "reward_std": 0.12412404268980026, "rewards/reward_fn/mean": 0.8563609719276428, "rewards/reward_fn/std": 0.12412404268980026, "step": 125, "step_time": 99.89914110297104 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 208.125, "completions/clipped_ratio": 0.0, "completions/max_length": 210.0, "completions/max_terminated_length": 210.0, "completions/mean_length": 208.125, "completions/mean_terminated_length": 208.125, "completions/min_length": 200.5, "completions/min_terminated_length": 200.5, "epoch": 0.84, "frac_reward_zero_std": 0.5, "grad_norm": 0.062068283557891846, "kl": 0.0, "learning_rate": 4.823368810567056e-05, "loss": -0.013048075139522552, "num_tokens": 1701455.0, "reward": 0.9390365481376648, "reward_std": 0.028855346143245697, "rewards/reward_fn/mean": 0.9390365481376648, "rewards/reward_fn/std": 0.028855346143245697, "step": 126, "step_time": 89.4614521729527 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 239.5625, "completions/clipped_ratio": 0.0, "completions/max_length": 247.5, "completions/max_terminated_length": 247.5, "completions/mean_length": 239.5625, "completions/mean_terminated_length": 239.5625, "completions/min_length": 201.5, "completions/min_terminated_length": 201.5, "epoch": 0.8466666666666667, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.817959636416969e-05, "loss": 0.0, "num_tokens": 1714064.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "step": 127, "step_time": 89.98527943901718 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 299.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 407.0, "completions/max_terminated_length": 407.0, "completions/mean_length": 299.6875, "completions/mean_terminated_length": 299.6875, "completions/min_length": 232.5, "completions/min_terminated_length": 232.5, "epoch": 0.8533333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 0.05948776379227638, "kl": 0.0, "learning_rate": 4.81247200794554e-05, "loss": 0.01649428904056549, "num_tokens": 1728195.0, "reward": 0.0764324888586998, "reward_std": 0.2988954664178891, "rewards/reward_fn/mean": 0.0764324888586998, "rewards/reward_fn/std": 0.2988954664178891, "step": 128, "step_time": 226.33545375603717 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 476.0, "completions/clipped_ratio": 0.0, "completions/max_length": 904.0, "completions/max_terminated_length": 904.0, "completions/mean_length": 476.0, "completions/mean_terminated_length": 476.0, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "epoch": 0.86, "frac_reward_zero_std": 0.0, "grad_norm": 0.10832423716783524, "kl": 0.0, "learning_rate": 4.806906110888606e-05, "loss": -0.07909408956766129, "num_tokens": 1745547.0, "reward": -0.04347759485244751, "reward_std": 0.7136133313179016, "rewards/reward_fn/mean": -0.04347759485244751, "rewards/reward_fn/std": 0.7136133313179016, "step": 129, "step_time": 249.15961863403209 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 195.0, "completions/clipped_ratio": 0.0, "completions/max_length": 222.5, "completions/max_terminated_length": 222.5, "completions/mean_length": 195.0, "completions/mean_terminated_length": 195.0, "completions/min_length": 162.0, "completions/min_terminated_length": 162.0, "epoch": 0.8666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.07652340829372406, "kl": 0.0, "learning_rate": 4.8012621336311016e-05, "loss": -0.023258736357092857, "num_tokens": 1759219.0, "reward": 0.5377570688724518, "reward_std": 0.21886634081602097, "rewards/reward_fn/mean": 0.5377570688724518, "rewards/reward_fn/std": 0.21886634081602097, "step": 130, "step_time": 77.46455274592154 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 870.0625, "completions/clipped_ratio": 0.1875, "completions/max_length": 1466.0, "completions/max_terminated_length": 1397.0, "completions/mean_length": 870.0625, "completions/mean_terminated_length": 654.8875122070312, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "epoch": 0.8733333333333333, "frac_reward_zero_std": 0.5, "grad_norm": 0.01964535377919674, "kl": 0.0, "learning_rate": 4.7955402672006854e-05, "loss": 0.19607774913311005, "num_tokens": 1782388.0, "reward": 0.2218644618988037, "reward_std": 0.6362914443016052, "rewards/reward_fn/mean": 0.2218644618988037, "rewards/reward_fn/std": 0.6362914443016052, "step": 131, "step_time": 409.15760393301025 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 352.25, "completions/clipped_ratio": 0.0, "completions/max_length": 441.0, "completions/max_terminated_length": 441.0, "completions/mean_length": 352.25, "completions/mean_terminated_length": 352.25, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "epoch": 0.88, "frac_reward_zero_std": 0.5, "grad_norm": 0.04634862020611763, "kl": 0.0, "learning_rate": 4.789740705261278e-05, "loss": 0.0035696476697921753, "num_tokens": 1797352.0, "reward": 0.5494393706321716, "reward_std": 0.2591811418533325, "rewards/reward_fn/mean": 0.5494393706321716, "rewards/reward_fn/std": 0.2591811418533325, "step": 132, "step_time": 164.5742819889565 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 352.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 600.0, "completions/max_terminated_length": 600.0, "completions/mean_length": 352.0625, "completions/mean_terminated_length": 352.0625, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "epoch": 0.8866666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 0.04685305058956146, "kl": 0.0, "learning_rate": 4.783863644106502e-05, "loss": -0.13369393348693848, "num_tokens": 1812129.0, "reward": 0.6904256045818329, "reward_std": 0.1820424646139145, "rewards/reward_fn/mean": 0.6904256045818329, "rewards/reward_fn/std": 0.1820424646139145, "step": 133, "step_time": 185.4885982349515 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 467.3125, "completions/clipped_ratio": 0.0, "completions/max_length": 832.5, "completions/max_terminated_length": 832.5, "completions/mean_length": 467.3125, "completions/mean_terminated_length": 467.3125, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "epoch": 0.8933333333333333, "frac_reward_zero_std": 0.5, "grad_norm": 0.05197684094309807, "kl": 0.0, "learning_rate": 4.777909282653042e-05, "loss": 0.02681293897330761, "num_tokens": 1828782.0, "reward": 0.6091441139578819, "reward_std": 0.21364624798297882, "rewards/reward_fn/mean": 0.6091441139578819, "rewards/reward_fn/std": 0.21364624798297882, "step": 134, "step_time": 412.91265689296415 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 368.375, "completions/clipped_ratio": 0.0, "completions/max_length": 580.5, "completions/max_terminated_length": 580.5, "completions/mean_length": 368.375, "completions/mean_terminated_length": 368.375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "epoch": 0.9, "frac_reward_zero_std": 0.0, "grad_norm": 0.1014743223786354, "kl": 0.0, "learning_rate": 4.771877822433911e-05, "loss": 0.07772844284772873, "num_tokens": 1844388.0, "reward": 0.7372333705425262, "reward_std": 0.02323738712834711, "rewards/reward_fn/mean": 0.7372333705425262, "rewards/reward_fn/std": 0.02323738712834711, "step": 135, "step_time": 168.2943005109555 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 364.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 605.0, "completions/max_terminated_length": 605.0, "completions/mean_length": 364.6875, "completions/mean_terminated_length": 364.6875, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "epoch": 0.9066666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 0.14220644533634186, "kl": 0.0, "learning_rate": 4.765769467591625e-05, "loss": 0.2252202183008194, "num_tokens": 1858895.0, "reward": 0.6875, "reward_std": 0.883883535861969, "rewards/reward_fn/mean": 0.6875, "rewards/reward_fn/std": 0.883883535861969, "step": 136, "step_time": 171.65731632098323 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 199.625, "completions/clipped_ratio": 0.0, "completions/max_length": 243.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 199.625, "completions/mean_terminated_length": 199.625, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "epoch": 0.9133333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.05413385108113289, "kl": 0.0, "learning_rate": 4.759584424871302e-05, "loss": -0.02340317703783512, "num_tokens": 1872657.0, "reward": 0.6003958731889725, "reward_std": 0.10894697532057762, "rewards/reward_fn/mean": 0.6003958731889725, "rewards/reward_fn/std": 0.10894697532057762, "step": 137, "step_time": 82.92960694391513 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 580.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 983.5, "completions/max_terminated_length": 983.5, "completions/mean_length": 580.9375, "completions/mean_terminated_length": 580.9375, "completions/min_length": 338.5, "completions/min_terminated_length": 338.5, "epoch": 0.92, "frac_reward_zero_std": 0.0, "grad_norm": 0.038850050419569016, "kl": 0.0, "learning_rate": 4.7533229036136553e-05, "loss": -0.07479565590620041, "num_tokens": 1892224.0, "reward": 0.30399832874536514, "reward_std": 0.08702933220774867, "rewards/reward_fn/mean": 0.30399832874536514, "rewards/reward_fn/std": 0.08702933220774867, "step": 138, "step_time": 276.3797710710205 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 335.75, "completions/clipped_ratio": 0.0, "completions/max_length": 382.0, "completions/max_terminated_length": 382.0, "completions/mean_length": 335.75, "completions/mean_terminated_length": 335.75, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "epoch": 0.9266666666666666, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.7469851157479177e-05, "loss": 0.0, "num_tokens": 1906212.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "step": 139, "step_time": 115.95142895297613 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 247.1875, "completions/clipped_ratio": 0.0, "completions/max_length": 251.5, "completions/max_terminated_length": 251.5, "completions/mean_length": 247.1875, "completions/mean_terminated_length": 247.1875, "completions/min_length": 233.5, "completions/min_terminated_length": 233.5, "epoch": 0.9333333333333333, "frac_reward_zero_std": 0.5, "grad_norm": 0.011438372544944286, "kl": 0.0, "learning_rate": 4.740571275784659e-05, "loss": -3.4224241971969604e-05, "num_tokens": 1918951.0, "reward": 0.5426880717277527, "reward_std": 1.0973985808959696e-05, "rewards/reward_fn/mean": 0.5426880717277527, "rewards/reward_fn/std": 1.0973985808959696e-05, "step": 140, "step_time": 95.01890190399718 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 521.0, "completions/clipped_ratio": 0.0, "completions/max_length": 757.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 521.0, "completions/mean_terminated_length": 521.0, "completions/min_length": 299.5, "completions/min_terminated_length": 299.5, "epoch": 0.94, "frac_reward_zero_std": 0.0, "grad_norm": 0.024509411305189133, "kl": 0.0, "learning_rate": 4.734081600808531e-05, "loss": -0.09396696835756302, "num_tokens": 1936599.0, "reward": 0.41570543497800827, "reward_std": 0.3383743853873966, "rewards/reward_fn/mean": 0.41570543497800827, "rewards/reward_fn/std": 0.3383743853873966, "step": 141, "step_time": 234.9987866890151 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 233.5, "completions/clipped_ratio": 0.0, "completions/max_length": 250.5, "completions/max_terminated_length": 250.5, "completions/mean_length": 233.5, "completions/mean_terminated_length": 233.5, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "epoch": 0.9466666666666667, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.72751631047092e-05, "loss": 0.0, "num_tokens": 1949183.0, "reward": 0.5619802176952362, "reward_std": 0.0, "rewards/reward_fn/mean": 0.5619802176952362, "rewards/reward_fn/std": 0.0, "step": 142, "step_time": 90.94075334299123 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 423.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 761.5, "completions/max_terminated_length": 761.5, "completions/mean_length": 423.0625, "completions/mean_terminated_length": 423.0625, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "epoch": 0.9533333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 0.1250963658094406, "kl": 0.0, "learning_rate": 4.7208756269825104e-05, "loss": 0.12352190911769867, "num_tokens": 1966080.0, "reward": 0.34782883524894714, "reward_std": 0.6882529407739639, "rewards/reward_fn/mean": 0.34782883524894714, "rewards/reward_fn/std": 0.6882529407739639, "step": 143, "step_time": 226.21472980699036 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 208.125, "completions/clipped_ratio": 0.0, "completions/max_length": 216.5, "completions/max_terminated_length": 216.5, "completions/mean_length": 208.125, "completions/mean_terminated_length": 208.125, "completions/min_length": 194.5, "completions/min_terminated_length": 194.5, "epoch": 0.96, "frac_reward_zero_std": 0.5, "grad_norm": 0.04892691969871521, "kl": 0.0, "learning_rate": 4.714159775105765e-05, "loss": -0.016497205942869186, "num_tokens": 1979202.0, "reward": 0.7357056438922882, "reward_std": 0.09705093502998352, "rewards/reward_fn/mean": 0.7357056438922882, "rewards/reward_fn/std": 0.09705093502998352, "step": 144, "step_time": 84.11677284108009 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 203.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 216.5, "completions/max_terminated_length": 216.5, "completions/mean_length": 203.6875, "completions/mean_terminated_length": 203.6875, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "epoch": 0.9666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.05953475832939148, "kl": 0.0, "learning_rate": 4.707368982147318e-05, "loss": -0.014895366504788399, "num_tokens": 1992269.0, "reward": 0.324895441532135, "reward_std": 0.049588216134793583, "rewards/reward_fn/mean": 0.324895441532135, "rewards/reward_fn/std": 0.049588216134793583, "step": 145, "step_time": 84.07973394100554 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 257.5, "completions/clipped_ratio": 0.0, "completions/max_length": 295.0, "completions/max_terminated_length": 295.0, "completions/mean_length": 257.5, "completions/mean_terminated_length": 257.5, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "epoch": 0.9733333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 0.06280596554279327, "kl": 0.0, "learning_rate": 4.700503477950278e-05, "loss": 0.003337457776069641, "num_tokens": 2007037.0, "reward": 0.49471792578697205, "reward_std": 0.21104929596185684, "rewards/reward_fn/mean": 0.49471792578697205, "rewards/reward_fn/std": 0.21104929596185684, "step": 146, "step_time": 96.53027527104132 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 498.8125, "completions/clipped_ratio": 0.0, "completions/max_length": 731.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 498.8125, "completions/mean_terminated_length": 498.8125, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "epoch": 0.98, "frac_reward_zero_std": 0.5, "grad_norm": 0.034298237413167953, "kl": 0.0, "learning_rate": 4.693563494886455e-05, "loss": -0.06357244402170181, "num_tokens": 2024218.0, "reward": 0.9669809341430664, "reward_std": 0.04393957555294037, "rewards/reward_fn/mean": 0.9669809341430664, "rewards/reward_fn/std": 0.04393957555294037, "step": 147, "step_time": 206.35053608001908 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 312.4375, "completions/clipped_ratio": 0.0, "completions/max_length": 378.5, "completions/max_terminated_length": 378.5, "completions/mean_length": 312.4375, "completions/mean_terminated_length": 312.4375, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "epoch": 0.9866666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 5.0839618779718876e-05, "kl": 0.0, "learning_rate": 4.6865492678484895e-05, "loss": 0.00010071517317555845, "num_tokens": 2038105.0, "reward": 0.5656213238835335, "reward_std": 4.5056875563886933e-08, "rewards/reward_fn/mean": 0.5656213238835335, "rewards/reward_fn/std": 4.5056875563886933e-08, "step": 148, "step_time": 124.25263723096577 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 567.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 914.5, "completions/max_terminated_length": 914.5, "completions/mean_length": 567.9375, "completions/mean_terminated_length": 567.9375, "completions/min_length": 354.5, "completions/min_terminated_length": 354.5, "epoch": 0.9933333333333333, "frac_reward_zero_std": 0.5, "grad_norm": 0.040333859622478485, "kl": 0.0, "learning_rate": 4.679461034241906e-05, "loss": -0.033196814358234406, "num_tokens": 2056408.0, "reward": 0.7618521153926849, "reward_std": 0.45696911215782166, "rewards/reward_fn/mean": 0.7618521153926849, "rewards/reward_fn/std": 0.45696911215782166, "step": 149, "step_time": 253.0286384250503 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 252.1875, "completions/clipped_ratio": 0.0, "completions/max_length": 302.5, "completions/max_terminated_length": 302.5, "completions/mean_length": 252.1875, "completions/mean_terminated_length": 252.1875, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "epoch": 1.0, "frac_reward_zero_std": 0.0, "grad_norm": 0.06843557208776474, "kl": 0.0, "learning_rate": 4.672299033977076e-05, "loss": 0.05354718863964081, "num_tokens": 2071083.0, "reward": 0.6244544088840485, "reward_std": 0.09683120995759964, "rewards/reward_fn/mean": 0.6244544088840485, "rewards/reward_fn/std": 0.09683120995759964, "step": 150, "step_time": 98.54597930196906 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 329.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 382.5, "completions/max_terminated_length": 382.5, "completions/mean_length": 329.9375, "completions/mean_terminated_length": 329.9375, "completions/min_length": 274.5, "completions/min_terminated_length": 274.5, "epoch": 1.0066666666666666, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.665063509461097e-05, "loss": 0.0, "num_tokens": 2085242.0, "reward": 0.9492384493350983, "reward_std": 0.0, "rewards/reward_fn/mean": 0.9492384493350983, "rewards/reward_fn/std": 0.0, "step": 151, "step_time": 125.676316802972 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 773.125, "completions/clipped_ratio": 0.0, "completions/max_length": 1563.5, "completions/max_terminated_length": 1563.5, "completions/mean_length": 773.125, "completions/mean_terminated_length": 773.125, "completions/min_length": 421.5, "completions/min_terminated_length": 421.5, "epoch": 1.0133333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 0.026009313762187958, "kl": 0.0, "learning_rate": 4.657754705589591e-05, "loss": -0.1481044441461563, "num_tokens": 2107260.0, "reward": 0.8665153086185455, "reward_std": 0.24713269983112696, "rewards/reward_fn/mean": 0.8665153086185455, "rewards/reward_fn/std": 0.24713269983112696, "step": 152, "step_time": 419.2542784839752 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 453.875, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 453.875, "completions/mean_terminated_length": 453.875, "completions/min_length": 310.5, "completions/min_terminated_length": 310.5, "epoch": 1.02, "frac_reward_zero_std": 0.5, "grad_norm": 0.00012790317123290151, "kl": 0.0, "learning_rate": 4.650372869738414e-05, "loss": -0.00028114268206991255, "num_tokens": 2123154.0, "reward": 0.9999999403953552, "reward_std": 1.2744008870413381e-07, "rewards/reward_fn/mean": 0.9999999403953552, "rewards/reward_fn/std": 1.2744008870413381e-07, "step": 153, "step_time": 161.3001378289773 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 255.3125, "completions/clipped_ratio": 0.0, "completions/max_length": 278.0, "completions/max_terminated_length": 278.0, "completions/mean_length": 255.3125, "completions/mean_terminated_length": 255.3125, "completions/min_length": 220.5, "completions/min_terminated_length": 220.5, "epoch": 1.0266666666666666, "frac_reward_zero_std": 0.5, "grad_norm": 0.0790843740105629, "kl": 0.0, "learning_rate": 4.642918251755281e-05, "loss": 0.010872147977352142, "num_tokens": 2136751.0, "reward": 0.695651650428772, "reward_std": 0.1743757575750351, "rewards/reward_fn/mean": 0.695651650428772, "rewards/reward_fn/std": 0.1743757575750351, "step": 154, "step_time": 90.34915228991304 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 333.5, "completions/clipped_ratio": 0.0, "completions/max_length": 378.0, "completions/max_terminated_length": 378.0, "completions/mean_length": 333.5, "completions/mean_terminated_length": 333.5, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "epoch": 1.0333333333333334, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.6353911039513145e-05, "loss": 0.0, "num_tokens": 2150983.0, "reward": 0.9483389258384705, "reward_std": 0.0, "rewards/reward_fn/mean": 0.9483389258384705, "rewards/reward_fn/std": 0.0, "step": 155, "step_time": 128.63163546600845 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 602.25, "completions/clipped_ratio": 0.0, "completions/max_length": 1006.5, "completions/max_terminated_length": 1006.5, "completions/mean_length": 602.25, "completions/mean_terminated_length": 602.25, "completions/min_length": 456.0, "completions/min_terminated_length": 456.0, "epoch": 1.04, "frac_reward_zero_std": 0.0, "grad_norm": 0.05929885059595108, "kl": 0.0, "learning_rate": 4.627791681092499e-05, "loss": -0.05129897594451904, "num_tokens": 2170827.0, "reward": 0.5733692049980164, "reward_std": 0.2317853346467018, "rewards/reward_fn/mean": 0.5733692049980164, "rewards/reward_fn/std": 0.2317853346467018, "step": 156, "step_time": 285.2149426349206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 537.0, "completions/clipped_ratio": 0.0, "completions/max_length": 688.0, "completions/max_terminated_length": 688.0, "completions/mean_length": 537.0, "completions/mean_terminated_length": 537.0, "completions/min_length": 419.5, "completions/min_terminated_length": 419.5, "epoch": 1.0466666666666666, "frac_reward_zero_std": 0.5, "grad_norm": 0.036035988479852676, "kl": 0.0, "learning_rate": 4.620120240391065e-05, "loss": -0.06904249638319016, "num_tokens": 2188515.0, "reward": 0.9940311908721924, "reward_std": 0.01009331364184618, "rewards/reward_fn/mean": 0.9940311908721924, "rewards/reward_fn/std": 0.01009331364184618, "step": 157, "step_time": 206.95500113192247 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 209.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 210.0, "completions/max_terminated_length": 210.0, "completions/mean_length": 209.6875, "completions/mean_terminated_length": 209.6875, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "epoch": 1.0533333333333332, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.612377041496776e-05, "loss": 0.0, "num_tokens": 2200886.0, "reward": 0.5638016313314438, "reward_std": 0.0, "rewards/reward_fn/mean": 0.5638016313314438, "rewards/reward_fn/std": 0.0, "step": 158, "step_time": 91.06828080205014 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 209.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 210.0, "completions/max_terminated_length": 210.0, "completions/mean_length": 209.9375, "completions/mean_terminated_length": 209.9375, "completions/min_length": 209.5, "completions/min_terminated_length": 209.5, "epoch": 1.06, "frac_reward_zero_std": 0.5, "grad_norm": 0.00034379371209070086, "kl": 0.0, "learning_rate": 4.604562346488144e-05, "loss": -3.690889570862055e-05, "num_tokens": 2213205.0, "reward": 0.5148106962442398, "reward_std": 4.6602090719716216e-07, "rewards/reward_fn/mean": 0.5148106962442398, "rewards/reward_fn/std": 4.6602090719716216e-07, "step": 159, "step_time": 91.57397655490786 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 654.3125, "completions/clipped_ratio": 0.0, "completions/max_length": 905.5, "completions/max_terminated_length": 905.5, "completions/mean_length": 654.3125, "completions/mean_terminated_length": 654.3125, "completions/min_length": 387.0, "completions/min_terminated_length": 387.0, "epoch": 1.0666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.02906476892530918, "kl": 0.0, "learning_rate": 4.5966764198635606e-05, "loss": 0.06294281035661697, "num_tokens": 2233018.0, "reward": 0.006258882582187653, "reward_std": 0.15943723261341347, "rewards/reward_fn/mean": 0.006258882582187653, "rewards/reward_fn/std": 0.15943723261341347, "step": 160, "step_time": 322.56884266098496 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 337.1875, "completions/clipped_ratio": 0.0, "completions/max_length": 500.5, "completions/max_terminated_length": 500.5, "completions/mean_length": 337.1875, "completions/mean_terminated_length": 337.1875, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "epoch": 1.0733333333333333, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.588719528532342e-05, "loss": 0.0, "num_tokens": 2247301.0, "reward": 0.5473424270749092, "reward_std": 0.0, "rewards/reward_fn/mean": 0.5473424270749092, "rewards/reward_fn/std": 0.0, "step": 161, "step_time": 156.99171748693334 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 703.9375, "completions/clipped_ratio": 0.0625, "completions/max_length": 1155.5, "completions/max_terminated_length": 913.0, "completions/mean_length": 703.9375, "completions/mean_terminated_length": 643.0357666015625, "completions/min_length": 487.0, "completions/min_terminated_length": 487.0, "epoch": 1.08, "frac_reward_zero_std": 0.0, "grad_norm": 0.066600002348423, "kl": 0.0, "learning_rate": 4.580691941805695e-05, "loss": 0.07555639743804932, "num_tokens": 2268652.0, "reward": 0.5355610847473145, "reward_std": 0.733004629611969, "rewards/reward_fn/mean": 0.5355610847473145, "rewards/reward_fn/std": 0.733004629611969, "step": 162, "step_time": 318.9592123919865 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 916.625, "completions/clipped_ratio": 0.125, "completions/max_length": 1129.0, "completions/max_terminated_length": 904.0, "completions/mean_length": 916.625, "completions/mean_terminated_length": 845.875, "completions/min_length": 699.5, "completions/min_terminated_length": 699.5, "epoch": 1.0866666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.015455679967999458, "kl": 0.0, "learning_rate": 4.572593931387604e-05, "loss": 0.0705941915512085, "num_tokens": 2292702.0, "reward": -0.2573578804731369, "reward_std": 0.42069763968612506, "rewards/reward_fn/mean": -0.2573578804731369, "rewards/reward_fn/std": 0.42069763968612506, "step": 163, "step_time": 332.97736496001016 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 331.5625, "completions/clipped_ratio": 0.0, "completions/max_length": 382.5, "completions/max_terminated_length": 382.5, "completions/mean_length": 331.5625, "completions/mean_terminated_length": 331.5625, "completions/min_length": 259.5, "completions/min_terminated_length": 259.5, "epoch": 1.0933333333333333, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.5644257713656356e-05, "loss": 0.0, "num_tokens": 2306919.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "step": 164, "step_time": 126.35907765099546 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 495.4375, "completions/clipped_ratio": 0.0, "completions/max_length": 933.0, "completions/max_terminated_length": 933.0, "completions/mean_length": 495.4375, "completions/mean_terminated_length": 495.4375, "completions/min_length": 323.5, "completions/min_terminated_length": 323.5, "epoch": 1.1, "frac_reward_zero_std": 0.0, "grad_norm": 0.05776069313287735, "kl": 0.0, "learning_rate": 4.556187738201656e-05, "loss": 0.006418749690055847, "num_tokens": 2324894.0, "reward": 0.7401622533798218, "reward_std": 0.410004697740078, "rewards/reward_fn/mean": 0.7401622533798218, "rewards/reward_fn/std": 0.410004697740078, "step": 165, "step_time": 318.11655589198926 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 368.5, "completions/clipped_ratio": 0.0, "completions/max_length": 424.0, "completions/max_terminated_length": 424.0, "completions/mean_length": 368.5, "completions/mean_terminated_length": 368.5, "completions/min_length": 297.5, "completions/min_terminated_length": 297.5, "epoch": 1.1066666666666667, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.54788011072248e-05, "loss": 0.0, "num_tokens": 2339502.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "step": 166, "step_time": 129.41105996404076 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 209.8125, "completions/clipped_ratio": 0.0, "completions/max_length": 210.0, "completions/max_terminated_length": 210.0, "completions/mean_length": 209.8125, "completions/mean_terminated_length": 209.8125, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "epoch": 1.1133333333333333, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.539503170110431e-05, "loss": 0.0, "num_tokens": 2351923.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "step": 167, "step_time": 96.18989022210008 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 608.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 873.5, "completions/max_terminated_length": 873.5, "completions/mean_length": 608.0625, "completions/mean_terminated_length": 608.0625, "completions/min_length": 387.0, "completions/min_terminated_length": 387.0, "epoch": 1.12, "frac_reward_zero_std": 0.5, "grad_norm": 0.0253182090818882, "kl": 0.0, "learning_rate": 4.531057199893824e-05, "loss": -0.0755884125828743, "num_tokens": 2371084.0, "reward": 0.6556649804115295, "reward_std": 0.1775762438774109, "rewards/reward_fn/mean": 0.6556649804115295, "rewards/reward_fn/std": 0.1775762438774109, "step": 168, "step_time": 257.9385768309585 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 351.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 391.5, "completions/max_terminated_length": 391.5, "completions/mean_length": 351.6875, "completions/mean_terminated_length": 351.6875, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "epoch": 1.1266666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 0.05709303170442581, "kl": 0.0, "learning_rate": 4.522542485937369e-05, "loss": -0.023601766675710678, "num_tokens": 2386415.0, "reward": 0.6813934743404388, "reward_std": 0.054284658282995224, "rewards/reward_fn/mean": 0.6813934743404388, "rewards/reward_fn/std": 0.054284658282995224, "step": 169, "step_time": 120.74896475899732 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 673.8125, "completions/clipped_ratio": 0.0, "completions/max_length": 923.5, "completions/max_terminated_length": 923.5, "completions/mean_length": 673.8125, "completions/mean_terminated_length": 673.8125, "completions/min_length": 403.5, "completions/min_terminated_length": 403.5, "epoch": 1.1333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.08081483095884323, "kl": 0.0, "learning_rate": 4.5139593164324986e-05, "loss": -0.06292446702718735, "num_tokens": 2407308.0, "reward": 0.5982428789138794, "reward_std": 0.30377645045518875, "rewards/reward_fn/mean": 0.5982428789138794, "rewards/reward_fn/std": 0.30377645045518875, "step": 170, "step_time": 268.2763004680164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 233.4375, "completions/clipped_ratio": 0.0, "completions/max_length": 234.0, "completions/max_terminated_length": 234.0, "completions/mean_length": 233.4375, "completions/mean_terminated_length": 233.4375, "completions/min_length": 232.5, "completions/min_terminated_length": 232.5, "epoch": 1.1400000000000001, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.50530798188761e-05, "loss": 0.0, "num_tokens": 2419827.0, "reward": 0.9509938955307007, "reward_std": 0.0, "rewards/reward_fn/mean": 0.9509938955307007, "rewards/reward_fn/std": 0.0, "step": 171, "step_time": 91.46864466898842 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 209.8125, "completions/clipped_ratio": 0.0, "completions/max_length": 210.0, "completions/max_terminated_length": 210.0, "completions/mean_length": 209.8125, "completions/mean_terminated_length": 209.8125, "completions/min_length": 209.5, "completions/min_terminated_length": 209.5, "epoch": 1.1466666666666667, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.496588775118232e-05, "loss": 0.0, "num_tokens": 2432200.0, "reward": 0.8966733813285828, "reward_std": 0.0, "rewards/reward_fn/mean": 0.8966733813285828, "rewards/reward_fn/std": 0.0, "step": 172, "step_time": 91.63760911300778 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 316.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.0, "completions/max_terminated_length": 319.0, "completions/mean_length": 316.0, "completions/mean_terminated_length": 316.0, "completions/min_length": 297.5, "completions/min_terminated_length": 297.5, "epoch": 1.1533333333333333, "frac_reward_zero_std": 0.5, "grad_norm": 0.036608047783374786, "kl": 0.0, "learning_rate": 4.48780199123712e-05, "loss": 0.00680706650018692, "num_tokens": 2446848.0, "reward": 0.619898796081543, "reward_std": 0.09930147230625153, "rewards/reward_fn/mean": 0.619898796081543, "rewards/reward_fn/std": 0.09930147230625153, "step": 173, "step_time": 102.21163495199289 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 724.875, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 724.875, "completions/mean_terminated_length": 724.875, "completions/min_length": 393.0, "completions/min_terminated_length": 393.0, "epoch": 1.16, "frac_reward_zero_std": 0.5, "grad_norm": 0.03367030993103981, "kl": 0.0, "learning_rate": 4.478947927644258e-05, "loss": 0.06719475984573364, "num_tokens": 2467862.0, "reward": 0.277934730052948, "reward_std": 0.3036574125289917, "rewards/reward_fn/mean": 0.277934730052948, "rewards/reward_fn/std": 0.3036574125289917, "step": 174, "step_time": 317.1425336949178 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 339.25, "completions/clipped_ratio": 0.0, "completions/max_length": 379.0, "completions/max_terminated_length": 379.0, "completions/mean_length": 339.25, "completions/mean_terminated_length": 339.25, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "epoch": 1.1666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.07057405263185501, "kl": 0.0, "learning_rate": 4.4700268840168045e-05, "loss": -0.013459604233503342, "num_tokens": 2483850.0, "reward": 0.4681895226240158, "reward_std": 0.23409754782915115, "rewards/reward_fn/mean": 0.4681895226240158, "rewards/reward_fn/std": 0.23409754782915115, "step": 175, "step_time": 119.56902152101975 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 435.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 508.5, "completions/max_terminated_length": 508.5, "completions/mean_length": 435.9375, "completions/mean_terminated_length": 435.9375, "completions/min_length": 390.5, "completions/min_terminated_length": 390.5, "epoch": 1.1733333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.07291749119758606, "kl": 0.0, "learning_rate": 4.4610391622989396e-05, "loss": 0.01635035127401352, "num_tokens": 2501393.0, "reward": 0.4043854922056198, "reward_std": 0.7844874858856201, "rewards/reward_fn/mean": 0.4043854922056198, "rewards/reward_fn/std": 0.7844874858856201, "step": 176, "step_time": 152.86159872601274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 385.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 490.5, "completions/max_terminated_length": 490.5, "completions/mean_length": 385.9375, "completions/mean_terminated_length": 385.9375, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "epoch": 1.18, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.4519850666916484e-05, "loss": 0.0, "num_tokens": 2516200.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "step": 177, "step_time": 144.09567704610527 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 482.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 489.0, "completions/max_terminated_length": 489.0, "completions/mean_length": 482.0625, "completions/mean_terminated_length": 482.0625, "completions/min_length": 479.5, "completions/min_terminated_length": 479.5, "epoch": 1.1866666666666668, "frac_reward_zero_std": 0.0, "grad_norm": 0.054842259734869, "kl": 0.0, "learning_rate": 4.442864903642428e-05, "loss": 0.004520110785961151, "num_tokens": 2534569.0, "reward": 0.48581717908382416, "reward_std": 0.5444052815437317, "rewards/reward_fn/mean": 0.48581717908382416, "rewards/reward_fn/std": 0.5444052815437317, "step": 178, "step_time": 148.80164092604537 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 209.5625, "completions/clipped_ratio": 0.0, "completions/max_length": 210.0, "completions/max_terminated_length": 210.0, "completions/mean_length": 209.5625, "completions/mean_terminated_length": 209.5625, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "epoch": 1.1933333333333334, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.43367898183491e-05, "loss": 0.0, "num_tokens": 2546914.0, "reward": 0.5261223241686821, "reward_std": 0.0, "rewards/reward_fn/mean": 0.5261223241686821, "rewards/reward_fn/std": 0.0, "step": 179, "step_time": 92.55844796006568 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 509.875, "completions/clipped_ratio": 0.0, "completions/max_length": 643.5, "completions/max_terminated_length": 643.5, "completions/mean_length": 509.875, "completions/mean_terminated_length": 509.875, "completions/min_length": 426.0, "completions/min_terminated_length": 426.0, "epoch": 1.2, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.4244276121784195e-05, "loss": 0.0, "num_tokens": 2563760.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "step": 180, "step_time": 184.21834510291228 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 372.75, "completions/clipped_ratio": 0.0, "completions/max_length": 387.5, "completions/max_terminated_length": 387.5, "completions/mean_length": 372.75, "completions/mean_terminated_length": 372.75, "completions/min_length": 369.0, "completions/min_terminated_length": 369.0, "epoch": 1.2066666666666666, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.415111107797445e-05, "loss": 0.0, "num_tokens": 2578380.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "step": 181, "step_time": 119.07426050008507 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 380.75, "completions/clipped_ratio": 0.0, "completions/max_length": 490.5, "completions/max_terminated_length": 490.5, "completions/mean_length": 380.75, "completions/mean_terminated_length": 380.75, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "epoch": 1.2133333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 0.10535330325365067, "kl": 0.0, "learning_rate": 4.405729784021046e-05, "loss": -0.029865777119994164, "num_tokens": 2593112.0, "reward": 0.625, "reward_std": 0.816463440656662, "rewards/reward_fn/mean": 0.625, "rewards/reward_fn/std": 0.816463440656662, "step": 182, "step_time": 144.44632358610397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 866.625, "completions/clipped_ratio": 0.125, "completions/max_length": 1213.0, "completions/max_terminated_length": 1204.0, "completions/mean_length": 866.625, "completions/mean_terminated_length": 758.5416870117188, "completions/min_length": 523.0, "completions/min_terminated_length": 523.0, "epoch": 1.22, "frac_reward_zero_std": 0.0, "grad_norm": 0.03957837074995041, "kl": 0.0, "learning_rate": 4.396283958372173e-05, "loss": 0.08437013626098633, "num_tokens": 2617154.0, "reward": -0.035997092723846436, "reward_std": 0.7678898870944977, "rewards/reward_fn/mean": -0.035997092723846436, "rewards/reward_fn/std": 0.7678898870944977, "step": 183, "step_time": 338.4485408520559 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 358.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 429.5, "completions/max_terminated_length": 429.5, "completions/mean_length": 358.9375, "completions/mean_terminated_length": 358.9375, "completions/min_length": 311.5, "completions/min_terminated_length": 311.5, "epoch": 1.2266666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 0.0457036979496479, "kl": 0.0, "learning_rate": 4.386773950556931e-05, "loss": 0.04372432082891464, "num_tokens": 2633545.0, "reward": 0.32335518300533295, "reward_std": 0.10355385020375252, "rewards/reward_fn/mean": 0.32335518300533295, "rewards/reward_fn/std": 0.10355385020375252, "step": 184, "step_time": 132.418771808967 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 559.0, "completions/clipped_ratio": 0.0, "completions/max_length": 729.5, "completions/max_terminated_length": 729.5, "completions/mean_length": 559.0, "completions/mean_terminated_length": 559.0, "completions/min_length": 411.0, "completions/min_terminated_length": 411.0, "epoch": 1.2333333333333334, "frac_reward_zero_std": 0.5, "grad_norm": 0.04097546264529228, "kl": 0.0, "learning_rate": 4.377200082453749e-05, "loss": 0.010574601590633392, "num_tokens": 2651889.0, "reward": 0.8245929479598999, "reward_std": 0.12071260064840317, "rewards/reward_fn/mean": 0.8245929479598999, "rewards/reward_fn/std": 0.12071260064840317, "step": 185, "step_time": 216.6957085240283 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 261.625, "completions/clipped_ratio": 0.0, "completions/max_length": 275.0, "completions/max_terminated_length": 275.0, "completions/mean_length": 261.625, "completions/mean_terminated_length": 261.625, "completions/min_length": 253.5, "completions/min_terminated_length": 253.5, "epoch": 1.24, "frac_reward_zero_std": 0.0, "grad_norm": 0.05110285058617592, "kl": 0.0, "learning_rate": 4.36756267810249e-05, "loss": -0.0024216175079345703, "num_tokens": 2665851.0, "reward": 0.3784840330481529, "reward_std": 0.10511010140180588, "rewards/reward_fn/mean": 0.3784840330481529, "rewards/reward_fn/std": 0.10511010140180588, "step": 186, "step_time": 103.37580211204477 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 316.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 372.0, "completions/max_terminated_length": 372.0, "completions/mean_length": 316.9375, "completions/mean_terminated_length": 316.9375, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "epoch": 1.2466666666666666, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.357862063693486e-05, "loss": 0.0, "num_tokens": 2679610.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "step": 187, "step_time": 113.27226198796416 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 361.875, "completions/clipped_ratio": 0.0, "completions/max_length": 427.5, "completions/max_terminated_length": 427.5, "completions/mean_length": 361.875, "completions/mean_terminated_length": 361.875, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "epoch": 1.2533333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 0.0703757181763649, "kl": 0.0, "learning_rate": 4.34809856755649e-05, "loss": 0.01859419420361519, "num_tokens": 2694040.0, "reward": 0.9870609939098358, "reward_std": 0.036594912206055596, "rewards/reward_fn/mean": 0.9870609939098358, "rewards/reward_fn/std": 0.036594912206055596, "step": 188, "step_time": 128.19551112595946 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 361.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 449.5, "completions/max_terminated_length": 449.5, "completions/mean_length": 361.9375, "completions/mean_terminated_length": 361.9375, "completions/min_length": 294.5, "completions/min_terminated_length": 294.5, "epoch": 1.26, "frac_reward_zero_std": 0.5, "grad_norm": 0.06226472556591034, "kl": 0.0, "learning_rate": 4.3382725201495723e-05, "loss": 0.0047224462032318115, "num_tokens": 2709535.0, "reward": 0.6252674013376236, "reward_std": 0.1955031156539917, "rewards/reward_fn/mean": 0.6252674013376236, "rewards/reward_fn/std": 0.1955031156539917, "step": 189, "step_time": 135.10929551004665 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 761.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 1120.5, "completions/max_terminated_length": 1120.5, "completions/mean_length": 761.6875, "completions/mean_terminated_length": 761.6875, "completions/min_length": 444.0, "completions/min_terminated_length": 444.0, "epoch": 1.2666666666666666, "frac_reward_zero_std": 0.5, "grad_norm": 0.0425034761428833, "kl": 0.0, "learning_rate": 4.3283842540479264e-05, "loss": -0.004420645534992218, "num_tokens": 2731914.0, "reward": 0.17015497386455536, "reward_std": 0.03678224980831146, "rewards/reward_fn/mean": 0.17015497386455536, "rewards/reward_fn/std": 0.03678224980831146, "step": 190, "step_time": 313.89839813805884 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 301.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 371.0, "completions/max_terminated_length": 371.0, "completions/mean_length": 301.0625, "completions/mean_terminated_length": 301.0625, "completions/min_length": 253.5, "completions/min_terminated_length": 253.5, "epoch": 1.2733333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 0.09773189574480057, "kl": 0.0, "learning_rate": 4.318434103932622e-05, "loss": 0.08142915368080139, "num_tokens": 2747275.0, "reward": 0.6540311872959137, "reward_std": 0.19406485185027122, "rewards/reward_fn/mean": 0.6540311872959137, "rewards/reward_fn/std": 0.19406485185027122, "step": 191, "step_time": 116.78903921408346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 482.0, "completions/clipped_ratio": 0.0, "completions/max_length": 730.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 482.0, "completions/mean_terminated_length": 482.0, "completions/min_length": 296.5, "completions/min_terminated_length": 296.5, "epoch": 1.28, "frac_reward_zero_std": 0.5, "grad_norm": 0.028390681371092796, "kl": 0.0, "learning_rate": 4.30842240657927e-05, "loss": 0.003453485667705536, "num_tokens": 2764083.0, "reward": 0.8803534507751465, "reward_std": 0.3384115397930145, "rewards/reward_fn/mean": 0.8803534507751465, "rewards/reward_fn/std": 0.3384115397930145, "step": 192, "step_time": 211.86879105505068 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 365.5625, "completions/clipped_ratio": 0.0, "completions/max_length": 472.0, "completions/max_terminated_length": 472.0, "completions/mean_length": 365.5625, "completions/mean_terminated_length": 365.5625, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "epoch": 1.2866666666666666, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.2983495008466276e-05, "loss": 0.0, "num_tokens": 2778700.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "step": 193, "step_time": 138.9105662080692 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 225.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 236.5, "completions/max_terminated_length": 236.5, "completions/mean_length": 225.6875, "completions/mean_terminated_length": 225.6875, "completions/min_length": 214.5, "completions/min_terminated_length": 214.5, "epoch": 1.2933333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 0.5753271579742432, "kl": 0.0, "learning_rate": 4.288215727665129e-05, "loss": -0.010599890723824501, "num_tokens": 2792055.0, "reward": 0.49704940617084503, "reward_std": 0.031217220712960625, "rewards/reward_fn/mean": 0.49704940617084503, "rewards/reward_fn/std": 0.031217220712960625, "step": 194, "step_time": 89.62762422900414 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 389.1875, "completions/clipped_ratio": 0.0, "completions/max_length": 607.0, "completions/max_terminated_length": 607.0, "completions/mean_length": 389.1875, "completions/mean_terminated_length": 389.1875, "completions/min_length": 259.5, "completions/min_terminated_length": 259.5, "epoch": 1.3, "frac_reward_zero_std": 0.5, "grad_norm": 0.05310267210006714, "kl": 0.0, "learning_rate": 4.278021430025343e-05, "loss": -0.12552142143249512, "num_tokens": 2807642.0, "reward": 0.3906529098749161, "reward_std": 0.24650515615940094, "rewards/reward_fn/mean": 0.3906529098749161, "rewards/reward_fn/std": 0.24650515615940094, "step": 195, "step_time": 185.09018681605812 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 385.3125, "completions/clipped_ratio": 0.0, "completions/max_length": 501.5, "completions/max_terminated_length": 501.5, "completions/mean_length": 385.3125, "completions/mean_terminated_length": 385.3125, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "epoch": 1.3066666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 0.027251476421952248, "kl": 0.0, "learning_rate": 4.267766952966369e-05, "loss": -0.006735360249876976, "num_tokens": 2823903.0, "reward": 0.9465951919555664, "reward_std": 0.04546740645120906, "rewards/reward_fn/mean": 0.9465951919555664, "rewards/reward_fn/std": 0.04546740645120906, "step": 196, "step_time": 148.99407557304949 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 627.375, "completions/clipped_ratio": 0.0, "completions/max_length": 968.0, "completions/max_terminated_length": 968.0, "completions/mean_length": 627.375, "completions/mean_terminated_length": 627.375, "completions/min_length": 425.0, "completions/min_terminated_length": 425.0, "epoch": 1.3133333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 0.06101449579000473, "kl": 0.0, "learning_rate": 4.257452643564155e-05, "loss": -0.060674093663692474, "num_tokens": 2843725.0, "reward": 0.4497956559062004, "reward_std": 0.6334844715893269, "rewards/reward_fn/mean": 0.4497956559062004, "rewards/reward_fn/std": 0.6334844715893269, "step": 197, "step_time": 271.5556913619512 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 466.8125, "completions/clipped_ratio": 0.0, "completions/max_length": 702.5, "completions/max_terminated_length": 702.5, "completions/mean_length": 466.8125, "completions/mean_terminated_length": 466.8125, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "epoch": 1.32, "frac_reward_zero_std": 0.5, "grad_norm": 0.05101463943719864, "kl": 0.0, "learning_rate": 4.24707885091975e-05, "loss": -0.07410165667533875, "num_tokens": 2860386.0, "reward": 0.7855673432350159, "reward_std": 0.14362023770809174, "rewards/reward_fn/mean": 0.7855673432350159, "rewards/reward_fn/std": 0.14362023770809174, "step": 198, "step_time": 198.9184314039885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 272.125, "completions/clipped_ratio": 0.0, "completions/max_length": 312.0, "completions/max_terminated_length": 312.0, "completions/mean_length": 272.125, "completions/mean_terminated_length": 272.125, "completions/min_length": 241.5, "completions/min_terminated_length": 241.5, "epoch": 1.3266666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 0.03716262802481651, "kl": 0.0, "learning_rate": 4.2366459261474933e-05, "loss": -0.00589386373758316, "num_tokens": 2874348.0, "reward": 0.9446012079715729, "reward_std": 0.022661298513412476, "rewards/reward_fn/mean": 0.9446012079715729, "rewards/reward_fn/std": 0.022661298513412476, "step": 199, "step_time": 99.15163314703386 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 247.1875, "completions/clipped_ratio": 0.0, "completions/max_length": 276.5, "completions/max_terminated_length": 276.5, "completions/mean_length": 247.1875, "completions/mean_terminated_length": 247.1875, "completions/min_length": 226.5, "completions/min_terminated_length": 226.5, "epoch": 1.3333333333333333, "frac_reward_zero_std": 0.5, "grad_norm": 0.05213377997279167, "kl": 0.0, "learning_rate": 4.226154222363124e-05, "loss": 0.055981483310461044, "num_tokens": 2888167.0, "reward": 0.822132408618927, "reward_std": 0.1446933150291443, "rewards/reward_fn/mean": 0.822132408618927, "rewards/reward_fn/std": 0.1446933150291443, "step": 200, "step_time": 99.35902613506187 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 394.25, "completions/clipped_ratio": 0.0, "completions/max_length": 477.5, "completions/max_terminated_length": 477.5, "completions/mean_length": 394.25, "completions/mean_terminated_length": 394.25, "completions/min_length": 305.5, "completions/min_terminated_length": 305.5, "epoch": 1.34, "frac_reward_zero_std": 0.0, "grad_norm": 0.07755188643932343, "kl": 0.0, "learning_rate": 4.215604094671835e-05, "loss": -0.05246041715145111, "num_tokens": 2904579.0, "reward": 0.5525769889354706, "reward_std": 0.588111624121666, "rewards/reward_fn/mean": 0.5525769889354706, "rewards/reward_fn/std": 0.588111624121666, "step": 201, "step_time": 144.34619156294502 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 401.75, "completions/clipped_ratio": 0.0, "completions/max_length": 550.5, "completions/max_terminated_length": 550.5, "completions/mean_length": 401.75, "completions/mean_terminated_length": 401.75, "completions/min_length": 251.5, "completions/min_terminated_length": 251.5, "epoch": 1.3466666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 0.044257309287786484, "kl": 0.0, "learning_rate": 4.2049959001562464e-05, "loss": -0.0892098918557167, "num_tokens": 2920423.0, "reward": 0.39934873580932617, "reward_std": 0.12950681149959564, "rewards/reward_fn/mean": 0.39934873580932617, "rewards/reward_fn/std": 0.12950681149959564, "step": 202, "step_time": 169.23466102196835 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 267.875, "completions/clipped_ratio": 0.0, "completions/max_length": 288.0, "completions/max_terminated_length": 288.0, "completions/mean_length": 267.875, "completions/mean_terminated_length": 267.875, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "epoch": 1.3533333333333333, "frac_reward_zero_std": 0.5, "grad_norm": 0.09341531991958618, "kl": 0.0, "learning_rate": 4.194329997864331e-05, "loss": 1.2942723515152466e-07, "num_tokens": 2933605.0, "reward": 0.5648120641708374, "reward_std": 0.011593026109039783, "rewards/reward_fn/mean": 0.5648120641708374, "rewards/reward_fn/std": 0.011593026109039783, "step": 203, "step_time": 98.69075074000284 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 421.125, "completions/clipped_ratio": 0.0, "completions/max_length": 471.5, "completions/max_terminated_length": 471.5, "completions/mean_length": 421.125, "completions/mean_terminated_length": 421.125, "completions/min_length": 351.5, "completions/min_terminated_length": 351.5, "epoch": 1.3599999999999999, "frac_reward_zero_std": 0.0, "grad_norm": 0.06170891597867012, "kl": 0.0, "learning_rate": 4.183606748797251e-05, "loss": -0.03707977384328842, "num_tokens": 2950479.0, "reward": 0.4111311808228493, "reward_std": 0.44924869388341904, "rewards/reward_fn/mean": 0.4111311808228493, "rewards/reward_fn/std": 0.44924869388341904, "step": 204, "step_time": 148.0194004449877 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 263.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 278.5, "completions/max_terminated_length": 278.5, "completions/mean_length": 263.0625, "completions/mean_terminated_length": 263.0625, "completions/min_length": 222.5, "completions/min_terminated_length": 222.5, "epoch": 1.3666666666666667, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.172826515897146e-05, "loss": 0.0, "num_tokens": 2963288.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "step": 205, "step_time": 89.47718564799288 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 498.125, "completions/clipped_ratio": 0.0, "completions/max_length": 709.5, "completions/max_terminated_length": 709.5, "completions/mean_length": 498.125, "completions/mean_terminated_length": 498.125, "completions/min_length": 416.5, "completions/min_terminated_length": 416.5, "epoch": 1.3733333333333333, "frac_reward_zero_std": 0.5, "grad_norm": 0.03950470685958862, "kl": 0.0, "learning_rate": 4.1619896640348445e-05, "loss": 0.005682090297341347, "num_tokens": 2980506.0, "reward": 0.5262927748262882, "reward_std": 0.1411045640707016, "rewards/reward_fn/mean": 0.5262927748262882, "rewards/reward_fn/std": 0.1411045640707016, "step": 206, "step_time": 207.79324868600816 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 489.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 656.0, "completions/max_terminated_length": 656.0, "completions/mean_length": 489.6875, "completions/mean_terminated_length": 489.6875, "completions/min_length": 413.0, "completions/min_terminated_length": 413.0, "epoch": 1.38, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.1510965599975196e-05, "loss": 0.0, "num_tokens": 2997557.0, "reward": 0.8645179867744446, "reward_std": 0.0, "rewards/reward_fn/mean": 0.8645179867744446, "rewards/reward_fn/std": 0.0, "step": 207, "step_time": 187.55295675288653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 528.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 661.0, "completions/max_terminated_length": 661.0, "completions/mean_length": 528.0625, "completions/mean_terminated_length": 528.0625, "completions/min_length": 395.5, "completions/min_terminated_length": 395.5, "epoch": 1.3866666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 0.07827558368444443, "kl": 0.0, "learning_rate": 4.140147572476268e-05, "loss": 0.026687294244766235, "num_tokens": 3015694.0, "reward": 0.7765228152275085, "reward_std": 0.4247249960899353, "rewards/reward_fn/mean": 0.7765228152275085, "rewards/reward_fn/std": 0.4247249960899353, "step": 208, "step_time": 202.78064414602704 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 440.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 606.5, "completions/max_terminated_length": 606.5, "completions/mean_length": 440.0625, "completions/mean_terminated_length": 440.0625, "completions/min_length": 328.5, "completions/min_terminated_length": 328.5, "epoch": 1.3933333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.0835847333073616, "kl": 0.0, "learning_rate": 4.129143072053638e-05, "loss": -0.08566160500049591, "num_tokens": 3032919.0, "reward": 0.6553326100111008, "reward_std": 0.08002846711315215, "rewards/reward_fn/mean": 0.6553326100111008, "rewards/reward_fn/std": 0.08002846711315215, "step": 209, "step_time": 179.81165817193687 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 528.5, "completions/clipped_ratio": 0.0625, "completions/max_length": 1155.5, "completions/max_terminated_length": 654.5, "completions/mean_length": 528.5, "completions/mean_terminated_length": 441.4464416503906, "completions/min_length": 273.5, "completions/min_terminated_length": 273.5, "epoch": 1.4, "frac_reward_zero_std": 0.0, "grad_norm": 0.044825032353401184, "kl": 0.0, "learning_rate": 4.118083431191081e-05, "loss": 0.21945233643054962, "num_tokens": 3051503.0, "reward": 0.45718520134687424, "reward_std": 0.586285712197423, "rewards/reward_fn/mean": 0.45718520134687424, "rewards/reward_fn/std": 0.586285712197423, "step": 210, "step_time": 320.2217711000703 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 457.5625, "completions/clipped_ratio": 0.0, "completions/max_length": 808.0, "completions/max_terminated_length": 808.0, "completions/mean_length": 457.5625, "completions/mean_terminated_length": 457.5625, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "epoch": 1.4066666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 0.050018809735774994, "kl": 0.0, "learning_rate": 4.1069690242163484e-05, "loss": -0.009994201362133026, "num_tokens": 3068272.0, "reward": 0.3910246044397354, "reward_std": 0.43388164043426514, "rewards/reward_fn/mean": 0.3910246044397354, "rewards/reward_fn/std": 0.43388164043426514, "step": 211, "step_time": 242.25502372306073 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 519.625, "completions/clipped_ratio": 0.0, "completions/max_length": 772.0, "completions/max_terminated_length": 772.0, "completions/mean_length": 519.625, "completions/mean_terminated_length": 519.625, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "epoch": 1.4133333333333333, "frac_reward_zero_std": 0.5, "grad_norm": 0.03964097052812576, "kl": 0.0, "learning_rate": 4.095800227310821e-05, "loss": 0.08257991820573807, "num_tokens": 3085970.0, "reward": 0.43319354951381683, "reward_std": 0.19981928169727325, "rewards/reward_fn/mean": 0.43319354951381683, "rewards/reward_fn/std": 0.19981928169727325, "step": 212, "step_time": 231.8354788030265 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 319.5625, "completions/clipped_ratio": 0.0, "completions/max_length": 383.0, "completions/max_terminated_length": 383.0, "completions/mean_length": 319.5625, "completions/mean_terminated_length": 319.5625, "completions/min_length": 282.5, "completions/min_terminated_length": 282.5, "epoch": 1.42, "frac_reward_zero_std": 0.5, "grad_norm": 0.03575386852025986, "kl": 0.0, "learning_rate": 4.0845774184967754e-05, "loss": 0.021894656121730804, "num_tokens": 3100747.0, "reward": 0.959542840719223, "reward_std": 0.023604698479175568, "rewards/reward_fn/mean": 0.959542840719223, "rewards/reward_fn/std": 0.023604698479175568, "step": 213, "step_time": 117.80390733794775 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 409.8125, "completions/clipped_ratio": 0.0, "completions/max_length": 700.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 409.8125, "completions/mean_terminated_length": 409.8125, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "epoch": 1.4266666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 0.06581282615661621, "kl": 0.0, "learning_rate": 4.073300977624594e-05, "loss": 0.1886315941810608, "num_tokens": 3116696.0, "reward": 0.896901398897171, "reward_std": 0.061386555433273315, "rewards/reward_fn/mean": 0.896901398897171, "rewards/reward_fn/std": 0.061386555433273315, "step": 214, "step_time": 209.5079359910451 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 539.1875, "completions/clipped_ratio": 0.0, "completions/max_length": 758.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 539.1875, "completions/mean_terminated_length": 539.1875, "completions/min_length": 383.5, "completions/min_terminated_length": 383.5, "epoch": 1.4333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.07676459103822708, "kl": 0.0, "learning_rate": 4.0619712863599e-05, "loss": -0.09422039985656738, "num_tokens": 3135171.0, "reward": 0.6873325705528259, "reward_std": 0.2853637635707855, "rewards/reward_fn/mean": 0.6873325705528259, "rewards/reward_fn/std": 0.2853637635707855, "step": 215, "step_time": 221.83494500699453 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 254.4375, "completions/clipped_ratio": 0.0, "completions/max_length": 276.0, "completions/max_terminated_length": 276.0, "completions/mean_length": 254.4375, "completions/mean_terminated_length": 254.4375, "completions/min_length": 227.5, "completions/min_terminated_length": 227.5, "epoch": 1.44, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.05058872817065e-05, "loss": 0.0, "num_tokens": 3148138.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "step": 216, "step_time": 97.23418708896497 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 318.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 415.0, "completions/max_terminated_length": 415.0, "completions/mean_length": 318.6875, "completions/mean_terminated_length": 318.6875, "completions/min_length": 273.5, "completions/min_terminated_length": 273.5, "epoch": 1.4466666666666668, "frac_reward_zero_std": 0.5, "grad_norm": 0.0035884876269847155, "kl": 0.0, "learning_rate": 4.039153688314145e-05, "loss": -0.0018375769723206758, "num_tokens": 3162613.0, "reward": 0.9999989867210388, "reward_std": 2.858738071154221e-06, "rewards/reward_fn/mean": 0.9999989867210388, "rewards/reward_fn/std": 2.858738071154221e-06, "step": 217, "step_time": 136.8080933529418 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 209.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 210.0, "completions/max_terminated_length": 210.0, "completions/mean_length": 209.9375, "completions/mean_terminated_length": 209.9375, "completions/min_length": 209.5, "completions/min_terminated_length": 209.5, "epoch": 1.4533333333333334, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.0276665538239996e-05, "loss": 0.0, "num_tokens": 3174996.0, "reward": 0.5221271961927414, "reward_std": 0.0, "rewards/reward_fn/mean": 0.5221271961927414, "rewards/reward_fn/std": 0.0, "step": 218, "step_time": 88.73153007001383 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 429.1875, "completions/clipped_ratio": 0.0, "completions/max_length": 592.0, "completions/max_terminated_length": 592.0, "completions/mean_length": 429.1875, "completions/mean_terminated_length": 429.1875, "completions/min_length": 334.5, "completions/min_terminated_length": 334.5, "epoch": 1.46, "frac_reward_zero_std": 0.0, "grad_norm": 0.10545268654823303, "kl": 0.0, "learning_rate": 4.0161277134970345e-05, "loss": -0.02452065981924534, "num_tokens": 3191567.0, "reward": 0.622944176197052, "reward_std": 0.289985753595829, "rewards/reward_fn/mean": 0.622944176197052, "rewards/reward_fn/std": 0.289985753595829, "step": 219, "step_time": 182.1568425160949 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 219.75, "completions/clipped_ratio": 0.0, "completions/max_length": 247.0, "completions/max_terminated_length": 247.0, "completions/mean_length": 219.75, "completions/mean_terminated_length": 219.75, "completions/min_length": 194.5, "completions/min_terminated_length": 194.5, "epoch": 1.4666666666666668, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4.0045375578801214e-05, "loss": 0.0, "num_tokens": 3203891.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "step": 220, "step_time": 90.31312085100217 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 332.0, "completions/clipped_ratio": 0.0, "completions/max_length": 417.5, "completions/max_terminated_length": 417.5, "completions/mean_length": 332.0, "completions/mean_terminated_length": 332.0, "completions/min_length": 258.5, "completions/min_terminated_length": 258.5, "epoch": 1.4733333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 0.06864330917596817, "kl": 0.0, "learning_rate": 3.9928964792569655e-05, "loss": -0.04521460831165314, "num_tokens": 3219307.0, "reward": 0.750563770532608, "reward_std": 0.2518393062055111, "rewards/reward_fn/mean": 0.750563770532608, "rewards/reward_fn/std": 0.2518393062055111, "step": 221, "step_time": 133.98197106306907 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 258.25, "completions/clipped_ratio": 0.0, "completions/max_length": 282.5, "completions/max_terminated_length": 282.5, "completions/mean_length": 258.25, "completions/mean_terminated_length": 258.25, "completions/min_length": 232.5, "completions/min_terminated_length": 232.5, "epoch": 1.48, "frac_reward_zero_std": 0.5, "grad_norm": 0.0495823435485363, "kl": 0.0, "learning_rate": 3.981204871634827e-05, "loss": -0.015496471896767616, "num_tokens": 3233079.0, "reward": 0.7546902000904083, "reward_std": 0.003863011719658971, "rewards/reward_fn/mean": 0.7546902000904083, "rewards/reward_fn/std": 0.003863011719658971, "step": 222, "step_time": 92.20567861292511 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 201.625, "completions/clipped_ratio": 0.0, "completions/max_length": 229.5, "completions/max_terminated_length": 229.5, "completions/mean_length": 201.625, "completions/mean_terminated_length": 201.625, "completions/min_length": 190.5, "completions/min_terminated_length": 190.5, "epoch": 1.4866666666666668, "frac_reward_zero_std": 0.5, "grad_norm": 6.255746848182753e-05, "kl": 0.0, "learning_rate": 3.969463130731183e-05, "loss": 0.0001317363348789513, "num_tokens": 3245049.0, "reward": 0.5408785976469517, "reward_std": 4.5056875563886933e-08, "rewards/reward_fn/mean": 0.5408785976469517, "rewards/reward_fn/std": 4.5056875563886933e-08, "step": 223, "step_time": 83.72366849990794 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 282.5625, "completions/clipped_ratio": 0.0, "completions/max_length": 307.5, "completions/max_terminated_length": 307.5, "completions/mean_length": 282.5625, "completions/mean_terminated_length": 282.5625, "completions/min_length": 254.5, "completions/min_terminated_length": 254.5, "epoch": 1.4933333333333334, "frac_reward_zero_std": 0.5, "grad_norm": 0.057080987840890884, "kl": 0.0, "learning_rate": 3.957671653960337e-05, "loss": -0.015905410051345825, "num_tokens": 3259186.0, "reward": 0.6227080449461937, "reward_std": 0.15797464549541473, "rewards/reward_fn/mean": 0.6227080449461937, "rewards/reward_fn/std": 0.15797464549541473, "step": 224, "step_time": 98.58942143298918 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 407.75, "completions/clipped_ratio": 0.0, "completions/max_length": 556.0, "completions/max_terminated_length": 556.0, "completions/mean_length": 407.75, "completions/mean_terminated_length": 407.75, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "epoch": 1.5, "frac_reward_zero_std": 0.0, "grad_norm": 0.0834810733795166, "kl": 0.0, "learning_rate": 3.945830840419966e-05, "loss": -0.08507833629846573, "num_tokens": 3275478.0, "reward": 0.7662567794322968, "reward_std": 0.24252163991332054, "rewards/reward_fn/mean": 0.7662567794322968, "rewards/reward_fn/std": 0.24252163991332054, "step": 225, "step_time": 167.97602311102673 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 223.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 236.5, "completions/max_terminated_length": 236.5, "completions/mean_length": 223.9375, "completions/mean_terminated_length": 223.9375, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "epoch": 1.5066666666666668, "frac_reward_zero_std": 0.5, "grad_norm": 0.05319474637508392, "kl": 0.0, "learning_rate": 3.933941090877615e-05, "loss": 0.030159292742609978, "num_tokens": 3288837.0, "reward": 0.47812579572200775, "reward_std": 0.04895319417119026, "rewards/reward_fn/mean": 0.47812579572200775, "rewards/reward_fn/std": 0.04895319417119026, "step": 226, "step_time": 88.83022219099803 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 271.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 291.5, "completions/max_terminated_length": 291.5, "completions/mean_length": 271.9375, "completions/mean_terminated_length": 271.9375, "completions/min_length": 261.5, "completions/min_terminated_length": 261.5, "epoch": 1.5133333333333332, "frac_reward_zero_std": 0.5, "grad_norm": 0.0350649356842041, "kl": 0.0, "learning_rate": 3.9220028077571295e-05, "loss": 0.001047968864440918, "num_tokens": 3302796.0, "reward": 0.9795733690261841, "reward_std": 0.007507155183702707, "rewards/reward_fn/mean": 0.9795733690261841, "rewards/reward_fn/std": 0.007507155183702707, "step": 227, "step_time": 94.09043034503702 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 216.375, "completions/clipped_ratio": 0.0, "completions/max_length": 216.5, "completions/max_terminated_length": 216.5, "completions/mean_length": 216.375, "completions/mean_terminated_length": 216.375, "completions/min_length": 215.5, "completions/min_terminated_length": 215.5, "epoch": 1.52, "frac_reward_zero_std": 0.5, "grad_norm": 0.0425180047750473, "kl": 0.0, "learning_rate": 3.910016395125037e-05, "loss": -0.00019904226064682007, "num_tokens": 3315978.0, "reward": 0.9242193102836609, "reward_std": 0.008535998873412609, "rewards/reward_fn/mean": 0.9242193102836609, "rewards/reward_fn/std": 0.008535998873412609, "step": 228, "step_time": 83.15773937094491 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 209.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 210.0, "completions/max_terminated_length": 210.0, "completions/mean_length": 209.9375, "completions/mean_terminated_length": 209.9375, "completions/min_length": 209.5, "completions/min_terminated_length": 209.5, "epoch": 1.5266666666666666, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 3.897982258676867e-05, "loss": 0.0, "num_tokens": 3328353.0, "reward": 0.09640464186668396, "reward_std": 0.0, "rewards/reward_fn/mean": 0.09640464186668396, "rewards/reward_fn/std": 0.0, "step": 229, "step_time": 92.9228468539659 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 271.125, "completions/clipped_ratio": 0.0, "completions/max_length": 288.0, "completions/max_terminated_length": 288.0, "completions/mean_length": 271.125, "completions/mean_terminated_length": 271.125, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "epoch": 1.5333333333333332, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 3.885900805723429e-05, "loss": 0.0, "num_tokens": 3341515.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "step": 230, "step_time": 103.87323606194695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 271.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.0, "completions/max_terminated_length": 289.0, "completions/mean_length": 271.0, "completions/mean_terminated_length": 271.0, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "epoch": 1.54, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 3.873772445177015e-05, "loss": 0.0, "num_tokens": 3354707.0, "reward": 0.5801214650273323, "reward_std": 0.0, "rewards/reward_fn/mean": 0.5801214650273323, "rewards/reward_fn/std": 0.0, "step": 231, "step_time": 104.76834018190857 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 351.875, "completions/clipped_ratio": 0.0, "completions/max_length": 449.5, "completions/max_terminated_length": 449.5, "completions/mean_length": 351.875, "completions/mean_terminated_length": 351.875, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "epoch": 1.5466666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 0.0679924488067627, "kl": 0.0, "learning_rate": 3.861597587537568e-05, "loss": -0.0012257262133061886, "num_tokens": 3370505.0, "reward": 0.7388543784618378, "reward_std": 0.04050896943635962, "rewards/reward_fn/mean": 0.7388543784618378, "rewards/reward_fn/std": 0.04050896943635962, "step": 232, "step_time": 135.8323201989988 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 450.5, "completions/clipped_ratio": 0.0, "completions/max_length": 612.0, "completions/max_terminated_length": 612.0, "completions/mean_length": 450.5, "completions/mean_terminated_length": 450.5, "completions/min_length": 369.0, "completions/min_terminated_length": 369.0, "epoch": 1.5533333333333332, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 3.8493766448787825e-05, "loss": 0.0, "num_tokens": 3386985.0, "reward": 0.40595003962516785, "reward_std": 0.0, "rewards/reward_fn/mean": 0.40595003962516785, "rewards/reward_fn/std": 0.0, "step": 233, "step_time": 180.66226357588312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 221.4375, "completions/clipped_ratio": 0.0, "completions/max_length": 236.5, "completions/max_terminated_length": 236.5, "completions/mean_length": 221.4375, "completions/mean_terminated_length": 221.4375, "completions/min_length": 216.5, "completions/min_terminated_length": 216.5, "epoch": 1.56, "frac_reward_zero_std": 0.5, "grad_norm": 0.06356774270534515, "kl": 0.0, "learning_rate": 3.837110030834161e-05, "loss": -0.02530668117105961, "num_tokens": 3400344.0, "reward": 0.8683105707168579, "reward_std": 0.07456143200397491, "rewards/reward_fn/mean": 0.8683105707168579, "rewards/reward_fn/std": 0.07456143200397491, "step": 234, "step_time": 91.35750043700682 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 277.3125, "completions/clipped_ratio": 0.0, "completions/max_length": 343.0, "completions/max_terminated_length": 343.0, "completions/mean_length": 277.3125, "completions/mean_terminated_length": 277.3125, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "epoch": 1.5666666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 6.303178815869614e-05, "kl": 0.0, "learning_rate": 3.824798160583012e-05, "loss": 0.00016298360424116254, "num_tokens": 3413677.0, "reward": 0.9469609558582306, "reward_std": 5.6321095343037086e-08, "rewards/reward_fn/mean": 0.9469609558582306, "rewards/reward_fn/std": 5.6321095343037086e-08, "step": 235, "step_time": 117.30590885004494 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 306.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 367.0, "completions/max_terminated_length": 367.0, "completions/mean_length": 306.9375, "completions/mean_terminated_length": 306.9375, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "epoch": 1.5733333333333333, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 3.8124414508364e-05, "loss": 0.0, "num_tokens": 3427252.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "step": 236, "step_time": 111.6909038599697 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 441.625, "completions/clipped_ratio": 0.0, "completions/max_length": 532.5, "completions/max_terminated_length": 532.5, "completions/mean_length": 441.625, "completions/mean_terminated_length": 441.625, "completions/min_length": 355.0, "completions/min_terminated_length": 355.0, "epoch": 1.58, "frac_reward_zero_std": 0.0, "grad_norm": 0.06842601299285889, "kl": 0.0, "learning_rate": 3.8000403198230387e-05, "loss": 0.025734104216098785, "num_tokens": 3444510.0, "reward": 0.4380612373352051, "reward_std": 0.2177828741259873, "rewards/reward_fn/mean": 0.4380612373352051, "rewards/reward_fn/std": 0.2177828741259873, "step": 237, "step_time": 162.99026800697902 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 426.8125, "completions/clipped_ratio": 0.0, "completions/max_length": 500.5, "completions/max_terminated_length": 500.5, "completions/mean_length": 426.8125, "completions/mean_terminated_length": 426.8125, "completions/min_length": 361.5, "completions/min_terminated_length": 361.5, "epoch": 1.5866666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 0.05216877907514572, "kl": 0.0, "learning_rate": 3.787595187275136e-05, "loss": -0.022159036248922348, "num_tokens": 3460707.0, "reward": 0.17322568595409393, "reward_std": 0.2979126274585724, "rewards/reward_fn/mean": 0.17322568595409393, "rewards/reward_fn/std": 0.2979126274585724, "step": 238, "step_time": 164.71848585800035 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 293.75, "completions/clipped_ratio": 0.0, "completions/max_length": 312.0, "completions/max_terminated_length": 312.0, "completions/mean_length": 293.75, "completions/mean_terminated_length": 293.75, "completions/min_length": 249.5, "completions/min_terminated_length": 249.5, "epoch": 1.5933333333333333, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 3.775106474414188e-05, "loss": 0.0, "num_tokens": 3474063.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "step": 239, "step_time": 98.01331254001707 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 210.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 210.5, "completions/max_terminated_length": 210.5, "completions/mean_length": 210.0625, "completions/mean_terminated_length": 210.0625, "completions/min_length": 210.0, "completions/min_terminated_length": 210.0, "epoch": 1.6, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 3.762574603936725e-05, "loss": 0.0, "num_tokens": 3486408.0, "reward": 0.8980219066143036, "reward_std": 0.0, "rewards/reward_fn/mean": 0.8980219066143036, "rewards/reward_fn/std": 0.0, "step": 240, "step_time": 90.27049679402262 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 335.625, "completions/clipped_ratio": 0.0, "completions/max_length": 411.0, "completions/max_terminated_length": 411.0, "completions/mean_length": 335.625, "completions/mean_terminated_length": 335.625, "completions/min_length": 298.5, "completions/min_terminated_length": 298.5, "epoch": 1.6066666666666667, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 3.7500000000000003e-05, "loss": 0.0, "num_tokens": 3501146.0, "reward": 0.5656213238835335, "reward_std": 0.0, "rewards/reward_fn/mean": 0.5656213238835335, "rewards/reward_fn/std": 0.0, "step": 241, "step_time": 160.73021261492977 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 295.5625, "completions/clipped_ratio": 0.0, "completions/max_length": 351.5, "completions/max_terminated_length": 351.5, "completions/mean_length": 295.5625, "completions/mean_terminated_length": 295.5625, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "epoch": 1.6133333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.11344259977340698, "kl": 0.0, "learning_rate": 3.7373830882076354e-05, "loss": 0.02902945876121521, "num_tokens": 3516555.0, "reward": 0.7674919366836548, "reward_std": 0.018040144525002688, "rewards/reward_fn/mean": 0.7674919366836548, "rewards/reward_fn/std": 0.018040144525002688, "step": 242, "step_time": 111.73577347001992 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 294.3125, "completions/clipped_ratio": 0.0, "completions/max_length": 334.5, "completions/max_terminated_length": 334.5, "completions/mean_length": 294.3125, "completions/mean_terminated_length": 294.3125, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "epoch": 1.62, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 3.7247242955952175e-05, "loss": 0.0, "num_tokens": 3529936.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "step": 243, "step_time": 103.81889916706132 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 209.9375, "completions/clipped_ratio": 0.0, "completions/max_length": 210.0, "completions/max_terminated_length": 210.0, "completions/mean_length": 209.9375, "completions/mean_terminated_length": 209.9375, "completions/min_length": 209.5, "completions/min_terminated_length": 209.5, "epoch": 1.6266666666666667, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 3.712024050615843e-05, "loss": 0.0, "num_tokens": 3542279.0, "reward": 0.5284403860569, "reward_std": 0.0, "rewards/reward_fn/mean": 0.5284403860569, "rewards/reward_fn/std": 0.0, "step": 244, "step_time": 96.00647811003728 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 313.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 373.5, "completions/max_terminated_length": 373.5, "completions/mean_length": 313.0625, "completions/mean_terminated_length": 313.0625, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "epoch": 1.6333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.07335827499628067, "kl": 0.0, "learning_rate": 3.699282783125616e-05, "loss": 0.0014421269297599792, "num_tokens": 3557960.0, "reward": 0.7668299674987793, "reward_std": 0.09549267427064478, "rewards/reward_fn/mean": 0.7668299674987793, "rewards/reward_fn/std": 0.09549267427064478, "step": 245, "step_time": 117.32995172601659 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 453.6875, "completions/clipped_ratio": 0.0, "completions/max_length": 613.0, "completions/max_terminated_length": 613.0, "completions/mean_length": 453.6875, "completions/mean_terminated_length": 453.6875, "completions/min_length": 325.5, "completions/min_terminated_length": 325.5, "epoch": 1.6400000000000001, "frac_reward_zero_std": 0.5, "grad_norm": 0.032244011759757996, "kl": 0.0, "learning_rate": 3.686500924369101e-05, "loss": 0.050328221172094345, "num_tokens": 3574691.0, "reward": 0.12802473455667496, "reward_std": 0.13411740958690643, "rewards/reward_fn/mean": 0.12802473455667496, "rewards/reward_fn/std": 0.13411740958690643, "step": 246, "step_time": 189.75963115796912 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 282.375, "completions/clipped_ratio": 0.0, "completions/max_length": 330.0, "completions/max_terminated_length": 330.0, "completions/mean_length": 282.375, "completions/mean_terminated_length": 282.375, "completions/min_length": 240.5, "completions/min_terminated_length": 240.5, "epoch": 1.6466666666666665, "frac_reward_zero_std": 0.5, "grad_norm": 0.00042099610436707735, "kl": 0.0, "learning_rate": 3.673678906964727e-05, "loss": 5.045448779128492e-05, "num_tokens": 3587825.0, "reward": 0.9999999105930328, "reward_std": 2.634479017160629e-07, "rewards/reward_fn/mean": 0.9999999105930328, "rewards/reward_fn/std": 2.634479017160629e-07, "step": 247, "step_time": 101.9842315640999 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 390.75, "completions/clipped_ratio": 0.0, "completions/max_length": 486.0, "completions/max_terminated_length": 486.0, "completions/mean_length": 390.75, "completions/mean_terminated_length": 390.75, "completions/min_length": 329.5, "completions/min_terminated_length": 329.5, "epoch": 1.6533333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.0787544846534729, "kl": 0.0, "learning_rate": 3.660817164890143e-05, "loss": 0.014489062130451202, "num_tokens": 3604285.0, "reward": 0.8284005224704742, "reward_std": 0.33756453602109104, "rewards/reward_fn/mean": 0.8284005224704742, "rewards/reward_fn/std": 0.33756453602109104, "step": 248, "step_time": 145.34176448197104 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 327.625, "completions/clipped_ratio": 0.0, "completions/max_length": 385.0, "completions/max_terminated_length": 385.0, "completions/mean_length": 327.625, "completions/mean_terminated_length": 327.625, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "epoch": 1.6600000000000001, "frac_reward_zero_std": 0.5, "grad_norm": 0.05011061951518059, "kl": 0.0, "learning_rate": 3.6479161334675296e-05, "loss": -0.005787290632724762, "num_tokens": 3619271.0, "reward": 0.9531709551811218, "reward_std": 0.004021522589027882, "rewards/reward_fn/mean": 0.9531709551811218, "rewards/reward_fn/std": 0.004021522589027882, "step": 249, "step_time": 118.20399923500372 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 359.5, "completions/clipped_ratio": 0.0, "completions/max_length": 442.5, "completions/max_terminated_length": 442.5, "completions/mean_length": 359.5, "completions/mean_terminated_length": 359.5, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "epoch": 1.6666666666666665, "frac_reward_zero_std": 0.5, "grad_norm": 0.05501338839530945, "kl": 0.0, "learning_rate": 3.634976249348867e-05, "loss": -0.006874836981296539, "num_tokens": 3635063.0, "reward": 0.9404093027114868, "reward_std": 0.16401344537734985, "rewards/reward_fn/mean": 0.9404093027114868, "rewards/reward_fn/std": 0.16401344537734985, "step": 250, "step_time": 154.596884167986 } ], "logging_steps": 1, "max_steps": 600, "num_input_tokens_seen": 3635063, "num_train_epochs": 4, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }