| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.26881720430107525, |
| "eval_steps": 500, |
| "global_step": 50, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio": 1.5976482927726465e-06, |
| "completions/clipped_ratio": 0.10546875, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4069.0, |
| "completions/mean_length": 1215.251953125, |
| "completions/mean_terminated_length": 883.8755493164062, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.005376344086021506, |
| "frac_reward_zero_std": 0.03125, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 687505.0, |
| "register/okay": 0.21875, |
| "register/okay_any": 0.40625, |
| "register/premium": 0.6799999999999999, |
| "register/verify_any": 0.375, |
| "reward": 0.396484375, |
| "reward_std": 0.4896455705165863, |
| "rewards/correct/mean": 0.396484375, |
| "rewards/correct/std": 0.4896455705165863, |
| "rollout/chars": 1732.3125, |
| "rollout/commit": 0.71875, |
| "rollout/reward": 0.46875, |
| "rollout/trunc": 0.03125, |
| "sampling/importance_sampling_ratio/max": 2.788275718688965, |
| "sampling/importance_sampling_ratio/mean": 0.9801996946334839, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.8374314308166504, |
| "sampling/sampling_logp_difference/mean": 0.00512130418792367, |
| "step": 1, |
| "step_time": 150.84468165802537 |
| }, |
| { |
| "epoch": 0.005376344086021506, |
| "grad_norm": 0.005536369979381561, |
| "learning_rate": 0.0, |
| "loss": -0.10681209713220596, |
| "step": 1 |
| }, |
| { |
| "clip_ratio": 3.647639914561296e-06, |
| "completions/clipped_ratio": 0.1171875, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4028.0, |
| "completions/mean_length": 1264.6640625, |
| "completions/mean_terminated_length": 906.8141479492188, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.010752688172043012, |
| "frac_reward_zero_std": 0.03125, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 1395861.0, |
| "register/okay": 0.25, |
| "register/okay_any": 0.3125, |
| "register/premium": 0.625, |
| "register/verify_any": 0.3125, |
| "reward": 0.4140625, |
| "reward_std": 0.49304109811782837, |
| "rewards/correct/mean": 0.4140625, |
| "rewards/correct/std": 0.49304109811782837, |
| "rollout/chars": 1863.84375, |
| "rollout/commit": 0.6875, |
| "rollout/reward": 0.53125, |
| "rollout/trunc": 0.03125, |
| "sampling/importance_sampling_ratio/max": 2.8976492881774902, |
| "sampling/importance_sampling_ratio/mean": 0.9634870290756226, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.6949191093444824, |
| "sampling/sampling_logp_difference/mean": 0.00507094943895936, |
| "step": 2, |
| "step_time": 126.88190104090609 |
| }, |
| { |
| "epoch": 0.010752688172043012, |
| "grad_norm": 0.004960422404110432, |
| "learning_rate": 2.0000000000000003e-06, |
| "loss": -0.06525798887014389, |
| "step": 2 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completions/clipped_ratio": 0.095703125, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4039.0, |
| "completions/mean_length": 1090.421875, |
| "completions/mean_terminated_length": 772.3369140625, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.016129032258064516, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 2006541.0, |
| "register/okay": 0.15625, |
| "register/okay_any": 0.28125, |
| "register/premium": 0.7407407407407407, |
| "register/verify_any": 0.28125, |
| "reward": 0.3984375, |
| "reward_std": 0.4900552034378052, |
| "rewards/correct/mean": 0.3984375, |
| "rewards/correct/std": 0.4900552034378052, |
| "rollout/chars": 2198.59375, |
| "rollout/commit": 0.53125, |
| "rollout/reward": 0.375, |
| "rollout/trunc": 0.03125, |
| "sampling/importance_sampling_ratio/max": 2.531585693359375, |
| "sampling/importance_sampling_ratio/mean": 0.9748052954673767, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.445171594619751, |
| "sampling/sampling_logp_difference/mean": 0.005298345349729061, |
| "step": 3, |
| "step_time": 133.05447170190746 |
| }, |
| { |
| "epoch": 0.016129032258064516, |
| "grad_norm": 0.006240344140678644, |
| "learning_rate": 4.000000000000001e-06, |
| "loss": -0.1190323755145073, |
| "step": 3 |
| }, |
| { |
| "clip_ratio": 2.7389637580199633e-06, |
| "completions/clipped_ratio": 0.1171875, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4058.0, |
| "completions/mean_length": 1281.017578125, |
| "completions/mean_terminated_length": 914.2079467773438, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.021505376344086023, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 2743142.0, |
| "register/okay": 0.3125, |
| "register/okay_any": 0.46875, |
| "register/premium": 0.2727272727272727, |
| "register/verify_any": 0.46875, |
| "reward": 0.3828125, |
| "reward_std": 0.486548513174057, |
| "rewards/correct/mean": 0.3828125, |
| "rewards/correct/std": 0.486548513174057, |
| "rollout/chars": 4298.34375, |
| "rollout/commit": 0.71875, |
| "rollout/reward": 0.3125, |
| "rollout/trunc": 0.25, |
| "sampling/importance_sampling_ratio/max": 2.8071212768554688, |
| "sampling/importance_sampling_ratio/mean": 0.9747036099433899, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.5947754383087158, |
| "sampling/sampling_logp_difference/mean": 0.005121023394167423, |
| "step": 4, |
| "step_time": 288.29422205802985 |
| }, |
| { |
| "epoch": 0.021505376344086023, |
| "grad_norm": 0.004777645226567984, |
| "learning_rate": 6e-06, |
| "loss": -0.08420069515705109, |
| "step": 4 |
| }, |
| { |
| "clip_ratio": 4.7834116173817165e-05, |
| "completions/clipped_ratio": 0.08984375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4077.0, |
| "completions/mean_length": 1166.685546875, |
| "completions/mean_terminated_length": 893.030029296875, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.026881720430107527, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 3396405.0, |
| "register/okay": 0.40625, |
| "register/okay_any": 0.65625, |
| "register/premium": 0.27530364372469635, |
| "register/verify_any": 0.6875, |
| "reward": 0.47265625, |
| "reward_std": 0.49974003434181213, |
| "rewards/correct/mean": 0.47265625, |
| "rewards/correct/std": 0.49974003434181213, |
| "rollout/chars": 5199.34375, |
| "rollout/commit": 0.6875, |
| "rollout/reward": 0.375, |
| "rollout/trunc": 0.21875, |
| "sampling/importance_sampling_ratio/max": 2.6029703617095947, |
| "sampling/importance_sampling_ratio/mean": 0.9715023040771484, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.4979090690612793, |
| "sampling/sampling_logp_difference/mean": 0.00514589948579669, |
| "step": 5, |
| "step_time": 118.11216273508035 |
| }, |
| { |
| "epoch": 0.026881720430107527, |
| "grad_norm": 0.006235240027308464, |
| "learning_rate": 8.000000000000001e-06, |
| "loss": -0.06412481516599655, |
| "step": 5 |
| }, |
| { |
| "clip_ratio": 0.0013401048065588839, |
| "completions/clipped_ratio": 0.130859375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4087.0, |
| "completions/mean_length": 1430.537109375, |
| "completions/mean_terminated_length": 1046.298828125, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.03225806451612903, |
| "frac_reward_zero_std": 0.03125, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 4183160.0, |
| "register/okay": 0.21875, |
| "register/okay_any": 0.5, |
| "register/premium": 0.4971428571428571, |
| "register/verify_any": 0.53125, |
| "reward": 0.41796875, |
| "reward_std": 0.4937073290348053, |
| "rewards/correct/mean": 0.41796875, |
| "rewards/correct/std": 0.4937073290348053, |
| "rollout/chars": 4344.03125, |
| "rollout/commit": 0.8125, |
| "rollout/reward": 0.46875, |
| "rollout/trunc": 0.125, |
| "sampling/importance_sampling_ratio/max": 2.8645777702331543, |
| "sampling/importance_sampling_ratio/mean": 0.9869682788848877, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.3402305841445923, |
| "sampling/sampling_logp_difference/mean": 0.005047679413110018, |
| "step": 6, |
| "step_time": 126.73521159705706 |
| }, |
| { |
| "epoch": 0.03225806451612903, |
| "grad_norm": 0.00521544786170125, |
| "learning_rate": 1e-05, |
| "loss": -0.09711194038391113, |
| "step": 6 |
| }, |
| { |
| "clip_ratio": 0.004071735407706001, |
| "completions/clipped_ratio": 0.119140625, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4087.0, |
| "completions/mean_length": 1213.759765625, |
| "completions/mean_terminated_length": 842.0376586914062, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.03763440860215054, |
| "frac_reward_zero_std": 0.0625, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 4863773.0, |
| "register/okay": 0.25, |
| "register/okay_any": 0.3125, |
| "register/premium": 0.2916666666666667, |
| "register/verify_any": 0.34375, |
| "reward": 0.37890625, |
| "reward_std": 0.4855891764163971, |
| "rewards/correct/mean": 0.37890625, |
| "rewards/correct/std": 0.4855891764163971, |
| "rollout/chars": 3508.96875, |
| "rollout/commit": 0.6875, |
| "rollout/reward": 0.15625, |
| "rollout/trunc": 0.125, |
| "sampling/importance_sampling_ratio/max": 2.994431257247925, |
| "sampling/importance_sampling_ratio/mean": 0.9885159134864807, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.8572158813476562, |
| "sampling/sampling_logp_difference/mean": 0.0053655486553907394, |
| "step": 7, |
| "step_time": 147.0131771651213 |
| }, |
| { |
| "epoch": 0.03763440860215054, |
| "grad_norm": 0.004825656767934561, |
| "learning_rate": 1e-05, |
| "loss": -0.07372352480888367, |
| "step": 7 |
| }, |
| { |
| "clip_ratio": 0.0074993749899476825, |
| "completions/clipped_ratio": 0.125, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4058.0, |
| "completions/mean_length": 1266.484375, |
| "completions/mean_terminated_length": 880.3058471679688, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.043010752688172046, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 5574261.0, |
| "register/okay": 0.25, |
| "register/okay_any": 0.40625, |
| "register/premium": 0.5, |
| "register/verify_any": 0.40625, |
| "reward": 0.404296875, |
| "reward_std": 0.4912354052066803, |
| "rewards/correct/mean": 0.404296875, |
| "rewards/correct/std": 0.4912354052066803, |
| "rollout/chars": 3730.25, |
| "rollout/commit": 0.6875, |
| "rollout/reward": 0.25, |
| "rollout/trunc": 0.03125, |
| "sampling/importance_sampling_ratio/max": 2.7395002841949463, |
| "sampling/importance_sampling_ratio/mean": 0.9624147415161133, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.5250225067138672, |
| "sampling/sampling_logp_difference/mean": 0.005175733007490635, |
| "step": 8, |
| "step_time": 141.7585549650248 |
| }, |
| { |
| "epoch": 0.043010752688172046, |
| "grad_norm": 0.0049206651747226715, |
| "learning_rate": 1e-05, |
| "loss": -0.09447586536407471, |
| "step": 8 |
| }, |
| { |
| "clip_ratio": 0.010272848696331494, |
| "completions/clipped_ratio": 0.109375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4093.0, |
| "completions/mean_length": 1234.017578125, |
| "completions/mean_terminated_length": 900.2850952148438, |
| "completions/min_length": 3.0, |
| "completions/min_terminated_length": 3.0, |
| "epoch": 0.04838709677419355, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 6267934.0, |
| "register/okay": 0.34375, |
| "register/okay_any": 0.53125, |
| "register/premium": 0.393939393939394, |
| "register/verify_any": 0.625, |
| "reward": 0.462890625, |
| "reward_std": 0.4991086423397064, |
| "rewards/correct/mean": 0.462890625, |
| "rewards/correct/std": 0.4991086423397064, |
| "rollout/chars": 5054.625, |
| "rollout/commit": 0.8125, |
| "rollout/reward": 0.46875, |
| "rollout/trunc": 0.125, |
| "sampling/importance_sampling_ratio/max": 2.8355865478515625, |
| "sampling/importance_sampling_ratio/mean": 0.9936923980712891, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 4.098477363586426, |
| "sampling/sampling_logp_difference/mean": 0.005666417069733143, |
| "step": 9, |
| "step_time": 144.23339352413313 |
| }, |
| { |
| "epoch": 0.04838709677419355, |
| "grad_norm": 0.0060818931087851524, |
| "learning_rate": 1e-05, |
| "loss": -0.12299580872058868, |
| "step": 9 |
| }, |
| { |
| "clip_ratio": 0.01380008689375245, |
| "completions/clipped_ratio": 0.087890625, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4091.0, |
| "completions/mean_length": 1173.78125, |
| "completions/mean_terminated_length": 892.197021484375, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.053763440860215055, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 6927438.0, |
| "register/okay": 0.5, |
| "register/okay_any": 0.625, |
| "register/premium": 0.5, |
| "register/verify_any": 0.59375, |
| "reward": 0.49609375, |
| "reward_std": 0.5004737377166748, |
| "rewards/correct/mean": 0.49609375, |
| "rewards/correct/std": 0.5004737377166748, |
| "rollout/chars": 2617.8125, |
| "rollout/commit": 0.78125, |
| "rollout/reward": 0.6875, |
| "rollout/trunc": 0.0625, |
| "sampling/importance_sampling_ratio/max": 2.481905221939087, |
| "sampling/importance_sampling_ratio/mean": 0.969363272190094, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.00347900390625, |
| "sampling/sampling_logp_difference/mean": 0.0053543029353022575, |
| "step": 10, |
| "step_time": 212.75371372816153 |
| }, |
| { |
| "epoch": 0.053763440860215055, |
| "grad_norm": 0.005507849156856537, |
| "learning_rate": 1e-05, |
| "loss": -0.09586161375045776, |
| "step": 10 |
| }, |
| { |
| "clip_ratio": 0.014677733306598384, |
| "completions/clipped_ratio": 0.109375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4023.0, |
| "completions/mean_length": 1157.3125, |
| "completions/mean_terminated_length": 805.37939453125, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.05913978494623656, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 7582542.0, |
| "register/okay": 0.0625, |
| "register/okay_any": 0.375, |
| "register/premium": 0.4666666666666667, |
| "register/verify_any": 0.40625, |
| "reward": 0.494140625, |
| "reward_std": 0.5004546642303467, |
| "rewards/correct/mean": 0.494140625, |
| "rewards/correct/std": 0.5004546642303467, |
| "rollout/chars": 2919.6875, |
| "rollout/commit": 0.8125, |
| "rollout/reward": 0.5625, |
| "rollout/trunc": 0.03125, |
| "sampling/importance_sampling_ratio/max": 2.9180502891540527, |
| "sampling/importance_sampling_ratio/mean": 0.9749065637588501, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.5184186697006226, |
| "sampling/sampling_logp_difference/mean": 0.005147980526089668, |
| "step": 11, |
| "step_time": 146.4343996279058 |
| }, |
| { |
| "epoch": 0.05913978494623656, |
| "grad_norm": 0.004948989488184452, |
| "learning_rate": 1e-05, |
| "loss": -0.07085411250591278, |
| "step": 11 |
| }, |
| { |
| "clip_ratio": 0.01596076832538529, |
| "completions/clipped_ratio": 0.083984375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4089.0, |
| "completions/mean_length": 1122.380859375, |
| "completions/mean_terminated_length": 866.9168701171875, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.06451612903225806, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 8216993.0, |
| "register/okay": 0.28125, |
| "register/okay_any": 0.5, |
| "register/premium": 0.47342995169082125, |
| "register/verify_any": 0.5, |
| "reward": 0.484375, |
| "reward_std": 0.5002445578575134, |
| "rewards/correct/mean": 0.484375, |
| "rewards/correct/std": 0.5002445578575134, |
| "rollout/chars": 5048.125, |
| "rollout/commit": 0.71875, |
| "rollout/reward": 0.4375, |
| "rollout/trunc": 0.15625, |
| "sampling/importance_sampling_ratio/max": 2.8764474391937256, |
| "sampling/importance_sampling_ratio/mean": 0.9590312838554382, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.4287886619567871, |
| "sampling/sampling_logp_difference/mean": 0.004995434079319239, |
| "step": 12, |
| "step_time": 121.18447578209452 |
| }, |
| { |
| "epoch": 0.06451612903225806, |
| "grad_norm": 0.0055179595947265625, |
| "learning_rate": 1e-05, |
| "loss": -0.10851699858903885, |
| "step": 12 |
| }, |
| { |
| "clip_ratio": 0.008574715991926496, |
| "completions/clipped_ratio": 0.162109375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4021.0, |
| "completions/mean_length": 1473.20703125, |
| "completions/mean_terminated_length": 1001.4149169921875, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.06989247311827956, |
| "frac_reward_zero_std": 0.125, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 9032427.0, |
| "register/okay": 0.34375, |
| "register/okay_any": 0.53125, |
| "register/premium": 0.4805194805194805, |
| "register/verify_any": 0.5625, |
| "reward": 0.4765625, |
| "reward_std": 0.49993884563446045, |
| "rewards/correct/mean": 0.4765625, |
| "rewards/correct/std": 0.49993884563446045, |
| "rollout/chars": 3908.8125, |
| "rollout/commit": 0.75, |
| "rollout/reward": 0.59375, |
| "rollout/trunc": 0.0625, |
| "sampling/importance_sampling_ratio/max": 2.951629161834717, |
| "sampling/importance_sampling_ratio/mean": 0.9317843317985535, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.5227203369140625, |
| "sampling/sampling_logp_difference/mean": 0.005291281268000603, |
| "step": 13, |
| "step_time": 191.56482845492428 |
| }, |
| { |
| "epoch": 0.06989247311827956, |
| "grad_norm": 0.0041601029224693775, |
| "learning_rate": 1e-05, |
| "loss": -0.06557237356901169, |
| "step": 13 |
| }, |
| { |
| "clip_ratio": 0.013084022144539631, |
| "completions/clipped_ratio": 0.109375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4094.0, |
| "completions/mean_length": 1318.9921875, |
| "completions/mean_terminated_length": 994.7105102539062, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.07526881720430108, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 9763207.0, |
| "register/okay": 0.40625, |
| "register/okay_any": 0.75, |
| "register/premium": 0.12145748987854255, |
| "register/verify_any": 0.8125, |
| "reward": 0.513671875, |
| "reward_std": 0.5003018379211426, |
| "rewards/correct/mean": 0.513671875, |
| "rewards/correct/std": 0.5003018379211426, |
| "rollout/chars": 8604.15625, |
| "rollout/commit": 0.5, |
| "rollout/reward": 0.3125, |
| "rollout/trunc": 0.46875, |
| "sampling/importance_sampling_ratio/max": 2.9212090969085693, |
| "sampling/importance_sampling_ratio/mean": 0.9688857197761536, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.9780625104904175, |
| "sampling/sampling_logp_difference/mean": 0.005228726658970118, |
| "step": 14, |
| "step_time": 357.421331763966 |
| }, |
| { |
| "epoch": 0.07526881720430108, |
| "grad_norm": 0.005021767690777779, |
| "learning_rate": 1e-05, |
| "loss": -0.06424752622842789, |
| "step": 14 |
| }, |
| { |
| "clip_ratio": 0.02033586335255677, |
| "completions/clipped_ratio": 0.111328125, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4048.0, |
| "completions/mean_length": 1445.634765625, |
| "completions/mean_terminated_length": 1114.4571533203125, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.08064516129032258, |
| "frac_reward_zero_std": 0.03125, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 10567756.0, |
| "register/okay": 0.25, |
| "register/okay_any": 0.34375, |
| "register/premium": 0.625, |
| "register/verify_any": 0.4375, |
| "reward": 0.50390625, |
| "reward_std": 0.5004737377166748, |
| "rewards/correct/mean": 0.50390625, |
| "rewards/correct/std": 0.5004737377166748, |
| "rollout/chars": 3988.84375, |
| "rollout/commit": 0.78125, |
| "rollout/reward": 0.40625, |
| "rollout/trunc": 0.09375, |
| "sampling/importance_sampling_ratio/max": 2.932295083999634, |
| "sampling/importance_sampling_ratio/mean": 0.9546321630477905, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.5548479557037354, |
| "sampling/sampling_logp_difference/mean": 0.005285207182168961, |
| "step": 15, |
| "step_time": 166.41254067909904 |
| }, |
| { |
| "epoch": 0.08064516129032258, |
| "grad_norm": 0.004748669918626547, |
| "learning_rate": 1e-05, |
| "loss": -0.07962360978126526, |
| "step": 15 |
| }, |
| { |
| "clip_ratio": 0.021063678299469757, |
| "completions/clipped_ratio": 0.064453125, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4073.0, |
| "completions/mean_length": 1114.67578125, |
| "completions/mean_terminated_length": 934.5532836914062, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.08602150537634409, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 11193990.0, |
| "register/okay": 0.375, |
| "register/okay_any": 0.46875, |
| "register/premium": 0.65, |
| "register/verify_any": 0.46875, |
| "reward": 0.537109375, |
| "reward_std": 0.4991086423397064, |
| "rewards/correct/mean": 0.537109375, |
| "rewards/correct/std": 0.4991086423397064, |
| "rollout/chars": 3170.40625, |
| "rollout/commit": 0.8125, |
| "rollout/reward": 0.59375, |
| "rollout/trunc": 0.03125, |
| "sampling/importance_sampling_ratio/max": 2.9447994232177734, |
| "sampling/importance_sampling_ratio/mean": 0.9693629741668701, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.8633896112442017, |
| "sampling/sampling_logp_difference/mean": 0.005218314006924629, |
| "step": 16, |
| "step_time": 119.10572393791517 |
| }, |
| { |
| "epoch": 0.08602150537634409, |
| "grad_norm": 0.005955121014267206, |
| "learning_rate": 1e-05, |
| "loss": -0.09967996180057526, |
| "step": 16 |
| }, |
| { |
| "clip_ratio": 0.010124757566700282, |
| "completions/clipped_ratio": 0.166015625, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4044.0, |
| "completions/mean_length": 1443.7734375, |
| "completions/mean_terminated_length": 943.2716064453125, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.0913978494623656, |
| "frac_reward_zero_std": 0.03125, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 11993010.0, |
| "register/okay": 0.25, |
| "register/okay_any": 0.4375, |
| "register/premium": 0.75, |
| "register/verify_any": 0.5625, |
| "reward": 0.47265625, |
| "reward_std": 0.49974003434181213, |
| "rewards/correct/mean": 0.47265625, |
| "rewards/correct/std": 0.49974003434181213, |
| "rollout/chars": 3285.5, |
| "rollout/commit": 0.8125, |
| "rollout/reward": 0.4375, |
| "rollout/trunc": 0.03125, |
| "sampling/importance_sampling_ratio/max": 2.907860517501831, |
| "sampling/importance_sampling_ratio/mean": 0.9712395668029785, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.7675509452819824, |
| "sampling/sampling_logp_difference/mean": 0.005249205976724625, |
| "step": 17, |
| "step_time": 169.88692099996842 |
| }, |
| { |
| "epoch": 0.0913978494623656, |
| "grad_norm": 0.005303401034325361, |
| "learning_rate": 1e-05, |
| "loss": -0.051697950810194016, |
| "step": 17 |
| }, |
| { |
| "clip_ratio": 0.016437006976047996, |
| "completions/clipped_ratio": 0.09375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4040.0, |
| "completions/mean_length": 1151.44140625, |
| "completions/mean_terminated_length": 864.4482421875, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.0967741935483871, |
| "frac_reward_zero_std": 0.03125, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 12633684.0, |
| "register/okay": 0.25, |
| "register/okay_any": 0.46875, |
| "register/premium": 0.6666666666666667, |
| "register/verify_any": 0.5, |
| "reward": 0.537109375, |
| "reward_std": 0.4991086423397064, |
| "rewards/correct/mean": 0.537109375, |
| "rewards/correct/std": 0.4991086423397064, |
| "rollout/chars": 1661.1875, |
| "rollout/commit": 0.71875, |
| "rollout/reward": 0.5, |
| "rollout/trunc": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.9537272453308105, |
| "sampling/importance_sampling_ratio/mean": 0.987904965877533, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.7457265853881836, |
| "sampling/sampling_logp_difference/mean": 0.005279700271785259, |
| "step": 18, |
| "step_time": 147.74140067375265 |
| }, |
| { |
| "epoch": 0.0967741935483871, |
| "grad_norm": 0.005066175479441881, |
| "learning_rate": 1e-05, |
| "loss": -0.09199890494346619, |
| "step": 18 |
| }, |
| { |
| "clip_ratio": 0.016431453828772646, |
| "completions/clipped_ratio": 0.111328125, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4067.0, |
| "completions/mean_length": 1284.71875, |
| "completions/mean_terminated_length": 955.4945678710938, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.10215053763440861, |
| "frac_reward_zero_std": 0.03125, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 13357524.0, |
| "register/okay": 0.3125, |
| "register/okay_any": 0.5625, |
| "register/premium": 0.6818181818181819, |
| "register/verify_any": 0.5625, |
| "reward": 0.46484375, |
| "reward_std": 0.49925029277801514, |
| "rewards/correct/mean": 0.46484375, |
| "rewards/correct/std": 0.49925029277801514, |
| "rollout/chars": 3002.90625, |
| "rollout/commit": 0.71875, |
| "rollout/reward": 0.53125, |
| "rollout/trunc": 0.0625, |
| "sampling/importance_sampling_ratio/max": 2.83459210395813, |
| "sampling/importance_sampling_ratio/mean": 0.9570035934448242, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.8500914573669434, |
| "sampling/sampling_logp_difference/mean": 0.005208856426179409, |
| "step": 19, |
| "step_time": 134.55601952417055 |
| }, |
| { |
| "epoch": 0.10215053763440861, |
| "grad_norm": 0.0050915940664708614, |
| "learning_rate": 1e-05, |
| "loss": -0.06887590885162354, |
| "step": 19 |
| }, |
| { |
| "clip_ratio": 0.015742540885185008, |
| "completions/clipped_ratio": 0.068359375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 3939.0, |
| "completions/mean_length": 1023.6875, |
| "completions/mean_terminated_length": 823.9559326171875, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.10752688172043011, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 13931684.0, |
| "register/okay": 0.375, |
| "register/okay_any": 0.5, |
| "register/premium": 0.4, |
| "register/verify_any": 0.5, |
| "reward": 0.533203125, |
| "reward_std": 0.4993842542171478, |
| "rewards/correct/mean": 0.533203125, |
| "rewards/correct/std": 0.4993842542171478, |
| "rollout/chars": 3815.75, |
| "rollout/commit": 0.8125, |
| "rollout/reward": 0.5, |
| "rollout/trunc": 0.09375, |
| "sampling/importance_sampling_ratio/max": 2.7999613285064697, |
| "sampling/importance_sampling_ratio/mean": 1.0000779628753662, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.5036005973815918, |
| "sampling/sampling_logp_difference/mean": 0.005762252025306225, |
| "step": 20, |
| "step_time": 113.17120536096627 |
| }, |
| { |
| "epoch": 0.10752688172043011, |
| "grad_norm": 0.007009309716522694, |
| "learning_rate": 1e-05, |
| "loss": -0.09315481781959534, |
| "step": 20 |
| }, |
| { |
| "clip_ratio": 0.00888046599811787, |
| "completions/clipped_ratio": 0.087890625, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4087.0, |
| "completions/mean_length": 1349.1796875, |
| "completions/mean_terminated_length": 1109.8543701171875, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.11290322580645161, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 14678448.0, |
| "register/okay": 0.40625, |
| "register/okay_any": 0.5, |
| "register/premium": 0.631578947368421, |
| "register/verify_any": 0.5625, |
| "reward": 0.548828125, |
| "reward_std": 0.498096764087677, |
| "rewards/correct/mean": 0.548828125, |
| "rewards/correct/std": 0.498096764087677, |
| "rollout/chars": 2420.65625, |
| "rollout/commit": 0.8125, |
| "rollout/reward": 0.625, |
| "rollout/trunc": 0.03125, |
| "sampling/importance_sampling_ratio/max": 2.826205253601074, |
| "sampling/importance_sampling_ratio/mean": 0.9306512475013733, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.6579908132553101, |
| "sampling/sampling_logp_difference/mean": 0.005072219762951136, |
| "step": 21, |
| "step_time": 136.9637122741551 |
| }, |
| { |
| "epoch": 0.11290322580645161, |
| "grad_norm": 0.004881712142378092, |
| "learning_rate": 1e-05, |
| "loss": -0.07977926731109619, |
| "step": 21 |
| }, |
| { |
| "clip_ratio": 0.006619919394324825, |
| "completions/clipped_ratio": 0.177734375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4033.0, |
| "completions/mean_length": 1557.130859375, |
| "completions/mean_terminated_length": 1008.3491821289062, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.11827956989247312, |
| "frac_reward_zero_std": 0.0625, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 15541459.0, |
| "register/okay": 0.375, |
| "register/okay_any": 0.65625, |
| "register/premium": 0.6, |
| "register/verify_any": 0.625, |
| "reward": 0.486328125, |
| "reward_std": 0.5003018379211426, |
| "rewards/correct/mean": 0.486328125, |
| "rewards/correct/std": 0.5003018379211426, |
| "rollout/chars": 4022.375, |
| "rollout/commit": 0.90625, |
| "rollout/reward": 0.625, |
| "rollout/trunc": 0.09375, |
| "sampling/importance_sampling_ratio/max": 2.895916223526001, |
| "sampling/importance_sampling_ratio/mean": 0.9804240465164185, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.6260337829589844, |
| "sampling/sampling_logp_difference/mean": 0.005503639578819275, |
| "step": 22, |
| "step_time": 166.04075432108948 |
| }, |
| { |
| "epoch": 0.11827956989247312, |
| "grad_norm": 0.004497839603573084, |
| "learning_rate": 1e-05, |
| "loss": -0.03457781299948692, |
| "step": 22 |
| }, |
| { |
| "clip_ratio": 0.008130440383865789, |
| "completions/clipped_ratio": 0.130859375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4082.0, |
| "completions/mean_length": 1388.0390625, |
| "completions/mean_terminated_length": 980.3236083984375, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.12365591397849462, |
| "frac_reward_zero_std": 0.0625, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.5, |
| "groups/live_frac": 0.5, |
| "groups/n": 2, |
| "num_tokens": 16304375.0, |
| "register/okay": 0.46875, |
| "register/okay_any": 0.71875, |
| "register/premium": -0.019607843137254943, |
| "register/verify_any": 0.84375, |
| "reward": 0.521484375, |
| "reward_std": 0.5000267624855042, |
| "rewards/correct/mean": 0.521484375, |
| "rewards/correct/std": 0.5000267624855042, |
| "rollout/chars": 7497.34375, |
| "rollout/commit": 0.5625, |
| "rollout/reward": 0.34375, |
| "rollout/trunc": 0.4375, |
| "sampling/importance_sampling_ratio/max": 2.9332101345062256, |
| "sampling/importance_sampling_ratio/mean": 0.9471936821937561, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.8108747005462646, |
| "sampling/sampling_logp_difference/mean": 0.0053689200431108475, |
| "step": 23, |
| "step_time": 153.59572070796276 |
| }, |
| { |
| "epoch": 0.12365591397849462, |
| "grad_norm": 0.005164750851690769, |
| "learning_rate": 1e-05, |
| "loss": -0.08419081568717957, |
| "step": 23 |
| }, |
| { |
| "clip_ratio": 0.010075022632008768, |
| "completions/clipped_ratio": 0.10546875, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4033.0, |
| "completions/mean_length": 1299.1953125, |
| "completions/mean_terminated_length": 969.4410400390625, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "epoch": 0.12903225806451613, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 17027259.0, |
| "register/okay": 0.3125, |
| "register/okay_any": 0.5, |
| "register/premium": 0.5454545454545454, |
| "register/verify_any": 0.53125, |
| "reward": 0.525390625, |
| "reward_std": 0.4998432695865631, |
| "rewards/correct/mean": 0.525390625, |
| "rewards/correct/std": 0.4998432695865631, |
| "rollout/chars": 2898.625, |
| "rollout/commit": 0.84375, |
| "rollout/reward": 0.625, |
| "rollout/trunc": 0.03125, |
| "sampling/importance_sampling_ratio/max": 2.7062180042266846, |
| "sampling/importance_sampling_ratio/mean": 0.9389370083808899, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.5482387542724609, |
| "sampling/sampling_logp_difference/mean": 0.005153043195605278, |
| "step": 24, |
| "step_time": 147.44872645987198 |
| }, |
| { |
| "epoch": 0.12903225806451613, |
| "grad_norm": 0.004218562971800566, |
| "learning_rate": 1e-05, |
| "loss": -0.0943070650100708, |
| "step": 24 |
| }, |
| { |
| "clip_ratio": 0.011164478141836298, |
| "completions/clipped_ratio": 0.14453125, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4049.0, |
| "completions/mean_length": 1490.28515625, |
| "completions/mean_terminated_length": 1059.365234375, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.13440860215053763, |
| "frac_reward_zero_std": 0.03125, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 17844301.0, |
| "register/okay": 0.625, |
| "register/okay_any": 0.75, |
| "register/premium": 0.6, |
| "register/verify_any": 0.8125, |
| "reward": 0.509765625, |
| "reward_std": 0.5003935098648071, |
| "rewards/correct/mean": 0.509765625, |
| "rewards/correct/std": 0.5003935098648071, |
| "rollout/chars": 5824.28125, |
| "rollout/commit": 0.78125, |
| "rollout/reward": 0.625, |
| "rollout/trunc": 0.09375, |
| "sampling/importance_sampling_ratio/max": 2.9208223819732666, |
| "sampling/importance_sampling_ratio/mean": 0.9381150007247925, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.46186184883117676, |
| "sampling/sampling_logp_difference/mean": 0.005271736532449722, |
| "step": 25, |
| "step_time": 175.4866898949258 |
| }, |
| { |
| "epoch": 0.13440860215053763, |
| "grad_norm": 0.004921833518892527, |
| "learning_rate": 1e-05, |
| "loss": -0.02945581078529358, |
| "step": 25 |
| }, |
| { |
| "clip_ratio": 0.008224556875575217, |
| "completions/clipped_ratio": 0.111328125, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4072.0, |
| "completions/mean_length": 1501.216796875, |
| "completions/mean_terminated_length": 1185.1275634765625, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.13978494623655913, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 18674172.0, |
| "register/okay": 0.28125, |
| "register/okay_any": 0.5, |
| "register/premium": 0.6521739130434783, |
| "register/verify_any": 0.625, |
| "reward": 0.53125, |
| "reward_std": 0.4995105266571045, |
| "rewards/correct/mean": 0.53125, |
| "rewards/correct/std": 0.4995105266571045, |
| "rollout/chars": 4819.0625, |
| "rollout/commit": 0.71875, |
| "rollout/reward": 0.53125, |
| "rollout/trunc": 0.125, |
| "sampling/importance_sampling_ratio/max": 2.943610429763794, |
| "sampling/importance_sampling_ratio/mean": 0.9341355562210083, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.6876890659332275, |
| "sampling/sampling_logp_difference/mean": 0.005215954966843128, |
| "step": 26, |
| "step_time": 167.32986848492874 |
| }, |
| { |
| "epoch": 0.13978494623655913, |
| "grad_norm": 0.005027454346418381, |
| "learning_rate": 1e-05, |
| "loss": -0.010844079777598381, |
| "step": 26 |
| }, |
| { |
| "clip_ratio": 0.011227854249227676, |
| "completions/clipped_ratio": 0.11328125, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4079.0, |
| "completions/mean_length": 1291.7265625, |
| "completions/mean_terminated_length": 951.308349609375, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.14516129032258066, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 19391184.0, |
| "register/okay": 0.46875, |
| "register/okay_any": 0.65625, |
| "register/premium": 0.5215686274509804, |
| "register/verify_any": 0.65625, |
| "reward": 0.560546875, |
| "reward_std": 0.49680593609809875, |
| "rewards/correct/mean": 0.560546875, |
| "rewards/correct/std": 0.49680593609809875, |
| "rollout/chars": 2457.03125, |
| "rollout/commit": 0.875, |
| "rollout/reward": 0.65625, |
| "rollout/trunc": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.8571557998657227, |
| "sampling/importance_sampling_ratio/mean": 0.9634403586387634, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.5841646194458008, |
| "sampling/sampling_logp_difference/mean": 0.005507075227797031, |
| "step": 27, |
| "step_time": 190.42925496015232 |
| }, |
| { |
| "epoch": 0.14516129032258066, |
| "grad_norm": 0.0052156648598611355, |
| "learning_rate": 1e-05, |
| "loss": -0.08258605003356934, |
| "step": 27 |
| }, |
| { |
| "clip_ratio": 0.007539385935615428, |
| "completions/clipped_ratio": 0.142578125, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 3857.0, |
| "completions/mean_length": 1458.96484375, |
| "completions/mean_terminated_length": 1029.77001953125, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.15053763440860216, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 20195006.0, |
| "register/okay": 0.4375, |
| "register/okay_any": 0.625, |
| "register/premium": 0.5, |
| "register/verify_any": 0.625, |
| "reward": 0.51953125, |
| "reward_std": 0.5001069903373718, |
| "rewards/correct/mean": 0.51953125, |
| "rewards/correct/std": 0.5001069903373718, |
| "rollout/chars": 3357.65625, |
| "rollout/commit": 0.875, |
| "rollout/reward": 0.71875, |
| "rollout/trunc": 0.03125, |
| "sampling/importance_sampling_ratio/max": 2.9831702709198, |
| "sampling/importance_sampling_ratio/mean": 0.964399516582489, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.4702794551849365, |
| "sampling/sampling_logp_difference/mean": 0.0050812773406505585, |
| "step": 28, |
| "step_time": 169.85662743984722 |
| }, |
| { |
| "epoch": 0.15053763440860216, |
| "grad_norm": 0.005234717857092619, |
| "learning_rate": 1e-05, |
| "loss": -0.03653641790151596, |
| "step": 28 |
| }, |
| { |
| "clip_ratio": 0.00900801509123994, |
| "completions/clipped_ratio": 0.140625, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4049.0, |
| "completions/mean_length": 1361.64453125, |
| "completions/mean_terminated_length": 915.3340454101562, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.15591397849462366, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 20957400.0, |
| "register/okay": 0.5, |
| "register/okay_any": 0.65625, |
| "register/premium": 0.4375, |
| "register/verify_any": 0.65625, |
| "reward": 0.509765625, |
| "reward_std": 0.5003935098648071, |
| "rewards/correct/mean": 0.509765625, |
| "rewards/correct/std": 0.5003935098648071, |
| "rollout/chars": 4667.1875, |
| "rollout/commit": 0.8125, |
| "rollout/reward": 0.46875, |
| "rollout/trunc": 0.15625, |
| "sampling/importance_sampling_ratio/max": 2.649383783340454, |
| "sampling/importance_sampling_ratio/mean": 0.925227165222168, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.4926145076751709, |
| "sampling/sampling_logp_difference/mean": 0.005465826950967312, |
| "step": 29, |
| "step_time": 179.66692242206773 |
| }, |
| { |
| "epoch": 0.15591397849462366, |
| "grad_norm": 0.0046501075848937035, |
| "learning_rate": 1e-05, |
| "loss": -0.01778368279337883, |
| "step": 29 |
| }, |
| { |
| "clip_ratio": 0.008490964444717974, |
| "completions/clipped_ratio": 0.12109375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4025.0, |
| "completions/mean_length": 1251.921875, |
| "completions/mean_terminated_length": 860.0711059570312, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.16129032258064516, |
| "frac_reward_zero_std": 0.03125, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 21658480.0, |
| "register/okay": 0.25, |
| "register/okay_any": 0.59375, |
| "register/premium": 0.4166666666666667, |
| "register/verify_any": 0.65625, |
| "reward": 0.537109375, |
| "reward_std": 0.4991086423397064, |
| "rewards/correct/mean": 0.537109375, |
| "rewards/correct/std": 0.4991086423397064, |
| "rollout/chars": 3536.9375, |
| "rollout/commit": 0.78125, |
| "rollout/reward": 0.5625, |
| "rollout/trunc": 0.09375, |
| "sampling/importance_sampling_ratio/max": 2.795497417449951, |
| "sampling/importance_sampling_ratio/mean": 0.9529191255569458, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.5288147926330566, |
| "sampling/sampling_logp_difference/mean": 0.005513270851224661, |
| "step": 30, |
| "step_time": 155.76831682719057 |
| }, |
| { |
| "epoch": 0.16129032258064516, |
| "grad_norm": 0.005014779977500439, |
| "learning_rate": 1e-05, |
| "loss": -0.05131439492106438, |
| "step": 30 |
| }, |
| { |
| "clip_ratio": 0.006336418812679767, |
| "completions/clipped_ratio": 0.125, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4083.0, |
| "completions/mean_length": 1301.068359375, |
| "completions/mean_terminated_length": 901.79248046875, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.16666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 22375811.0, |
| "register/okay": 0.40625, |
| "register/okay_any": 0.59375, |
| "register/premium": 0.5546558704453441, |
| "register/verify_any": 0.59375, |
| "reward": 0.560546875, |
| "reward_std": 0.49680593609809875, |
| "rewards/correct/mean": 0.560546875, |
| "rewards/correct/std": 0.49680593609809875, |
| "rollout/chars": 3711.875, |
| "rollout/commit": 0.84375, |
| "rollout/reward": 0.59375, |
| "rollout/trunc": 0.03125, |
| "sampling/importance_sampling_ratio/max": 2.959876775741577, |
| "sampling/importance_sampling_ratio/mean": 0.9501587748527527, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.3710575103759766, |
| "sampling/sampling_logp_difference/mean": 0.005541889928281307, |
| "step": 31, |
| "step_time": 131.88608792208834 |
| }, |
| { |
| "epoch": 0.16666666666666666, |
| "grad_norm": 0.0051728710532188416, |
| "learning_rate": 1e-05, |
| "loss": -0.024730639532208443, |
| "step": 31 |
| }, |
| { |
| "clip_ratio": 0.005722886576222663, |
| "completions/clipped_ratio": 0.13671875, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 3969.0, |
| "completions/mean_length": 1450.232421875, |
| "completions/mean_terminated_length": 1031.219482421875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "epoch": 0.17204301075268819, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 23181034.0, |
| "register/okay": 0.3125, |
| "register/okay_any": 0.53125, |
| "register/premium": 0.5, |
| "register/verify_any": 0.53125, |
| "reward": 0.525390625, |
| "reward_std": 0.4998432695865631, |
| "rewards/correct/mean": 0.525390625, |
| "rewards/correct/std": 0.4998432695865631, |
| "rollout/chars": 1586.09375, |
| "rollout/commit": 0.875, |
| "rollout/reward": 0.65625, |
| "rollout/trunc": 0.03125, |
| "sampling/importance_sampling_ratio/max": 2.913379192352295, |
| "sampling/importance_sampling_ratio/mean": 0.9456106424331665, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.5503673553466797, |
| "sampling/sampling_logp_difference/mean": 0.0051387008279562, |
| "step": 32, |
| "step_time": 166.45166590507142 |
| }, |
| { |
| "epoch": 0.17204301075268819, |
| "grad_norm": 0.004503391217440367, |
| "learning_rate": 1e-05, |
| "loss": -0.05110776051878929, |
| "step": 32 |
| }, |
| { |
| "clip_ratio": 0.00871692816963332, |
| "completions/clipped_ratio": 0.080078125, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4093.0, |
| "completions/mean_length": 1226.68359375, |
| "completions/mean_terminated_length": 976.9130249023438, |
| "completions/min_length": 3.0, |
| "completions/min_terminated_length": 3.0, |
| "epoch": 0.1774193548387097, |
| "frac_reward_zero_std": 0.03125, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 23870856.0, |
| "register/okay": 0.40625, |
| "register/okay_any": 0.59375, |
| "register/premium": 0.631578947368421, |
| "register/verify_any": 0.59375, |
| "reward": 0.595703125, |
| "reward_std": 0.4912354052066803, |
| "rewards/correct/mean": 0.595703125, |
| "rewards/correct/std": 0.4912354052066803, |
| "rollout/chars": 2484.125, |
| "rollout/commit": 0.8125, |
| "rollout/reward": 0.625, |
| "rollout/trunc": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.6280722618103027, |
| "sampling/importance_sampling_ratio/mean": 0.948501467704773, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.574876070022583, |
| "sampling/sampling_logp_difference/mean": 0.005526247434318066, |
| "step": 33, |
| "step_time": 150.0948413569713 |
| }, |
| { |
| "epoch": 0.1774193548387097, |
| "grad_norm": 0.005056716967374086, |
| "learning_rate": 1e-05, |
| "loss": -0.08073143661022186, |
| "step": 33 |
| }, |
| { |
| "clip_ratio": 0.011121355913019215, |
| "completions/clipped_ratio": 0.126953125, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4096.0, |
| "completions/mean_length": 1472.6640625, |
| "completions/mean_terminated_length": 1109.4608154296875, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.1827956989247312, |
| "frac_reward_zero_std": 0.03125, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 24677244.0, |
| "register/okay": 0.375, |
| "register/okay_any": 0.5625, |
| "register/premium": 0.5, |
| "register/verify_any": 0.59375, |
| "reward": 0.552734375, |
| "reward_std": 0.4976975917816162, |
| "rewards/correct/mean": 0.552734375, |
| "rewards/correct/std": 0.4976975917816162, |
| "rollout/chars": 5478.375, |
| "rollout/commit": 0.71875, |
| "rollout/reward": 0.4375, |
| "rollout/trunc": 0.1875, |
| "sampling/importance_sampling_ratio/max": 2.9688801765441895, |
| "sampling/importance_sampling_ratio/mean": 0.9284334778785706, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.5757560729980469, |
| "sampling/sampling_logp_difference/mean": 0.004951587412506342, |
| "step": 34, |
| "step_time": 136.5056155999191 |
| }, |
| { |
| "epoch": 0.1827956989247312, |
| "grad_norm": 0.0041446140967309475, |
| "learning_rate": 1e-05, |
| "loss": -0.07102659344673157, |
| "step": 34 |
| }, |
| { |
| "clip_ratio": 0.010774545253298129, |
| "completions/clipped_ratio": 0.103515625, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4085.0, |
| "completions/mean_length": 1302.275390625, |
| "completions/mean_terminated_length": 986.1568603515625, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.1881720430107527, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 25401865.0, |
| "register/okay": 0.28125, |
| "register/okay_any": 0.4375, |
| "register/premium": 0.5217391304347826, |
| "register/verify_any": 0.4375, |
| "reward": 0.55078125, |
| "reward_std": 0.497901052236557, |
| "rewards/correct/mean": 0.55078125, |
| "rewards/correct/std": 0.497901052236557, |
| "rollout/chars": 1811.84375, |
| "rollout/commit": 0.96875, |
| "rollout/reward": 0.625, |
| "rollout/trunc": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.8669655323028564, |
| "sampling/importance_sampling_ratio/mean": 0.9753417372703552, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.639234721660614, |
| "sampling/sampling_logp_difference/mean": 0.005423135124146938, |
| "step": 35, |
| "step_time": 143.07610333099728 |
| }, |
| { |
| "epoch": 0.1881720430107527, |
| "grad_norm": 0.005199028179049492, |
| "learning_rate": 1e-05, |
| "loss": -0.10055884718894958, |
| "step": 35 |
| }, |
| { |
| "clip_ratio": 0.008821962311230891, |
| "completions/clipped_ratio": 0.09765625, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4078.0, |
| "completions/mean_length": 1438.88671875, |
| "completions/mean_terminated_length": 1151.3203125, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.1935483870967742, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 26195471.0, |
| "register/okay": 0.40625, |
| "register/okay_any": 0.59375, |
| "register/premium": 0.35222672064777333, |
| "register/verify_any": 0.625, |
| "reward": 0.603515625, |
| "reward_std": 0.4896455705165863, |
| "rewards/correct/mean": 0.603515625, |
| "rewards/correct/std": 0.4896455705165863, |
| "rollout/chars": 4416.875, |
| "rollout/commit": 0.625, |
| "rollout/reward": 0.40625, |
| "rollout/trunc": 0.25, |
| "sampling/importance_sampling_ratio/max": 2.8039920330047607, |
| "sampling/importance_sampling_ratio/mean": 0.9573048949241638, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.043118953704834, |
| "sampling/sampling_logp_difference/mean": 0.005074475891888142, |
| "step": 36, |
| "step_time": 133.7379490098101 |
| }, |
| { |
| "epoch": 0.1935483870967742, |
| "grad_norm": 0.004406781867146492, |
| "learning_rate": 1e-05, |
| "loss": -0.1069176122546196, |
| "step": 36 |
| }, |
| { |
| "clip_ratio": 0.006849257575595402, |
| "completions/clipped_ratio": 0.115234375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4060.0, |
| "completions/mean_length": 1347.248046875, |
| "completions/mean_terminated_length": 1007.0419311523438, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.1989247311827957, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 26947182.0, |
| "register/okay": 0.40625, |
| "register/okay_any": 0.59375, |
| "register/premium": 0.6356275303643725, |
| "register/verify_any": 0.5625, |
| "reward": 0.556640625, |
| "reward_std": 0.49726733565330505, |
| "rewards/correct/mean": 0.556640625, |
| "rewards/correct/std": 0.49726733565330505, |
| "rollout/chars": 3451.28125, |
| "rollout/commit": 0.84375, |
| "rollout/reward": 0.46875, |
| "rollout/trunc": 0.0625, |
| "sampling/importance_sampling_ratio/max": 2.5512001514434814, |
| "sampling/importance_sampling_ratio/mean": 0.9806995391845703, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.6822056770324707, |
| "sampling/sampling_logp_difference/mean": 0.005445747636258602, |
| "step": 37, |
| "step_time": 152.8698032599641 |
| }, |
| { |
| "epoch": 0.1989247311827957, |
| "grad_norm": 0.005230598617345095, |
| "learning_rate": 1e-05, |
| "loss": -0.025864053517580032, |
| "step": 37 |
| }, |
| { |
| "clip_ratio": 0.006331337699066353, |
| "completions/clipped_ratio": 0.09375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 3999.0, |
| "completions/mean_length": 1394.583984375, |
| "completions/mean_terminated_length": 1123.70263671875, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "epoch": 0.20430107526881722, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 27718505.0, |
| "register/okay": 0.59375, |
| "register/okay_any": 0.71875, |
| "register/premium": 0.17004048582995945, |
| "register/verify_any": 0.75, |
| "reward": 0.599609375, |
| "reward_std": 0.4904567301273346, |
| "rewards/correct/mean": 0.599609375, |
| "rewards/correct/std": 0.4904567301273346, |
| "rollout/chars": 4813.65625, |
| "rollout/commit": 0.78125, |
| "rollout/reward": 0.5625, |
| "rollout/trunc": 0.25, |
| "sampling/importance_sampling_ratio/max": 2.8853559494018555, |
| "sampling/importance_sampling_ratio/mean": 0.9194067716598511, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.876793622970581, |
| "sampling/sampling_logp_difference/mean": 0.005094734486192465, |
| "step": 38, |
| "step_time": 154.3992670099833 |
| }, |
| { |
| "epoch": 0.20430107526881722, |
| "grad_norm": 0.004638301208615303, |
| "learning_rate": 1e-05, |
| "loss": -0.08557215332984924, |
| "step": 38 |
| }, |
| { |
| "clip_ratio": 0.0064660851248845574, |
| "completions/clipped_ratio": 0.115234375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4018.0, |
| "completions/mean_length": 1461.224609375, |
| "completions/mean_terminated_length": 1136.0728759765625, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.20967741935483872, |
| "frac_reward_zero_std": 0.03125, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 28517884.0, |
| "register/okay": 0.375, |
| "register/okay_any": 0.625, |
| "register/premium": 0.3666666666666666, |
| "register/verify_any": 0.65625, |
| "reward": 0.56640625, |
| "reward_std": 0.4960552453994751, |
| "rewards/correct/mean": 0.56640625, |
| "rewards/correct/std": 0.4960552453994751, |
| "rollout/chars": 3073.21875, |
| "rollout/commit": 0.875, |
| "rollout/reward": 0.6875, |
| "rollout/trunc": 0.03125, |
| "sampling/importance_sampling_ratio/max": 2.964137077331543, |
| "sampling/importance_sampling_ratio/mean": 0.9699006080627441, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.7239212989807129, |
| "sampling/sampling_logp_difference/mean": 0.00531615037471056, |
| "step": 39, |
| "step_time": 165.53434173611458 |
| }, |
| { |
| "epoch": 0.20967741935483872, |
| "grad_norm": 0.004682127386331558, |
| "learning_rate": 1e-05, |
| "loss": -0.07406766712665558, |
| "step": 39 |
| }, |
| { |
| "clip_ratio": 0.00743856916869845, |
| "completions/clipped_ratio": 0.1875, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4065.0, |
| "completions/mean_length": 1560.82421875, |
| "completions/mean_terminated_length": 988.454345703125, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.21505376344086022, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 29374962.0, |
| "register/okay": 0.375, |
| "register/okay_any": 0.65625, |
| "register/premium": 0.44999999999999996, |
| "register/verify_any": 0.65625, |
| "reward": 0.5234375, |
| "reward_std": 0.49993884563446045, |
| "rewards/correct/mean": 0.5234375, |
| "rewards/correct/std": 0.49993884563446045, |
| "rollout/chars": 4500.40625, |
| "rollout/commit": 0.84375, |
| "rollout/reward": 0.71875, |
| "rollout/trunc": 0.09375, |
| "sampling/importance_sampling_ratio/max": 2.669318199157715, |
| "sampling/importance_sampling_ratio/mean": 0.9309188723564148, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.1114287376403809, |
| "sampling/sampling_logp_difference/mean": 0.005226319655776024, |
| "step": 40, |
| "step_time": 165.58794789918466 |
| }, |
| { |
| "epoch": 0.21505376344086022, |
| "grad_norm": 0.004323912318795919, |
| "learning_rate": 1e-05, |
| "loss": -0.0437479205429554, |
| "step": 40 |
| }, |
| { |
| "clip_ratio": 0.008895926593140757, |
| "completions/clipped_ratio": 0.068359375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 3994.0, |
| "completions/mean_length": 1094.49609375, |
| "completions/mean_terminated_length": 899.8070678710938, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.22043010752688172, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 29994576.0, |
| "register/okay": 0.46875, |
| "register/okay_any": 0.65625, |
| "register/premium": 0.5294117647058824, |
| "register/verify_any": 0.6875, |
| "reward": 0.60546875, |
| "reward_std": 0.4892277717590332, |
| "rewards/correct/mean": 0.60546875, |
| "rewards/correct/std": 0.4892277717590332, |
| "rollout/chars": 4548.75, |
| "rollout/commit": 0.875, |
| "rollout/reward": 0.71875, |
| "rollout/trunc": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.583728313446045, |
| "sampling/importance_sampling_ratio/mean": 0.9422752857208252, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.7799820899963379, |
| "sampling/sampling_logp_difference/mean": 0.00539364293217659, |
| "step": 41, |
| "step_time": 130.81026078393916 |
| }, |
| { |
| "epoch": 0.22043010752688172, |
| "grad_norm": 0.004999271593987942, |
| "learning_rate": 1e-05, |
| "loss": -0.09776099026203156, |
| "step": 41 |
| }, |
| { |
| "clip_ratio": 0.008774890723543649, |
| "completions/clipped_ratio": 0.119140625, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4048.0, |
| "completions/mean_length": 1410.099609375, |
| "completions/mean_terminated_length": 1046.818115234375, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.22580645161290322, |
| "frac_reward_zero_std": 0.0625, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 30777635.0, |
| "register/okay": 0.5, |
| "register/okay_any": 0.75, |
| "register/premium": 0.5, |
| "register/verify_any": 0.78125, |
| "reward": 0.572265625, |
| "reward_std": 0.4952339828014374, |
| "rewards/correct/mean": 0.572265625, |
| "rewards/correct/std": 0.4952339828014374, |
| "rollout/chars": 2519.5, |
| "rollout/commit": 0.90625, |
| "rollout/reward": 0.6875, |
| "rollout/trunc": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.90441632270813, |
| "sampling/importance_sampling_ratio/mean": 0.9259432554244995, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.7504162788391113, |
| "sampling/sampling_logp_difference/mean": 0.005266791209578514, |
| "step": 42, |
| "step_time": 182.9241154739866 |
| }, |
| { |
| "epoch": 0.22580645161290322, |
| "grad_norm": 0.004195198882371187, |
| "learning_rate": 1e-05, |
| "loss": -0.08093886822462082, |
| "step": 42 |
| }, |
| { |
| "clip_ratio": 0.003392930002974026, |
| "completions/clipped_ratio": 0.138671875, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4073.0, |
| "completions/mean_length": 1570.396484375, |
| "completions/mean_terminated_length": 1182.2109375, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "epoch": 0.23118279569892472, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 31636510.0, |
| "register/okay": 0.53125, |
| "register/okay_any": 0.8125, |
| "register/premium": 0.4313725490196078, |
| "register/verify_any": 0.8125, |
| "reward": 0.552734375, |
| "reward_std": 0.4976975917816162, |
| "rewards/correct/mean": 0.552734375, |
| "rewards/correct/std": 0.4976975917816162, |
| "rollout/chars": 7332.375, |
| "rollout/commit": 0.71875, |
| "rollout/reward": 0.5625, |
| "rollout/trunc": 0.25, |
| "sampling/importance_sampling_ratio/max": 2.9183237552642822, |
| "sampling/importance_sampling_ratio/mean": 0.9194202423095703, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.0795272588729858, |
| "sampling/sampling_logp_difference/mean": 0.005144014023244381, |
| "step": 43, |
| "step_time": 154.99469502893044 |
| }, |
| { |
| "epoch": 0.23118279569892472, |
| "grad_norm": 0.004625420551747084, |
| "learning_rate": 1e-05, |
| "loss": -0.04634333774447441, |
| "step": 43 |
| }, |
| { |
| "clip_ratio": 0.005952214763055963, |
| "completions/clipped_ratio": 0.169921875, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4095.0, |
| "completions/mean_length": 1643.234375, |
| "completions/mean_terminated_length": 1150.73876953125, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "epoch": 0.23655913978494625, |
| "frac_reward_zero_std": 0.03125, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 32532246.0, |
| "register/okay": 0.5, |
| "register/okay_any": 0.75, |
| "register/premium": 0.25, |
| "register/verify_any": 0.75, |
| "reward": 0.591796875, |
| "reward_std": 0.49198177456855774, |
| "rewards/correct/mean": 0.591796875, |
| "rewards/correct/std": 0.49198177456855774, |
| "rollout/chars": 7713.09375, |
| "rollout/commit": 0.5625, |
| "rollout/reward": 0.125, |
| "rollout/trunc": 0.4375, |
| "sampling/importance_sampling_ratio/max": 2.849212646484375, |
| "sampling/importance_sampling_ratio/mean": 0.9619724750518799, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.8619403839111328, |
| "sampling/sampling_logp_difference/mean": 0.0050093987956643105, |
| "step": 44, |
| "step_time": 210.83732629718725 |
| }, |
| { |
| "epoch": 0.23655913978494625, |
| "grad_norm": 0.004370514303445816, |
| "learning_rate": 1e-05, |
| "loss": -0.024845460429787636, |
| "step": 44 |
| }, |
| { |
| "clip_ratio": 0.005458997462028492, |
| "completions/clipped_ratio": 0.0859375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4088.0, |
| "completions/mean_length": 1299.287109375, |
| "completions/mean_terminated_length": 1036.348388671875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "epoch": 0.24193548387096775, |
| "frac_reward_zero_std": 0.0625, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 33250953.0, |
| "register/okay": 0.4375, |
| "register/okay_any": 0.59375, |
| "register/premium": 0.373015873015873, |
| "register/verify_any": 0.59375, |
| "reward": 0.640625, |
| "reward_std": 0.48028653860092163, |
| "rewards/correct/mean": 0.640625, |
| "rewards/correct/std": 0.48028653860092163, |
| "rollout/chars": 3825.34375, |
| "rollout/commit": 0.84375, |
| "rollout/reward": 0.71875, |
| "rollout/trunc": 0.0625, |
| "sampling/importance_sampling_ratio/max": 2.873892068862915, |
| "sampling/importance_sampling_ratio/mean": 0.9793621301651001, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.9061906337738037, |
| "sampling/sampling_logp_difference/mean": 0.00526129174977541, |
| "step": 45, |
| "step_time": 140.2460780791589 |
| }, |
| { |
| "epoch": 0.24193548387096775, |
| "grad_norm": 0.005457329098135233, |
| "learning_rate": 1e-05, |
| "loss": -0.10607008635997772, |
| "step": 45 |
| }, |
| { |
| "clip_ratio": 0.010226659814179584, |
| "completions/clipped_ratio": 0.09375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 3943.0, |
| "completions/mean_length": 1321.83984375, |
| "completions/mean_terminated_length": 1034.8577880859375, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.24731182795698925, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 33977671.0, |
| "register/okay": 0.625, |
| "register/okay_any": 0.75, |
| "register/premium": 0.13333333333333336, |
| "register/verify_any": 0.75, |
| "reward": 0.6484375, |
| "reward_std": 0.4779251217842102, |
| "rewards/correct/mean": 0.6484375, |
| "rewards/correct/std": 0.4779251217842102, |
| "rollout/chars": 6811.09375, |
| "rollout/commit": 0.625, |
| "rollout/reward": 0.5, |
| "rollout/trunc": 0.21875, |
| "sampling/importance_sampling_ratio/max": 2.659039258956909, |
| "sampling/importance_sampling_ratio/mean": 0.9590080976486206, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.9100682735443115, |
| "sampling/sampling_logp_difference/mean": 0.005064851604402065, |
| "step": 46, |
| "step_time": 143.94649956107605 |
| }, |
| { |
| "epoch": 0.24731182795698925, |
| "grad_norm": 0.004310651682317257, |
| "learning_rate": 1e-05, |
| "loss": -0.061595045030117035, |
| "step": 46 |
| }, |
| { |
| "clip_ratio": 0.006405189862107363, |
| "completions/clipped_ratio": 0.123046875, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4082.0, |
| "completions/mean_length": 1577.265625, |
| "completions/mean_terminated_length": 1229.140380859375, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.25268817204301075, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 34842799.0, |
| "register/okay": 0.4375, |
| "register/okay_any": 0.65625, |
| "register/premium": 0.6190476190476191, |
| "register/verify_any": 0.71875, |
| "reward": 0.595703125, |
| "reward_std": 0.4912354052066803, |
| "rewards/correct/mean": 0.595703125, |
| "rewards/correct/std": 0.4912354052066803, |
| "rollout/chars": 4768.96875, |
| "rollout/commit": 0.625, |
| "rollout/reward": 0.4375, |
| "rollout/trunc": 0.1875, |
| "sampling/importance_sampling_ratio/max": 2.9756243228912354, |
| "sampling/importance_sampling_ratio/mean": 0.9440420269966125, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 0.9075045585632324, |
| "sampling/sampling_logp_difference/mean": 0.005123917944729328, |
| "step": 47, |
| "step_time": 161.0352724850527 |
| }, |
| { |
| "epoch": 0.25268817204301075, |
| "grad_norm": 0.004760539624840021, |
| "learning_rate": 1e-05, |
| "loss": -0.07464903593063354, |
| "step": 47 |
| }, |
| { |
| "clip_ratio": 0.00796015004743822, |
| "completions/clipped_ratio": 0.13671875, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4063.0, |
| "completions/mean_length": 1538.916015625, |
| "completions/mean_terminated_length": 1133.947998046875, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.25806451612903225, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 35684628.0, |
| "register/okay": 0.5, |
| "register/okay_any": 0.59375, |
| "register/premium": 0.4375, |
| "register/verify_any": 0.65625, |
| "reward": 0.595703125, |
| "reward_std": 0.4912354052066803, |
| "rewards/correct/mean": 0.595703125, |
| "rewards/correct/std": 0.4912354052066803, |
| "rollout/chars": 3902.28125, |
| "rollout/commit": 0.8125, |
| "rollout/reward": 0.71875, |
| "rollout/trunc": 0.0625, |
| "sampling/importance_sampling_ratio/max": 2.9745614528656006, |
| "sampling/importance_sampling_ratio/mean": 0.923351526260376, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.1928386688232422, |
| "sampling/sampling_logp_difference/mean": 0.004994367249310017, |
| "step": 48, |
| "step_time": 152.8169575711945 |
| }, |
| { |
| "epoch": 0.25806451612903225, |
| "grad_norm": 0.004467393737286329, |
| "learning_rate": 1e-05, |
| "loss": -0.037505295127630234, |
| "step": 48 |
| }, |
| { |
| "clip_ratio": 0.008797254737146432, |
| "completions/clipped_ratio": 0.115234375, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4058.0, |
| "completions/mean_length": 1352.208984375, |
| "completions/mean_terminated_length": 1037.169921875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "epoch": 0.26344086021505375, |
| "frac_reward_zero_std": 0.0, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 36437887.0, |
| "register/okay": 0.625, |
| "register/okay_any": 0.75, |
| "register/premium": 0.4166666666666667, |
| "register/verify_any": 0.71875, |
| "reward": 0.6015625, |
| "reward_std": 0.4900552034378052, |
| "rewards/correct/mean": 0.6015625, |
| "rewards/correct/std": 0.4900552034378052, |
| "rollout/chars": 4644.5625, |
| "rollout/commit": 0.78125, |
| "rollout/reward": 0.59375, |
| "rollout/trunc": 0.125, |
| "sampling/importance_sampling_ratio/max": 2.9898746013641357, |
| "sampling/importance_sampling_ratio/mean": 0.9452433586120605, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.5847194194793701, |
| "sampling/sampling_logp_difference/mean": 0.005605063866823912, |
| "step": 49, |
| "step_time": 146.1176283750101 |
| }, |
| { |
| "epoch": 0.26344086021505375, |
| "grad_norm": 0.005190466996282339, |
| "learning_rate": 1e-05, |
| "loss": -0.052763931453228, |
| "step": 49 |
| }, |
| { |
| "clip_ratio": 0.009056178430000728, |
| "completions/clipped_ratio": 0.1171875, |
| "completions/max_length": 4096.0, |
| "completions/max_terminated_length": 4052.0, |
| "completions/mean_length": 1506.935546875, |
| "completions/mean_terminated_length": 1172.281005859375, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.26881720430107525, |
| "frac_reward_zero_std": 0.03125, |
| "groups/all_right": 0.0, |
| "groups/all_wrong": 0.0, |
| "groups/live_frac": 1.0, |
| "groups/n": 2, |
| "num_tokens": 37264702.0, |
| "register/okay": 0.53125, |
| "register/okay_any": 0.625, |
| "register/premium": 0.40784313725490196, |
| "register/verify_any": 0.65625, |
| "reward": 0.59765625, |
| "reward_std": 0.4908501207828522, |
| "rewards/correct/mean": 0.59765625, |
| "rewards/correct/std": 0.4908501207828522, |
| "rollout/chars": 3848.09375, |
| "rollout/commit": 0.875, |
| "rollout/reward": 0.75, |
| "rollout/trunc": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.8477158546447754, |
| "sampling/importance_sampling_ratio/mean": 0.9113643765449524, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.059030532836914, |
| "sampling/sampling_logp_difference/mean": 0.005309573840349913, |
| "step": 50, |
| "step_time": 146.82213812193368 |
| }, |
| { |
| "epoch": 0.26881720430107525, |
| "grad_norm": 0.004830545280128717, |
| "learning_rate": 1e-05, |
| "loss": -0.0666024386882782, |
| "step": 50 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 300, |
| "num_input_tokens_seen": 37264702, |
| "num_train_epochs": 2, |
| "save_steps": 25, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
