{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.7045454545454546, "eval_steps": 500, "global_step": 150, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio": 2.6910373094324314e-06, "completions/clipped_ratio": 0.36328125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4087.0, "completions/mean_length": 2821.140625, "completions/mean_terminated_length": 2098.60107421875, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "epoch": 0.011363636363636364, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 1549064.0, "register/okay": 0.0, "register/okay_any": 0.9375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.5703125, "reward_std": 0.4955156147480011, "rewards/correct/mean": 0.5703125, "rewards/correct/std": 0.4955156147480011, "rollout/chars": 7862.0, "rollout/commit": 0.84375, "rollout/reward": 0.6875, "rollout/trunc": 0.28125, "sampling/importance_sampling_ratio/max": 2.773446559906006, "sampling/importance_sampling_ratio/mean": 0.9391552209854126, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.223902702331543, "sampling/sampling_logp_difference/mean": 0.004874078091233969, "step": 1, "step_time": 262.9772285820072 }, { "epoch": 0.011363636363636364, "grad_norm": 0.0036677950993180275, "learning_rate": 0.0, "loss": 0.07016487419605255, "step": 1 }, { "clip_ratio": 2.384185791015625e-06, "completions/clipped_ratio": 0.353515625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4077.0, "completions/mean_length": 2826.212890625, "completions/mean_terminated_length": 2131.85791015625, "completions/min_length": 440.0, "completions/min_terminated_length": 440.0, "epoch": 0.022727272727272728, "frac_reward_zero_std": 0.09375, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 3072933.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.583984375, "reward_std": 0.493378221988678, "rewards/correct/mean": 0.583984375, "rewards/correct/std": 0.493378221988678, "rollout/chars": 6436.875, "rollout/commit": 0.875, "rollout/reward": 0.78125, "rollout/trunc": 0.1875, "sampling/importance_sampling_ratio/max": 2.993257999420166, "sampling/importance_sampling_ratio/mean": 0.9715690016746521, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5188601016998291, "sampling/sampling_logp_difference/mean": 0.00453161308541894, "step": 2, "step_time": 234.78294498598916 }, { "epoch": 0.022727272727272728, "grad_norm": 0.0031095428857952356, "learning_rate": 2.0000000000000003e-06, "loss": 0.03657037392258644, "step": 2 }, { "clip_ratio": 1.0142219366571226e-06, "completions/clipped_ratio": 0.3125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4082.0, "completions/mean_length": 2454.72265625, "completions/mean_terminated_length": 1708.6875, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "epoch": 0.03409090909090909, "frac_reward_zero_std": 0.125, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 4395431.0, "register/okay": 0.03125, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.630859375, "reward_std": 0.4830440282821655, "rewards/correct/mean": 0.630859375, "rewards/correct/std": 0.4830440282821655, "rollout/chars": 6361.1875, "rollout/commit": 0.875, "rollout/reward": 0.875, "rollout/trunc": 0.125, "sampling/importance_sampling_ratio/max": 2.924746513366699, "sampling/importance_sampling_ratio/mean": 0.9988688826560974, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3344054222106934, "sampling/sampling_logp_difference/mean": 0.004491552244871855, "step": 3, "step_time": 239.0937386670048 }, { "epoch": 0.03409090909090909, "grad_norm": 0.003723646979779005, "learning_rate": 4.000000000000001e-06, "loss": 0.034827012568712234, "step": 3 }, { "clip_ratio": 1.9186707049811957e-06, "completions/clipped_ratio": 0.373046875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4087.0, "completions/mean_length": 2846.294921875, "completions/mean_terminated_length": 2102.700927734375, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "epoch": 0.045454545454545456, "frac_reward_zero_std": 0.09375, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 5927854.0, "register/okay": 0.0, "register/okay_any": 0.9375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.5625, "reward_std": 0.49656352400779724, "rewards/correct/mean": 0.5625, "rewards/correct/std": 0.49656352400779724, "rollout/chars": 9865.4375, "rollout/commit": 0.75, "rollout/reward": 0.65625, "rollout/trunc": 0.3125, "sampling/importance_sampling_ratio/max": 2.9617812633514404, "sampling/importance_sampling_ratio/mean": 0.9208488464355469, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.0877385139465332, "sampling/sampling_logp_difference/mean": 0.004612394608557224, "step": 4, "step_time": 227.59205132999705 }, { "epoch": 0.045454545454545456, "grad_norm": 0.0031638338696211576, "learning_rate": 6e-06, "loss": 0.052659276872873306, "step": 4 }, { "clip_ratio": 1.481114054513455e-06, "completions/clipped_ratio": 0.298828125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4078.0, "completions/mean_length": 2491.001953125, "completions/mean_terminated_length": 1806.9776611328125, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "epoch": 0.056818181818181816, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 7270063.0, "register/okay": 0.0625, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.646484375, "reward_std": 0.47852855920791626, "rewards/correct/mean": 0.646484375, "rewards/correct/std": 0.47852855920791626, "rollout/chars": 9426.375, "rollout/commit": 0.59375, "rollout/reward": 0.59375, "rollout/trunc": 0.40625, "sampling/importance_sampling_ratio/max": 2.7546863555908203, "sampling/importance_sampling_ratio/mean": 0.9406723380088806, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.0478997230529785, "sampling/sampling_logp_difference/mean": 0.0044736433774232864, "step": 5, "step_time": 209.38289929599705 }, { "epoch": 0.056818181818181816, "grad_norm": 0.003155597485601902, "learning_rate": 8.000000000000001e-06, "loss": 0.036301881074905396, "step": 5 }, { "clip_ratio": 2.491965801709739e-06, "completions/clipped_ratio": 0.443359375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4055.0, "completions/mean_length": 3193.939453125, "completions/mean_terminated_length": 2475.456298828125, "completions/min_length": 572.0, "completions/min_terminated_length": 572.0, "epoch": 0.06818181818181818, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 8985328.0, "register/okay": 0.0, "register/okay_any": 0.8125, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.509765625, "reward_std": 0.5003935098648071, "rewards/correct/mean": 0.509765625, "rewards/correct/std": 0.5003935098648071, "rollout/chars": 12321.75, "rollout/commit": 0.34375, "rollout/reward": 0.15625, "rollout/trunc": 0.8125, "sampling/importance_sampling_ratio/max": 2.806901693344116, "sampling/importance_sampling_ratio/mean": 0.9583785533905029, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.0830349922180176, "sampling/sampling_logp_difference/mean": 0.004532902501523495, "step": 6, "step_time": 232.05504834799285 }, { "epoch": 0.06818181818181818, "grad_norm": 0.0032948916777968407, "learning_rate": 1e-05, "loss": 0.022378744557499886, "step": 6 }, { "clip_ratio": 1.430511474609375e-06, "completions/clipped_ratio": 0.30859375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4077.0, "completions/mean_length": 2478.0546875, "completions/mean_terminated_length": 1755.9208984375, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "epoch": 0.07954545454545454, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 10316972.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.6328125, "reward_std": 0.48250964283943176, "rewards/correct/mean": 0.6328125, "rewards/correct/std": 0.48250964283943176, "rollout/chars": 4690.90625, "rollout/commit": 0.96875, "rollout/reward": 0.96875, "rollout/trunc": 0.03125, "sampling/importance_sampling_ratio/max": 2.98211932182312, "sampling/importance_sampling_ratio/mean": 0.9605857133865356, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7990617752075195, "sampling/sampling_logp_difference/mean": 0.00447505246847868, "step": 7, "step_time": 225.70014225300474 }, { "epoch": 0.07954545454545454, "grad_norm": 0.003252506721764803, "learning_rate": 1e-05, "loss": 0.03394366055727005, "step": 7 }, { "clip_ratio": 4.0595539871901565e-06, "completions/clipped_ratio": 0.287109375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4086.0, "completions/mean_length": 2682.54296875, "completions/mean_terminated_length": 2113.287841796875, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "epoch": 0.09090909090909091, "frac_reward_zero_std": 0.0625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 11759986.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.6484375, "reward_std": 0.4779251217842102, "rewards/correct/mean": 0.6484375, "rewards/correct/std": 0.4779251217842102, "rollout/chars": 7723.875, "rollout/commit": 0.84375, "rollout/reward": 0.75, "rollout/trunc": 0.21875, "sampling/importance_sampling_ratio/max": 2.72625732421875, "sampling/importance_sampling_ratio/mean": 0.9662811160087585, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5347617864608765, "sampling/sampling_logp_difference/mean": 0.004520789720118046, "step": 8, "step_time": 228.13665069599665 }, { "epoch": 0.09090909090909091, "grad_norm": 0.0035843115765601397, "learning_rate": 1e-05, "loss": 0.06731684505939484, "step": 8 }, { "clip_ratio": 1.1685516028592247e-05, "completions/clipped_ratio": 0.302734375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4077.0, "completions/mean_length": 2592.853515625, "completions/mean_terminated_length": 1940.2269287109375, "completions/min_length": 365.0, "completions/min_terminated_length": 365.0, "epoch": 0.10227272727272728, "frac_reward_zero_std": 0.09375, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 13154615.0, "register/okay": 0.0625, "register/okay_any": 0.8125, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.623046875, "reward_std": 0.4850969910621643, "rewards/correct/mean": 0.623046875, "rewards/correct/std": 0.4850969910621643, "rollout/chars": 6270.625, "rollout/commit": 0.96875, "rollout/reward": 0.84375, "rollout/trunc": 0.09375, "sampling/importance_sampling_ratio/max": 2.6416101455688477, "sampling/importance_sampling_ratio/mean": 0.9010199308395386, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5122027397155762, "sampling/sampling_logp_difference/mean": 0.00482128094881773, "step": 9, "step_time": 230.5110018680025 }, { "epoch": 0.10227272727272728, "grad_norm": 0.003008148865774274, "learning_rate": 1e-05, "loss": 0.020274115726351738, "step": 9 }, { "clip_ratio": 1.5000192377101484e-05, "completions/clipped_ratio": 0.294921875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4018.0, "completions/mean_length": 2688.658203125, "completions/mean_terminated_length": 2100.030517578125, "completions/min_length": 428.0, "completions/min_terminated_length": 428.0, "epoch": 0.11363636363636363, "frac_reward_zero_std": 0.0625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 14602376.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.642578125, "reward_std": 0.4797092080116272, "rewards/correct/mean": 0.642578125, "rewards/correct/std": 0.4797092080116272, "rollout/chars": 5943.53125, "rollout/commit": 0.8125, "rollout/reward": 0.71875, "rollout/trunc": 0.1875, "sampling/importance_sampling_ratio/max": 2.8662335872650146, "sampling/importance_sampling_ratio/mean": 0.9329501986503601, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.4294443130493164, "sampling/sampling_logp_difference/mean": 0.005068408325314522, "step": 10, "step_time": 221.11443298699305 }, { "epoch": 0.11363636363636363, "grad_norm": 0.004120167810469866, "learning_rate": 1e-05, "loss": 0.08815088868141174, "step": 10 }, { "clip_ratio": 1.3279116956255166e-05, "completions/clipped_ratio": 0.32421875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4055.0, "completions/mean_length": 2560.62890625, "completions/mean_terminated_length": 1824.0057373046875, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "epoch": 0.125, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 15977178.0, "register/okay": 0.0, "register/okay_any": 0.90625, "register/premium": -0.16129032258064513, "register/verify_any": 0.96875, "reward": 0.625, "reward_std": 0.4845963716506958, "rewards/correct/mean": 0.625, "rewards/correct/std": 0.4845963716506958, "rollout/chars": 6113.28125, "rollout/commit": 0.84375, "rollout/reward": 0.84375, "rollout/trunc": 0.125, "sampling/importance_sampling_ratio/max": 2.96163010597229, "sampling/importance_sampling_ratio/mean": 0.9653053283691406, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.036353349685669, "sampling/sampling_logp_difference/mean": 0.004582596477121115, "step": 11, "step_time": 238.2820661850019 }, { "epoch": 0.125, "grad_norm": 0.003486036090180278, "learning_rate": 1e-05, "loss": 0.06840801239013672, "step": 11 }, { "clip_ratio": 2.348333373447531e-05, "completions/clipped_ratio": 0.3515625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4083.0, "completions/mean_length": 2810.513671875, "completions/mean_terminated_length": 2113.563232421875, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "epoch": 0.13636363636363635, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 17479505.0, "register/okay": 0.03125, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.623046875, "reward_std": 0.4850969910621643, "rewards/correct/mean": 0.623046875, "rewards/correct/std": 0.4850969910621643, "rollout/chars": 7909.25, "rollout/commit": 0.65625, "rollout/reward": 0.65625, "rollout/trunc": 0.34375, "sampling/importance_sampling_ratio/max": 2.941793918609619, "sampling/importance_sampling_ratio/mean": 0.9080376625061035, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1040374040603638, "sampling/sampling_logp_difference/mean": 0.004863664973527193, "step": 12, "step_time": 223.75293871599933 }, { "epoch": 0.13636363636363635, "grad_norm": 0.003054369240999222, "learning_rate": 1e-05, "loss": 0.04905109480023384, "step": 12 }, { "clip_ratio": 3.613838339333597e-05, "completions/clipped_ratio": 0.251953125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4096.0, "completions/mean_length": 2390.146484375, "completions/mean_terminated_length": 1815.590087890625, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "epoch": 0.14772727272727273, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.5, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 18765740.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": -0.5483870967741935, "register/verify_any": 0.96875, "reward": 0.662109375, "reward_std": 0.4734536409378052, "rewards/correct/mean": 0.662109375, "rewards/correct/std": 0.4734536409378052, "rollout/chars": 6902.8125, "rollout/commit": 0.625, "rollout/reward": 0.46875, "rollout/trunc": 0.40625, "sampling/importance_sampling_ratio/max": 2.6642820835113525, "sampling/importance_sampling_ratio/mean": 0.9322664141654968, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.39106273651123047, "sampling/sampling_logp_difference/mean": 0.0050008250400424, "step": 13, "step_time": 204.52620178501456 }, { "epoch": 0.14772727272727273, "grad_norm": 0.0034245674032717943, "learning_rate": 1e-05, "loss": 0.03518375754356384, "step": 13 }, { "clip_ratio": 3.3756232539872144e-05, "completions/clipped_ratio": 0.384765625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4080.0, "completions/mean_length": 2794.69921875, "completions/mean_terminated_length": 1980.869873046875, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "epoch": 0.1590909090909091, "frac_reward_zero_std": 0.09375, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 20267250.0, "register/okay": 0.0, "register/okay_any": 0.84375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.533203125, "reward_std": 0.4993842542171478, "rewards/correct/mean": 0.533203125, "rewards/correct/std": 0.4993842542171478, "rollout/chars": 6717.75, "rollout/commit": 0.5625, "rollout/reward": 0.53125, "rollout/trunc": 0.40625, "sampling/importance_sampling_ratio/max": 2.9956328868865967, "sampling/importance_sampling_ratio/mean": 0.9270258545875549, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.949026346206665, "sampling/sampling_logp_difference/mean": 0.0053371889516711235, "step": 14, "step_time": 241.89416026500476 }, { "epoch": 0.1590909090909091, "grad_norm": 0.0032463697716593742, "learning_rate": 1e-05, "loss": 0.04247649013996124, "step": 14 }, { "clip_ratio": 4.435342742681314e-05, "completions/clipped_ratio": 0.197265625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4089.0, "completions/mean_length": 2334.177734375, "completions/mean_terminated_length": 1901.223876953125, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "epoch": 0.17045454545454544, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 21532909.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.76171875, "reward_std": 0.42644867300987244, "rewards/correct/mean": 0.76171875, "rewards/correct/std": 0.42644867300987244, "rollout/chars": 8942.15625, "rollout/commit": 0.8125, "rollout/reward": 0.6875, "rollout/trunc": 0.3125, "sampling/importance_sampling_ratio/max": 2.972348928451538, "sampling/importance_sampling_ratio/mean": 0.9189268350601196, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5567886829376221, "sampling/sampling_logp_difference/mean": 0.004913019947707653, "step": 15, "step_time": 223.23767583300287 }, { "epoch": 0.17045454545454544, "grad_norm": 0.0037231335882097483, "learning_rate": 1e-05, "loss": 0.05135633423924446, "step": 15 }, { "clip_ratio": 4.653011211530611e-05, "completions/clipped_ratio": 0.279296875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4088.0, "completions/mean_length": 2597.017578125, "completions/mean_terminated_length": 2016.1112060546875, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "epoch": 0.18181818181818182, "frac_reward_zero_std": 0.125, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 22936550.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.65234375, "reward_std": 0.47669193148612976, "rewards/correct/mean": 0.65234375, "rewards/correct/std": 0.47669193148612976, "rollout/chars": 5921.5, "rollout/commit": 0.96875, "rollout/reward": 0.96875, "rollout/trunc": 0.03125, "sampling/importance_sampling_ratio/max": 2.970635175704956, "sampling/importance_sampling_ratio/mean": 0.9122588038444519, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.617943286895752, "sampling/sampling_logp_difference/mean": 0.004951866343617439, "step": 16, "step_time": 213.3137985569956 }, { "epoch": 0.18181818181818182, "grad_norm": 0.002896440913900733, "learning_rate": 1e-05, "loss": 0.046279728412628174, "step": 16 }, { "clip_ratio": 4.9786167437559925e-05, "completions/clipped_ratio": 0.130859375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4090.0, "completions/mean_length": 1984.505859375, "completions/mean_terminated_length": 1666.5955810546875, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "epoch": 0.19318181818181818, "frac_reward_zero_std": 0.0625, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 24023145.0, "register/okay": 0.0, "register/okay_any": 0.84375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.765625, "reward_std": 0.42402184009552, "rewards/correct/mean": 0.765625, "rewards/correct/std": 0.42402184009552, "rollout/chars": 6103.28125, "rollout/commit": 0.9375, "rollout/reward": 0.84375, "rollout/trunc": 0.125, "sampling/importance_sampling_ratio/max": 2.654937267303467, "sampling/importance_sampling_ratio/mean": 0.9073137640953064, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5429407358169556, "sampling/sampling_logp_difference/mean": 0.0048791998997330666, "step": 17, "step_time": 198.31722998900295 }, { "epoch": 0.19318181818181818, "grad_norm": 0.004091218579560518, "learning_rate": 1e-05, "loss": 0.05541734769940376, "step": 17 }, { "clip_ratio": 5.2998024330008775e-05, "completions/clipped_ratio": 0.36328125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4088.0, "completions/mean_length": 2802.869140625, "completions/mean_terminated_length": 2065.070556640625, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "epoch": 0.20454545454545456, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 25519670.0, "register/okay": 0.0, "register/okay_any": 0.90625, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.578125, "reward_std": 0.49434176087379456, "rewards/correct/mean": 0.578125, "rewards/correct/std": 0.49434176087379456, "rollout/chars": 3893.9375, "rollout/commit": 0.90625, "rollout/reward": 0.90625, "rollout/trunc": 0.03125, "sampling/importance_sampling_ratio/max": 2.7872073650360107, "sampling/importance_sampling_ratio/mean": 0.9027870893478394, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.47285139560699463, "sampling/sampling_logp_difference/mean": 0.004941096995025873, "step": 18, "step_time": 244.71225519200016 }, { "epoch": 0.20454545454545456, "grad_norm": 0.0028813090175390244, "learning_rate": 1e-05, "loss": 0.029713578522205353, "step": 18 }, { "clip_ratio": 4.2906002590825665e-05, "completions/clipped_ratio": 0.291015625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4077.0, "completions/mean_length": 2633.5078125, "completions/mean_terminated_length": 2033.201171875, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "epoch": 0.2159090909090909, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 26949290.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.66796875, "reward_std": 0.47140273451805115, "rewards/correct/mean": 0.66796875, "rewards/correct/std": 0.47140273451805115, "rollout/chars": 9273.59375, "rollout/commit": 0.5, "rollout/reward": 0.46875, "rollout/trunc": 0.5, "sampling/importance_sampling_ratio/max": 2.972933530807495, "sampling/importance_sampling_ratio/mean": 0.8908332586288452, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4295125007629395, "sampling/sampling_logp_difference/mean": 0.004852948244661093, "step": 19, "step_time": 210.78038853899852 }, { "epoch": 0.2159090909090909, "grad_norm": 0.003327382495626807, "learning_rate": 1e-05, "loss": 0.06632369756698608, "step": 19 }, { "clip_ratio": 5.486932650455856e-05, "completions/clipped_ratio": 0.427734375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4071.0, "completions/mean_length": 3086.205078125, "completions/mean_terminated_length": 2331.443603515625, "completions/min_length": 704.0, "completions/min_terminated_length": 704.0, "epoch": 0.22727272727272727, "frac_reward_zero_std": 0.0625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 28610611.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.5234375, "reward_std": 0.49993884563446045, "rewards/correct/mean": 0.5234375, "rewards/correct/std": 0.49993884563446045, "rollout/chars": 9274.625, "rollout/commit": 0.53125, "rollout/reward": 0.375, "rollout/trunc": 0.5, "sampling/importance_sampling_ratio/max": 2.9439356327056885, "sampling/importance_sampling_ratio/mean": 0.9422547221183777, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.7304327487945557, "sampling/sampling_logp_difference/mean": 0.004762661177664995, "step": 20, "step_time": 241.5823824050094 }, { "epoch": 0.22727272727272727, "grad_norm": 0.003230234608054161, "learning_rate": 1e-05, "loss": 0.0657850056886673, "step": 20 }, { "clip_ratio": 5.2212915420568606e-05, "completions/clipped_ratio": 0.328125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4057.0, "completions/mean_length": 2670.73046875, "completions/mean_terminated_length": 1974.6685791015625, "completions/min_length": 406.0, "completions/min_terminated_length": 406.0, "epoch": 0.23863636363636365, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 30047289.0, "register/okay": 0.03125, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.6171875, "reward_std": 0.486548513174057, "rewards/correct/mean": 0.6171875, "rewards/correct/std": 0.486548513174057, "rollout/chars": 8537.1875, "rollout/commit": 0.625, "rollout/reward": 0.53125, "rollout/trunc": 0.4375, "sampling/importance_sampling_ratio/max": 2.9177842140197754, "sampling/importance_sampling_ratio/mean": 0.9250489473342896, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.4100780487060547, "sampling/sampling_logp_difference/mean": 0.005068977363407612, "step": 21, "step_time": 230.3553039509934 }, { "epoch": 0.23863636363636365, "grad_norm": 0.003675194690003991, "learning_rate": 1e-05, "loss": 0.059521663933992386, "step": 21 }, { "clip_ratio": 6.675696363345196e-05, "completions/clipped_ratio": 0.255859375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4076.0, "completions/mean_length": 2579.5390625, "completions/mean_terminated_length": 2058.13134765625, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "epoch": 0.25, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 31434285.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.6796875, "reward_std": 0.4670529365539551, "rewards/correct/mean": 0.6796875, "rewards/correct/std": 0.4670529365539551, "rollout/chars": 6334.8125, "rollout/commit": 0.65625, "rollout/reward": 0.59375, "rollout/trunc": 0.40625, "sampling/importance_sampling_ratio/max": 2.882204294204712, "sampling/importance_sampling_ratio/mean": 0.936687707901001, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2923583984375, "sampling/sampling_logp_difference/mean": 0.004951627925038338, "step": 22, "step_time": 196.23515353100447 }, { "epoch": 0.25, "grad_norm": 0.003399793989956379, "learning_rate": 1e-05, "loss": 0.036319706588983536, "step": 22 }, { "clip_ratio": 5.6475377618880884e-05, "completions/clipped_ratio": 0.26171875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4096.0, "completions/mean_length": 2351.69140625, "completions/mean_terminated_length": 1733.338623046875, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "epoch": 0.26136363636363635, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.5, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 32720095.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.6484375, "reward_std": 0.4779251217842102, "rewards/correct/mean": 0.6484375, "rewards/correct/std": 0.4779251217842102, "rollout/chars": 11748.21875, "rollout/commit": 0.15625, "rollout/reward": 0.0625, "rollout/trunc": 0.875, "sampling/importance_sampling_ratio/max": 2.905789852142334, "sampling/importance_sampling_ratio/mean": 0.9464510083198547, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.0502471923828125, "sampling/sampling_logp_difference/mean": 0.005386264994740486, "step": 23, "step_time": 234.36768228199435 }, { "epoch": 0.26136363636363635, "grad_norm": 0.004066686611622572, "learning_rate": 1e-05, "loss": 0.07233693450689316, "step": 23 }, { "clip_ratio": 5.326732753019314e-05, "completions/clipped_ratio": 0.421875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4090.0, "completions/mean_length": 3116.578125, "completions/mean_terminated_length": 2401.9560546875, "completions/min_length": 509.0, "completions/min_terminated_length": 509.0, "epoch": 0.2727272727272727, "frac_reward_zero_std": 0.0625, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 34398855.0, "register/okay": 0.0, "register/okay_any": 0.90625, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.52734375, "reward_std": 0.49974003434181213, "rewards/correct/mean": 0.52734375, "rewards/correct/std": 0.49974003434181213, "rollout/chars": 7776.96875, "rollout/commit": 0.71875, "rollout/reward": 0.71875, "rollout/trunc": 0.28125, "sampling/importance_sampling_ratio/max": 2.990600109100342, "sampling/importance_sampling_ratio/mean": 0.8955240249633789, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.8885411024093628, "sampling/sampling_logp_difference/mean": 0.004893308971077204, "step": 24, "step_time": 248.16172832200755 }, { "epoch": 0.2727272727272727, "grad_norm": 0.003279327414929867, "learning_rate": 1e-05, "loss": 0.03035794571042061, "step": 24 }, { "clip_ratio": 4.4846863659131486e-05, "completions/clipped_ratio": 0.32421875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4039.0, "completions/mean_length": 2545.95703125, "completions/mean_terminated_length": 1802.2947998046875, "completions/min_length": 200.0, "completions/min_terminated_length": 200.0, "epoch": 0.2840909090909091, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 35778657.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.619140625, "reward_std": 0.48607301712036133, "rewards/correct/mean": 0.619140625, "rewards/correct/std": 0.48607301712036133, "rollout/chars": 7533.4375, "rollout/commit": 0.59375, "rollout/reward": 0.46875, "rollout/trunc": 0.28125, "sampling/importance_sampling_ratio/max": 2.9910829067230225, "sampling/importance_sampling_ratio/mean": 0.9145698547363281, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5044717788696289, "sampling/sampling_logp_difference/mean": 0.005069740582257509, "step": 25, "step_time": 230.56244620599318 }, { "epoch": 0.2840909090909091, "grad_norm": 0.0035241309087723494, "learning_rate": 1e-05, "loss": 0.04239988327026367, "step": 25 }, { "clip_ratio": 6.0805309203715296e-05, "completions/clipped_ratio": 0.310546875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4089.0, "completions/mean_length": 2703.7734375, "completions/mean_terminated_length": 2076.679931640625, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "epoch": 0.29545454545454547, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 37239005.0, "register/okay": 0.0, "register/okay_any": 0.84375, "register/premium": -0.06451612903225812, "register/verify_any": 0.96875, "reward": 0.658203125, "reward_std": 0.4747757613658905, "rewards/correct/mean": 0.658203125, "rewards/correct/std": 0.4747757613658905, "rollout/chars": 3381.46875, "rollout/commit": 1.0, "rollout/reward": 0.9375, "rollout/trunc": 0.0, "sampling/importance_sampling_ratio/max": 2.837442636489868, "sampling/importance_sampling_ratio/mean": 0.8787897229194641, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.8435883522033691, "sampling/sampling_logp_difference/mean": 0.0050262752920389175, "step": 26, "step_time": 233.77161700500073 }, { "epoch": 0.29545454545454547, "grad_norm": 0.0030401181429624557, "learning_rate": 1e-05, "loss": 0.019256848841905594, "step": 26 }, { "clip_ratio": 4.174492096353788e-05, "completions/clipped_ratio": 0.27734375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4070.0, "completions/mean_length": 2551.8125, "completions/mean_terminated_length": 1959.178466796875, "completions/min_length": 338.0, "completions/min_terminated_length": 338.0, "epoch": 0.3068181818181818, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 38608157.0, "register/okay": 0.03125, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.650390625, "reward_std": 0.47731292247772217, "rewards/correct/mean": 0.650390625, "rewards/correct/std": 0.47731292247772217, "rollout/chars": 10290.8125, "rollout/commit": 0.21875, "rollout/reward": 0.15625, "rollout/trunc": 0.84375, "sampling/importance_sampling_ratio/max": 2.992258071899414, "sampling/importance_sampling_ratio/mean": 0.9573348164558411, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.7723093032836914, "sampling/sampling_logp_difference/mean": 0.004969725385308266, "step": 27, "step_time": 224.3992258420003 }, { "epoch": 0.3068181818181818, "grad_norm": 0.0036496196407824755, "learning_rate": 1e-05, "loss": 0.04576047509908676, "step": 27 }, { "clip_ratio": 4.57169376204547e-05, "completions/clipped_ratio": 0.279296875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4050.0, "completions/mean_length": 2679.435546875, "completions/mean_terminated_length": 2130.468994140625, "completions/min_length": 420.0, "completions/min_terminated_length": 420.0, "epoch": 0.3181818181818182, "frac_reward_zero_std": 0.0625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 40047020.0, "register/okay": 0.0, "register/okay_any": 0.84375, "register/premium": 0.8387096774193549, "register/verify_any": 0.96875, "reward": 0.646484375, "reward_std": 0.47852855920791626, "rewards/correct/mean": 0.646484375, "rewards/correct/std": 0.47852855920791626, "rollout/chars": 6973.1875, "rollout/commit": 0.9375, "rollout/reward": 0.8125, "rollout/trunc": 0.0625, "sampling/importance_sampling_ratio/max": 2.9695754051208496, "sampling/importance_sampling_ratio/mean": 0.8856563568115234, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3932266235351562, "sampling/sampling_logp_difference/mean": 0.004716905765235424, "step": 28, "step_time": 233.92598559500766 }, { "epoch": 0.3181818181818182, "grad_norm": 0.003118283813819289, "learning_rate": 1e-05, "loss": 0.03405182063579559, "step": 28 }, { "clip_ratio": 6.522296916955383e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4092.0, "completions/mean_length": 2800.392578125, "completions/mean_terminated_length": 2294.535400390625, "completions/min_length": 163.0, "completions/min_terminated_length": 163.0, "epoch": 0.32954545454545453, "frac_reward_zero_std": 0.0625, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 41565365.0, "register/okay": 0.0, "register/okay_any": 0.90625, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.673828125, "reward_std": 0.4692695140838623, "rewards/correct/mean": 0.673828125, "rewards/correct/std": 0.4692695140838623, "rollout/chars": 6729.40625, "rollout/commit": 0.90625, "rollout/reward": 0.875, "rollout/trunc": 0.09375, "sampling/importance_sampling_ratio/max": 2.92781662940979, "sampling/importance_sampling_ratio/mean": 0.8985303640365601, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.8867032527923584, "sampling/sampling_logp_difference/mean": 0.00486964825540781, "step": 29, "step_time": 238.03783438399478 }, { "epoch": 0.32954545454545453, "grad_norm": 0.0032935026101768017, "learning_rate": 1e-05, "loss": 0.052372947335243225, "step": 29 }, { "clip_ratio": 8.097707109300245e-05, "completions/clipped_ratio": 0.25390625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4058.0, "completions/mean_length": 2527.5546875, "completions/mean_terminated_length": 1993.7906494140625, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "epoch": 0.3409090909090909, "frac_reward_zero_std": 0.03125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 42921617.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.619140625, "reward_std": 0.48607301712036133, "rewards/correct/mean": 0.619140625, "rewards/correct/std": 0.48607301712036133, "rollout/chars": 6342.75, "rollout/commit": 0.8125, "rollout/reward": 0.5625, "rollout/trunc": 0.28125, "sampling/importance_sampling_ratio/max": 2.988611936569214, "sampling/importance_sampling_ratio/mean": 0.8912668228149414, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.7639431953430176, "sampling/sampling_logp_difference/mean": 0.0050128307193517685, "step": 30, "step_time": 224.68840066700432 }, { "epoch": 0.3409090909090909, "grad_norm": 0.003287514206022024, "learning_rate": 1e-05, "loss": 0.04755740240216255, "step": 30 }, { "clip_ratio": 7.465987437171862e-05, "completions/clipped_ratio": 0.3046875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4068.0, "completions/mean_length": 2656.521484375, "completions/mean_terminated_length": 2025.73876953125, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "epoch": 0.3522727272727273, "frac_reward_zero_std": 0.0625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 44355548.0, "register/okay": 0.0, "register/okay_any": 0.9375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.55859375, "reward_std": 0.4970405399799347, "rewards/correct/mean": 0.55859375, "rewards/correct/std": 0.4970405399799347, "rollout/chars": 4952.1875, "rollout/commit": 0.90625, "rollout/reward": 0.5, "rollout/trunc": 0.03125, "sampling/importance_sampling_ratio/max": 2.961225748062134, "sampling/importance_sampling_ratio/mean": 0.8841115236282349, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.722623348236084, "sampling/sampling_logp_difference/mean": 0.005319459363818169, "step": 31, "step_time": 213.5908911309998 }, { "epoch": 0.3522727272727273, "grad_norm": 0.0033462513238191605, "learning_rate": 1e-05, "loss": 0.04553345590829849, "step": 31 }, { "clip_ratio": 6.642126277256466e-05, "completions/clipped_ratio": 0.333984375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4091.0, "completions/mean_length": 2756.259765625, "completions/mean_terminated_length": 2084.42529296875, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "epoch": 0.36363636363636365, "frac_reward_zero_std": 0.0625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 45837921.0, "register/okay": 0.0, "register/okay_any": 0.875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.572265625, "reward_std": 0.4952339828014374, "rewards/correct/mean": 0.572265625, "rewards/correct/std": 0.4952339828014374, "rollout/chars": 7447.9375, "rollout/commit": 0.875, "rollout/reward": 0.71875, "rollout/trunc": 0.21875, "sampling/importance_sampling_ratio/max": 2.9116132259368896, "sampling/importance_sampling_ratio/mean": 0.9092390537261963, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.0500521659851074, "sampling/sampling_logp_difference/mean": 0.004691188223659992, "step": 32, "step_time": 235.9118720449951 }, { "epoch": 0.36363636363636365, "grad_norm": 0.003006890183314681, "learning_rate": 1e-05, "loss": 0.03242364525794983, "step": 32 }, { "clip_ratio": 5.475359353113163e-05, "completions/clipped_ratio": 0.29296875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4088.0, "completions/mean_length": 2574.208984375, "completions/mean_terminated_length": 1943.6326904296875, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "epoch": 0.375, "frac_reward_zero_std": 0.0625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 47223516.0, "register/okay": 0.0, "register/okay_any": 0.9375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.640625, "reward_std": 0.48028653860092163, "rewards/correct/mean": 0.640625, "rewards/correct/std": 0.48028653860092163, "rollout/chars": 6007.3125, "rollout/commit": 0.625, "rollout/reward": 0.53125, "rollout/trunc": 0.4375, "sampling/importance_sampling_ratio/max": 2.9517362117767334, "sampling/importance_sampling_ratio/mean": 0.9390583634376526, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.7950553894042969, "sampling/sampling_logp_difference/mean": 0.004755455069243908, "step": 33, "step_time": 206.17870632700215 }, { "epoch": 0.375, "grad_norm": 0.003763378830626607, "learning_rate": 1e-05, "loss": 0.06001519411802292, "step": 33 }, { "clip_ratio": 4.351094924004428e-05, "completions/clipped_ratio": 0.298828125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4083.0, "completions/mean_length": 2655.25390625, "completions/mean_terminated_length": 2041.231201171875, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "epoch": 0.38636363636363635, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 48646558.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.6328125, "reward_std": 0.48250964283943176, "rewards/correct/mean": 0.6328125, "rewards/correct/std": 0.48250964283943176, "rollout/chars": 9312.15625, "rollout/commit": 0.75, "rollout/reward": 0.6875, "rollout/trunc": 0.3125, "sampling/importance_sampling_ratio/max": 2.745772123336792, "sampling/importance_sampling_ratio/mean": 0.8967394828796387, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.7830262780189514, "sampling/sampling_logp_difference/mean": 0.004709095694124699, "step": 34, "step_time": 209.81075684600728 }, { "epoch": 0.38636363636363635, "grad_norm": 0.0032068060245364904, "learning_rate": 1e-05, "loss": 0.05563589930534363, "step": 34 }, { "clip_ratio": 7.661448302087592e-05, "completions/clipped_ratio": 0.1953125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4045.0, "completions/mean_length": 2417.951171875, "completions/mean_terminated_length": 2010.6578369140625, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "epoch": 0.3977272727272727, "frac_reward_zero_std": 0.09375, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 49943429.0, "register/okay": 0.03125, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.7109375, "reward_std": 0.45377036929130554, "rewards/correct/mean": 0.7109375, "rewards/correct/std": 0.45377036929130554, "rollout/chars": 5155.21875, "rollout/commit": 0.96875, "rollout/reward": 0.84375, "rollout/trunc": 0.03125, "sampling/importance_sampling_ratio/max": 2.74753737449646, "sampling/importance_sampling_ratio/mean": 0.9276185035705566, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.055890440940857, "sampling/sampling_logp_difference/mean": 0.004778267350047827, "step": 35, "step_time": 206.7543564430016 }, { "epoch": 0.3977272727272727, "grad_norm": 0.003544215578585863, "learning_rate": 1e-05, "loss": 0.04388181120157242, "step": 35 }, { "clip_ratio": 5.353016700837543e-05, "completions/clipped_ratio": 0.2890625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4087.0, "completions/mean_length": 2484.22265625, "completions/mean_terminated_length": 1828.8846435546875, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "epoch": 0.4090909090909091, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 51278839.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.59375, "reward_std": 0.49161264300346375, "rewards/correct/mean": 0.59375, "rewards/correct/std": 0.49161264300346375, "rollout/chars": 6862.0625, "rollout/commit": 0.59375, "rollout/reward": 0.53125, "rollout/trunc": 0.375, "sampling/importance_sampling_ratio/max": 2.991304874420166, "sampling/importance_sampling_ratio/mean": 0.9369009733200073, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5460691452026367, "sampling/sampling_logp_difference/mean": 0.005069076083600521, "step": 36, "step_time": 206.1250407339976 }, { "epoch": 0.4090909090909091, "grad_norm": 0.003753333119675517, "learning_rate": 1e-05, "loss": 0.03270333260297775, "step": 36 }, { "clip_ratio": 4.700995953044185e-05, "completions/clipped_ratio": 0.294921875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4056.0, "completions/mean_length": 2447.91796875, "completions/mean_terminated_length": 1758.553955078125, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "epoch": 0.42045454545454547, "frac_reward_zero_std": 0.0625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 52603213.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.611328125, "reward_std": 0.4879252314567566, "rewards/correct/mean": 0.611328125, "rewards/correct/std": 0.4879252314567566, "rollout/chars": 7591.4375, "rollout/commit": 0.65625, "rollout/reward": 0.5625, "rollout/trunc": 0.4375, "sampling/importance_sampling_ratio/max": 2.9310998916625977, "sampling/importance_sampling_ratio/mean": 0.9377585649490356, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.7711648941040039, "sampling/sampling_logp_difference/mean": 0.005007673520594835, "step": 37, "step_time": 229.42526798800827 }, { "epoch": 0.42045454545454547, "grad_norm": 0.0034484269563108683, "learning_rate": 1e-05, "loss": 0.06802354753017426, "step": 37 }, { "clip_ratio": 3.721059533745574e-05, "completions/clipped_ratio": 0.193359375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4057.0, "completions/mean_length": 2208.328125, "completions/mean_terminated_length": 1755.83544921875, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "epoch": 0.4318181818181818, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 53789989.0, "register/okay": 0.0, "register/okay_any": 0.90625, "register/premium": -0.19354838709677424, "register/verify_any": 0.96875, "reward": 0.75390625, "reward_std": 0.4311550557613373, "rewards/correct/mean": 0.75390625, "rewards/correct/std": 0.4311550557613373, "rollout/chars": 4564.59375, "rollout/commit": 0.84375, "rollout/reward": 0.8125, "rollout/trunc": 0.0625, "sampling/importance_sampling_ratio/max": 2.7628633975982666, "sampling/importance_sampling_ratio/mean": 0.9271668195724487, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.45870041847229004, "sampling/sampling_logp_difference/mean": 0.004600748419761658, "step": 38, "step_time": 186.41656695598795 }, { "epoch": 0.4318181818181818, "grad_norm": 0.003216351615265012, "learning_rate": 1e-05, "loss": 0.022610703483223915, "step": 38 }, { "clip_ratio": 6.092197099860641e-05, "completions/clipped_ratio": 0.2578125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4096.0, "completions/mean_length": 2408.974609375, "completions/mean_terminated_length": 1822.955322265625, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "epoch": 0.4431818181818182, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 55082712.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.662109375, "reward_std": 0.4734536409378052, "rewards/correct/mean": 0.662109375, "rewards/correct/std": 0.4734536409378052, "rollout/chars": 7649.53125, "rollout/commit": 0.875, "rollout/reward": 0.75, "rollout/trunc": 0.1875, "sampling/importance_sampling_ratio/max": 2.948693037033081, "sampling/importance_sampling_ratio/mean": 0.9154539108276367, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.4746341705322266, "sampling/sampling_logp_difference/mean": 0.005190804600715637, "step": 39, "step_time": 208.00693321699873 }, { "epoch": 0.4431818181818182, "grad_norm": 0.0036431732587516308, "learning_rate": 1e-05, "loss": 0.03268931061029434, "step": 39 }, { "clip_ratio": 4.239076474732428e-05, "completions/clipped_ratio": 0.19140625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4095.0, "completions/mean_length": 2321.060546875, "completions/mean_terminated_length": 1900.90576171875, "completions/min_length": 158.0, "completions/min_terminated_length": 158.0, "epoch": 0.45454545454545453, "frac_reward_zero_std": 0.09375, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 56331847.0, "register/okay": 0.0, "register/okay_any": 0.9375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.7265625, "reward_std": 0.4461594223976135, "rewards/correct/mean": 0.7265625, "rewards/correct/std": 0.4461594223976135, "rollout/chars": 6050.65625, "rollout/commit": 0.90625, "rollout/reward": 0.875, "rollout/trunc": 0.09375, "sampling/importance_sampling_ratio/max": 2.9441235065460205, "sampling/importance_sampling_ratio/mean": 0.9550347328186035, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5566263198852539, "sampling/sampling_logp_difference/mean": 0.004640746861696243, "step": 40, "step_time": 232.20252943600644 }, { "epoch": 0.45454545454545453, "grad_norm": 0.003447507508099079, "learning_rate": 1e-05, "loss": 0.040430452674627304, "step": 40 }, { "clip_ratio": 4.128463280039796e-05, "completions/clipped_ratio": 0.23046875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4061.0, "completions/mean_length": 2350.501953125, "completions/mean_terminated_length": 1827.738525390625, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "epoch": 0.4659090909090909, "frac_reward_zero_std": 0.09375, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 57599928.0, "register/okay": 0.0, "register/okay_any": 0.6875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.658203125, "reward_std": 0.4747757613658905, "rewards/correct/mean": 0.658203125, "rewards/correct/std": 0.4747757613658905, "rollout/chars": 6086.21875, "rollout/commit": 0.90625, "rollout/reward": 0.84375, "rollout/trunc": 0.09375, "sampling/importance_sampling_ratio/max": 2.7746007442474365, "sampling/importance_sampling_ratio/mean": 0.9619885683059692, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.888315200805664, "sampling/sampling_logp_difference/mean": 0.004755809437483549, "step": 41, "step_time": 234.87280439599635 }, { "epoch": 0.4659090909090909, "grad_norm": 0.003600901225581765, "learning_rate": 1e-05, "loss": 0.03945820778608322, "step": 41 }, { "clip_ratio": 4.081156771462702e-05, "completions/clipped_ratio": 0.240234375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4091.0, "completions/mean_length": 2359.01171875, "completions/mean_terminated_length": 1809.7840576171875, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "epoch": 0.4772727272727273, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 58867902.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.677734375, "reward_std": 0.46780112385749817, "rewards/correct/mean": 0.677734375, "rewards/correct/std": 0.46780112385749817, "rollout/chars": 9607.8125, "rollout/commit": 0.71875, "rollout/reward": 0.40625, "rollout/trunc": 0.375, "sampling/importance_sampling_ratio/max": 2.874417304992676, "sampling/importance_sampling_ratio/mean": 0.9376507997512817, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5530819892883301, "sampling/sampling_logp_difference/mean": 0.004874600097537041, "step": 42, "step_time": 207.32737054100653 }, { "epoch": 0.4772727272727273, "grad_norm": 0.003691115416586399, "learning_rate": 1e-05, "loss": 0.035165682435035706, "step": 42 }, { "clip_ratio": 4.7922681119416666e-05, "completions/clipped_ratio": 0.2109375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4095.0, "completions/mean_length": 2121.927734375, "completions/mean_terminated_length": 1594.2054443359375, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "epoch": 0.48863636363636365, "frac_reward_zero_std": 0.3125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 60021737.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.751953125, "reward_std": 0.4323015511035919, "rewards/correct/mean": 0.751953125, "rewards/correct/std": 0.4323015511035919, "rollout/chars": 7986.0, "rollout/commit": 0.625, "rollout/reward": 0.59375, "rollout/trunc": 0.375, "sampling/importance_sampling_ratio/max": 2.9013590812683105, "sampling/importance_sampling_ratio/mean": 0.9236944317817688, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5296273231506348, "sampling/sampling_logp_difference/mean": 0.004975184798240662, "step": 43, "step_time": 222.71566601800077 }, { "epoch": 0.48863636363636365, "grad_norm": 0.0032141930423676968, "learning_rate": 1e-05, "loss": 0.031207803636789322, "step": 43 }, { "clip_ratio": 4.502689955643291e-05, "completions/clipped_ratio": 0.2734375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4081.0, "completions/mean_length": 2710.71484375, "completions/mean_terminated_length": 2189.370849609375, "completions/min_length": 399.0, "completions/min_terminated_length": 399.0, "epoch": 0.5, "frac_reward_zero_std": 0.09375, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 61482727.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.65234375, "reward_std": 0.47669193148612976, "rewards/correct/mean": 0.65234375, "rewards/correct/std": 0.47669193148612976, "rollout/chars": 13552.09375, "rollout/commit": 0.40625, "rollout/reward": 0.25, "rollout/trunc": 0.625, "sampling/importance_sampling_ratio/max": 2.9828054904937744, "sampling/importance_sampling_ratio/mean": 0.9278571009635925, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.7164409160614014, "sampling/sampling_logp_difference/mean": 0.005057974718511105, "step": 44, "step_time": 205.39486570699955 }, { "epoch": 0.5, "grad_norm": 0.0035538969095796347, "learning_rate": 1e-05, "loss": 0.04245685040950775, "step": 44 }, { "clip_ratio": 3.6159511012101575e-05, "completions/clipped_ratio": 0.208984375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4075.0, "completions/mean_length": 2432.134765625, "completions/mean_terminated_length": 1992.5457763671875, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "epoch": 0.5113636363636364, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 62793372.0, "register/okay": 0.0, "register/okay_any": 0.9375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.7421875, "reward_std": 0.43785804510116577, "rewards/correct/mean": 0.7421875, "rewards/correct/std": 0.43785804510116577, "rollout/chars": 6969.3125, "rollout/commit": 0.8125, "rollout/reward": 0.8125, "rollout/trunc": 0.1875, "sampling/importance_sampling_ratio/max": 2.979311943054199, "sampling/importance_sampling_ratio/mean": 0.9486267566680908, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.080000877380371, "sampling/sampling_logp_difference/mean": 0.004703765735030174, "step": 45, "step_time": 213.5226564270124 }, { "epoch": 0.5113636363636364, "grad_norm": 0.004064072854816914, "learning_rate": 1e-05, "loss": 0.06307149678468704, "step": 45 }, { "clip_ratio": 6.182664350262712e-05, "completions/clipped_ratio": 0.251953125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4064.0, "completions/mean_length": 2435.552734375, "completions/mean_terminated_length": 1876.289794921875, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "epoch": 0.5227272727272727, "frac_reward_zero_std": 0.0625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 64113431.0, "register/okay": 0.09375, "register/okay_any": 0.90625, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.6328125, "reward_std": 0.48250964283943176, "rewards/correct/mean": 0.6328125, "rewards/correct/std": 0.48250964283943176, "rollout/chars": 7549.25, "rollout/commit": 0.90625, "rollout/reward": 0.8125, "rollout/trunc": 0.15625, "sampling/importance_sampling_ratio/max": 2.989046573638916, "sampling/importance_sampling_ratio/mean": 0.8794365525245667, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0649478435516357, "sampling/sampling_logp_difference/mean": 0.005066515412181616, "step": 46, "step_time": 238.6869710050014 }, { "epoch": 0.5227272727272727, "grad_norm": 0.0037526381202042103, "learning_rate": 1e-05, "loss": 0.05647933483123779, "step": 46 }, { "clip_ratio": 4.784553817671622e-05, "completions/clipped_ratio": 0.17578125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4086.0, "completions/mean_length": 2084.8671875, "completions/mean_terminated_length": 1655.95263671875, "completions/min_length": 180.0, "completions/min_terminated_length": 180.0, "epoch": 0.5340909090909091, "frac_reward_zero_std": 0.21875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 65256483.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.76171875, "reward_std": 0.42644867300987244, "rewards/correct/mean": 0.76171875, "rewards/correct/std": 0.42644867300987244, "rollout/chars": 3682.75, "rollout/commit": 0.90625, "rollout/reward": 0.8125, "rollout/trunc": 0.0625, "sampling/importance_sampling_ratio/max": 2.84405779838562, "sampling/importance_sampling_ratio/mean": 0.9413059949874878, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5454630255699158, "sampling/sampling_logp_difference/mean": 0.005082877352833748, "step": 47, "step_time": 215.25980535299823 }, { "epoch": 0.5340909090909091, "grad_norm": 0.003414996201172471, "learning_rate": 1e-05, "loss": 0.04025837779045105, "step": 47 }, { "clip_ratio": 4.7273616303300514e-05, "completions/clipped_ratio": 0.365234375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4090.0, "completions/mean_length": 2880.806640625, "completions/mean_terminated_length": 2181.60302734375, "completions/min_length": 430.0, "completions/min_terminated_length": 430.0, "epoch": 0.5454545454545454, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 66795920.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.56640625, "reward_std": 0.4960552453994751, "rewards/correct/mean": 0.56640625, "rewards/correct/std": 0.4960552453994751, "rollout/chars": 7730.71875, "rollout/commit": 0.71875, "rollout/reward": 0.65625, "rollout/trunc": 0.28125, "sampling/importance_sampling_ratio/max": 2.967653751373291, "sampling/importance_sampling_ratio/mean": 0.8913018703460693, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.6522915363311768, "sampling/sampling_logp_difference/mean": 0.004959162324666977, "step": 48, "step_time": 232.4684189019972 }, { "epoch": 0.5454545454545454, "grad_norm": 0.0030148394871503115, "learning_rate": 1e-05, "loss": 0.03712388128042221, "step": 48 }, { "clip_ratio": 4.2036647414533945e-05, "completions/clipped_ratio": 0.275390625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4095.0, "completions/mean_length": 2501.763671875, "completions/mean_terminated_length": 1895.867919921875, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "epoch": 0.5568181818181818, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 68155639.0, "register/okay": 0.0625, "register/okay_any": 0.9375, "register/premium": -0.06451612903225812, "register/verify_any": 0.96875, "reward": 0.65234375, "reward_std": 0.47669193148612976, "rewards/correct/mean": 0.65234375, "rewards/correct/std": 0.47669193148612976, "rollout/chars": 4332.3125, "rollout/commit": 1.0, "rollout/reward": 0.9375, "rollout/trunc": 0.0, "sampling/importance_sampling_ratio/max": 2.96474289894104, "sampling/importance_sampling_ratio/mean": 0.9402225017547607, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1063487529754639, "sampling/sampling_logp_difference/mean": 0.004922760184854269, "step": 49, "step_time": 242.5428965280007 }, { "epoch": 0.5568181818181818, "grad_norm": 0.003150244476273656, "learning_rate": 1e-05, "loss": 0.02683938853442669, "step": 49 }, { "clip_ratio": 5.262581521492393e-05, "completions/clipped_ratio": 0.181640625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4083.0, "completions/mean_length": 2219.458984375, "completions/mean_terminated_length": 1802.9571533203125, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "epoch": 0.5681818181818182, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 69365058.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.716796875, "reward_std": 0.4509948492050171, "rewards/correct/mean": 0.716796875, "rewards/correct/std": 0.4509948492050171, "rollout/chars": 7746.8125, "rollout/commit": 0.65625, "rollout/reward": 0.46875, "rollout/trunc": 0.375, "sampling/importance_sampling_ratio/max": 2.8404057025909424, "sampling/importance_sampling_ratio/mean": 0.9204064607620239, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.4059598445892334, "sampling/sampling_logp_difference/mean": 0.004588037263602018, "step": 50, "step_time": 194.41973535799843 }, { "epoch": 0.5681818181818182, "grad_norm": 0.003299945965409279, "learning_rate": 1e-05, "loss": 0.02735048718750477, "step": 50 }, { "clip_ratio": 4.658469151763711e-05, "completions/clipped_ratio": 0.28515625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4088.0, "completions/mean_length": 2728.69921875, "completions/mean_terminated_length": 2183.273193359375, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "epoch": 0.5795454545454546, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 70823400.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.6484375, "reward_std": 0.4779251217842102, "rewards/correct/mean": 0.6484375, "rewards/correct/std": 0.4779251217842102, "rollout/chars": 7130.25, "rollout/commit": 0.8125, "rollout/reward": 0.65625, "rollout/trunc": 0.28125, "sampling/importance_sampling_ratio/max": 2.8543965816497803, "sampling/importance_sampling_ratio/mean": 0.9431490898132324, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4018628597259521, "sampling/sampling_logp_difference/mean": 0.004403980448842049, "step": 51, "step_time": 217.6025530260049 }, { "epoch": 0.5795454545454546, "grad_norm": 0.002995844464749098, "learning_rate": 1e-05, "loss": 0.0240937490016222, "step": 51 }, { "clip_ratio": 2.9246679787320318e-05, "completions/clipped_ratio": 0.279296875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4079.0, "completions/mean_length": 2497.025390625, "completions/mean_terminated_length": 1878.0352783203125, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "epoch": 0.5909090909090909, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 72188197.0, "register/okay": 0.0, "register/okay_any": 0.875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.64453125, "reward_std": 0.47912323474884033, "rewards/correct/mean": 0.64453125, "rewards/correct/std": 0.47912323474884033, "rollout/chars": 9988.96875, "rollout/commit": 0.5, "rollout/reward": 0.34375, "rollout/trunc": 0.53125, "sampling/importance_sampling_ratio/max": 2.9232101440429688, "sampling/importance_sampling_ratio/mean": 0.9172062873840332, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.8242874145507812, "sampling/sampling_logp_difference/mean": 0.005016656592488289, "step": 52, "step_time": 214.57408631000362 }, { "epoch": 0.5909090909090909, "grad_norm": 0.003411246929317713, "learning_rate": 1e-05, "loss": 0.0576096810400486, "step": 52 }, { "clip_ratio": 5.054653161096212e-05, "completions/clipped_ratio": 0.2265625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4092.0, "completions/mean_length": 2143.57421875, "completions/mean_terminated_length": 1571.6514892578125, "completions/min_length": 195.0, "completions/min_terminated_length": 195.0, "epoch": 0.6022727272727273, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 73356299.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.69140625, "reward_std": 0.4623647928237915, "rewards/correct/mean": 0.69140625, "rewards/correct/std": 0.4623647928237915, "rollout/chars": 6858.78125, "rollout/commit": 0.78125, "rollout/reward": 0.59375, "rollout/trunc": 0.3125, "sampling/importance_sampling_ratio/max": 2.9921696186065674, "sampling/importance_sampling_ratio/mean": 0.9181144833564758, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.030383586883545, "sampling/sampling_logp_difference/mean": 0.004691462032496929, "step": 53, "step_time": 181.63221877400065 }, { "epoch": 0.6022727272727273, "grad_norm": 0.003647678066045046, "learning_rate": 1e-05, "loss": 0.04092782735824585, "step": 53 }, { "clip_ratio": 4.249565392910881e-05, "completions/clipped_ratio": 0.27734375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4086.0, "completions/mean_length": 2531.005859375, "completions/mean_terminated_length": 1930.386474609375, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "epoch": 0.6136363636363636, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 74714142.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.654296875, "reward_std": 0.4760620892047882, "rewards/correct/mean": 0.654296875, "rewards/correct/std": 0.4760620892047882, "rollout/chars": 9815.0625, "rollout/commit": 0.75, "rollout/reward": 0.625, "rollout/trunc": 0.34375, "sampling/importance_sampling_ratio/max": 2.764146327972412, "sampling/importance_sampling_ratio/mean": 0.946114182472229, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.4311070442199707, "sampling/sampling_logp_difference/mean": 0.0047568269073963165, "step": 54, "step_time": 219.72718010098833 }, { "epoch": 0.6136363636363636, "grad_norm": 0.003797076875343919, "learning_rate": 1e-05, "loss": 0.08744138479232788, "step": 54 }, { "clip_ratio": 5.191841631813077e-05, "completions/clipped_ratio": 0.265625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4093.0, "completions/mean_length": 2389.51171875, "completions/mean_terminated_length": 1772.271240234375, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "epoch": 0.625, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 75994788.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.642578125, "reward_std": 0.4797092080116272, "rewards/correct/mean": 0.642578125, "rewards/correct/std": 0.4797092080116272, "rollout/chars": 5294.71875, "rollout/commit": 0.8125, "rollout/reward": 0.59375, "rollout/trunc": 0.28125, "sampling/importance_sampling_ratio/max": 2.9695074558258057, "sampling/importance_sampling_ratio/mean": 0.9630504846572876, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.6024425029754639, "sampling/sampling_logp_difference/mean": 0.004764664452522993, "step": 55, "step_time": 202.55114150499867 }, { "epoch": 0.625, "grad_norm": 0.0037811100482940674, "learning_rate": 1e-05, "loss": 0.05018414929509163, "step": 55 }, { "clip_ratio": 3.818530814214682e-05, "completions/clipped_ratio": 0.271484375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4093.0, "completions/mean_length": 2400.95703125, "completions/mean_terminated_length": 1769.292236328125, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "epoch": 0.6363636363636364, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 77289662.0, "register/okay": 0.0, "register/okay_any": 0.9375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.6640625, "reward_std": 0.4727790653705597, "rewards/correct/mean": 0.6640625, "rewards/correct/std": 0.4727790653705597, "rollout/chars": 7601.21875, "rollout/commit": 0.6875, "rollout/reward": 0.4375, "rollout/trunc": 0.34375, "sampling/importance_sampling_ratio/max": 2.9998912811279297, "sampling/importance_sampling_ratio/mean": 0.9442312717437744, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.8419792652130127, "sampling/sampling_logp_difference/mean": 0.004754454363137484, "step": 56, "step_time": 229.20321971600424 }, { "epoch": 0.6363636363636364, "grad_norm": 0.003686784068122506, "learning_rate": 1e-05, "loss": 0.05815686285495758, "step": 56 }, { "clip_ratio": 4.021892334549193e-05, "completions/clipped_ratio": 0.177734375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4083.0, "completions/mean_length": 1999.87109375, "completions/mean_terminated_length": 1546.7886962890625, "completions/min_length": 118.0, "completions/min_terminated_length": 118.0, "epoch": 0.6477272727272727, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 78373484.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.751953125, "reward_std": 0.4323015511035919, "rewards/correct/mean": 0.751953125, "rewards/correct/std": 0.4323015511035919, "rollout/chars": 9423.875, "rollout/commit": 0.46875, "rollout/reward": 0.4375, "rollout/trunc": 0.5625, "sampling/importance_sampling_ratio/max": 2.8716530799865723, "sampling/importance_sampling_ratio/mean": 0.9636626243591309, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.7181296348571777, "sampling/sampling_logp_difference/mean": 0.004822447896003723, "step": 57, "step_time": 207.26324840899179 }, { "epoch": 0.6477272727272727, "grad_norm": 0.003879693103954196, "learning_rate": 1e-05, "loss": 0.04891563206911087, "step": 57 }, { "clip_ratio": 4.452647993957726e-05, "completions/clipped_ratio": 0.255859375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4094.0, "completions/mean_length": 2422.7890625, "completions/mean_terminated_length": 1848.577392578125, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "epoch": 0.6590909090909091, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 79684544.0, "register/okay": 0.0, "register/okay_any": 0.75, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.658203125, "reward_std": 0.4747757613658905, "rewards/correct/mean": 0.658203125, "rewards/correct/std": 0.4747757613658905, "rollout/chars": 8886.1875, "rollout/commit": 0.8125, "rollout/reward": 0.5625, "rollout/trunc": 0.3125, "sampling/importance_sampling_ratio/max": 2.7507035732269287, "sampling/importance_sampling_ratio/mean": 0.9232572913169861, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.0612422227859497, "sampling/sampling_logp_difference/mean": 0.004986472427845001, "step": 58, "step_time": 227.0520191610085 }, { "epoch": 0.6590909090909091, "grad_norm": 0.0036361583042889833, "learning_rate": 1e-05, "loss": 0.03166275471448898, "step": 58 }, { "clip_ratio": 4.475837192785548e-05, "completions/clipped_ratio": 0.296875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4091.0, "completions/mean_length": 2580.60546875, "completions/mean_terminated_length": 1940.772216796875, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "epoch": 0.6704545454545454, "frac_reward_zero_std": 0.0625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 81072166.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.630859375, "reward_std": 0.4830440282821655, "rewards/correct/mean": 0.630859375, "rewards/correct/std": 0.4830440282821655, "rollout/chars": 9337.8125, "rollout/commit": 0.8125, "rollout/reward": 0.59375, "rollout/trunc": 0.25, "sampling/importance_sampling_ratio/max": 2.8947973251342773, "sampling/importance_sampling_ratio/mean": 0.9524353742599487, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.6679909229278564, "sampling/sampling_logp_difference/mean": 0.004624797031283379, "step": 59, "step_time": 226.3665666429988 }, { "epoch": 0.6704545454545454, "grad_norm": 0.003573889844119549, "learning_rate": 1e-05, "loss": 0.05335336923599243, "step": 59 }, { "clip_ratio": 5.3456279715646815e-05, "completions/clipped_ratio": 0.189453125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4049.0, "completions/mean_length": 2079.482421875, "completions/mean_terminated_length": 1608.15185546875, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "epoch": 0.6818181818181818, "frac_reward_zero_std": 0.21875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 82194877.0, "register/okay": 0.09375, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.728515625, "reward_std": 0.44516023993492126, "rewards/correct/mean": 0.728515625, "rewards/correct/std": 0.44516023993492126, "rollout/chars": 9809.5625, "rollout/commit": 0.75, "rollout/reward": 0.625, "rollout/trunc": 0.28125, "sampling/importance_sampling_ratio/max": 2.8220648765563965, "sampling/importance_sampling_ratio/mean": 0.9266712665557861, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5561995506286621, "sampling/sampling_logp_difference/mean": 0.004602561239153147, "step": 60, "step_time": 202.86149891199238 }, { "epoch": 0.6818181818181818, "grad_norm": 0.0034001499880105257, "learning_rate": 1e-05, "loss": 0.039109375327825546, "step": 60 }, { "clip_ratio": 3.886056003921112e-05, "completions/clipped_ratio": 0.26953125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4096.0, "completions/mean_length": 2465.1796875, "completions/mean_terminated_length": 1864.7166748046875, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "epoch": 0.6931818181818182, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 83540089.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.669921875, "reward_std": 0.47070086002349854, "rewards/correct/mean": 0.669921875, "rewards/correct/std": 0.47070086002349854, "rollout/chars": 7354.40625, "rollout/commit": 0.75, "rollout/reward": 0.625, "rollout/trunc": 0.34375, "sampling/importance_sampling_ratio/max": 2.9583966732025146, "sampling/importance_sampling_ratio/mean": 0.9647207260131836, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3499653339385986, "sampling/sampling_logp_difference/mean": 0.0047961147502064705, "step": 61, "step_time": 245.6543137339977 }, { "epoch": 0.6931818181818182, "grad_norm": 0.0036566774360835552, "learning_rate": 1e-05, "loss": 0.04430537670850754, "step": 61 }, { "clip_ratio": 4.715258347687268e-05, "completions/clipped_ratio": 0.22265625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4094.0, "completions/mean_length": 2493.732421875, "completions/mean_terminated_length": 2034.7913818359375, "completions/min_length": 384.0, "completions/min_terminated_length": 384.0, "epoch": 0.7045454545454546, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 84883168.0, "register/okay": 0.0, "register/okay_any": 0.8125, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.71875, "reward_std": 0.45004892349243164, "rewards/correct/mean": 0.71875, "rewards/correct/std": 0.45004892349243164, "rollout/chars": 11460.28125, "rollout/commit": 0.71875, "rollout/reward": 0.59375, "rollout/trunc": 0.3125, "sampling/importance_sampling_ratio/max": 2.996396064758301, "sampling/importance_sampling_ratio/mean": 0.927265465259552, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3536334037780762, "sampling/sampling_logp_difference/mean": 0.004705195315182209, "step": 62, "step_time": 217.31840921299954 }, { "epoch": 0.7045454545454546, "grad_norm": 0.0036497486289590597, "learning_rate": 1e-05, "loss": 0.0523228719830513, "step": 62 }, { "clip_ratio": 5.232278874700569e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4072.0, "completions/mean_length": 2206.771484375, "completions/mean_terminated_length": 1677.7874755859375, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "epoch": 0.7159090909090909, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 86075979.0, "register/okay": 0.0, "register/okay_any": 0.90625, "register/premium": -0.4137931034482759, "register/verify_any": 0.90625, "reward": 0.73046875, "reward_std": 0.44415023922920227, "rewards/correct/mean": 0.73046875, "rewards/correct/std": 0.44415023922920227, "rollout/chars": 6005.75, "rollout/commit": 0.65625, "rollout/reward": 0.625, "rollout/trunc": 0.3125, "sampling/importance_sampling_ratio/max": 2.9880001544952393, "sampling/importance_sampling_ratio/mean": 0.914750874042511, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5109350681304932, "sampling/sampling_logp_difference/mean": 0.004886114038527012, "step": 63, "step_time": 212.4865788399984 }, { "epoch": 0.7159090909090909, "grad_norm": 0.003489773953333497, "learning_rate": 1e-05, "loss": 0.0597803108394146, "step": 63 }, { "clip_ratio": 3.697571492011775e-05, "completions/clipped_ratio": 0.21484375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4095.0, "completions/mean_length": 2341.447265625, "completions/mean_terminated_length": 1861.345703125, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "epoch": 0.7272727272727273, "frac_reward_zero_std": 0.21875, "groups/all_right": 1.0, "groups/all_wrong": 0.0, "groups/live_frac": 0.0, "groups/n": 2, "num_tokens": 87347744.0, "register/okay": 0.0, "register/okay_any": 0.9375, "register/premium": 0.0, "register/verify_any": 0.96875, "reward": 0.693359375, "reward_std": 0.4615498185157776, "rewards/correct/mean": 0.693359375, "rewards/correct/std": 0.4615498185157776, "rollout/chars": 2203.65625, "rollout/commit": 1.0, "rollout/reward": 1.0, "rollout/trunc": 0.0, "sampling/importance_sampling_ratio/max": 2.9642345905303955, "sampling/importance_sampling_ratio/mean": 0.9727215766906738, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.9567770957946777, "sampling/sampling_logp_difference/mean": 0.004946683067828417, "step": 64, "step_time": 212.92553410700566 }, { "epoch": 0.7272727272727273, "grad_norm": 0.004189362283796072, "learning_rate": 1e-05, "loss": 0.07004909217357635, "step": 64 }, { "clip_ratio": 3.796882333517715e-05, "completions/clipped_ratio": 0.2578125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4076.0, "completions/mean_length": 2461.544921875, "completions/mean_terminated_length": 1893.786865234375, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "epoch": 0.7386363636363636, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 88670167.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.658203125, "reward_std": 0.4747757613658905, "rewards/correct/mean": 0.658203125, "rewards/correct/std": 0.4747757613658905, "rollout/chars": 8104.09375, "rollout/commit": 0.53125, "rollout/reward": 0.40625, "rollout/trunc": 0.5625, "sampling/importance_sampling_ratio/max": 2.8981740474700928, "sampling/importance_sampling_ratio/mean": 0.9541049003601074, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.915899395942688, "sampling/sampling_logp_difference/mean": 0.0045292191207408905, "step": 65, "step_time": 205.35146746500322 }, { "epoch": 0.7386363636363636, "grad_norm": 0.0034650671295821667, "learning_rate": 1e-05, "loss": 0.055638059973716736, "step": 65 }, { "clip_ratio": 5.986664916690643e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4048.0, "completions/mean_length": 2107.029296875, "completions/mean_terminated_length": 1648.0361328125, "completions/min_length": 108.0, "completions/min_terminated_length": 108.0, "epoch": 0.75, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 89814678.0, "register/okay": 0.0, "register/okay_any": 0.78125, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.748046875, "reward_std": 0.43455907702445984, "rewards/correct/mean": 0.748046875, "rewards/correct/std": 0.43455907702445984, "rollout/chars": 3176.59375, "rollout/commit": 0.96875, "rollout/reward": 0.96875, "rollout/trunc": 0.0, "sampling/importance_sampling_ratio/max": 2.9320180416107178, "sampling/importance_sampling_ratio/mean": 0.949730396270752, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.11814022064209, "sampling/sampling_logp_difference/mean": 0.005125499330461025, "step": 66, "step_time": 215.50733105100517 }, { "epoch": 0.75, "grad_norm": 0.0042817238718271255, "learning_rate": 1e-05, "loss": 0.056593768298625946, "step": 66 }, { "clip_ratio": 4.7780752197468246e-05, "completions/clipped_ratio": 0.17578125, "completions/max_length": 4096.0, "completions/max_terminated_length": 3997.0, "completions/mean_length": 1993.0234375, "completions/mean_terminated_length": 1544.5213623046875, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "epoch": 0.7613636363636364, "frac_reward_zero_std": 0.25, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 90900642.0, "register/okay": 0.03125, "register/okay_any": 0.90625, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.763671875, "reward_std": 0.42524150013923645, "rewards/correct/mean": 0.763671875, "rewards/correct/std": 0.42524150013923645, "rollout/chars": 3227.375, "rollout/commit": 1.0, "rollout/reward": 0.96875, "rollout/trunc": 0.0, "sampling/importance_sampling_ratio/max": 2.9239091873168945, "sampling/importance_sampling_ratio/mean": 0.9536864757537842, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.4159538745880127, "sampling/sampling_logp_difference/mean": 0.004675320349633694, "step": 67, "step_time": 213.83992650399523 }, { "epoch": 0.7613636363636364, "grad_norm": 0.003478818107396364, "learning_rate": 1e-05, "loss": 0.04430510476231575, "step": 67 }, { "clip_ratio": 2.2938071538192162e-05, "completions/clipped_ratio": 0.224609375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4096.0, "completions/mean_length": 2450.83203125, "completions/mean_terminated_length": 1974.27197265625, "completions/min_length": 578.0, "completions/min_terminated_length": 578.0, "epoch": 0.7727272727272727, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 92216028.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.69140625, "reward_std": 0.4623647928237915, "rewards/correct/mean": 0.69140625, "rewards/correct/std": 0.4623647928237915, "rollout/chars": 7057.125, "rollout/commit": 0.59375, "rollout/reward": 0.5, "rollout/trunc": 0.375, "sampling/importance_sampling_ratio/max": 2.927105188369751, "sampling/importance_sampling_ratio/mean": 0.9140702486038208, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.6576094627380371, "sampling/sampling_logp_difference/mean": 0.0042533790692687035, "step": 68, "step_time": 201.8367882020102 }, { "epoch": 0.7727272727272727, "grad_norm": 0.003300267970189452, "learning_rate": 1e-05, "loss": 0.05456356331706047, "step": 68 }, { "clip_ratio": 2.7818967055281973e-05, "completions/clipped_ratio": 0.24609375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4028.0, "completions/mean_length": 2318.80078125, "completions/mean_terminated_length": 1738.6787109375, "completions/min_length": 165.0, "completions/min_terminated_length": 165.0, "epoch": 0.7840909090909091, "frac_reward_zero_std": 0.25, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 93470742.0, "register/okay": 0.03125, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.689453125, "reward_std": 0.46317005157470703, "rewards/correct/mean": 0.689453125, "rewards/correct/std": 0.46317005157470703, "rollout/chars": 3183.625, "rollout/commit": 0.9375, "rollout/reward": 0.9375, "rollout/trunc": 0.0625, "sampling/importance_sampling_ratio/max": 2.8213675022125244, "sampling/importance_sampling_ratio/mean": 0.9475071430206299, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.982326865196228, "sampling/sampling_logp_difference/mean": 0.0047585004940629005, "step": 69, "step_time": 213.00111944902164 }, { "epoch": 0.7840909090909091, "grad_norm": 0.00274608819745481, "learning_rate": 1e-05, "loss": 0.02519788220524788, "step": 69 }, { "clip_ratio": 3.960314887763161e-05, "completions/clipped_ratio": 0.193359375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4095.0, "completions/mean_length": 2265.169921875, "completions/mean_terminated_length": 1826.302734375, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "epoch": 0.7954545454545454, "frac_reward_zero_std": 0.09375, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 94695453.0, "register/okay": 0.0, "register/okay_any": 0.90625, "register/premium": -0.033333333333333326, "register/verify_any": 0.9375, "reward": 0.712890625, "reward_std": 0.45285552740097046, "rewards/correct/mean": 0.712890625, "rewards/correct/std": 0.45285552740097046, "rollout/chars": 3913.0625, "rollout/commit": 0.96875, "rollout/reward": 0.96875, "rollout/trunc": 0.03125, "sampling/importance_sampling_ratio/max": 2.8199970722198486, "sampling/importance_sampling_ratio/mean": 0.9569797515869141, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2235708236694336, "sampling/sampling_logp_difference/mean": 0.004677018616348505, "step": 70, "step_time": 226.57580704100837 }, { "epoch": 0.7954545454545454, "grad_norm": 0.0036871617194265127, "learning_rate": 1e-05, "loss": 0.06538701057434082, "step": 70 }, { "clip_ratio": 4.43938058083404e-05, "completions/clipped_ratio": 0.16015625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4085.0, "completions/mean_length": 2226.521484375, "completions/mean_terminated_length": 1870.0162353515625, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "epoch": 0.8068181818181818, "frac_reward_zero_std": 0.09375, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 95895464.0, "register/okay": 0.0, "register/okay_any": 0.90625, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.748046875, "reward_std": 0.43455907702445984, "rewards/correct/mean": 0.748046875, "rewards/correct/std": 0.43455907702445984, "rollout/chars": 5632.53125, "rollout/commit": 0.90625, "rollout/reward": 0.8125, "rollout/trunc": 0.0625, "sampling/importance_sampling_ratio/max": 2.657371997833252, "sampling/importance_sampling_ratio/mean": 0.954735517501831, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.4114289879798889, "sampling/sampling_logp_difference/mean": 0.004413054324686527, "step": 71, "step_time": 203.12368171299022 }, { "epoch": 0.8068181818181818, "grad_norm": 0.003891103435307741, "learning_rate": 1e-05, "loss": 0.06480266153812408, "step": 71 }, { "clip_ratio": 3.642644270485107e-05, "completions/clipped_ratio": 0.158203125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4067.0, "completions/mean_length": 2170.19140625, "completions/mean_terminated_length": 1808.264404296875, "completions/min_length": 354.0, "completions/min_terminated_length": 354.0, "epoch": 0.8181818181818182, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 97069322.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.712890625, "reward_std": 0.45285552740097046, "rewards/correct/mean": 0.712890625, "rewards/correct/std": 0.45285552740097046, "rollout/chars": 5475.28125, "rollout/commit": 0.96875, "rollout/reward": 0.53125, "rollout/trunc": 0.03125, "sampling/importance_sampling_ratio/max": 2.9911015033721924, "sampling/importance_sampling_ratio/mean": 0.9561970829963684, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.6782810688018799, "sampling/sampling_logp_difference/mean": 0.004562602378427982, "step": 72, "step_time": 179.52364713398129 }, { "epoch": 0.8181818181818182, "grad_norm": 0.003477900754660368, "learning_rate": 1e-05, "loss": 0.04381885007023811, "step": 72 }, { "clip_ratio": 4.090962750069593e-05, "completions/clipped_ratio": 0.20703125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4046.0, "completions/mean_length": 2193.423828125, "completions/mean_terminated_length": 1696.692138671875, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "epoch": 0.8295454545454546, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 98250371.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.7265625, "reward_std": 0.4461594223976135, "rewards/correct/mean": 0.7265625, "rewards/correct/std": 0.4461594223976135, "rollout/chars": 10409.375, "rollout/commit": 0.5625, "rollout/reward": 0.375, "rollout/trunc": 0.5625, "sampling/importance_sampling_ratio/max": 2.8946011066436768, "sampling/importance_sampling_ratio/mean": 0.9556382894515991, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.9020514488220215, "sampling/sampling_logp_difference/mean": 0.004409998655319214, "step": 73, "step_time": 218.21551849300158 }, { "epoch": 0.8295454545454546, "grad_norm": 0.003277865005657077, "learning_rate": 1e-05, "loss": 0.026875026524066925, "step": 73 }, { "clip_ratio": 4.706709427182432e-05, "completions/clipped_ratio": 0.1796875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4079.0, "completions/mean_length": 2186.923828125, "completions/mean_terminated_length": 1768.745361328125, "completions/min_length": 162.0, "completions/min_terminated_length": 162.0, "epoch": 0.8409090909090909, "frac_reward_zero_std": 0.09375, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 99445244.0, "register/okay": 0.0, "register/okay_any": 0.84375, "register/premium": -0.32258064516129037, "register/verify_any": 0.96875, "reward": 0.712890625, "reward_std": 0.45285552740097046, "rewards/correct/mean": 0.712890625, "rewards/correct/std": 0.45285552740097046, "rollout/chars": 7441.53125, "rollout/commit": 0.78125, "rollout/reward": 0.6875, "rollout/trunc": 0.21875, "sampling/importance_sampling_ratio/max": 2.9914798736572266, "sampling/importance_sampling_ratio/mean": 0.9681503772735596, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2401676177978516, "sampling/sampling_logp_difference/mean": 0.004476681351661682, "step": 74, "step_time": 213.6933170320117 }, { "epoch": 0.8409090909090909, "grad_norm": 0.003938047681003809, "learning_rate": 1e-05, "loss": 0.03928064554929733, "step": 74 }, { "clip_ratio": 4.989902947727387e-05, "completions/clipped_ratio": 0.126953125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4058.0, "completions/mean_length": 1884.677734375, "completions/mean_terminated_length": 1563.120849609375, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "epoch": 0.8522727272727273, "frac_reward_zero_std": 0.125, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 100473111.0, "register/okay": 0.0, "register/okay_any": 0.9375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.759765625, "reward_std": 0.4276435375213623, "rewards/correct/mean": 0.759765625, "rewards/correct/std": 0.4276435375213623, "rollout/chars": 3591.84375, "rollout/commit": 1.0, "rollout/reward": 0.59375, "rollout/trunc": 0.0, "sampling/importance_sampling_ratio/max": 2.8167707920074463, "sampling/importance_sampling_ratio/mean": 0.9793204665184021, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.745476722717285, "sampling/sampling_logp_difference/mean": 0.0047586713917553425, "step": 75, "step_time": 165.5286567089861 }, { "epoch": 0.8522727272727273, "grad_norm": 0.0047437818720936775, "learning_rate": 1e-05, "loss": 0.03664234280586243, "step": 75 }, { "clip_ratio": 4.0110857980835135e-05, "completions/clipped_ratio": 0.19140625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4070.0, "completions/mean_length": 2181.103515625, "completions/mean_terminated_length": 1727.81884765625, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "epoch": 0.8636363636363636, "frac_reward_zero_std": 0.1875, "groups/all_right": 1.0, "groups/all_wrong": 0.0, "groups/live_frac": 0.0, "groups/n": 2, "num_tokens": 101649804.0, "register/okay": 0.03125, "register/okay_any": 1.0, "register/premium": 0.0, "register/verify_any": 0.9375, "reward": 0.734375, "reward_std": 0.44209739565849304, "rewards/correct/mean": 0.734375, "rewards/correct/std": 0.44209739565849304, "rollout/chars": 2777.03125, "rollout/commit": 1.0, "rollout/reward": 1.0, "rollout/trunc": 0.0, "sampling/importance_sampling_ratio/max": 2.9395360946655273, "sampling/importance_sampling_ratio/mean": 0.9469085931777954, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5487773418426514, "sampling/sampling_logp_difference/mean": 0.004658767022192478, "step": 76, "step_time": 174.94837079101126 }, { "epoch": 0.8636363636363636, "grad_norm": 0.0036041205748915672, "learning_rate": 1e-05, "loss": 0.021135743707418442, "step": 76 }, { "clip_ratio": 4.031354455946712e-05, "completions/clipped_ratio": 0.248046875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4088.0, "completions/mean_length": 2441.080078125, "completions/mean_terminated_length": 1895.17138671875, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "epoch": 0.875, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 102975829.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.634765625, "reward_std": 0.4819667339324951, "rewards/correct/mean": 0.634765625, "rewards/correct/std": 0.4819667339324951, "rollout/chars": 7108.96875, "rollout/commit": 0.9375, "rollout/reward": 0.65625, "rollout/trunc": 0.09375, "sampling/importance_sampling_ratio/max": 2.995331287384033, "sampling/importance_sampling_ratio/mean": 0.9539910554885864, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.8611626625061035, "sampling/sampling_logp_difference/mean": 0.004847717937082052, "step": 77, "step_time": 219.1072637529869 }, { "epoch": 0.875, "grad_norm": 0.003495191689580679, "learning_rate": 1e-05, "loss": 0.04030180722475052, "step": 77 }, { "clip_ratio": 3.0680181964726216e-05, "completions/clipped_ratio": 0.1796875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4071.0, "completions/mean_length": 2049.79296875, "completions/mean_terminated_length": 1601.5762939453125, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "epoch": 0.8863636363636364, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 104091467.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.75390625, "reward_std": 0.4311550557613373, "rewards/correct/mean": 0.75390625, "rewards/correct/std": 0.4311550557613373, "rollout/chars": 4061.25, "rollout/commit": 0.84375, "rollout/reward": 0.6875, "rollout/trunc": 0.15625, "sampling/importance_sampling_ratio/max": 2.8890159130096436, "sampling/importance_sampling_ratio/mean": 0.9771742820739746, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.6568914651870728, "sampling/sampling_logp_difference/mean": 0.004542468581348658, "step": 78, "step_time": 175.01377814499574 }, { "epoch": 0.8863636363636364, "grad_norm": 0.003674002829939127, "learning_rate": 1e-05, "loss": 0.039506569504737854, "step": 78 }, { "clip_ratio": 2.8853452761268272e-05, "completions/clipped_ratio": 0.181640625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4061.0, "completions/mean_length": 1899.384765625, "completions/mean_terminated_length": 1411.83056640625, "completions/min_length": 182.0, "completions/min_terminated_length": 182.0, "epoch": 0.8977272727272727, "frac_reward_zero_std": 0.3125, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 105127920.0, "register/okay": 0.0, "register/okay_any": 0.9375, "register/premium": 0.33333333333333337, "register/verify_any": 0.90625, "reward": 0.771484375, "reward_std": 0.4202871024608612, "rewards/correct/mean": 0.771484375, "rewards/correct/std": 0.4202871024608612, "rollout/chars": 2620.625, "rollout/commit": 1.0, "rollout/reward": 0.96875, "rollout/trunc": 0.0, "sampling/importance_sampling_ratio/max": 2.538262128829956, "sampling/importance_sampling_ratio/mean": 0.9482463002204895, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.38710737228393555, "sampling/sampling_logp_difference/mean": 0.004496150184422731, "step": 79, "step_time": 213.3325628570019 }, { "epoch": 0.8977272727272727, "grad_norm": 0.0037710790056735277, "learning_rate": 1e-05, "loss": 0.02028367854654789, "step": 79 }, { "clip_ratio": 4.6364476474991534e-05, "completions/clipped_ratio": 0.2109375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4078.0, "completions/mean_length": 2310.5234375, "completions/mean_terminated_length": 1833.2177734375, "completions/min_length": 401.0, "completions/min_terminated_length": 401.0, "epoch": 0.9090909090909091, "frac_reward_zero_std": 0.0625, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 106390988.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.689453125, "reward_std": 0.46317005157470703, "rewards/correct/mean": 0.689453125, "rewards/correct/std": 0.46317005157470703, "rollout/chars": 6785.6875, "rollout/commit": 1.0, "rollout/reward": 0.8125, "rollout/trunc": 0.0625, "sampling/importance_sampling_ratio/max": 2.758918046951294, "sampling/importance_sampling_ratio/mean": 0.9405354261398315, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.229443073272705, "sampling/sampling_logp_difference/mean": 0.004641121719032526, "step": 80, "step_time": 216.44453251499363 }, { "epoch": 0.9090909090909091, "grad_norm": 0.0037098946049809456, "learning_rate": 1e-05, "loss": 0.06528665125370026, "step": 80 }, { "clip_ratio": 3.082327532411e-05, "completions/clipped_ratio": 0.259765625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4060.0, "completions/mean_length": 2549.537109375, "completions/mean_terminated_length": 2006.8470458984375, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "epoch": 0.9204545454545454, "frac_reward_zero_std": 0.21875, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 107762415.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.67578125, "reward_std": 0.4685399830341339, "rewards/correct/mean": 0.67578125, "rewards/correct/std": 0.4685399830341339, "rollout/chars": 7743.03125, "rollout/commit": 0.6875, "rollout/reward": 0.65625, "rollout/trunc": 0.34375, "sampling/importance_sampling_ratio/max": 2.976248025894165, "sampling/importance_sampling_ratio/mean": 0.9721881151199341, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4394348859786987, "sampling/sampling_logp_difference/mean": 0.004551028832793236, "step": 81, "step_time": 227.58272692199535 }, { "epoch": 0.9204545454545454, "grad_norm": 0.003331459127366543, "learning_rate": 1e-05, "loss": 0.03878581523895264, "step": 81 }, { "clip_ratio": 4.367932081095205e-05, "completions/clipped_ratio": 0.169921875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4082.0, "completions/mean_length": 2035.5703125, "completions/mean_terminated_length": 1613.7882080078125, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "epoch": 0.9318181818181818, "frac_reward_zero_std": 0.25, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 108876835.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": -0.06451612903225812, "register/verify_any": 0.96875, "reward": 0.75, "reward_std": 0.43343618512153625, "rewards/correct/mean": 0.75, "rewards/correct/std": 0.43343618512153625, "rollout/chars": 3646.9375, "rollout/commit": 1.0, "rollout/reward": 0.9375, "rollout/trunc": 0.0, "sampling/importance_sampling_ratio/max": 2.9806768894195557, "sampling/importance_sampling_ratio/mean": 0.9658198356628418, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1863908767700195, "sampling/sampling_logp_difference/mean": 0.004391004331409931, "step": 82, "step_time": 215.4745593230109 }, { "epoch": 0.9318181818181818, "grad_norm": 0.0037595191970467567, "learning_rate": 1e-05, "loss": 0.03050805814564228, "step": 82 }, { "clip_ratio": 4.5627527583746996e-05, "completions/clipped_ratio": 0.201171875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4092.0, "completions/mean_length": 2278.6953125, "completions/mean_terminated_length": 1821.0367431640625, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "epoch": 0.9431818181818182, "frac_reward_zero_std": 0.25, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 110110983.0, "register/okay": 0.0, "register/okay_any": 0.875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.703125, "reward_std": 0.45732781291007996, "rewards/correct/mean": 0.703125, "rewards/correct/std": 0.45732781291007996, "rollout/chars": 7487.9375, "rollout/commit": 0.71875, "rollout/reward": 0.5625, "rollout/trunc": 0.28125, "sampling/importance_sampling_ratio/max": 2.9907960891723633, "sampling/importance_sampling_ratio/mean": 0.9532361030578613, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.028660535812378, "sampling/sampling_logp_difference/mean": 0.004711423069238663, "step": 83, "step_time": 203.9813289429876 }, { "epoch": 0.9431818181818182, "grad_norm": 0.0036984889302402735, "learning_rate": 1e-05, "loss": 0.05200440436601639, "step": 83 }, { "clip_ratio": 4.000065536047259e-05, "completions/clipped_ratio": 0.185546875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4091.0, "completions/mean_length": 2399.474609375, "completions/mean_terminated_length": 2012.97607421875, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "epoch": 0.9545454545454546, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 111412874.0, "register/okay": 0.0, "register/okay_any": 0.90625, "register/premium": -0.3870967741935484, "register/verify_any": 0.96875, "reward": 0.71484375, "reward_std": 0.45193037390708923, "rewards/correct/mean": 0.71484375, "rewards/correct/std": 0.45193037390708923, "rollout/chars": 5195.65625, "rollout/commit": 0.8125, "rollout/reward": 0.625, "rollout/trunc": 0.25, "sampling/importance_sampling_ratio/max": 2.4650051593780518, "sampling/importance_sampling_ratio/mean": 0.9181221127510071, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.371954441070557, "sampling/sampling_logp_difference/mean": 0.004593420773744583, "step": 84, "step_time": 194.1572900240135 }, { "epoch": 0.9545454545454546, "grad_norm": 0.003423116635531187, "learning_rate": 1e-05, "loss": 0.02327781170606613, "step": 84 }, { "clip_ratio": 3.935570055091375e-05, "completions/clipped_ratio": 0.216796875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4087.0, "completions/mean_length": 2182.353515625, "completions/mean_terminated_length": 1660.06494140625, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "epoch": 0.9659090909090909, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 112594607.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.724609375, "reward_std": 0.44714778661727905, "rewards/correct/mean": 0.724609375, "rewards/correct/std": 0.44714778661727905, "rollout/chars": 6484.03125, "rollout/commit": 0.875, "rollout/reward": 0.84375, "rollout/trunc": 0.15625, "sampling/importance_sampling_ratio/max": 2.973640203475952, "sampling/importance_sampling_ratio/mean": 0.9410937428474426, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.407167911529541, "sampling/sampling_logp_difference/mean": 0.004774660337716341, "step": 85, "step_time": 224.1207649390126 }, { "epoch": 0.9659090909090909, "grad_norm": 0.0036640476901084185, "learning_rate": 1e-05, "loss": 0.04924885183572769, "step": 85 }, { "clip_ratio": 2.7320112621964654e-05, "completions/clipped_ratio": 0.208984375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4065.0, "completions/mean_length": 2155.9296875, "completions/mean_terminated_length": 1643.367919921875, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "epoch": 0.9772727272727273, "frac_reward_zero_std": 0.28125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 113764891.0, "register/okay": 0.0, "register/okay_any": 0.90625, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.72265625, "reward_std": 0.4481254518032074, "rewards/correct/mean": 0.72265625, "rewards/correct/std": 0.4481254518032074, "rollout/chars": 6559.96875, "rollout/commit": 0.78125, "rollout/reward": 0.71875, "rollout/trunc": 0.25, "sampling/importance_sampling_ratio/max": 2.988853931427002, "sampling/importance_sampling_ratio/mean": 0.9482577443122864, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.490507125854492, "sampling/sampling_logp_difference/mean": 0.0044696033000946045, "step": 86, "step_time": 204.2120325049982 }, { "epoch": 0.9772727272727273, "grad_norm": 0.0037808024790138006, "learning_rate": 1e-05, "loss": 0.027897892519831657, "step": 86 }, { "clip_ratio": 4.0730785940468195e-05, "completions/clipped_ratio": 0.1796875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4049.0, "completions/mean_length": 2207.6015625, "completions/mean_terminated_length": 1793.952392578125, "completions/min_length": 392.0, "completions/min_terminated_length": 392.0, "epoch": 0.9886363636363636, "frac_reward_zero_std": 0.125, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 114959919.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.708984375, "reward_std": 0.45467492938041687, "rewards/correct/mean": 0.708984375, "rewards/correct/std": 0.45467492938041687, "rollout/chars": 6403.25, "rollout/commit": 0.9375, "rollout/reward": 0.75, "rollout/trunc": 0.0625, "sampling/importance_sampling_ratio/max": 2.8789844512939453, "sampling/importance_sampling_ratio/mean": 0.961101770401001, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.7220220565795898, "sampling/sampling_logp_difference/mean": 0.004631407558917999, "step": 87, "step_time": 204.78965708398755 }, { "epoch": 0.9886363636363636, "grad_norm": 0.0037596486508846283, "learning_rate": 1e-05, "loss": 0.06505006551742554, "step": 87 }, { "clip_ratio": 2.3238894073074334e-05, "completions/clipped_ratio": 0.10546875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4077.0, "completions/mean_length": 1959.783203125, "completions/mean_terminated_length": 1707.9149169921875, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "epoch": 1.0, "frac_reward_zero_std": 0.21875, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 116021648.0, "register/okay": 0.0, "register/okay_any": 0.90625, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.833984375, "reward_std": 0.3724585771560669, "rewards/correct/mean": 0.833984375, "rewards/correct/std": 0.3724585771560669, "rollout/chars": 4463.15625, "rollout/commit": 0.96875, "rollout/reward": 0.9375, "rollout/trunc": 0.0625, "sampling/importance_sampling_ratio/max": 2.903125762939453, "sampling/importance_sampling_ratio/mean": 1.0001908540725708, "sampling/importance_sampling_ratio/min": 0.0427229143679142, "sampling/sampling_logp_difference/max": 0.527714729309082, "sampling/sampling_logp_difference/mean": 0.004416049458086491, "step": 88, "step_time": 171.5405668849926 }, { "epoch": 1.0, "grad_norm": 0.0036739709321409464, "learning_rate": 1e-05, "loss": 0.03120698407292366, "step": 88 }, { "clip_ratio": 4.8754361273495306e-05, "completions/clipped_ratio": 0.1953125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4069.0, "completions/mean_length": 2059.328125, "completions/mean_terminated_length": 1564.9903564453125, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "epoch": 1.0113636363636365, "frac_reward_zero_std": 0.21875, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 117141832.0, "register/okay": 0.0, "register/okay_any": 0.9375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.73046875, "reward_std": 0.44415023922920227, "rewards/correct/mean": 0.73046875, "rewards/correct/std": 0.44415023922920227, "rollout/chars": 5413.90625, "rollout/commit": 0.84375, "rollout/reward": 0.625, "rollout/trunc": 0.15625, "sampling/importance_sampling_ratio/max": 2.9950687885284424, "sampling/importance_sampling_ratio/mean": 0.9584047794342041, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.6936482191085815, "sampling/sampling_logp_difference/mean": 0.004635064862668514, "step": 89, "step_time": 213.23433205298352 }, { "epoch": 1.0113636363636365, "grad_norm": 0.0035795806907117367, "learning_rate": 1e-05, "loss": 0.04079495742917061, "step": 89 }, { "clip_ratio": 3.5283550914755324e-05, "completions/clipped_ratio": 0.140625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4078.0, "completions/mean_length": 2145.16796875, "completions/mean_terminated_length": 1825.9407958984375, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "epoch": 1.0227272727272727, "frac_reward_zero_std": 0.28125, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 118315342.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.76953125, "reward_std": 0.42154473066329956, "rewards/correct/mean": 0.76953125, "rewards/correct/std": 0.42154473066329956, "rollout/chars": 4900.375, "rollout/commit": 0.9375, "rollout/reward": 0.875, "rollout/trunc": 0.09375, "sampling/importance_sampling_ratio/max": 2.9338715076446533, "sampling/importance_sampling_ratio/mean": 0.9188010692596436, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7511227130889893, "sampling/sampling_logp_difference/mean": 0.004697825293987989, "step": 90, "step_time": 173.5616934130012 }, { "epoch": 1.0227272727272727, "grad_norm": 0.003947510849684477, "learning_rate": 1e-05, "loss": 0.02850821241736412, "step": 90 }, { "clip_ratio": 3.4324143314279354e-05, "completions/clipped_ratio": 0.216796875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4067.0, "completions/mean_length": 2320.306640625, "completions/mean_terminated_length": 1828.7806396484375, "completions/min_length": 200.0, "completions/min_terminated_length": 200.0, "epoch": 1.0340909090909092, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 119577099.0, "register/okay": 0.0, "register/okay_any": 0.9375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.7265625, "reward_std": 0.4461594223976135, "rewards/correct/mean": 0.7265625, "rewards/correct/std": 0.4461594223976135, "rollout/chars": 6038.375, "rollout/commit": 0.78125, "rollout/reward": 0.75, "rollout/trunc": 0.21875, "sampling/importance_sampling_ratio/max": 2.961963415145874, "sampling/importance_sampling_ratio/mean": 0.9410706758499146, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.6066663265228271, "sampling/sampling_logp_difference/mean": 0.0046050529927015305, "step": 91, "step_time": 194.22084415600693 }, { "epoch": 1.0340909090909092, "grad_norm": 0.003745067398995161, "learning_rate": 1e-05, "loss": 0.05161101371049881, "step": 91 }, { "clip_ratio": 2.8153965445198992e-05, "completions/clipped_ratio": 0.14453125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4070.0, "completions/mean_length": 2106.283203125, "completions/mean_terminated_length": 1770.1209716796875, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "epoch": 1.0454545454545454, "frac_reward_zero_std": 0.25, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 120717372.0, "register/okay": 0.0, "register/okay_any": 0.84375, "register/premium": -0.06451612903225812, "register/verify_any": 0.96875, "reward": 0.783203125, "reward_std": 0.4124660789966583, "rewards/correct/mean": 0.783203125, "rewards/correct/std": 0.4124660789966583, "rollout/chars": 4021.875, "rollout/commit": 0.9375, "rollout/reward": 0.9375, "rollout/trunc": 0.0625, "sampling/importance_sampling_ratio/max": 2.950380563735962, "sampling/importance_sampling_ratio/mean": 0.9628072381019592, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3171067237854004, "sampling/sampling_logp_difference/mean": 0.004503586329519749, "step": 92, "step_time": 185.98780072100635 }, { "epoch": 1.0454545454545454, "grad_norm": 0.0037332919891923666, "learning_rate": 1e-05, "loss": 0.05408789962530136, "step": 92 }, { "clip_ratio": 3.645778167538083e-05, "completions/clipped_ratio": 0.17578125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4078.0, "completions/mean_length": 2234.03125, "completions/mean_terminated_length": 1836.928955078125, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "epoch": 1.0568181818181819, "frac_reward_zero_std": 0.21875, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 121923036.0, "register/okay": 0.03125, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.736328125, "reward_std": 0.4410543739795685, "rewards/correct/mean": 0.736328125, "rewards/correct/std": 0.4410543739795685, "rollout/chars": 6789.5625, "rollout/commit": 0.78125, "rollout/reward": 0.6875, "rollout/trunc": 0.25, "sampling/importance_sampling_ratio/max": 2.825411796569824, "sampling/importance_sampling_ratio/mean": 0.9296578168869019, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4022775888442993, "sampling/sampling_logp_difference/mean": 0.00476218992844224, "step": 93, "step_time": 176.75329591200716 }, { "epoch": 1.0568181818181819, "grad_norm": 0.0036660602781921625, "learning_rate": 1e-05, "loss": 0.042269282042980194, "step": 93 }, { "clip_ratio": 3.608566220236753e-05, "completions/clipped_ratio": 0.138671875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4095.0, "completions/mean_length": 2214.705078125, "completions/mean_terminated_length": 1911.8616943359375, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "epoch": 1.0681818181818181, "frac_reward_zero_std": 0.21875, "groups/all_right": 1.0, "groups/all_wrong": 0.0, "groups/live_frac": 0.0, "groups/n": 2, "num_tokens": 123131589.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": 0.0, "register/verify_any": 0.96875, "reward": 0.7890625, "reward_std": 0.4083731174468994, "rewards/correct/mean": 0.7890625, "rewards/correct/std": 0.4083731174468994, "rollout/chars": 4254.34375, "rollout/commit": 1.0, "rollout/reward": 1.0, "rollout/trunc": 0.0, "sampling/importance_sampling_ratio/max": 2.7271571159362793, "sampling/importance_sampling_ratio/mean": 0.9117441773414612, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.763913154602051, "sampling/sampling_logp_difference/mean": 0.004605266265571117, "step": 94, "step_time": 198.71478324200143 }, { "epoch": 1.0681818181818181, "grad_norm": 0.0034007634967565536, "learning_rate": 1e-05, "loss": 0.037059236317873, "step": 94 }, { "clip_ratio": 2.708482827529224e-05, "completions/clipped_ratio": 0.20703125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4095.0, "completions/mean_length": 2148.421875, "completions/mean_terminated_length": 1639.94091796875, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "epoch": 1.0795454545454546, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 124295837.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": -0.12903225806451613, "register/verify_any": 0.96875, "reward": 0.712890625, "reward_std": 0.45285552740097046, "rewards/correct/mean": 0.712890625, "rewards/correct/std": 0.45285552740097046, "rollout/chars": 5562.3125, "rollout/commit": 0.90625, "rollout/reward": 0.875, "rollout/trunc": 0.09375, "sampling/importance_sampling_ratio/max": 2.9818408489227295, "sampling/importance_sampling_ratio/mean": 0.956209659576416, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.4998002052307129, "sampling/sampling_logp_difference/mean": 0.004824258387088776, "step": 95, "step_time": 244.76602041400474 }, { "epoch": 1.0795454545454546, "grad_norm": 0.00329650123603642, "learning_rate": 1e-05, "loss": 0.04946717619895935, "step": 95 }, { "clip_ratio": 2.7690322895068675e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4083.0, "completions/mean_length": 1963.8203125, "completions/mean_terminated_length": 1568.9722900390625, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "epoch": 1.0909090909090908, "frac_reward_zero_std": 0.25, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 125372961.0, "register/okay": 0.0, "register/okay_any": 0.875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.767578125, "reward_std": 0.42278963327407837, "rewards/correct/mean": 0.767578125, "rewards/correct/std": 0.42278963327407837, "rollout/chars": 9976.40625, "rollout/commit": 0.65625, "rollout/reward": 0.46875, "rollout/trunc": 0.40625, "sampling/importance_sampling_ratio/max": 2.9282562732696533, "sampling/importance_sampling_ratio/mean": 0.959475040435791, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.6861233711242676, "sampling/sampling_logp_difference/mean": 0.004683908075094223, "step": 96, "step_time": 175.66822066700115 }, { "epoch": 1.0909090909090908, "grad_norm": 0.004028100520372391, "learning_rate": 1e-05, "loss": 0.06202996149659157, "step": 96 }, { "clip_ratio": 1.776730141500593e-05, "completions/clipped_ratio": 0.103515625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4089.0, "completions/mean_length": 1788.0859375, "completions/mean_terminated_length": 1521.5947265625, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "epoch": 1.1022727272727273, "frac_reward_zero_std": 0.40625, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 126353197.0, "register/okay": 0.03125, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.845703125, "reward_std": 0.36158639192581177, "rewards/correct/mean": 0.845703125, "rewards/correct/std": 0.36158639192581177, "rollout/chars": 5476.25, "rollout/commit": 0.96875, "rollout/reward": 0.96875, "rollout/trunc": 0.03125, "sampling/importance_sampling_ratio/max": 2.615661382675171, "sampling/importance_sampling_ratio/mean": 0.9756219387054443, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.34112077951431274, "sampling/sampling_logp_difference/mean": 0.004467233084142208, "step": 97, "step_time": 188.14648345100431 }, { "epoch": 1.1022727272727273, "grad_norm": 0.003993295598775148, "learning_rate": 1e-05, "loss": 0.0466451421380043, "step": 97 }, { "clip_ratio": 3.808449798725633e-05, "completions/clipped_ratio": 0.19140625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4065.0, "completions/mean_length": 2329.572265625, "completions/mean_terminated_length": 1911.432373046875, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "epoch": 1.1136363636363635, "frac_reward_zero_std": 0.03125, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 127619138.0, "register/okay": 0.0, "register/okay_any": 0.71875, "register/premium": -0.0714285714285714, "register/verify_any": 0.875, "reward": 0.732421875, "reward_std": 0.4431293308734894, "rewards/correct/mean": 0.732421875, "rewards/correct/std": 0.4431293308734894, "rollout/chars": 2204.1875, "rollout/commit": 0.96875, "rollout/reward": 0.9375, "rollout/trunc": 0.0, "sampling/importance_sampling_ratio/max": 2.9139654636383057, "sampling/importance_sampling_ratio/mean": 0.9507982730865479, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.6805757284164429, "sampling/sampling_logp_difference/mean": 0.004579736851155758, "step": 98, "step_time": 186.86527647100593 }, { "epoch": 1.1136363636363635, "grad_norm": 0.004190219100564718, "learning_rate": 1e-05, "loss": 0.06765235960483551, "step": 98 }, { "clip_ratio": 2.735300012091102e-05, "completions/clipped_ratio": 0.162109375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4094.0, "completions/mean_length": 2326.181640625, "completions/mean_terminated_length": 1983.769287109375, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "epoch": 1.125, "frac_reward_zero_std": 0.09375, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 128891231.0, "register/okay": 0.0, "register/okay_any": 0.90625, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.740234375, "reward_std": 0.4389347732067108, "rewards/correct/mean": 0.740234375, "rewards/correct/std": 0.4389347732067108, "rollout/chars": 9092.28125, "rollout/commit": 0.71875, "rollout/reward": 0.5625, "rollout/trunc": 0.34375, "sampling/importance_sampling_ratio/max": 2.9636125564575195, "sampling/importance_sampling_ratio/mean": 0.9375482797622681, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.7035832405090332, "sampling/sampling_logp_difference/mean": 0.0049150558188557625, "step": 99, "step_time": 194.6006933720273 }, { "epoch": 1.125, "grad_norm": 0.003842512611299753, "learning_rate": 1e-05, "loss": 0.05795514956116676, "step": 99 }, { "clip_ratio": 2.080607362131559e-05, "completions/clipped_ratio": 0.13671875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4091.0, "completions/mean_length": 2086.37109375, "completions/mean_terminated_length": 1768.4661865234375, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "epoch": 1.1363636363636362, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 130031853.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.765625, "reward_std": 0.42402184009552, "rewards/correct/mean": 0.765625, "rewards/correct/std": 0.42402184009552, "rollout/chars": 8263.5625, "rollout/commit": 0.625, "rollout/reward": 0.625, "rollout/trunc": 0.375, "sampling/importance_sampling_ratio/max": 2.9234774112701416, "sampling/importance_sampling_ratio/mean": 0.9527230262756348, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.8298144340515137, "sampling/sampling_logp_difference/mean": 0.00444570928812027, "step": 100, "step_time": 205.96869300399703 }, { "epoch": 1.1363636363636362, "grad_norm": 0.0036328760907053947, "learning_rate": 1e-05, "loss": 0.05291016399860382, "step": 100 }, { "clip_ratio": 3.829837811508696e-05, "completions/clipped_ratio": 0.123046875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4058.0, "completions/mean_length": 2088.583984375, "completions/mean_terminated_length": 1806.919921875, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "epoch": 1.1477272727272727, "frac_reward_zero_std": 0.21875, "groups/all_right": 1.0, "groups/all_wrong": 0.0, "groups/live_frac": 0.0, "groups/n": 2, "num_tokens": 131166728.0, "register/okay": 0.03125, "register/okay_any": 0.875, "register/premium": 0.0, "register/verify_any": 0.96875, "reward": 0.72265625, "reward_std": 0.4481254518032074, "rewards/correct/mean": 0.72265625, "rewards/correct/std": 0.4481254518032074, "rollout/chars": 4316.625, "rollout/commit": 1.0, "rollout/reward": 1.0, "rollout/trunc": 0.0, "sampling/importance_sampling_ratio/max": 2.8553688526153564, "sampling/importance_sampling_ratio/mean": 0.9402266144752502, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1525399684906006, "sampling/sampling_logp_difference/mean": 0.004410602618008852, "step": 101, "step_time": 175.65274109602615 }, { "epoch": 1.1477272727272727, "grad_norm": 0.004013084806501865, "learning_rate": 1e-05, "loss": 0.0298526082187891, "step": 101 }, { "clip_ratio": 2.9813903609010595e-05, "completions/clipped_ratio": 0.16796875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4092.0, "completions/mean_length": 2126.2421875, "completions/mean_terminated_length": 1728.591552734375, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "epoch": 1.1590909090909092, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 132319092.0, "register/okay": 0.0, "register/okay_any": 0.8125, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.73828125, "reward_std": 0.44000017642974854, "rewards/correct/mean": 0.73828125, "rewards/correct/std": 0.44000017642974854, "rollout/chars": 8251.5625, "rollout/commit": 0.75, "rollout/reward": 0.71875, "rollout/trunc": 0.25, "sampling/importance_sampling_ratio/max": 2.6589834690093994, "sampling/importance_sampling_ratio/mean": 0.9480547904968262, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.180695056915283, "sampling/sampling_logp_difference/mean": 0.004475418012589216, "step": 102, "step_time": 172.173375399012 }, { "epoch": 1.1590909090909092, "grad_norm": 0.003630053484812379, "learning_rate": 1e-05, "loss": 0.037708982825279236, "step": 102 }, { "clip_ratio": 3.311573766495712e-05, "completions/clipped_ratio": 0.099609375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4005.0, "completions/mean_length": 1942.947265625, "completions/mean_terminated_length": 1704.757080078125, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "epoch": 1.1704545454545454, "frac_reward_zero_std": 0.09375, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 133378153.0, "register/okay": 0.0, "register/okay_any": 0.90625, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.78125, "reward_std": 0.41380295157432556, "rewards/correct/mean": 0.78125, "rewards/correct/std": 0.41380295157432556, "rollout/chars": 9598.75, "rollout/commit": 0.78125, "rollout/reward": 0.625, "rollout/trunc": 0.34375, "sampling/importance_sampling_ratio/max": 2.914154052734375, "sampling/importance_sampling_ratio/mean": 0.9925707578659058, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.049931287765503, "sampling/sampling_logp_difference/mean": 0.004874754697084427, "step": 103, "step_time": 180.64307162597834 }, { "epoch": 1.1704545454545454, "grad_norm": 0.004443930462002754, "learning_rate": 1e-05, "loss": 0.03272289037704468, "step": 103 }, { "clip_ratio": 3.371821759401428e-05, "completions/clipped_ratio": 0.21484375, "completions/max_length": 4096.0, "completions/max_terminated_length": 3901.0, "completions/mean_length": 2214.423828125, "completions/mean_terminated_length": 1699.564697265625, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "epoch": 1.1818181818181819, "frac_reward_zero_std": 0.28125, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 134576242.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.6796875, "reward_std": 0.4670529365539551, "rewards/correct/mean": 0.6796875, "rewards/correct/std": 0.4670529365539551, "rollout/chars": 5086.625, "rollout/commit": 0.90625, "rollout/reward": 0.84375, "rollout/trunc": 0.09375, "sampling/importance_sampling_ratio/max": 2.8436055183410645, "sampling/importance_sampling_ratio/mean": 0.954440176486969, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1737936735153198, "sampling/sampling_logp_difference/mean": 0.004643854685127735, "step": 104, "step_time": 222.77558530599345 }, { "epoch": 1.1818181818181819, "grad_norm": 0.003324542660266161, "learning_rate": 1e-05, "loss": 0.027721669524908066, "step": 104 }, { "clip_ratio": 2.527395594142945e-05, "completions/clipped_ratio": 0.19921875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4083.0, "completions/mean_length": 2366.494140625, "completions/mean_terminated_length": 1936.226806640625, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "epoch": 1.1931818181818181, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 135851599.0, "register/okay": 0.0, "register/okay_any": 0.84375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.650390625, "reward_std": 0.47731292247772217, "rewards/correct/mean": 0.650390625, "rewards/correct/std": 0.47731292247772217, "rollout/chars": 5585.625, "rollout/commit": 0.84375, "rollout/reward": 0.71875, "rollout/trunc": 0.15625, "sampling/importance_sampling_ratio/max": 2.707390785217285, "sampling/importance_sampling_ratio/mean": 0.9411368370056152, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.9681485891342163, "sampling/sampling_logp_difference/mean": 0.004678533412516117, "step": 105, "step_time": 198.77671564600314 }, { "epoch": 1.1931818181818181, "grad_norm": 0.0038325313944369555, "learning_rate": 1e-05, "loss": 0.040182776749134064, "step": 105 }, { "clip_ratio": 2.9666035970876692e-05, "completions/clipped_ratio": 0.19140625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4082.0, "completions/mean_length": 2156.87109375, "completions/mean_terminated_length": 1698.0989990234375, "completions/min_length": 377.0, "completions/min_terminated_length": 377.0, "epoch": 1.2045454545454546, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 137025469.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.720703125, "reward_std": 0.44909247756004333, "rewards/correct/mean": 0.720703125, "rewards/correct/std": 0.44909247756004333, "rollout/chars": 7301.75, "rollout/commit": 0.625, "rollout/reward": 0.625, "rollout/trunc": 0.375, "sampling/importance_sampling_ratio/max": 2.785024881362915, "sampling/importance_sampling_ratio/mean": 0.9441764950752258, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.7113728523254395, "sampling/sampling_logp_difference/mean": 0.004587256349623203, "step": 106, "step_time": 195.86932415099 }, { "epoch": 1.2045454545454546, "grad_norm": 0.004103610757738352, "learning_rate": 1e-05, "loss": 0.061051007360219955, "step": 106 }, { "clip_ratio": 3.602866661367443e-05, "completions/clipped_ratio": 0.080078125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4057.0, "completions/mean_length": 1713.435546875, "completions/mean_terminated_length": 1506.0361328125, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "epoch": 1.2159090909090908, "frac_reward_zero_std": 0.25, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 137969420.0, "register/okay": 0.03125, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.8671875, "reward_std": 0.33970388770103455, "rewards/correct/mean": 0.8671875, "rewards/correct/std": 0.33970388770103455, "rollout/chars": 6246.75, "rollout/commit": 0.75, "rollout/reward": 0.625, "rollout/trunc": 0.25, "sampling/importance_sampling_ratio/max": 2.5106136798858643, "sampling/importance_sampling_ratio/mean": 0.9642152786254883, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.208148002624512, "sampling/sampling_logp_difference/mean": 0.004733750596642494, "step": 107, "step_time": 144.30355614399014 }, { "epoch": 1.2159090909090908, "grad_norm": 0.0035058525390923023, "learning_rate": 1e-05, "loss": 0.038567136973142624, "step": 107 }, { "clip_ratio": 3.512928589088915e-05, "completions/clipped_ratio": 0.158203125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4096.0, "completions/mean_length": 2106.703125, "completions/mean_terminated_length": 1732.844482421875, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "epoch": 1.2272727272727273, "frac_reward_zero_std": 0.09375, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 139111844.0, "register/okay": 0.0, "register/okay_any": 0.71875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.71875, "reward_std": 0.45004892349243164, "rewards/correct/mean": 0.71875, "rewards/correct/std": 0.45004892349243164, "rollout/chars": 3310.15625, "rollout/commit": 0.96875, "rollout/reward": 0.90625, "rollout/trunc": 0.0, "sampling/importance_sampling_ratio/max": 2.728463649749756, "sampling/importance_sampling_ratio/mean": 0.9584470987319946, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.4000037908554077, "sampling/sampling_logp_difference/mean": 0.004530446603894234, "step": 108, "step_time": 199.19784780900227 }, { "epoch": 1.2272727272727273, "grad_norm": 0.003907789476215839, "learning_rate": 1e-05, "loss": 0.07161826640367508, "step": 108 }, { "clip_ratio": 2.5208430713519192e-05, "completions/clipped_ratio": 0.1328125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4042.0, "completions/mean_length": 1998.94140625, "completions/mean_terminated_length": 1677.770263671875, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "epoch": 1.2386363636363638, "frac_reward_zero_std": 0.21875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 140209558.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.783203125, "reward_std": 0.4124660789966583, "rewards/correct/mean": 0.783203125, "rewards/correct/std": 0.4124660789966583, "rollout/chars": 4694.71875, "rollout/commit": 0.90625, "rollout/reward": 0.78125, "rollout/trunc": 0.15625, "sampling/importance_sampling_ratio/max": 2.8777048587799072, "sampling/importance_sampling_ratio/mean": 0.960885763168335, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.8307828903198242, "sampling/sampling_logp_difference/mean": 0.004563405644148588, "step": 109, "step_time": 195.02158101601526 }, { "epoch": 1.2386363636363638, "grad_norm": 0.004254871513694525, "learning_rate": 1e-05, "loss": 0.06813612580299377, "step": 109 }, { "clip_ratio": 2.873683217785583e-05, "completions/clipped_ratio": 0.169921875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4079.0, "completions/mean_length": 2205.048828125, "completions/mean_terminated_length": 1817.9599609375, "completions/min_length": 402.0, "completions/min_terminated_length": 402.0, "epoch": 1.25, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 141405311.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.748046875, "reward_std": 0.43455907702445984, "rewards/correct/mean": 0.748046875, "rewards/correct/std": 0.43455907702445984, "rollout/chars": 7274.875, "rollout/commit": 0.59375, "rollout/reward": 0.4375, "rollout/trunc": 0.4375, "sampling/importance_sampling_ratio/max": 2.993765354156494, "sampling/importance_sampling_ratio/mean": 0.9446817636489868, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7587637901306152, "sampling/sampling_logp_difference/mean": 0.004816094879060984, "step": 110, "step_time": 180.07329708900943 }, { "epoch": 1.25, "grad_norm": 0.0034592526499181986, "learning_rate": 1e-05, "loss": 0.04061869531869888, "step": 110 }, { "clip_ratio": 2.828031472290604e-05, "completions/clipped_ratio": 0.12890625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4084.0, "completions/mean_length": 2103.15625, "completions/mean_terminated_length": 1808.251220703125, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "epoch": 1.2613636363636362, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 142557135.0, "register/okay": 0.03125, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.736328125, "reward_std": 0.4410543739795685, "rewards/correct/mean": 0.736328125, "rewards/correct/std": 0.4410543739795685, "rollout/chars": 9438.71875, "rollout/commit": 0.75, "rollout/reward": 0.28125, "rollout/trunc": 0.375, "sampling/importance_sampling_ratio/max": 2.8664064407348633, "sampling/importance_sampling_ratio/mean": 0.9498780369758606, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.6922615766525269, "sampling/sampling_logp_difference/mean": 0.004735993687063456, "step": 111, "step_time": 180.82312986201578 }, { "epoch": 1.2613636363636362, "grad_norm": 0.003941928502172232, "learning_rate": 1e-05, "loss": 0.049756407737731934, "step": 111 }, { "clip_ratio": 2.3861995373408718e-05, "completions/clipped_ratio": 0.109375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4073.0, "completions/mean_length": 1873.53515625, "completions/mean_terminated_length": 1600.600830078125, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "epoch": 1.2727272727272727, "frac_reward_zero_std": 0.28125, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 143585425.0, "register/okay": 0.0, "register/okay_any": 0.9375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.78515625, "reward_std": 0.4111155867576599, "rewards/correct/mean": 0.78515625, "rewards/correct/std": 0.4111155867576599, "rollout/chars": 5517.3125, "rollout/commit": 0.96875, "rollout/reward": 0.875, "rollout/trunc": 0.03125, "sampling/importance_sampling_ratio/max": 2.882420063018799, "sampling/importance_sampling_ratio/mean": 0.9606696367263794, "sampling/importance_sampling_ratio/min": 0.11609238386154175, "sampling/sampling_logp_difference/max": 1.085385799407959, "sampling/sampling_logp_difference/mean": 0.004776550456881523, "step": 112, "step_time": 184.76188581599854 }, { "epoch": 1.2727272727272727, "grad_norm": 0.004367031157016754, "learning_rate": 1e-05, "loss": 0.03861969709396362, "step": 112 }, { "clip_ratio": 2.1232936433079885e-05, "completions/clipped_ratio": 0.158203125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4094.0, "completions/mean_length": 1893.74609375, "completions/mean_terminated_length": 1479.8653564453125, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "epoch": 1.2840909090909092, "frac_reward_zero_std": 0.21875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 144630175.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": -0.25806451612903225, "register/verify_any": 0.96875, "reward": 0.763671875, "reward_std": 0.42524150013923645, "rewards/correct/mean": 0.763671875, "rewards/correct/std": 0.42524150013923645, "rollout/chars": 4554.65625, "rollout/commit": 0.78125, "rollout/reward": 0.75, "rollout/trunc": 0.1875, "sampling/importance_sampling_ratio/max": 2.8746249675750732, "sampling/importance_sampling_ratio/mean": 0.9863653182983398, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.9550256729125977, "sampling/sampling_logp_difference/mean": 0.004798654466867447, "step": 113, "step_time": 186.8795310879941 }, { "epoch": 1.2840909090909092, "grad_norm": 0.0044394368305802345, "learning_rate": 1e-05, "loss": 0.049711160361766815, "step": 113 }, { "clip_ratio": 4.15757208429568e-05, "completions/clipped_ratio": 0.267578125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4090.0, "completions/mean_length": 2487.701171875, "completions/mean_terminated_length": 1900.135986328125, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "epoch": 1.2954545454545454, "frac_reward_zero_std": 0.125, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 145977974.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.58203125, "reward_std": 0.4937073290348053, "rewards/correct/mean": 0.58203125, "rewards/correct/std": 0.4937073290348053, "rollout/chars": 2982.34375, "rollout/commit": 0.96875, "rollout/reward": 0.96875, "rollout/trunc": 0.03125, "sampling/importance_sampling_ratio/max": 2.9935050010681152, "sampling/importance_sampling_ratio/mean": 0.9447262287139893, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.6688656806945801, "sampling/sampling_logp_difference/mean": 0.0048196567222476006, "step": 114, "step_time": 224.4956963980003 }, { "epoch": 1.2954545454545454, "grad_norm": 0.0037473472766578197, "learning_rate": 1e-05, "loss": 0.004939033649861813, "step": 114 }, { "clip_ratio": 2.229895432037665e-05, "completions/clipped_ratio": 0.19140625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4072.0, "completions/mean_length": 2172.037109375, "completions/mean_terminated_length": 1716.606201171875, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "epoch": 1.3068181818181819, "frac_reward_zero_std": 0.25, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 147158425.0, "register/okay": 0.0, "register/okay_any": 0.875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.70703125, "reward_std": 0.455569326877594, "rewards/correct/mean": 0.70703125, "rewards/correct/std": 0.455569326877594, "rollout/chars": 8236.1875, "rollout/commit": 0.75, "rollout/reward": 0.59375, "rollout/trunc": 0.40625, "sampling/importance_sampling_ratio/max": 2.964155435562134, "sampling/importance_sampling_ratio/mean": 0.9155155420303345, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5859681367874146, "sampling/sampling_logp_difference/mean": 0.0047963084653019905, "step": 115, "step_time": 207.03050563798752 }, { "epoch": 1.3068181818181819, "grad_norm": 0.0031300601549446583, "learning_rate": 1e-05, "loss": 0.006877028848975897, "step": 115 }, { "clip_ratio": 3.457510467796965e-05, "completions/clipped_ratio": 0.17578125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4046.0, "completions/mean_length": 2214.63671875, "completions/mean_terminated_length": 1813.398193359375, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "epoch": 1.3181818181818181, "frac_reward_zero_std": 0.09375, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 148360175.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": -0.19354838709677424, "register/verify_any": 0.96875, "reward": 0.673828125, "reward_std": 0.4692695140838623, "rewards/correct/mean": 0.673828125, "rewards/correct/std": 0.4692695140838623, "rollout/chars": 6260.71875, "rollout/commit": 0.875, "rollout/reward": 0.8125, "rollout/trunc": 0.15625, "sampling/importance_sampling_ratio/max": 2.6160268783569336, "sampling/importance_sampling_ratio/mean": 0.9045487642288208, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.643240451812744, "sampling/sampling_logp_difference/mean": 0.00476405955851078, "step": 116, "step_time": 214.0483556869949 }, { "epoch": 1.3181818181818181, "grad_norm": 0.0035981147084385157, "learning_rate": 1e-05, "loss": 0.02397262305021286, "step": 116 }, { "clip_ratio": 1.480037298051684e-05, "completions/clipped_ratio": 0.177734375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4084.0, "completions/mean_length": 2261.462890625, "completions/mean_terminated_length": 1864.924072265625, "completions/min_length": 402.0, "completions/min_terminated_length": 402.0, "epoch": 1.3295454545454546, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 149580716.0, "register/okay": 0.03125, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.732421875, "reward_std": 0.4431293308734894, "rewards/correct/mean": 0.732421875, "rewards/correct/std": 0.4431293308734894, "rollout/chars": 5766.625, "rollout/commit": 0.9375, "rollout/reward": 0.875, "rollout/trunc": 0.09375, "sampling/importance_sampling_ratio/max": 2.8465805053710938, "sampling/importance_sampling_ratio/mean": 0.9381433129310608, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.9152131080627441, "sampling/sampling_logp_difference/mean": 0.005000153556466103, "step": 117, "step_time": 201.5918023599952 }, { "epoch": 1.3295454545454546, "grad_norm": 0.003709086449816823, "learning_rate": 1e-05, "loss": 0.03645225614309311, "step": 117 }, { "clip_ratio": 1.7631352534408506e-05, "completions/clipped_ratio": 0.189453125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4058.0, "completions/mean_length": 2231.15234375, "completions/mean_terminated_length": 1795.2723388671875, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "epoch": 1.3409090909090908, "frac_reward_zero_std": 0.25, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 150792794.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.728515625, "reward_std": 0.44516023993492126, "rewards/correct/mean": 0.728515625, "rewards/correct/std": 0.44516023993492126, "rollout/chars": 9116.71875, "rollout/commit": 0.625, "rollout/reward": 0.4375, "rollout/trunc": 0.4375, "sampling/importance_sampling_ratio/max": 2.9848761558532715, "sampling/importance_sampling_ratio/mean": 0.954424262046814, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.732245683670044, "sampling/sampling_logp_difference/mean": 0.004473987966775894, "step": 118, "step_time": 184.61766383699432 }, { "epoch": 1.3409090909090908, "grad_norm": 0.003316386602818966, "learning_rate": 1e-05, "loss": 0.03109358623623848, "step": 118 }, { "clip_ratio": 2.351750583784451e-05, "completions/clipped_ratio": 0.140625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4052.0, "completions/mean_length": 1980.19921875, "completions/mean_terminated_length": 1633.9771728515625, "completions/min_length": 187.0, "completions/min_terminated_length": 187.0, "epoch": 1.3522727272727273, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 151884928.0, "register/okay": 0.0, "register/okay_any": 0.90625, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.75, "reward_std": 0.43343618512153625, "rewards/correct/mean": 0.75, "rewards/correct/std": 0.43343618512153625, "rollout/chars": 10327.125, "rollout/commit": 0.65625, "rollout/reward": 0.3125, "rollout/trunc": 0.40625, "sampling/importance_sampling_ratio/max": 2.889343023300171, "sampling/importance_sampling_ratio/mean": 0.947662353515625, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.7009620666503906, "sampling/sampling_logp_difference/mean": 0.00471032690256834, "step": 119, "step_time": 216.69066300101986 }, { "epoch": 1.3522727272727273, "grad_norm": 0.003888605162501335, "learning_rate": 1e-05, "loss": 0.02938293293118477, "step": 119 }, { "clip_ratio": 2.466431749326148e-05, "completions/clipped_ratio": 0.11328125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4074.0, "completions/mean_length": 1929.3671875, "completions/mean_terminated_length": 1652.5726318359375, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "epoch": 1.3636363636363638, "frac_reward_zero_std": 0.25, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 152949132.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.7890625, "reward_std": 0.4083731174468994, "rewards/correct/mean": 0.7890625, "rewards/correct/std": 0.4083731174468994, "rollout/chars": 6741.5, "rollout/commit": 0.78125, "rollout/reward": 0.78125, "rollout/trunc": 0.21875, "sampling/importance_sampling_ratio/max": 2.862010955810547, "sampling/importance_sampling_ratio/mean": 0.9365251660346985, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.4940710663795471, "sampling/sampling_logp_difference/mean": 0.004855409264564514, "step": 120, "step_time": 187.2340369799931 }, { "epoch": 1.3636363636363638, "grad_norm": 0.003466185415163636, "learning_rate": 1e-05, "loss": 0.026897814124822617, "step": 120 }, { "clip_ratio": 4.0080947542264767e-05, "completions/clipped_ratio": 0.1015625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4044.0, "completions/mean_length": 1797.08984375, "completions/mean_terminated_length": 1537.2130126953125, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "epoch": 1.375, "frac_reward_zero_std": 0.28125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 153936602.0, "register/okay": 0.0, "register/okay_any": 0.9375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.83984375, "reward_std": 0.3671095669269562, "rewards/correct/mean": 0.83984375, "rewards/correct/std": 0.3671095669269562, "rollout/chars": 7900.625, "rollout/commit": 0.75, "rollout/reward": 0.59375, "rollout/trunc": 0.28125, "sampling/importance_sampling_ratio/max": 2.991859197616577, "sampling/importance_sampling_ratio/mean": 0.9488269686698914, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.6493926048278809, "sampling/sampling_logp_difference/mean": 0.0050193979404866695, "step": 121, "step_time": 158.4213080249683 }, { "epoch": 1.375, "grad_norm": 0.0036867305170744658, "learning_rate": 1e-05, "loss": 0.046038560569286346, "step": 121 }, { "clip_ratio": 2.4638323793624295e-05, "completions/clipped_ratio": 0.1484375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4094.0, "completions/mean_length": 2046.83984375, "completions/mean_terminated_length": 1689.646728515625, "completions/min_length": 180.0, "completions/min_terminated_length": 180.0, "epoch": 1.3863636363636362, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 155046920.0, "register/okay": 0.0, "register/okay_any": 0.90625, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.763671875, "reward_std": 0.42524150013923645, "rewards/correct/mean": 0.763671875, "rewards/correct/std": 0.42524150013923645, "rollout/chars": 4279.625, "rollout/commit": 0.96875, "rollout/reward": 0.90625, "rollout/trunc": 0.03125, "sampling/importance_sampling_ratio/max": 2.6585822105407715, "sampling/importance_sampling_ratio/mean": 0.9457110166549683, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.2984497547149658, "sampling/sampling_logp_difference/mean": 0.004718760959804058, "step": 122, "step_time": 168.27481706099934 }, { "epoch": 1.3863636363636362, "grad_norm": 0.0035898289643228054, "learning_rate": 1e-05, "loss": 0.03144431486725807, "step": 122 }, { "clip_ratio": 2.7891743286545534e-05, "completions/clipped_ratio": 0.083984375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4000.0, "completions/mean_length": 1619.361328125, "completions/mean_terminated_length": 1392.2921142578125, "completions/min_length": 130.0, "completions/min_terminated_length": 130.0, "epoch": 1.3977272727272727, "frac_reward_zero_std": 0.21875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 155949441.0, "register/okay": 0.0, "register/okay_any": 0.9375, "register/premium": 0.4838709677419355, "register/verify_any": 0.96875, "reward": 0.837890625, "reward_std": 0.3689115643501282, "rewards/correct/mean": 0.837890625, "rewards/correct/std": 0.3689115643501282, "rollout/chars": 8953.5, "rollout/commit": 0.75, "rollout/reward": 0.46875, "rollout/trunc": 0.3125, "sampling/importance_sampling_ratio/max": 2.7981770038604736, "sampling/importance_sampling_ratio/mean": 0.9692623019218445, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.961540699005127, "sampling/sampling_logp_difference/mean": 0.005073338281363249, "step": 123, "step_time": 187.87035358900903 }, { "epoch": 1.3977272727272727, "grad_norm": 0.004081201273947954, "learning_rate": 1e-05, "loss": 0.022434189915657043, "step": 123 }, { "clip_ratio": 1.4497935524104832e-05, "completions/clipped_ratio": 0.126953125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4068.0, "completions/mean_length": 2070.365234375, "completions/mean_terminated_length": 1775.809814453125, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "epoch": 1.4090909090909092, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 157079244.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.77734375, "reward_std": 0.41643625497817993, "rewards/correct/mean": 0.77734375, "rewards/correct/std": 0.41643625497817993, "rollout/chars": 5278.53125, "rollout/commit": 1.0, "rollout/reward": 0.96875, "rollout/trunc": 0.0, "sampling/importance_sampling_ratio/max": 2.9108169078826904, "sampling/importance_sampling_ratio/mean": 0.9524581432342529, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.0120267868041992, "sampling/sampling_logp_difference/mean": 0.004769218619912863, "step": 124, "step_time": 201.4901939239935 }, { "epoch": 1.4090909090909092, "grad_norm": 0.0036955843679606915, "learning_rate": 1e-05, "loss": 0.007700637448579073, "step": 124 }, { "clip_ratio": 3.58198747676397e-05, "completions/clipped_ratio": 0.16015625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4068.0, "completions/mean_length": 2125.9375, "completions/mean_terminated_length": 1750.2510986328125, "completions/min_length": 158.0, "completions/min_terminated_length": 158.0, "epoch": 1.4204545454545454, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 158234860.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.7421875, "reward_std": 0.43785804510116577, "rewards/correct/mean": 0.7421875, "rewards/correct/std": 0.43785804510116577, "rollout/chars": 5290.46875, "rollout/commit": 0.96875, "rollout/reward": 0.65625, "rollout/trunc": 0.0625, "sampling/importance_sampling_ratio/max": 2.905568838119507, "sampling/importance_sampling_ratio/mean": 0.9495104551315308, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5693403482437134, "sampling/sampling_logp_difference/mean": 0.004894970916211605, "step": 125, "step_time": 196.06127513899992 }, { "epoch": 1.4204545454545454, "grad_norm": 0.003997082356363535, "learning_rate": 1e-05, "loss": 0.05033978074789047, "step": 125 }, { "clip_ratio": 2.0383582352678786e-05, "completions/clipped_ratio": 0.1640625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4071.0, "completions/mean_length": 2206.013671875, "completions/mean_terminated_length": 1835.0816650390625, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "epoch": 1.4318181818181819, "frac_reward_zero_std": 0.25, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 159425267.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.689453125, "reward_std": 0.46317005157470703, "rewards/correct/mean": 0.689453125, "rewards/correct/std": 0.46317005157470703, "rollout/chars": 5433.8125, "rollout/commit": 0.96875, "rollout/reward": 0.875, "rollout/trunc": 0.0625, "sampling/importance_sampling_ratio/max": 2.9743244647979736, "sampling/importance_sampling_ratio/mean": 0.9571348428726196, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.642356872558594, "sampling/sampling_logp_difference/mean": 0.004836567211896181, "step": 126, "step_time": 198.8079241899759 }, { "epoch": 1.4318181818181819, "grad_norm": 0.0037744869478046894, "learning_rate": 1e-05, "loss": 0.02012319304049015, "step": 126 }, { "clip_ratio": 1.2340648595454695e-05, "completions/clipped_ratio": 0.169921875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4030.0, "completions/mean_length": 2166.1640625, "completions/mean_terminated_length": 1771.115234375, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "epoch": 1.4431818181818181, "frac_reward_zero_std": 0.28125, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 160598311.0, "register/okay": 0.0, "register/okay_any": 0.9375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.765625, "reward_std": 0.42402184009552, "rewards/correct/mean": 0.765625, "rewards/correct/std": 0.42402184009552, "rollout/chars": 4611.0625, "rollout/commit": 0.96875, "rollout/reward": 0.78125, "rollout/trunc": 0.0625, "sampling/importance_sampling_ratio/max": 2.652622699737549, "sampling/importance_sampling_ratio/mean": 0.9411092400550842, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.6841281652450562, "sampling/sampling_logp_difference/mean": 0.004764620680361986, "step": 127, "step_time": 214.5418794480065 }, { "epoch": 1.4431818181818181, "grad_norm": 0.003572101006284356, "learning_rate": 1e-05, "loss": 0.031996577978134155, "step": 127 }, { "clip_ratio": 2.6374917467819614e-05, "completions/clipped_ratio": 0.123046875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4093.0, "completions/mean_length": 2007.900390625, "completions/mean_terminated_length": 1714.9154052734375, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "epoch": 1.4545454545454546, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 161686932.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.75, "reward_std": 0.43343618512153625, "rewards/correct/mean": 0.75, "rewards/correct/std": 0.43343618512153625, "rollout/chars": 8982.4375, "rollout/commit": 0.875, "rollout/reward": 0.8125, "rollout/trunc": 0.1875, "sampling/importance_sampling_ratio/max": 2.870851993560791, "sampling/importance_sampling_ratio/mean": 0.9417217969894409, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.523747444152832, "sampling/sampling_logp_difference/mean": 0.005140463821589947, "step": 128, "step_time": 159.59821970000485 }, { "epoch": 1.4545454545454546, "grad_norm": 0.004223524127155542, "learning_rate": 1e-05, "loss": 0.011089015752077103, "step": 128 }, { "clip_ratio": 2.314268590453139e-05, "completions/clipped_ratio": 0.146484375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4089.0, "completions/mean_length": 2078.8125, "completions/mean_terminated_length": 1732.6131591796875, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "epoch": 1.4659090909090908, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 162813716.0, "register/okay": 0.03125, "register/okay_any": 0.9375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.78125, "reward_std": 0.41380295157432556, "rewards/correct/mean": 0.78125, "rewards/correct/std": 0.41380295157432556, "rollout/chars": 8222.84375, "rollout/commit": 0.78125, "rollout/reward": 0.71875, "rollout/trunc": 0.25, "sampling/importance_sampling_ratio/max": 2.910055637359619, "sampling/importance_sampling_ratio/mean": 0.9330929517745972, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1694750785827637, "sampling/sampling_logp_difference/mean": 0.004776017740368843, "step": 129, "step_time": 217.74412064300122 }, { "epoch": 1.4659090909090908, "grad_norm": 0.003418103326112032, "learning_rate": 1e-05, "loss": 0.030996738001704216, "step": 129 }, { "clip_ratio": 1.2713959790744411e-05, "completions/clipped_ratio": 0.11328125, "completions/max_length": 4096.0, "completions/max_terminated_length": 3944.0, "completions/mean_length": 1975.078125, "completions/mean_terminated_length": 1704.132080078125, "completions/min_length": 366.0, "completions/min_terminated_length": 366.0, "epoch": 1.4772727272727273, "frac_reward_zero_std": 0.28125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 163901084.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.7890625, "reward_std": 0.4083731174468994, "rewards/correct/mean": 0.7890625, "rewards/correct/std": 0.4083731174468994, "rollout/chars": 8696.625, "rollout/commit": 0.75, "rollout/reward": 0.5625, "rollout/trunc": 0.34375, "sampling/importance_sampling_ratio/max": 2.9872753620147705, "sampling/importance_sampling_ratio/mean": 0.9960354566574097, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.445742130279541, "sampling/sampling_logp_difference/mean": 0.004737896844744682, "step": 130, "step_time": 171.1068513739956 }, { "epoch": 1.4772727272727273, "grad_norm": 0.004176270216703415, "learning_rate": 1e-05, "loss": 0.039051882922649384, "step": 130 }, { "clip_ratio": 2.2550460528236727e-05, "completions/clipped_ratio": 0.13671875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4023.0, "completions/mean_length": 2035.076171875, "completions/mean_terminated_length": 1708.685546875, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "epoch": 1.4886363636363638, "frac_reward_zero_std": 0.09375, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 165013859.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.783203125, "reward_std": 0.4124660789966583, "rewards/correct/mean": 0.783203125, "rewards/correct/std": 0.4124660789966583, "rollout/chars": 5514.71875, "rollout/commit": 0.96875, "rollout/reward": 0.78125, "rollout/trunc": 0.03125, "sampling/importance_sampling_ratio/max": 2.7517294883728027, "sampling/importance_sampling_ratio/mean": 0.9417818784713745, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.7179064750671387, "sampling/sampling_logp_difference/mean": 0.004685908555984497, "step": 131, "step_time": 170.83824848699442 }, { "epoch": 1.4886363636363638, "grad_norm": 0.0036217328161001205, "learning_rate": 1e-05, "loss": 0.049398284405469894, "step": 131 }, { "clip_ratio": 1.985159246942203e-05, "completions/clipped_ratio": 0.15234375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4034.0, "completions/mean_length": 2045.060546875, "completions/mean_terminated_length": 1676.45849609375, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "epoch": 1.5, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 166135714.0, "register/okay": 0.0, "register/okay_any": 0.90625, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.732421875, "reward_std": 0.4431293308734894, "rewards/correct/mean": 0.732421875, "rewards/correct/std": 0.4431293308734894, "rollout/chars": 5996.9375, "rollout/commit": 0.8125, "rollout/reward": 0.75, "rollout/trunc": 0.21875, "sampling/importance_sampling_ratio/max": 2.9704782962799072, "sampling/importance_sampling_ratio/mean": 0.9610373973846436, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8830090761184692, "sampling/sampling_logp_difference/mean": 0.004899895749986172, "step": 132, "step_time": 184.38072705898958 }, { "epoch": 1.5, "grad_norm": 0.003977455198764801, "learning_rate": 1e-05, "loss": 0.003057589754462242, "step": 132 }, { "clip_ratio": 2.8942866265424527e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4047.0, "completions/mean_length": 1929.57421875, "completions/mean_terminated_length": 1528.38427734375, "completions/min_length": 184.0, "completions/min_terminated_length": 184.0, "epoch": 1.5113636363636362, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 167186872.0, "register/okay": 0.09375, "register/okay_any": 0.96875, "register/premium": -0.16129032258064513, "register/verify_any": 0.96875, "reward": 0.76171875, "reward_std": 0.42644867300987244, "rewards/correct/mean": 0.76171875, "rewards/correct/std": 0.42644867300987244, "rollout/chars": 3257.4375, "rollout/commit": 0.90625, "rollout/reward": 0.84375, "rollout/trunc": 0.09375, "sampling/importance_sampling_ratio/max": 2.804478406906128, "sampling/importance_sampling_ratio/mean": 0.9501476287841797, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.7517213821411133, "sampling/sampling_logp_difference/mean": 0.004789308644831181, "step": 133, "step_time": 235.25031293300708 }, { "epoch": 1.5113636363636362, "grad_norm": 0.0042426697909832, "learning_rate": 1e-05, "loss": 0.01811092719435692, "step": 133 }, { "clip_ratio": 2.275647773331002e-05, "completions/clipped_ratio": 0.142578125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4067.0, "completions/mean_length": 2147.28125, "completions/mean_terminated_length": 1823.2347412109375, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "epoch": 1.5227272727272727, "frac_reward_zero_std": 0.125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 168349688.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.75390625, "reward_std": 0.4311550557613373, "rewards/correct/mean": 0.75390625, "rewards/correct/std": 0.4311550557613373, "rollout/chars": 8072.6875, "rollout/commit": 0.96875, "rollout/reward": 0.5, "rollout/trunc": 0.09375, "sampling/importance_sampling_ratio/max": 2.7769110202789307, "sampling/importance_sampling_ratio/mean": 0.9160979986190796, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.6156325340271, "sampling/sampling_logp_difference/mean": 0.005172723904252052, "step": 134, "step_time": 204.1730504259831 }, { "epoch": 1.5227272727272727, "grad_norm": 0.004009219352155924, "learning_rate": 1e-05, "loss": 0.039975959807634354, "step": 134 }, { "clip_ratio": 1.8736977210664918e-05, "completions/clipped_ratio": 0.138671875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4000.0, "completions/mean_length": 1974.154296875, "completions/mean_terminated_length": 1632.5419921875, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "epoch": 1.5340909090909092, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 169425367.0, "register/okay": 0.03125, "register/okay_any": 0.78125, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.7421875, "reward_std": 0.43785804510116577, "rewards/correct/mean": 0.7421875, "rewards/correct/std": 0.43785804510116577, "rollout/chars": 6957.8125, "rollout/commit": 0.90625, "rollout/reward": 0.625, "rollout/trunc": 0.1875, "sampling/importance_sampling_ratio/max": 2.92814302444458, "sampling/importance_sampling_ratio/mean": 0.9607374668121338, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.41289329528808594, "sampling/sampling_logp_difference/mean": 0.004686000756919384, "step": 135, "step_time": 163.23045756699867 }, { "epoch": 1.5340909090909092, "grad_norm": 0.004250047728419304, "learning_rate": 1e-05, "loss": 0.015198467299342155, "step": 135 }, { "clip_ratio": 2.9312342803677893e-05, "completions/clipped_ratio": 0.177734375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4087.0, "completions/mean_length": 2036.111328125, "completions/mean_terminated_length": 1590.9715576171875, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "epoch": 1.5454545454545454, "frac_reward_zero_std": 0.21875, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 170537584.0, "register/okay": 0.0, "register/okay_any": 0.9375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.6875, "reward_std": 0.4639657139778137, "rewards/correct/mean": 0.6875, "rewards/correct/std": 0.4639657139778137, "rollout/chars": 5529.53125, "rollout/commit": 1.0, "rollout/reward": 0.6875, "rollout/trunc": 0.0, "sampling/importance_sampling_ratio/max": 2.942291259765625, "sampling/importance_sampling_ratio/mean": 0.9462506175041199, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.47405147552490234, "sampling/sampling_logp_difference/mean": 0.004781082738190889, "step": 136, "step_time": 233.96211808899534 }, { "epoch": 1.5454545454545454, "grad_norm": 0.0034462152980268, "learning_rate": 1e-05, "loss": 0.021424032747745514, "step": 136 }, { "clip_ratio": 1.868830662488108e-05, "completions/clipped_ratio": 0.095703125, "completions/max_length": 4096.0, "completions/max_terminated_length": 4051.0, "completions/mean_length": 1850.978515625, "completions/mean_terminated_length": 1613.3843994140625, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "epoch": 1.5568181818181817, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 171555781.0, "register/okay": 0.0, "register/okay_any": 0.90625, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.791015625, "reward_std": 0.40698084235191345, "rewards/correct/mean": 0.791015625, "rewards/correct/std": 0.40698084235191345, "rollout/chars": 6718.75, "rollout/commit": 0.84375, "rollout/reward": 0.5625, "rollout/trunc": 0.25, "sampling/importance_sampling_ratio/max": 2.8837714195251465, "sampling/importance_sampling_ratio/mean": 0.9316428899765015, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6558165550231934, "sampling/sampling_logp_difference/mean": 0.005001377779990435, "step": 137, "step_time": 152.97258339201653 }, { "epoch": 1.5568181818181817, "grad_norm": 0.003695861203595996, "learning_rate": 1e-05, "loss": 0.03669123351573944, "step": 137 }, { "clip_ratio": 1.818037424072827e-05, "completions/clipped_ratio": 0.09765625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4095.0, "completions/mean_length": 1802.67578125, "completions/mean_terminated_length": 1554.48046875, "completions/min_length": 366.0, "completions/min_terminated_length": 366.0, "epoch": 1.5681818181818183, "frac_reward_zero_std": 0.28125, "groups/all_right": 1.0, "groups/all_wrong": 0.0, "groups/live_frac": 0.0, "groups/n": 2, "num_tokens": 172539055.0, "register/okay": 0.03125, "register/okay_any": 0.9375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.80859375, "reward_std": 0.3937928080558777, "rewards/correct/mean": 0.80859375, "rewards/correct/std": 0.3937928080558777, "rollout/chars": 3608.8125, "rollout/commit": 1.0, "rollout/reward": 1.0, "rollout/trunc": 0.0, "sampling/importance_sampling_ratio/max": 2.8864824771881104, "sampling/importance_sampling_ratio/mean": 0.9788761734962463, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5558171272277832, "sampling/sampling_logp_difference/mean": 0.004682690836489201, "step": 138, "step_time": 214.24637426000118 }, { "epoch": 1.5681818181818183, "grad_norm": 0.004144175909459591, "learning_rate": 1e-05, "loss": 0.04392539709806442, "step": 138 }, { "clip_ratio": 2.474812504260626e-05, "completions/clipped_ratio": 0.1171875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4028.0, "completions/mean_length": 1803.603515625, "completions/mean_terminated_length": 1499.3031005859375, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "epoch": 1.5795454545454546, "frac_reward_zero_std": 0.21875, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 173542228.0, "register/okay": 0.03125, "register/okay_any": 0.84375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.72265625, "reward_std": 0.4481254518032074, "rewards/correct/mean": 0.72265625, "rewards/correct/std": 0.4481254518032074, "rollout/chars": 6108.75, "rollout/commit": 1.0, "rollout/reward": 0.875, "rollout/trunc": 0.03125, "sampling/importance_sampling_ratio/max": 2.847048044204712, "sampling/importance_sampling_ratio/mean": 0.9677566289901733, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5240151882171631, "sampling/sampling_logp_difference/mean": 0.0049498314037919044, "step": 139, "step_time": 225.22986474300706 }, { "epoch": 1.5795454545454546, "grad_norm": 0.003815558971837163, "learning_rate": 1e-05, "loss": -0.004730212967842817, "step": 139 }, { "clip_ratio": 1.5471826372959185e-05, "completions/clipped_ratio": 0.12109375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4093.0, "completions/mean_length": 2082.767578125, "completions/mean_terminated_length": 1805.388916015625, "completions/min_length": 437.0, "completions/min_terminated_length": 437.0, "epoch": 1.5909090909090908, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 174686141.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.736328125, "reward_std": 0.4410543739795685, "rewards/correct/mean": 0.736328125, "rewards/correct/std": 0.4410543739795685, "rollout/chars": 5149.78125, "rollout/commit": 0.96875, "rollout/reward": 0.78125, "rollout/trunc": 0.03125, "sampling/importance_sampling_ratio/max": 2.9328415393829346, "sampling/importance_sampling_ratio/mean": 0.9844332933425903, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1944245100021362, "sampling/sampling_logp_difference/mean": 0.005010742694139481, "step": 140, "step_time": 215.20142498500354 }, { "epoch": 1.5909090909090908, "grad_norm": 0.0041898805648088455, "learning_rate": 1e-05, "loss": 0.0373423807322979, "step": 140 }, { "clip_ratio": 2.6342355454289645e-05, "completions/clipped_ratio": 0.1015625, "completions/max_length": 4096.0, "completions/max_terminated_length": 3959.0, "completions/mean_length": 1706.4921875, "completions/mean_terminated_length": 1436.3739013671875, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "epoch": 1.6022727272727273, "frac_reward_zero_std": 0.28125, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 175632729.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.802734375, "reward_std": 0.3983237147331238, "rewards/correct/mean": 0.802734375, "rewards/correct/std": 0.3983237147331238, "rollout/chars": 5041.46875, "rollout/commit": 0.90625, "rollout/reward": 0.84375, "rollout/trunc": 0.125, "sampling/importance_sampling_ratio/max": 2.957944393157959, "sampling/importance_sampling_ratio/mean": 0.982999861240387, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8703277111053467, "sampling/sampling_logp_difference/mean": 0.005144694820046425, "step": 141, "step_time": 205.38766283299628 }, { "epoch": 1.6022727272727273, "grad_norm": 0.004375872202217579, "learning_rate": 1e-05, "loss": 0.05617167800664902, "step": 141 }, { "clip_ratio": 1.7171806860005745e-05, "completions/clipped_ratio": 0.099609375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4020.0, "completions/mean_length": 1890.771484375, "completions/mean_terminated_length": 1646.80908203125, "completions/min_length": 171.0, "completions/min_terminated_length": 171.0, "epoch": 1.6136363636363638, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 176660356.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.79296875, "reward_std": 0.40557438135147095, "rewards/correct/mean": 0.79296875, "rewards/correct/std": 0.40557438135147095, "rollout/chars": 6519.875, "rollout/commit": 0.90625, "rollout/reward": 0.875, "rollout/trunc": 0.125, "sampling/importance_sampling_ratio/max": 2.9895527362823486, "sampling/importance_sampling_ratio/mean": 0.9870668649673462, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5390865802764893, "sampling/sampling_logp_difference/mean": 0.004857347346842289, "step": 142, "step_time": 154.58443782302493 }, { "epoch": 1.6136363636363638, "grad_norm": 0.003908894956111908, "learning_rate": 1e-05, "loss": 0.0386885330080986, "step": 142 }, { "clip_ratio": 1.2467410158478742e-05, "completions/clipped_ratio": 0.083984375, "completions/max_length": 4096.0, "completions/max_terminated_length": 3990.0, "completions/mean_length": 1888.029296875, "completions/mean_terminated_length": 1685.5927734375, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "epoch": 1.625, "frac_reward_zero_std": 0.3125, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 177707091.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.80859375, "reward_std": 0.3937928080558777, "rewards/correct/mean": 0.80859375, "rewards/correct/std": 0.3937928080558777, "rollout/chars": 5737.03125, "rollout/commit": 0.96875, "rollout/reward": 0.65625, "rollout/trunc": 0.03125, "sampling/importance_sampling_ratio/max": 2.972224473953247, "sampling/importance_sampling_ratio/mean": 0.9599653482437134, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.866624116897583, "sampling/sampling_logp_difference/mean": 0.005074259825050831, "step": 143, "step_time": 156.3355814299939 }, { "epoch": 1.625, "grad_norm": 0.0038852300494909286, "learning_rate": 1e-05, "loss": 0.04118692874908447, "step": 143 }, { "clip_ratio": 3.606023904012545e-05, "completions/clipped_ratio": 0.130859375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4037.0, "completions/mean_length": 1858.986328125, "completions/mean_terminated_length": 1522.177490234375, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "epoch": 1.6363636363636362, "frac_reward_zero_std": 0.25, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 178728780.0, "register/okay": 0.0, "register/okay_any": 0.96875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.771484375, "reward_std": 0.4202871024608612, "rewards/correct/mean": 0.771484375, "rewards/correct/std": 0.4202871024608612, "rollout/chars": 7302.6875, "rollout/commit": 0.78125, "rollout/reward": 0.59375, "rollout/trunc": 0.3125, "sampling/importance_sampling_ratio/max": 2.9573657512664795, "sampling/importance_sampling_ratio/mean": 0.9615711569786072, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.4679912030696869, "sampling/sampling_logp_difference/mean": 0.004910617135465145, "step": 144, "step_time": 216.71988409100595 }, { "epoch": 1.6363636363636362, "grad_norm": 0.004278936423361301, "learning_rate": 1e-05, "loss": 0.052996911108493805, "step": 144 }, { "clip_ratio": 4.0485839519988076e-05, "completions/clipped_ratio": 0.076171875, "completions/max_length": 4096.0, "completions/max_terminated_length": 4091.0, "completions/mean_length": 1779.390625, "completions/mean_terminated_length": 1588.3804931640625, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "epoch": 1.6477272727272727, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 179706468.0, "register/okay": 0.03125, "register/okay_any": 0.90625, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.798828125, "reward_std": 0.4012683033943176, "rewards/correct/mean": 0.798828125, "rewards/correct/std": 0.4012683033943176, "rollout/chars": 4772.625, "rollout/commit": 0.9375, "rollout/reward": 0.78125, "rollout/trunc": 0.0625, "sampling/importance_sampling_ratio/max": 2.9420127868652344, "sampling/importance_sampling_ratio/mean": 0.9421902894973755, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2159937620162964, "sampling/sampling_logp_difference/mean": 0.005003421567380428, "step": 145, "step_time": 168.59408735701436 }, { "epoch": 1.6477272727272727, "grad_norm": 0.004511984530836344, "learning_rate": 1e-05, "loss": 0.05151772499084473, "step": 145 }, { "clip_ratio": 2.549199274426428e-05, "completions/clipped_ratio": 0.109375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4080.0, "completions/mean_length": 1864.126953125, "completions/mean_terminated_length": 1590.037353515625, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "epoch": 1.6590909090909092, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 180730597.0, "register/okay": 0.03125, "register/okay_any": 0.96875, "register/premium": -0.5, "register/verify_any": 0.9375, "reward": 0.73828125, "reward_std": 0.44000017642974854, "rewards/correct/mean": 0.73828125, "rewards/correct/std": 0.44000017642974854, "rollout/chars": 4061.25, "rollout/commit": 0.96875, "rollout/reward": 0.53125, "rollout/trunc": 0.03125, "sampling/importance_sampling_ratio/max": 2.9968650341033936, "sampling/importance_sampling_ratio/mean": 0.9596326351165771, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.9261562824249268, "sampling/sampling_logp_difference/mean": 0.0048238798044621944, "step": 146, "step_time": 198.43136818302446 }, { "epoch": 1.6590909090909092, "grad_norm": 0.004029472358524799, "learning_rate": 1e-05, "loss": 0.047014422714710236, "step": 146 }, { "clip_ratio": 3.068867385991325e-05, "completions/clipped_ratio": 0.16015625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4076.0, "completions/mean_length": 2147.10546875, "completions/mean_terminated_length": 1775.455810546875, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "epoch": 1.6704545454545454, "frac_reward_zero_std": 0.15625, "groups/all_right": 0.5, "groups/all_wrong": 0.0, "groups/live_frac": 0.5, "groups/n": 2, "num_tokens": 181907515.0, "register/okay": 0.03125, "register/okay_any": 0.875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.712890625, "reward_std": 0.45285552740097046, "rewards/correct/mean": 0.712890625, "rewards/correct/std": 0.45285552740097046, "rollout/chars": 5609.4375, "rollout/commit": 0.9375, "rollout/reward": 0.9375, "rollout/trunc": 0.0625, "sampling/importance_sampling_ratio/max": 2.961355447769165, "sampling/importance_sampling_ratio/mean": 0.9459095001220703, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.49617528915405273, "sampling/sampling_logp_difference/mean": 0.0052390312775969505, "step": 147, "step_time": 223.08501683599025 }, { "epoch": 1.6704545454545454, "grad_norm": 0.0040755197405815125, "learning_rate": 1e-05, "loss": 0.021937264129519463, "step": 147 }, { "clip_ratio": 1.8726599023466406e-05, "completions/clipped_ratio": 0.177734375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4075.0, "completions/mean_length": 2447.134765625, "completions/mean_terminated_length": 2090.729248046875, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "epoch": 1.6818181818181817, "frac_reward_zero_std": 0.09375, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 183239152.0, "register/okay": 0.0, "register/okay_any": 1.0, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.677734375, "reward_std": 0.46780112385749817, "rewards/correct/mean": 0.677734375, "rewards/correct/std": 0.46780112385749817, "rollout/chars": 7010.5625, "rollout/commit": 0.84375, "rollout/reward": 0.5625, "rollout/trunc": 0.15625, "sampling/importance_sampling_ratio/max": 2.9554741382598877, "sampling/importance_sampling_ratio/mean": 0.923827052116394, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.6381275653839111, "sampling/sampling_logp_difference/mean": 0.005033084657043219, "step": 148, "step_time": 207.17772821201652 }, { "epoch": 1.6818181818181817, "grad_norm": 0.003825137857347727, "learning_rate": 1e-05, "loss": 0.03941674530506134, "step": 148 }, { "clip_ratio": 1.7952673943000264e-05, "completions/clipped_ratio": 0.09765625, "completions/max_length": 4096.0, "completions/max_terminated_length": 4073.0, "completions/mean_length": 1881.142578125, "completions/mean_terminated_length": 1641.439453125, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "epoch": 1.6931818181818183, "frac_reward_zero_std": 0.28125, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 184268185.0, "register/okay": 0.0, "register/okay_any": 0.9375, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.822265625, "reward_std": 0.3826628625392914, "rewards/correct/mean": 0.822265625, "rewards/correct/std": 0.3826628625392914, "rollout/chars": 6702.1875, "rollout/commit": 0.71875, "rollout/reward": 0.65625, "rollout/trunc": 0.3125, "sampling/importance_sampling_ratio/max": 2.7608203887939453, "sampling/importance_sampling_ratio/mean": 0.9840610027313232, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.4484715461730957, "sampling/sampling_logp_difference/mean": 0.004564035218209028, "step": 149, "step_time": 157.57826894000027 }, { "epoch": 1.6931818181818183, "grad_norm": 0.003846439765766263, "learning_rate": 1e-05, "loss": 0.03307729959487915, "step": 149 }, { "clip_ratio": 2.669748795369742e-05, "completions/clipped_ratio": 0.162109375, "completions/max_length": 4096.0, "completions/max_terminated_length": 4095.0, "completions/mean_length": 1977.0234375, "completions/mean_terminated_length": 1567.058349609375, "completions/min_length": 369.0, "completions/min_terminated_length": 369.0, "epoch": 1.7045454545454546, "frac_reward_zero_std": 0.1875, "groups/all_right": 0.0, "groups/all_wrong": 0.0, "groups/live_frac": 1.0, "groups/n": 2, "num_tokens": 185348085.0, "register/okay": 0.0, "register/okay_any": 0.875, "register/premium": NaN, "register/verify_any": 1.0, "reward": 0.740234375, "reward_std": 0.4389347732067108, "rewards/correct/mean": 0.740234375, "rewards/correct/std": 0.4389347732067108, "rollout/chars": 5565.4375, "rollout/commit": 1.0, "rollout/reward": 0.78125, "rollout/trunc": 0.0, "sampling/importance_sampling_ratio/max": 2.7462289333343506, "sampling/importance_sampling_ratio/mean": 0.9060184955596924, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.952881097793579, "sampling/sampling_logp_difference/mean": 0.0051615480333566666, "step": 150, "step_time": 218.41358454099827 }, { "epoch": 1.7045454545454546, "grad_norm": 0.0036400691606104374, "learning_rate": 1e-05, "loss": 0.036437757313251495, "step": 150 } ], "logging_steps": 1, "max_steps": 300, "num_input_tokens_seen": 185348085, "num_train_epochs": 4, "save_steps": 25, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }