diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,4381 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5816066884769174, + "eval_steps": 200.0, + "global_step": 800, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 223.25, + "completions/mean_length": 184.6875, + "completions/min_length": 154.75, + "epoch": 0.0007270083605961468, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2505969216612862, + "kl": 0.0, + "learning_rate": 1.4492753623188406e-08, + "loss": -8.940696716308594e-08, + "reward": 1.800000011920929, + "reward_std": 0.16622530855238438, + "rewards/MveiAccuracyReward/mean": 0.96875, + "rewards/MveiAccuracyReward/std": 0.0883883461356163, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.33124999701976776, + "rewards/MveiLLMConsistencyReward/std": 0.11712000705301762, + "step": 1, + "step_time": 88.10392547957599 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 204.9375, + "completions/mean_length": 179.125, + "completions/min_length": 158.4375, + "epoch": 0.0036350418029807343, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.324666224992261, + "kl": 7.218225368887943e-05, + "learning_rate": 7.246376811594203e-08, + "loss": 2.7120113372802734e-06, + "reward": 1.9343749806284904, + "reward_std": 0.22219385765492916, + "rewards/MveiAccuracyReward/mean": 0.96875, + "rewards/MveiAccuracyReward/std": 0.07312605157494545, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.46562500670552254, + "rewards/MveiLLMConsistencyReward/std": 0.18109685834497213, + "step": 5, + "step_time": 80.04274107748643 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 203.9, + "completions/mean_length": 177.2, + "completions/min_length": 154.55, + "epoch": 0.007270083605961469, + "frac_reward_zero_std": 0.1, + "grad_norm": 1.3475511420920507, + "kl": 9.369138433612534e-05, + "learning_rate": 1.4492753623188405e-07, + "loss": 3.824383020401001e-06, + "reward": 1.7812500029802323, + "reward_std": 0.17669749669730664, + "rewards/MveiAccuracyReward/mean": 0.85, + "rewards/MveiAccuracyReward/std": 0.07891046851873398, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.4312500014901161, + "rewards/MveiLLMConsistencyReward/std": 0.1377484068274498, + "step": 10, + "step_time": 81.79588728807866 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.5, + "completions/mean_length": 188.91875, + "completions/min_length": 163.95, + "epoch": 0.010905125408942203, + "frac_reward_zero_std": 0.1, + "grad_norm": 1.162502802224763, + "kl": 8.805416291579604e-05, + "learning_rate": 2.1739130434782607e-07, + "loss": 3.489851951599121e-06, + "reward": 1.7925000011920929, + "reward_std": 0.2501345627009869, + "rewards/MveiAccuracyReward/mean": 0.9125, + "rewards/MveiAccuracyReward/std": 0.1473084270954132, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.380000002682209, + "rewards/MveiLLMConsistencyReward/std": 0.14857573956251144, + "step": 15, + "step_time": 78.87650614492595 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 217.85, + "completions/mean_length": 184.3125, + "completions/min_length": 159.75, + "epoch": 0.014540167211922937, + "frac_reward_zero_std": 0.2, + "grad_norm": 1.3559448301270443, + "kl": 9.812471289478708e-05, + "learning_rate": 2.898550724637681e-07, + "loss": 3.826618194580078e-06, + "reward": 1.8037499845027924, + "reward_std": 0.1779584601521492, + "rewards/MveiAccuracyReward/mean": 0.9625, + "rewards/MveiAccuracyReward/std": 0.09385617971420288, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.34125000759959223, + "rewards/MveiLLMConsistencyReward/std": 0.10352658182382583, + "step": 20, + "step_time": 78.4549627777189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 222.8, + "completions/mean_length": 179.50625, + "completions/min_length": 153.45, + "epoch": 0.01817520901490367, + "frac_reward_zero_std": 0.15, + "grad_norm": 1.073699055517611, + "kl": 0.00010079531693918398, + "learning_rate": 3.6231884057971015e-07, + "loss": 4.154443740844727e-06, + "reward": 1.8262500017881393, + "reward_std": 0.2465540524572134, + "rewards/MveiAccuracyReward/mean": 0.85, + "rewards/MveiAccuracyReward/std": 0.11845555454492569, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.4762500025331974, + "rewards/MveiLLMConsistencyReward/std": 0.173562653362751, + "step": 25, + "step_time": 83.93649914860725 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 206.25, + "completions/mean_length": 178.7, + "completions/min_length": 154.7, + "epoch": 0.021810250817884406, + "frac_reward_zero_std": 0.1, + "grad_norm": 1.2915238732276229, + "kl": 9.987151315726805e-05, + "learning_rate": 4.3478260869565214e-07, + "loss": 3.7729740142822264e-06, + "reward": 1.7924999475479126, + "reward_std": 0.15293546877801417, + "rewards/MveiAccuracyReward/mean": 0.88125, + "rewards/MveiAccuracyReward/std": 0.025877460837364197, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.41125001236796377, + "rewards/MveiLLMConsistencyReward/std": 0.14563345164060593, + "step": 30, + "step_time": 74.32066762559116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 209.6, + "completions/mean_length": 181.1, + "completions/min_length": 159.25, + "epoch": 0.02544529262086514, + "frac_reward_zero_std": 0.05, + "grad_norm": 1.2323526862059118, + "kl": 9.222824437529197e-05, + "learning_rate": 5.072463768115942e-07, + "loss": 3.7275254726409913e-06, + "reward": 2.002499985694885, + "reward_std": 0.16539971344172955, + "rewards/MveiAccuracyReward/mean": 0.98125, + "rewards/MveiAccuracyReward/std": 0.05303300768136978, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.5212500058114529, + "rewards/MveiLLMConsistencyReward/std": 0.14334528222680093, + "step": 35, + "step_time": 81.8711244918406 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 207.85, + "completions/mean_length": 179.81875, + "completions/min_length": 159.1, + "epoch": 0.029080334423845874, + "frac_reward_zero_std": 0.05, + "grad_norm": 1.3340606496373275, + "kl": 9.716968615975929e-05, + "learning_rate": 5.797101449275362e-07, + "loss": 3.904104232788086e-06, + "reward": 1.9056250035762787, + "reward_std": 0.19380234889686107, + "rewards/MveiAccuracyReward/mean": 0.9375, + "rewards/MveiAccuracyReward/std": 0.0816463440656662, + "rewards/MveiFormatReward/mean": 0.99375, + "rewards/MveiFormatReward/std": 0.01767766922712326, + "rewards/MveiLLMConsistencyReward/mean": 0.47124999910593035, + "rewards/MveiLLMConsistencyReward/std": 0.1763294253498316, + "step": 40, + "step_time": 81.85793585814535 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 209.55, + "completions/mean_length": 184.9875, + "completions/min_length": 160.55, + "epoch": 0.03271537622682661, + "frac_reward_zero_std": 0.25, + "grad_norm": 1.2263462985241742, + "kl": 0.00014888912246533437, + "learning_rate": 6.521739130434782e-07, + "loss": 5.656480789184571e-06, + "reward": 1.9612499594688415, + "reward_std": 0.1264342725276947, + "rewards/MveiAccuracyReward/mean": 0.99375, + "rewards/MveiAccuracyReward/std": 0.01767766922712326, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.467500015348196, + "rewards/MveiLLMConsistencyReward/std": 0.12135031558573246, + "step": 45, + "step_time": 79.34563678354024 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.65, + "completions/mean_length": 184.98125, + "completions/min_length": 161.9, + "epoch": 0.03635041802980734, + "frac_reward_zero_std": 0.05, + "grad_norm": 1.3899616519162392, + "kl": 0.00016749764145060909, + "learning_rate": 7.246376811594203e-07, + "loss": 6.7442655563354496e-06, + "reward": 1.9762499809265137, + "reward_std": 0.18204652182757855, + "rewards/MveiAccuracyReward/mean": 0.96875, + "rewards/MveiAccuracyReward/std": 0.06123279929161072, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.5075000047683715, + "rewards/MveiLLMConsistencyReward/std": 0.17018421925604343, + "step": 50, + "step_time": 90.00479119606317 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 202.05, + "completions/mean_length": 177.2625, + "completions/min_length": 154.5, + "epoch": 0.039985459832788076, + "frac_reward_zero_std": 0.05, + "grad_norm": 1.3463806058504582, + "kl": 0.00022680436522932724, + "learning_rate": 7.971014492753623e-07, + "loss": 8.958578109741212e-06, + "reward": 1.718749988079071, + "reward_std": 0.2583191357553005, + "rewards/MveiAccuracyReward/mean": 0.76875, + "rewards/MveiAccuracyReward/std": 0.11973364055156707, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.45000000223517417, + "rewards/MveiLLMConsistencyReward/std": 0.17364961095154285, + "step": 55, + "step_time": 83.08356610722839 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 204.4, + "completions/mean_length": 180.9375, + "completions/min_length": 158.5, + "epoch": 0.04362050163576881, + "frac_reward_zero_std": 0.15, + "grad_norm": 1.0034107978808375, + "kl": 0.0002776149965939112, + "learning_rate": 8.695652173913043e-07, + "loss": 1.1054426431655884e-05, + "reward": 1.869999971985817, + "reward_std": 0.21551885977387428, + "rewards/MveiAccuracyReward/mean": 0.8375, + "rewards/MveiAccuracyReward/std": 0.08984613567590713, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.5325000062584877, + "rewards/MveiLLMConsistencyReward/std": 0.1620310701429844, + "step": 60, + "step_time": 75.35761900916695 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 201.9, + "completions/mean_length": 177.61875, + "completions/min_length": 157.7, + "epoch": 0.04725554343874955, + "frac_reward_zero_std": 0.05, + "grad_norm": 1.2967341052700547, + "kl": 0.0006032844808942173, + "learning_rate": 9.420289855072463e-07, + "loss": 2.4043023586273193e-05, + "reward": 1.8174999833106995, + "reward_std": 0.22795652821660042, + "rewards/MveiAccuracyReward/mean": 0.88125, + "rewards/MveiAccuracyReward/std": 0.1075238049030304, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.4362500086426735, + "rewards/MveiLLMConsistencyReward/std": 0.16589595302939414, + "step": 65, + "step_time": 69.4659463264048 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 227.7, + "completions/mean_length": 187.575, + "completions/min_length": 162.35, + "epoch": 0.05089058524173028, + "frac_reward_zero_std": 0.15, + "grad_norm": 1.2025105509627323, + "kl": 0.0006610367418034002, + "learning_rate": 9.999985555970398e-07, + "loss": 2.6616454124450684e-05, + "reward": 1.693750038743019, + "reward_std": 0.21350354701280594, + "rewards/MveiAccuracyReward/mean": 0.7625, + "rewards/MveiAccuracyReward/std": 0.19401475489139558, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.4312499947845936, + "rewards/MveiLLMConsistencyReward/std": 0.18215158805251122, + "step": 70, + "step_time": 82.89062785804272 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 210.3, + "completions/mean_length": 183.86875, + "completions/min_length": 160.4, + "epoch": 0.05452562704471101, + "frac_reward_zero_std": 0.05, + "grad_norm": 1.274740408207357, + "kl": 0.000759283323714044, + "learning_rate": 9.999480023696746e-07, + "loss": 3.0159205198287965e-05, + "reward": 1.7981249630451202, + "reward_std": 0.20472086891531943, + "rewards/MveiAccuracyReward/mean": 0.90625, + "rewards/MveiAccuracyReward/std": 0.09932401329278946, + "rewards/MveiFormatReward/mean": 0.99375, + "rewards/MveiFormatReward/std": 0.01767766922712326, + "rewards/MveiLLMConsistencyReward/mean": 0.395000009983778, + "rewards/MveiLLMConsistencyReward/std": 0.13778294138610364, + "step": 75, + "step_time": 84.61769868656992 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.5, + "completions/mean_length": 185.40625, + "completions/min_length": 160.45, + "epoch": 0.05816066884769175, + "frac_reward_zero_std": 0.05, + "grad_norm": 1.2841827611979246, + "kl": 0.0011498910709633492, + "learning_rate": 9.998252373392842e-07, + "loss": 4.5952200889587405e-05, + "reward": 1.974999985098839, + "reward_std": 0.18783026114106177, + "rewards/MveiAccuracyReward/mean": 0.9375, + "rewards/MveiAccuracyReward/std": 0.05850084125995636, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.5375000052154064, + "rewards/MveiLLMConsistencyReward/std": 0.15051332265138626, + "step": 80, + "step_time": 83.98414503261446 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 229.2, + "completions/mean_length": 181.86875, + "completions/min_length": 156.8, + "epoch": 0.061795710650672485, + "frac_reward_zero_std": 0.15, + "grad_norm": 0.9487729565385968, + "kl": 0.0016302011121297256, + "learning_rate": 9.996302782378808e-07, + "loss": 6.515383720397949e-05, + "reward": 1.924374994635582, + "reward_std": 0.18220550268888475, + "rewards/MveiAccuracyReward/mean": 0.9125, + "rewards/MveiAccuracyReward/std": 0.06208146214485168, + "rewards/MveiFormatReward/mean": 0.99375, + "rewards/MveiFormatReward/std": 0.01767766922712326, + "rewards/MveiLLMConsistencyReward/mean": 0.5150000043213367, + "rewards/MveiLLMConsistencyReward/std": 0.14921745620667934, + "step": 85, + "step_time": 84.90599675662816 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 219.6, + "completions/mean_length": 185.33125, + "completions/min_length": 157.2, + "epoch": 0.06543075245365322, + "frac_reward_zero_std": 0.1, + "grad_norm": 1.2833217364089817, + "kl": 0.0021225117961876094, + "learning_rate": 9.993631532250894e-07, + "loss": 8.470714092254639e-05, + "reward": 2.0143749833106996, + "reward_std": 0.20510876551270485, + "rewards/MveiAccuracyReward/mean": 0.975, + "rewards/MveiAccuracyReward/std": 0.026726123690605164, + "rewards/MveiFormatReward/mean": 0.99375, + "rewards/MveiFormatReward/std": 0.01767766922712326, + "rewards/MveiLLMConsistencyReward/mean": 0.5425000056624413, + "rewards/MveiLLMConsistencyReward/std": 0.1708848036825657, + "step": 90, + "step_time": 88.06912175342441 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.95, + "completions/mean_length": 184.525, + "completions/min_length": 161.65, + "epoch": 0.06906579425663395, + "frac_reward_zero_std": 0.25, + "grad_norm": 1.3513061878234627, + "kl": 0.002421390629024245, + "learning_rate": 9.990239008840802e-07, + "loss": 9.694695472717285e-05, + "reward": 1.7700000196695327, + "reward_std": 0.20398809798061848, + "rewards/MveiAccuracyReward/mean": 0.85, + "rewards/MveiAccuracyReward/std": 0.046291005611419675, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.41999999806284904, + "rewards/MveiLLMConsistencyReward/std": 0.15986914485692977, + "step": 95, + "step_time": 82.67934676595033 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 212.55, + "completions/mean_length": 187.35625, + "completions/min_length": 162.25, + "epoch": 0.07270083605961468, + "frac_reward_zero_std": 0.2, + "grad_norm": 1.0309971338238968, + "kl": 0.004161263263085857, + "learning_rate": 9.986125702159954e-07, + "loss": 0.00016637742519378662, + "reward": 1.9337499886751175, + "reward_std": 0.19087109677493572, + "rewards/MveiAccuracyReward/mean": 0.93125, + "rewards/MveiAccuracyReward/std": 0.06123279929161072, + "rewards/MveiFormatReward/mean": 0.9875, + "rewards/MveiFormatReward/std": 0.03535533845424652, + "rewards/MveiLLMConsistencyReward/mean": 0.5087500065565109, + "rewards/MveiLLMConsistencyReward/std": 0.16509965918958186, + "step": 100, + "step_time": 80.46456340812146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 217.95, + "completions/mean_length": 188.29375, + "completions/min_length": 161.45, + "epoch": 0.07633587786259542, + "frac_reward_zero_std": 0.1, + "grad_norm": 1.1190120423614052, + "kl": 0.0048583348863758145, + "learning_rate": 9.98129220632872e-07, + "loss": 0.00019429922103881837, + "reward": 1.9524999916553498, + "reward_std": 0.234458002820611, + "rewards/MveiAccuracyReward/mean": 0.8875, + "rewards/MveiAccuracyReward/std": 0.08711026012897491, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.5650000050663948, + "rewards/MveiLLMConsistencyReward/std": 0.18389294520020485, + "step": 105, + "step_time": 78.62311269156635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 228.0, + "completions/mean_length": 195.41875, + "completions/min_length": 166.15, + "epoch": 0.07997091966557615, + "frac_reward_zero_std": 0.15, + "grad_norm": 1.2835284316608364, + "kl": 0.005961388075957075, + "learning_rate": 9.975739219490601e-07, + "loss": 0.00023860633373260499, + "reward": 1.969374990463257, + "reward_std": 0.20738761276006698, + "rewards/MveiAccuracyReward/mean": 0.925, + "rewards/MveiAccuracyReward/std": 0.0816463440656662, + "rewards/MveiFormatReward/mean": 0.99375, + "rewards/MveiFormatReward/std": 0.01767766922712326, + "rewards/MveiLLMConsistencyReward/mean": 0.5475000068545341, + "rewards/MveiLLMConsistencyReward/std": 0.17429551668465137, + "step": 110, + "step_time": 78.5548978485167 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.75, + "completions/mean_length": 193.85625, + "completions/min_length": 165.65, + "epoch": 0.0836059614685569, + "frac_reward_zero_std": 0.1, + "grad_norm": 1.1174049789751839, + "kl": 0.007999175693839788, + "learning_rate": 9.969467543711395e-07, + "loss": 0.00031993985176086425, + "reward": 1.8487499833106995, + "reward_std": 0.31319191604852675, + "rewards/MveiAccuracyReward/mean": 0.8625, + "rewards/MveiAccuracyReward/std": 0.17233722507953644, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.4862500101327896, + "rewards/MveiLLMConsistencyReward/std": 0.2090558473020792, + "step": 115, + "step_time": 81.70319018065929 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 225.25, + "completions/mean_length": 189.41875, + "completions/min_length": 162.3, + "epoch": 0.08724100327153762, + "frac_reward_zero_std": 0.05, + "grad_norm": 1.4027297985697493, + "kl": 0.009565949090756476, + "learning_rate": 9.962478084863336e-07, + "loss": 0.00038267150521278384, + "reward": 1.8074999898672104, + "reward_std": 0.2678529404103756, + "rewards/MveiAccuracyReward/mean": 0.775, + "rewards/MveiAccuracyReward/std": 0.07071067690849304, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.5325000025331974, + "rewards/MveiLLMConsistencyReward/std": 0.23568386361002922, + "step": 120, + "step_time": 78.01621430702508 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 273.2, + "completions/mean_length": 197.70625, + "completions/min_length": 168.55, + "epoch": 0.09087604507451835, + "frac_reward_zero_std": 0.15, + "grad_norm": 1.2807616775097996, + "kl": 0.011278474261052907, + "learning_rate": 9.954771852494262e-07, + "loss": 0.0004509925842285156, + "reward": 1.7581249564886092, + "reward_std": 0.2670029353350401, + "rewards/MveiAccuracyReward/mean": 0.73125, + "rewards/MveiAccuracyReward/std": 0.1075238049030304, + "rewards/MveiFormatReward/mean": 0.99375, + "rewards/MveiFormatReward/std": 0.01767766922712326, + "rewards/MveiLLMConsistencyReward/mean": 0.5300000093877315, + "rewards/MveiLLMConsistencyReward/std": 0.21534912884235383, + "step": 125, + "step_time": 83.60016283094883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.2, + "completions/mean_length": 197.24375, + "completions/min_length": 169.25, + "epoch": 0.0945110868774991, + "frac_reward_zero_std": 0.2, + "grad_norm": 1.1290708719557516, + "kl": 0.01565639809705317, + "learning_rate": 9.946349959681802e-07, + "loss": 0.0006264910101890564, + "reward": 1.9900000154972077, + "reward_std": 0.18564736619591712, + "rewards/MveiAccuracyReward/mean": 0.9, + "rewards/MveiAccuracyReward/std": 0.03535533845424652, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.5899999991059304, + "rewards/MveiLLMConsistencyReward/std": 0.17345385067164898, + "step": 130, + "step_time": 76.15535179227591 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 228.25, + "completions/mean_length": 189.63125, + "completions/min_length": 163.95, + "epoch": 0.09814612868047982, + "frac_reward_zero_std": 0.1, + "grad_norm": 1.0272810605279092, + "kl": 0.019541497249156235, + "learning_rate": 9.937213622872584e-07, + "loss": 0.0007814794778823852, + "reward": 2.0331250190734864, + "reward_std": 0.25848171301186085, + "rewards/MveiAccuracyReward/mean": 0.88125, + "rewards/MveiAccuracyReward/std": 0.0408231720328331, + "rewards/MveiFormatReward/mean": 0.99375, + "rewards/MveiFormatReward/std": 0.01767766922712326, + "rewards/MveiLLMConsistencyReward/mean": 0.6549999989569187, + "rewards/MveiLLMConsistencyReward/std": 0.22547860480844975, + "step": 135, + "step_time": 83.00242526158691 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 228.95, + "completions/mean_length": 194.59375, + "completions/min_length": 163.6, + "epoch": 0.10178117048346055, + "frac_reward_zero_std": 0.1, + "grad_norm": 1.2551395871215463, + "kl": 0.0215371529571712, + "learning_rate": 9.927364161706555e-07, + "loss": 0.0008617550134658814, + "reward": 1.8775000065565108, + "reward_std": 0.2053673155605793, + "rewards/MveiAccuracyReward/mean": 0.70625, + "rewards/MveiAccuracyReward/std": 0.049022963643074034, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.6712500013411045, + "rewards/MveiLLMConsistencyReward/std": 0.1611116800457239, + "step": 140, + "step_time": 84.6476160030812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.8, + "completions/mean_length": 198.46875, + "completions/min_length": 167.4, + "epoch": 0.1054162122864413, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.6100930813380608, + "kl": 0.02497743829153478, + "learning_rate": 9.916802998826365e-07, + "loss": 0.0009988397359848022, + "reward": 1.9999999761581422, + "reward_std": 0.17097588442265987, + "rewards/MveiAccuracyReward/mean": 0.88125, + "rewards/MveiAccuracyReward/std": 0.025877460837364197, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.6187500074505806, + "rewards/MveiLLMConsistencyReward/std": 0.1500001411885023, + "step": 145, + "step_time": 81.75836359225214 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 237.1, + "completions/mean_length": 195.95, + "completions/min_length": 165.7, + "epoch": 0.10905125408942203, + "frac_reward_zero_std": 0.05, + "grad_norm": 1.344354463877031, + "kl": 0.027870298735797404, + "learning_rate": 9.905531659671875e-07, + "loss": 0.001115068793296814, + "reward": 2.109999990463257, + "reward_std": 0.2530000418424606, + "rewards/MveiAccuracyReward/mean": 0.9125, + "rewards/MveiAccuracyReward/std": 0.051754921674728394, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.6975000023841857, + "rewards/MveiLLMConsistencyReward/std": 0.2356499671936035, + "step": 150, + "step_time": 74.11532084159553 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 234.25, + "completions/mean_length": 201.20625, + "completions/min_length": 172.3, + "epoch": 0.11268629589240277, + "frac_reward_zero_std": 0.1, + "grad_norm": 0.9416623465394275, + "kl": 0.028954134788364172, + "learning_rate": 9.89355177225984e-07, + "loss": 0.0011582642793655396, + "reward": 2.078749990463257, + "reward_std": 0.2853988204151392, + "rewards/MveiAccuracyReward/mean": 0.8875, + "rewards/MveiAccuracyReward/std": 0.0667006328701973, + "rewards/MveiFormatReward/mean": 0.9875, + "rewards/MveiFormatReward/std": 0.03535533845424652, + "rewards/MveiLLMConsistencyReward/mean": 0.6975000001490116, + "rewards/MveiLLMConsistencyReward/std": 0.22299464270472527, + "step": 155, + "step_time": 74.52814088463784 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 251.55, + "completions/mean_length": 204.425, + "completions/min_length": 172.3, + "epoch": 0.1163213376953835, + "frac_reward_zero_std": 0.1, + "grad_norm": 1.3168534501650018, + "kl": 0.03070834055542946, + "learning_rate": 9.880865066948748e-07, + "loss": 0.0012282460927963256, + "reward": 2.093749976158142, + "reward_std": 0.2688323128968477, + "rewards/MveiAccuracyReward/mean": 0.875, + "rewards/MveiAccuracyReward/std": 0.026726123690605164, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.7187500111758709, + "rewards/MveiLLMConsistencyReward/std": 0.2422573085874319, + "step": 160, + "step_time": 75.55069470070302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 249.7, + "completions/mean_length": 201.4125, + "completions/min_length": 168.95, + "epoch": 0.11995637949836423, + "frac_reward_zero_std": 0.15, + "grad_norm": 1.1277864654194623, + "kl": 0.034754920098930595, + "learning_rate": 9.867473376188896e-07, + "loss": 0.001390466094017029, + "reward": 2.108750009536743, + "reward_std": 0.2290588106960058, + "rewards/MveiAccuracyReward/mean": 0.85625, + "rewards/MveiAccuracyReward/std": 0.06943259090185165, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.7525000058114528, + "rewards/MveiLLMConsistencyReward/std": 0.17643247917294502, + "step": 165, + "step_time": 75.28065769933164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.55, + "completions/mean_length": 204.075, + "completions/min_length": 174.8, + "epoch": 0.12359142130134497, + "frac_reward_zero_std": 0.1, + "grad_norm": 1.3030370810276026, + "kl": 0.03521493105217814, + "learning_rate": 9.853378634257708e-07, + "loss": 0.0014090657234191895, + "reward": 2.0225000351667406, + "reward_std": 0.25071537867188454, + "rewards/MveiAccuracyReward/mean": 0.76875, + "rewards/MveiAccuracyReward/std": 0.09658813774585724, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.753749991953373, + "rewards/MveiLLMConsistencyReward/std": 0.20075847208499908, + "step": 170, + "step_time": 80.61941720545292 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 255.5, + "completions/mean_length": 199.24375, + "completions/min_length": 163.35, + "epoch": 0.1272264631043257, + "frac_reward_zero_std": 0.35, + "grad_norm": 1.105875333905631, + "kl": 0.03157349769026041, + "learning_rate": 9.838582876980358e-07, + "loss": 0.0012627072632312776, + "reward": 2.086249992251396, + "reward_std": 0.15274532735347748, + "rewards/MveiAccuracyReward/mean": 0.83125, + "rewards/MveiAccuracyReward/std": 0.0408231720328331, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.7550000041723252, + "rewards/MveiLLMConsistencyReward/std": 0.126476688683033, + "step": 175, + "step_time": 70.45618718527257 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 275.3, + "completions/mean_length": 205.85, + "completions/min_length": 171.55, + "epoch": 0.13086150490730644, + "frac_reward_zero_std": 0.25, + "grad_norm": 1.0071507022324566, + "kl": 0.0391253319568932, + "learning_rate": 9.823088241435714e-07, + "loss": 0.0015652745962142945, + "reward": 2.213750022649765, + "reward_std": 0.19908951558172702, + "rewards/MveiAccuracyReward/mean": 0.86875, + "rewards/MveiAccuracyReward/std": 0.044403792917728425, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.8449999928474426, + "rewards/MveiLLMConsistencyReward/std": 0.16771102957427503, + "step": 180, + "step_time": 80.93950719051062 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 243.0, + "completions/mean_length": 201.80625, + "completions/min_length": 169.9, + "epoch": 0.13449654671028716, + "frac_reward_zero_std": 0.1, + "grad_norm": 1.3398320846525047, + "kl": 0.03748435387387872, + "learning_rate": 9.806896965647657e-07, + "loss": 0.0014994919300079345, + "reward": 2.163750022649765, + "reward_std": 0.1716399945318699, + "rewards/MveiAccuracyReward/mean": 0.85, + "rewards/MveiAccuracyReward/std": 0.0, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.8137499943375588, + "rewards/MveiLLMConsistencyReward/std": 0.17163998819887638, + "step": 185, + "step_time": 75.46520342752338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.95, + "completions/mean_length": 205.61875, + "completions/min_length": 176.3, + "epoch": 0.1381315885132679, + "frac_reward_zero_std": 0.35, + "grad_norm": 0.9078611397382459, + "kl": 0.03967118514701724, + "learning_rate": 9.790011388261832e-07, + "loss": 0.0015863895416259766, + "reward": 2.1424999982118607, + "reward_std": 0.15116989873349668, + "rewards/MveiAccuracyReward/mean": 0.85, + "rewards/MveiAccuracyReward/std": 0.03535533845424652, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.7925000041723251, + "rewards/MveiLLMConsistencyReward/std": 0.14499584771692753, + "step": 190, + "step_time": 72.58420044183731 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.55, + "completions/mean_length": 204.53125, + "completions/min_length": 179.4, + "epoch": 0.14176663031624864, + "frac_reward_zero_std": 0.15, + "grad_norm": 1.1314213484288542, + "kl": 0.03606957383453846, + "learning_rate": 9.772433948207853e-07, + "loss": 0.0014427393674850465, + "reward": 2.2512499928474425, + "reward_std": 0.13642010278999805, + "rewards/MveiAccuracyReward/mean": 0.8375, + "rewards/MveiAccuracyReward/std": 0.03535533845424652, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9137500062584877, + "rewards/MveiLLMConsistencyReward/std": 0.10877882838249206, + "step": 195, + "step_time": 66.37393386811019 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.7, + "completions/mean_length": 201.05625, + "completions/min_length": 173.85, + "epoch": 0.14540167211922936, + "frac_reward_zero_std": 0.15, + "grad_norm": 1.0024507743569822, + "kl": 0.04934713859111071, + "learning_rate": 9.754167184347025e-07, + "loss": 0.0019738376140594482, + "reward": 2.355000001192093, + "reward_std": 0.17032154574990271, + "rewards/MveiAccuracyReward/mean": 0.975, + "rewards/MveiAccuracyReward/std": 0.046291005611419675, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.8800000011920929, + "rewards/MveiLLMConsistencyReward/std": 0.15183652900159358, + "step": 200, + "step_time": 70.0125329516828 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 281.15, + "completions/mean_length": 212.16875, + "completions/min_length": 172.35, + "epoch": 0.1490367139222101, + "frac_reward_zero_std": 0.4, + "grad_norm": 1.00198513572539, + "kl": 0.0418359481729567, + "learning_rate": 9.73521373510564e-07, + "loss": 0.0016731560230255128, + "reward": 2.2337499856948853, + "reward_std": 0.15645107291638852, + "rewards/MveiAccuracyReward/mean": 0.8375, + "rewards/MveiAccuracyReward/std": 0.049871626496315005, + "rewards/MveiFormatReward/mean": 0.9875, + "rewards/MveiFormatReward/std": 0.03535533845424652, + "rewards/MveiLLMConsistencyReward/mean": 0.9025000005960464, + "rewards/MveiLLMConsistencyReward/std": 0.11298311166465283, + "step": 205, + "step_time": 65.98886915110052 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 262.6, + "completions/mean_length": 206.9, + "completions/min_length": 172.75, + "epoch": 0.15267175572519084, + "frac_reward_zero_std": 0.15, + "grad_norm": 1.2981419714343267, + "kl": 0.0480030654463917, + "learning_rate": 9.715576338093886e-07, + "loss": 0.0019203543663024903, + "reward": 2.1325000047683718, + "reward_std": 0.2264687493443489, + "rewards/MveiAccuracyReward/mean": 0.85625, + "rewards/MveiAccuracyReward/std": 0.06123279929161072, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.7762500017881393, + "rewards/MveiLLMConsistencyReward/std": 0.1965201873332262, + "step": 210, + "step_time": 71.89682666286826 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.15, + "completions/mean_length": 194.93125, + "completions/min_length": 167.65, + "epoch": 0.1563067975281716, + "frac_reward_zero_std": 0.45, + "grad_norm": 0.06489088743601187, + "kl": 0.04637978160753846, + "learning_rate": 9.69525782971042e-07, + "loss": 0.0018558710813522338, + "reward": 2.3125000417232515, + "reward_std": 0.11380274556577205, + "rewards/MveiAccuracyReward/mean": 0.88125, + "rewards/MveiAccuracyReward/std": 0.025877460837364197, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9312499910593033, + "rewards/MveiLLMConsistencyReward/std": 0.0888317558914423, + "step": 215, + "step_time": 65.11796665452421 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.65, + "completions/mean_length": 210.25625, + "completions/min_length": 179.9, + "epoch": 0.1599418393311523, + "frac_reward_zero_std": 0.2, + "grad_norm": 1.2863226714929106, + "kl": 0.046673784218728545, + "learning_rate": 9.6742611447327e-07, + "loss": 0.0018677100539207458, + "reward": 2.2906250178813936, + "reward_std": 0.17234257869422437, + "rewards/MveiAccuracyReward/mean": 0.925, + "rewards/MveiAccuracyReward/std": 0.0667006328701973, + "rewards/MveiFormatReward/mean": 0.99375, + "rewards/MveiFormatReward/std": 0.01767766922712326, + "rewards/MveiLLMConsistencyReward/mean": 0.86875, + "rewards/MveiLLMConsistencyReward/std": 0.13413323648273945, + "step": 220, + "step_time": 72.660677921772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 250.25, + "completions/mean_length": 201.9875, + "completions/min_length": 168.8, + "epoch": 0.16357688113413305, + "frac_reward_zero_std": 0.25, + "grad_norm": 1.2082426371244996, + "kl": 0.046049478184431794, + "learning_rate": 9.652589315893074e-07, + "loss": 0.0018424808979034424, + "reward": 2.238749998807907, + "reward_std": 0.16750994548201562, + "rewards/MveiAccuracyReward/mean": 0.84375, + "rewards/MveiAccuracyReward/std": 0.05260358452796936, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.8950000017881393, + "rewards/MveiLLMConsistencyReward/std": 0.11923237554728985, + "step": 225, + "step_time": 75.52490624561906 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.3, + "completions/mean_length": 198.7375, + "completions/min_length": 170.0, + "epoch": 0.1672119229371138, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.9236178019080928, + "kl": 0.04613093789666891, + "learning_rate": 9.630245473440742e-07, + "loss": 0.0018455177545547486, + "reward": 2.2575000166893004, + "reward_std": 0.1700163245201111, + "rewards/MveiAccuracyReward/mean": 0.86875, + "rewards/MveiAccuracyReward/std": 0.06123279929161072, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.8887499965727329, + "rewards/MveiLLMConsistencyReward/std": 0.11767456233501435, + "step": 230, + "step_time": 69.37349548526109 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 239.5, + "completions/mean_length": 203.8125, + "completions/min_length": 172.9, + "epoch": 0.1708469647400945, + "frac_reward_zero_std": 0.3, + "grad_norm": 0.9845440231907046, + "kl": 0.05328067895025015, + "learning_rate": 9.607232844689633e-07, + "loss": 0.0021315254271030426, + "reward": 2.2650000005960464, + "reward_std": 0.1732545718550682, + "rewards/MveiAccuracyReward/mean": 0.89375, + "rewards/MveiAccuracyReward/std": 0.05260358452796936, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.8712499976158142, + "rewards/MveiLLMConsistencyReward/std": 0.1268510937690735, + "step": 235, + "step_time": 70.66867304816842 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.65, + "completions/mean_length": 204.45625, + "completions/min_length": 175.95, + "epoch": 0.17448200654307525, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.9490655950526524, + "kl": 0.04163944832980633, + "learning_rate": 9.583554753552244e-07, + "loss": 0.0016658216714859008, + "reward": 2.3737499892711638, + "reward_std": 0.10231012664735317, + "rewards/MveiAccuracyReward/mean": 0.9625, + "rewards/MveiAccuracyReward/std": 0.051754921674728394, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9112500041723252, + "rewards/MveiLLMConsistencyReward/std": 0.08913064040243626, + "step": 240, + "step_time": 76.31128382161259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.6, + "completions/mean_length": 203.68125, + "completions/min_length": 172.85, + "epoch": 0.178117048346056, + "frac_reward_zero_std": 0.25, + "grad_norm": 1.0975017411676056, + "kl": 0.05294170556589961, + "learning_rate": 9.559214620059548e-07, + "loss": 0.002118104696273804, + "reward": 2.1712500035762785, + "reward_std": 0.15000207386910916, + "rewards/MveiAccuracyReward/mean": 0.79375, + "rewards/MveiAccuracyReward/std": 0.01767766922712326, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.8775000005960465, + "rewards/MveiLLMConsistencyReward/std": 0.13570776283740998, + "step": 245, + "step_time": 74.231950693205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 241.6, + "completions/mean_length": 204.40625, + "completions/min_length": 175.65, + "epoch": 0.1817520901490367, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.610729868694356, + "kl": 0.04313134495168924, + "learning_rate": 9.534215959867009e-07, + "loss": 0.001725362241268158, + "reward": 2.318124997615814, + "reward_std": 0.17341354563832284, + "rewards/MveiAccuracyReward/mean": 0.9, + "rewards/MveiAccuracyReward/std": 0.0667006328701973, + "rewards/MveiFormatReward/mean": 0.99375, + "rewards/MveiFormatReward/std": 0.01767766922712326, + "rewards/MveiLLMConsistencyReward/mean": 0.9212500035762787, + "rewards/MveiLLMConsistencyReward/std": 0.11625524051487446, + "step": 250, + "step_time": 69.31044210270048 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.55, + "completions/mean_length": 197.3875, + "completions/min_length": 169.3, + "epoch": 0.18538713195201745, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.5939950616635872, + "kl": 0.047632408887147905, + "learning_rate": 9.508562383746782e-07, + "loss": 0.0019053682684898377, + "reward": 2.359999978542328, + "reward_std": 0.09763411059975624, + "rewards/MveiAccuracyReward/mean": 0.91875, + "rewards/MveiAccuracyReward/std": 0.025877460837364197, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.941250005364418, + "rewards/MveiLLMConsistencyReward/std": 0.07558601722121239, + "step": 255, + "step_time": 68.55646804682911 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.65, + "completions/mean_length": 199.56875, + "completions/min_length": 170.8, + "epoch": 0.1890221737549982, + "frac_reward_zero_std": 0.35, + "grad_norm": 1.4180883091743905, + "kl": 0.04739540405571461, + "learning_rate": 9.482257597066178e-07, + "loss": 0.0018957346677780152, + "reward": 2.372499978542328, + "reward_std": 0.08922924064099788, + "rewards/MveiAccuracyReward/mean": 0.94375, + "rewards/MveiAccuracyReward/std": 0.01767766922712326, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.928750005364418, + "rewards/MveiLLMConsistencyReward/std": 0.08588050492107868, + "step": 260, + "step_time": 68.8033996541053 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 234.25, + "completions/mean_length": 201.45625, + "completions/min_length": 174.3, + "epoch": 0.1926572155579789, + "frac_reward_zero_std": 0.3, + "grad_norm": 1.0886200490742217, + "kl": 0.04904018035158515, + "learning_rate": 9.455305399252467e-07, + "loss": 0.0019615098834037782, + "reward": 2.272499996423721, + "reward_std": 0.18130445517599583, + "rewards/MveiAccuracyReward/mean": 0.85625, + "rewards/MveiAccuracyReward/std": 0.05260358452796936, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9162499994039536, + "rewards/MveiLLMConsistencyReward/std": 0.14327077865600585, + "step": 265, + "step_time": 68.2446885280311 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 246.45, + "completions/mean_length": 210.21875, + "completions/min_length": 180.85, + "epoch": 0.19629225736095965, + "frac_reward_zero_std": 0.25, + "grad_norm": 1.2902017380177078, + "kl": 0.05741015672683716, + "learning_rate": 9.427709683244098e-07, + "loss": 0.0022964745759963988, + "reward": 2.280000013113022, + "reward_std": 0.12487307526171207, + "rewards/MveiAccuracyReward/mean": 0.88125, + "rewards/MveiAccuracyReward/std": 0.025877460837364197, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.8987499967217445, + "rewards/MveiLLMConsistencyReward/std": 0.1037971641868353, + "step": 270, + "step_time": 83.63584203645587 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 234.05, + "completions/mean_length": 199.225, + "completions/min_length": 169.35, + "epoch": 0.1999272991639404, + "frac_reward_zero_std": 0.3, + "grad_norm": 1.076003319834389, + "kl": 0.04970582015812397, + "learning_rate": 9.3994744349284e-07, + "loss": 0.0019883021712303163, + "reward": 2.387499988079071, + "reward_std": 0.12927508167922497, + "rewards/MveiAccuracyReward/mean": 0.98125, + "rewards/MveiAccuracyReward/std": 0.0408231720328331, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9062500029802323, + "rewards/MveiLLMConsistencyReward/std": 0.09720045439898968, + "step": 275, + "step_time": 78.91972422488034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 256.75, + "completions/mean_length": 209.825, + "completions/min_length": 177.8, + "epoch": 0.2035623409669211, + "frac_reward_zero_std": 0.3, + "grad_norm": 0.8446332609465718, + "kl": 0.04920104956254363, + "learning_rate": 9.370603732565874e-07, + "loss": 0.001968240737915039, + "reward": 2.2399999797344208, + "reward_std": 0.21542084217071533, + "rewards/MveiAccuracyReward/mean": 0.8625, + "rewards/MveiAccuracyReward/std": 0.07490042448043824, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.8775000035762787, + "rewards/MveiLLMConsistencyReward/std": 0.15429446138441563, + "step": 280, + "step_time": 73.99278201758861 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 234.55, + "completions/mean_length": 198.93125, + "completions/min_length": 169.65, + "epoch": 0.20719738276990185, + "frac_reward_zero_std": 0.3, + "grad_norm": 1.1007666161198268, + "kl": 0.050530897360295056, + "learning_rate": 9.341101746201128e-07, + "loss": 0.0020215705037117003, + "reward": 2.188749998807907, + "reward_std": 0.16432357504963874, + "rewards/MveiAccuracyReward/mean": 0.78125, + "rewards/MveiAccuracyReward/std": 0.07617851048707962, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9074999988079071, + "rewards/MveiLLMConsistencyReward/std": 0.12360709756612778, + "step": 285, + "step_time": 76.36430631838739 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 267.8, + "completions/mean_length": 204.09375, + "completions/min_length": 170.05, + "epoch": 0.2108324245728826, + "frac_reward_zero_std": 0.35, + "grad_norm": 1.1056192913802685, + "kl": 0.05204110499471426, + "learning_rate": 9.310972737060565e-07, + "loss": 0.002081531286239624, + "reward": 2.2899999916553497, + "reward_std": 0.1566681332886219, + "rewards/MveiAccuracyReward/mean": 0.84375, + "rewards/MveiAccuracyReward/std": 0.10435850620269775, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9462500005960465, + "rewards/MveiLLMConsistencyReward/std": 0.07905281074345112, + "step": 290, + "step_time": 79.71511269360781 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/mean_length": 199.08125, + "completions/min_length": 170.95, + "epoch": 0.21446746637586334, + "frac_reward_zero_std": 0.25, + "grad_norm": 1.1126091469502415, + "kl": 0.04745836248621345, + "learning_rate": 9.280221056936899e-07, + "loss": 0.0018991455435752869, + "reward": 2.301874989271164, + "reward_std": 0.11462128087878228, + "rewards/MveiAccuracyReward/mean": 0.86875, + "rewards/MveiAccuracyReward/std": 0.025877460837364197, + "rewards/MveiFormatReward/mean": 0.99375, + "rewards/MveiFormatReward/std": 0.01767766922712326, + "rewards/MveiLLMConsistencyReward/mean": 0.9362500011920929, + "rewards/MveiLLMConsistencyReward/std": 0.08925197273492813, + "step": 295, + "step_time": 71.66709435358644 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.4, + "completions/mean_length": 199.03125, + "completions/min_length": 169.55, + "epoch": 0.21810250817884405, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.8923720692263183, + "kl": 0.05621665976941585, + "learning_rate": 9.248851147560584e-07, + "loss": 0.002249002456665039, + "reward": 2.3031250119209288, + "reward_std": 0.13510362580418586, + "rewards/MveiAccuracyReward/mean": 0.875, + "rewards/MveiAccuracyReward/std": 0.07848104536533355, + "rewards/MveiFormatReward/mean": 0.99375, + "rewards/MveiFormatReward/std": 0.01767766922712326, + "rewards/MveiLLMConsistencyReward/mean": 0.9312499940395356, + "rewards/MveiLLMConsistencyReward/std": 0.08236783966422082, + "step": 300, + "step_time": 73.25305524095893 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.15, + "completions/mean_length": 200.5125, + "completions/min_length": 174.9, + "epoch": 0.2217375499818248, + "frac_reward_zero_std": 0.35, + "grad_norm": 1.0861069784142643, + "kl": 0.049988660216331485, + "learning_rate": 9.216867539958258e-07, + "loss": 0.0019994348287582396, + "reward": 2.3087499916553496, + "reward_std": 0.15251614190638066, + "rewards/MveiAccuracyReward/mean": 0.89375, + "rewards/MveiAccuracyReward/std": 0.06396867483854293, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9150000005960465, + "rewards/MveiLLMConsistencyReward/std": 0.10046111457049847, + "step": 305, + "step_time": 69.34589745849371 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 255.45, + "completions/mean_length": 214.9375, + "completions/min_length": 183.35, + "epoch": 0.22537259178480554, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.6395480649768335, + "kl": 0.04638021681457758, + "learning_rate": 9.184274853798293e-07, + "loss": 0.0018548578023910522, + "reward": 2.2462499499320985, + "reward_std": 0.19342792518436908, + "rewards/MveiAccuracyReward/mean": 0.86875, + "rewards/MveiAccuracyReward/std": 0.09069479852914811, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.8775000095367431, + "rewards/MveiLLMConsistencyReward/std": 0.1270702950656414, + "step": 310, + "step_time": 70.56883726418019 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.3, + "completions/mean_length": 200.3, + "completions/min_length": 172.2, + "epoch": 0.22900763358778625, + "frac_reward_zero_std": 0.55, + "grad_norm": 0.5671343833841904, + "kl": 0.0430236023850739, + "learning_rate": 9.151077796723523e-07, + "loss": 0.0017208054661750794, + "reward": 2.3950000166893006, + "reward_std": 0.11348208114504814, + "rewards/MveiAccuracyReward/mean": 0.95625, + "rewards/MveiAccuracyReward/std": 0.07216846644878387, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9387499958276748, + "rewards/MveiLLMConsistencyReward/std": 0.06905492618680001, + "step": 315, + "step_time": 76.92188999690116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 222.5, + "completions/mean_length": 194.75625, + "completions/min_length": 167.65, + "epoch": 0.232642675390767, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.7242691547319087, + "kl": 0.050981131847947836, + "learning_rate": 9.117281163671293e-07, + "loss": 0.0020397506654262544, + "reward": 2.1637499928474426, + "reward_std": 0.1011626586318016, + "rewards/MveiAccuracyReward/mean": 0.7375, + "rewards/MveiAccuracyReward/std": 0.05850084125995636, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9262500017881393, + "rewards/MveiLLMConsistencyReward/std": 0.07830574735999107, + "step": 320, + "step_time": 72.86219071298838 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 239.75, + "completions/mean_length": 199.225, + "completions/min_length": 169.05, + "epoch": 0.23627771719374774, + "frac_reward_zero_std": 0.2, + "grad_norm": 1.2222972612905987, + "kl": 0.05500196311622858, + "learning_rate": 9.082889836180878e-07, + "loss": 0.0022002458572387695, + "reward": 2.2662500202655793, + "reward_std": 0.1764941532164812, + "rewards/MveiAccuracyReward/mean": 0.8375, + "rewards/MveiAccuracyReward/std": 0.08984613567590713, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9287499934434891, + "rewards/MveiLLMConsistencyReward/std": 0.10632340535521508, + "step": 325, + "step_time": 82.92368378974497 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 226.8, + "completions/mean_length": 199.74375, + "completions/min_length": 174.25, + "epoch": 0.23991275899672845, + "frac_reward_zero_std": 0.45, + "grad_norm": 0.8967205014832655, + "kl": 0.047722284123301505, + "learning_rate": 9.047908781688403e-07, + "loss": 0.0019087165594100953, + "reward": 2.3562499821186065, + "reward_std": 0.09245501905679702, + "rewards/MveiAccuracyReward/mean": 0.90625, + "rewards/MveiAccuracyReward/std": 0.05260358452796936, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.95, + "rewards/MveiLLMConsistencyReward/std": 0.06289107538759708, + "step": 330, + "step_time": 71.50175745785236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 239.85, + "completions/mean_length": 203.53125, + "completions/min_length": 174.35, + "epoch": 0.2435478007997092, + "frac_reward_zero_std": 0.55, + "grad_norm": 0.5817438894302167, + "kl": 0.04887179965153336, + "learning_rate": 9.012343052809345e-07, + "loss": 0.0019553512334823608, + "reward": 2.4512499928474427, + "reward_std": 0.07724135890603065, + "rewards/MveiAccuracyReward/mean": 0.99375, + "rewards/MveiAccuracyReward/std": 0.01767766922712326, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9575000017881393, + "rewards/MveiLLMConsistencyReward/std": 0.06460157074034215, + "step": 335, + "step_time": 75.22226010747254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.2, + "completions/mean_length": 206.1, + "completions/min_length": 174.35, + "epoch": 0.24718284260268994, + "frac_reward_zero_std": 0.35, + "grad_norm": 1.066274323490832, + "kl": 0.04784417673945427, + "learning_rate": 8.976197786608755e-07, + "loss": 0.001913994550704956, + "reward": 2.183124992251396, + "reward_std": 0.21415966041386128, + "rewards/MveiAccuracyReward/mean": 0.8375, + "rewards/MveiAccuracyReward/std": 0.1165722593665123, + "rewards/MveiFormatReward/mean": 0.99375, + "rewards/MveiFormatReward/std": 0.01767766922712326, + "rewards/MveiLLMConsistencyReward/mean": 0.848750001937151, + "rewards/MveiLLMConsistencyReward/std": 0.12318442650139332, + "step": 340, + "step_time": 69.83715037330985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 243.15, + "completions/mean_length": 201.46875, + "completions/min_length": 174.45, + "epoch": 0.25081788440567065, + "frac_reward_zero_std": 0.3, + "grad_norm": 1.221705888191254, + "kl": 0.0555890429764986, + "learning_rate": 8.939478203859252e-07, + "loss": 0.0022241178900003432, + "reward": 2.307500010728836, + "reward_std": 0.1992207895964384, + "rewards/MveiAccuracyReward/mean": 0.9125, + "rewards/MveiAccuracyReward/std": 0.0974368005990982, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.894999997317791, + "rewards/MveiLLMConsistencyReward/std": 0.11989384666085243, + "step": 345, + "step_time": 83.6939534559846 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.55, + "completions/mean_length": 201.30625, + "completions/min_length": 173.6, + "epoch": 0.2544529262086514, + "frac_reward_zero_std": 0.4, + "grad_norm": 1.0766859278224827, + "kl": 0.04659942863509059, + "learning_rate": 8.902189608286955e-07, + "loss": 0.0018643148243427277, + "reward": 2.2662499874830244, + "reward_std": 0.13397842943668364, + "rewards/MveiAccuracyReward/mean": 0.83125, + "rewards/MveiAccuracyReward/std": 0.07617851048707962, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9350000053644181, + "rewards/MveiLLMConsistencyReward/std": 0.06851454712450504, + "step": 350, + "step_time": 64.66634232662618 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.9, + "completions/mean_length": 201.6375, + "completions/min_length": 175.4, + "epoch": 0.25808796801163214, + "frac_reward_zero_std": 0.1, + "grad_norm": 1.2979092849263858, + "kl": 0.04944706056267023, + "learning_rate": 8.864337385805406e-07, + "loss": 0.0019773632287979127, + "reward": 2.3549999713897707, + "reward_std": 0.1313345231115818, + "rewards/MveiAccuracyReward/mean": 0.93125, + "rewards/MveiAccuracyReward/std": 0.0408231720328331, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9237500101327896, + "rewards/MveiLLMConsistencyReward/std": 0.1102304808795452, + "step": 355, + "step_time": 66.65488817803562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.4, + "completions/mean_length": 198.9375, + "completions/min_length": 165.75, + "epoch": 0.2617230098146129, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.5964610310424787, + "kl": 0.050580203905701636, + "learning_rate": 8.825927003737652e-07, + "loss": 0.0020235240459442138, + "reward": 2.2137500196695328, + "reward_std": 0.14760382063686847, + "rewards/MveiAccuracyReward/mean": 0.80625, + "rewards/MveiAccuracyReward/std": 0.0925780937075615, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.907499996572733, + "rewards/MveiLLMConsistencyReward/std": 0.07478510178625583, + "step": 360, + "step_time": 70.3552474875003 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 239.4, + "completions/mean_length": 206.4875, + "completions/min_length": 179.8, + "epoch": 0.2653580516175936, + "frac_reward_zero_std": 0.3, + "grad_norm": 1.0660771631170882, + "kl": 0.05044060843065381, + "learning_rate": 8.786964010026541e-07, + "loss": 0.002017676830291748, + "reward": 2.343749976158142, + "reward_std": 0.12506617233157158, + "rewards/MveiAccuracyReward/mean": 0.9, + "rewards/MveiAccuracyReward/std": 0.07071067690849304, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9437500089406967, + "rewards/MveiLLMConsistencyReward/std": 0.08580026850104332, + "step": 365, + "step_time": 77.13768363855779 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.9, + "completions/mean_length": 208.29375, + "completions/min_length": 180.6, + "epoch": 0.2689930934205743, + "frac_reward_zero_std": 0.3, + "grad_norm": 0.8630473476120927, + "kl": 0.05125547675415874, + "learning_rate": 8.747454032433386e-07, + "loss": 0.0020504653453826903, + "reward": 2.267499989271164, + "reward_std": 0.19351629428565503, + "rewards/MveiAccuracyReward/mean": 0.8625, + "rewards/MveiAccuracyReward/std": 0.13698188662528993, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9049999982118606, + "rewards/MveiLLMConsistencyReward/std": 0.11784395799040795, + "step": 370, + "step_time": 65.95042751803994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.65, + "completions/mean_length": 205.79375, + "completions/min_length": 174.7, + "epoch": 0.27262813522355506, + "frac_reward_zero_std": 0.25, + "grad_norm": 1.2324165165629464, + "kl": 0.05260043805465102, + "learning_rate": 8.707402777725103e-07, + "loss": 0.002104008197784424, + "reward": 2.3274999976158144, + "reward_std": 0.1802256189286709, + "rewards/MveiAccuracyReward/mean": 0.9125, + "rewards/MveiAccuracyReward/std": 0.06208146214485168, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9150000005960465, + "rewards/MveiLLMConsistencyReward/std": 0.13712091296911239, + "step": 375, + "step_time": 64.55796298235654 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 228.65, + "completions/mean_length": 199.41875, + "completions/min_length": 171.9, + "epoch": 0.2762631770265358, + "frac_reward_zero_std": 0.3, + "grad_norm": 0.06080417156751461, + "kl": 0.055795913003385066, + "learning_rate": 8.666816030849932e-07, + "loss": 0.002232480049133301, + "reward": 2.3762500047683717, + "reward_std": 0.1070314772427082, + "rewards/MveiAccuracyReward/mean": 0.94375, + "rewards/MveiAccuracyReward/std": 0.049022963643074034, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9324999988079071, + "rewards/MveiLLMConsistencyReward/std": 0.07924907505512238, + "step": 380, + "step_time": 63.71235692538321 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 251.4, + "completions/mean_length": 198.98125, + "completions/min_length": 169.25, + "epoch": 0.27989821882951654, + "frac_reward_zero_std": 0.55, + "grad_norm": 1.1386499479267451, + "kl": 0.04871965404599905, + "learning_rate": 8.625699654101863e-07, + "loss": 0.0019491560757160186, + "reward": 2.441250002384186, + "reward_std": 0.08268089033663273, + "rewards/MveiAccuracyReward/mean": 0.975, + "rewards/MveiAccuracyReward/std": 0.043555130064487454, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9662499994039535, + "rewards/MveiLLMConsistencyReward/std": 0.05244523510336876, + "step": 385, + "step_time": 63.30250987969339 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 228.25, + "completions/mean_length": 194.51875, + "completions/min_length": 167.4, + "epoch": 0.2835332606324973, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.9166107345765797, + "kl": 0.056752162799239156, + "learning_rate": 8.584059586273904e-07, + "loss": 0.002271050214767456, + "reward": 2.4256250143051146, + "reward_std": 0.12021541744470596, + "rewards/MveiAccuracyReward/mean": 0.9625, + "rewards/MveiAccuracyReward/std": 0.0667006328701973, + "rewards/MveiFormatReward/mean": 0.99375, + "rewards/MveiFormatReward/std": 0.01767766922712326, + "rewards/MveiLLMConsistencyReward/mean": 0.9662499964237213, + "rewards/MveiLLMConsistencyReward/std": 0.06097796447575092, + "step": 390, + "step_time": 62.594334272667766 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.25, + "completions/mean_length": 202.8625, + "completions/min_length": 174.25, + "epoch": 0.28716830243547803, + "frac_reward_zero_std": 0.25, + "grad_norm": 1.2235684517287873, + "kl": 0.05721313580870628, + "learning_rate": 8.541901841800276e-07, + "loss": 0.002288556098937988, + "reward": 2.2050000011920927, + "reward_std": 0.17486557103693484, + "rewards/MveiAccuracyReward/mean": 0.7625, + "rewards/MveiAccuracyReward/std": 0.12477205097675323, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9424999952316284, + "rewards/MveiLLMConsistencyReward/std": 0.07683403715491295, + "step": 395, + "step_time": 63.35783043056726 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.6, + "completions/mean_length": 206.1375, + "completions/min_length": 174.05, + "epoch": 0.2908033442384587, + "frac_reward_zero_std": 0.3, + "grad_norm": 1.3283157079171533, + "kl": 0.05699926046654582, + "learning_rate": 8.499232509887711e-07, + "loss": 0.0022801071405410766, + "reward": 2.312500011920929, + "reward_std": 0.15544865280389786, + "rewards/MveiAccuracyReward/mean": 0.8625, + "rewards/MveiAccuracyReward/std": 0.09342675656080246, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.95, + "rewards/MveiLLMConsistencyReward/std": 0.07170455940067769, + "step": 400, + "step_time": 62.2062132999301 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 220.8, + "completions/mean_length": 190.46875, + "completions/min_length": 162.9, + "epoch": 0.29443838604143946, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.5493762494591856, + "kl": 0.05629263436421752, + "learning_rate": 8.456057753635922e-07, + "loss": 0.0022518396377563476, + "reward": 2.3249999821186065, + "reward_std": 0.10868542343378067, + "rewards/MveiAccuracyReward/mean": 0.85625, + "rewards/MveiAccuracyReward/std": 0.06943259090185165, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.96875, + "rewards/MveiLLMConsistencyReward/std": 0.045820656791329385, + "step": 405, + "step_time": 64.3292087059468 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 229.6, + "completions/mean_length": 197.7375, + "completions/min_length": 173.05, + "epoch": 0.2980734278444202, + "frac_reward_zero_std": 0.45, + "grad_norm": 0.9019815140117805, + "kl": 0.059184264950454236, + "learning_rate": 8.412383809147421e-07, + "loss": 0.0023672252893447874, + "reward": 2.304999977350235, + "reward_std": 0.11598805524408817, + "rewards/MveiAccuracyReward/mean": 0.85, + "rewards/MveiAccuracyReward/std": 0.0667006328701973, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9550000041723251, + "rewards/MveiLLMConsistencyReward/std": 0.06418436132371426, + "step": 410, + "step_time": 65.96730939485133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 241.1, + "completions/mean_length": 200.85, + "completions/min_length": 168.2, + "epoch": 0.30170846964740095, + "frac_reward_zero_std": 0.55, + "grad_norm": 0.872319679944253, + "kl": 0.049759996309876445, + "learning_rate": 8.368216984626787e-07, + "loss": 0.0019900724291801453, + "reward": 2.310000002384186, + "reward_std": 0.09225410297513008, + "rewards/MveiAccuracyReward/mean": 0.85625, + "rewards/MveiAccuracyReward/std": 0.0408231720328331, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9537499994039536, + "rewards/MveiLLMConsistencyReward/std": 0.05662188455462456, + "step": 415, + "step_time": 60.074932384863494 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.95, + "completions/mean_length": 208.09375, + "completions/min_length": 179.6, + "epoch": 0.3053435114503817, + "frac_reward_zero_std": 0.3, + "grad_norm": 1.0594661866937356, + "kl": 0.06661088336259127, + "learning_rate": 8.323563659469512e-07, + "loss": 0.002664998173713684, + "reward": 2.3474999964237213, + "reward_std": 0.1417607918381691, + "rewards/MveiAccuracyReward/mean": 0.89375, + "rewards/MveiAccuracyReward/std": 0.08795892298221589, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9537500023841858, + "rewards/MveiLLMConsistencyReward/std": 0.07330291867256164, + "step": 420, + "step_time": 71.8682894039899 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 225.5, + "completions/mean_length": 197.33125, + "completions/min_length": 173.6, + "epoch": 0.30897855325336243, + "frac_reward_zero_std": 0.2, + "grad_norm": 1.4046658369688225, + "kl": 0.055291316099464895, + "learning_rate": 8.278430283340573e-07, + "loss": 0.002212509512901306, + "reward": 2.2975000202655793, + "reward_std": 0.11464236341416836, + "rewards/MveiAccuracyReward/mean": 0.85, + "rewards/MveiAccuracyReward/std": 0.05850084125995636, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9474999934434891, + "rewards/MveiLLMConsistencyReward/std": 0.07296968623995781, + "step": 425, + "step_time": 63.44632751494646 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 253.85, + "completions/mean_length": 200.4625, + "completions/min_length": 170.0, + "epoch": 0.3126135950563432, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.9073856219255105, + "kl": 0.04947709441184998, + "learning_rate": 8.232823375242847e-07, + "loss": 0.0019794270396232606, + "reward": 2.3512500166893004, + "reward_std": 0.12073596306145191, + "rewards/MveiAccuracyReward/mean": 0.9, + "rewards/MveiAccuracyReward/std": 0.05850084125995636, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9512499958276749, + "rewards/MveiLLMConsistencyReward/std": 0.0663618292659521, + "step": 430, + "step_time": 66.05351043827832 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.8, + "completions/mean_length": 203.58125, + "completions/min_length": 174.95, + "epoch": 0.31624863685932386, + "frac_reward_zero_std": 0.55, + "grad_norm": 0.5903155787768735, + "kl": 0.057411357387900354, + "learning_rate": 8.186749522575522e-07, + "loss": 0.0022960007190704346, + "reward": 2.278749966621399, + "reward_std": 0.10757340379059314, + "rewards/MveiAccuracyReward/mean": 0.825, + "rewards/MveiAccuracyReward/std": 0.0667006328701973, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9537500023841858, + "rewards/MveiLLMConsistencyReward/std": 0.06164647303521633, + "step": 435, + "step_time": 63.88984635472298 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.95, + "completions/mean_length": 201.425, + "completions/min_length": 171.05, + "epoch": 0.3198836786623046, + "frac_reward_zero_std": 0.35, + "grad_norm": 1.2139861427067402, + "kl": 0.050447986274957654, + "learning_rate": 8.140215380182616e-07, + "loss": 0.002018173038959503, + "reward": 2.4462499976158143, + "reward_std": 0.104108402505517, + "rewards/MveiAccuracyReward/mean": 0.98125, + "rewards/MveiAccuracyReward/std": 0.0408231720328331, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9650000005960464, + "rewards/MveiLLMConsistencyReward/std": 0.06714780479669571, + "step": 440, + "step_time": 58.520873974263665 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.6, + "completions/mean_length": 203.875, + "completions/min_length": 176.65, + "epoch": 0.32351872046528535, + "frac_reward_zero_std": 0.45, + "grad_norm": 1.1225455396596835, + "kl": 0.0539106716401875, + "learning_rate": 8.093227669391765e-07, + "loss": 0.0021564602851867677, + "reward": 2.3699999928474424, + "reward_std": 0.11594206914305687, + "rewards/MveiAccuracyReward/mean": 0.90625, + "rewards/MveiAccuracyReward/std": 0.06943259090185165, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9637500017881393, + "rewards/MveiLLMConsistencyReward/std": 0.05517797470092774, + "step": 445, + "step_time": 65.62710179761052 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 241.7, + "completions/mean_length": 204.7875, + "completions/min_length": 177.05, + "epoch": 0.3271537622682661, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.9484777674770357, + "kl": 0.05883749946951866, + "learning_rate": 8.045793177043398e-07, + "loss": 0.002353560924530029, + "reward": 2.1237499833106996, + "reward_std": 0.17556292489171027, + "rewards/MveiAccuracyReward/mean": 0.74375, + "rewards/MveiAccuracyReward/std": 0.06396867483854293, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.8800000071525573, + "rewards/MveiLLMConsistencyReward/std": 0.12470418699085713, + "step": 450, + "step_time": 60.537814708426595 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.25, + "completions/mean_length": 202.1625, + "completions/min_length": 172.5, + "epoch": 0.33078880407124683, + "frac_reward_zero_std": 0.35, + "grad_norm": 1.3318374564649673, + "kl": 0.05984954619780183, + "learning_rate": 7.997918754510466e-07, + "loss": 0.0023946911096572876, + "reward": 2.231249988079071, + "reward_std": 0.15656850002706052, + "rewards/MveiAccuracyReward/mean": 0.825, + "rewards/MveiAccuracyReward/std": 0.043555130064487454, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.90625, + "rewards/MveiLLMConsistencyReward/std": 0.1252195317298174, + "step": 455, + "step_time": 70.18436226099729 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.6, + "completions/mean_length": 191.96875, + "completions/min_length": 164.25, + "epoch": 0.3344238458742276, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.5471439970028221, + "kl": 0.05672888103872538, + "learning_rate": 7.949611316708826e-07, + "loss": 0.002269802987575531, + "reward": 2.4125, + "reward_std": 0.060308949649333955, + "rewards/MveiAccuracyReward/mean": 0.94375, + "rewards/MveiAccuracyReward/std": 0.01767766922712326, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.96875, + "rewards/MveiLLMConsistencyReward/std": 0.046318995952606204, + "step": 460, + "step_time": 66.42560303509235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 228.5, + "completions/mean_length": 196.05, + "completions/min_length": 169.5, + "epoch": 0.33805888767720826, + "frac_reward_zero_std": 0.2, + "grad_norm": 1.2116747042882259, + "kl": 0.05652013067156077, + "learning_rate": 7.900877841098465e-07, + "loss": 0.0022601976990699766, + "reward": 2.328749990463257, + "reward_std": 0.1699168708175421, + "rewards/MveiAccuracyReward/mean": 0.925, + "rewards/MveiAccuracyReward/std": 0.07301712930202484, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9037500053644181, + "rewards/MveiLLMConsistencyReward/std": 0.11530957035720349, + "step": 465, + "step_time": 64.05582083277405 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 243.25, + "completions/mean_length": 207.1, + "completions/min_length": 174.7, + "epoch": 0.341693929480189, + "frac_reward_zero_std": 0.3, + "grad_norm": 1.1597501308228664, + "kl": 0.05327767413109541, + "learning_rate": 7.85172536667569e-07, + "loss": 0.002130722999572754, + "reward": 2.252499985694885, + "reward_std": 0.11413553021848202, + "rewards/MveiAccuracyReward/mean": 0.83125, + "rewards/MveiAccuracyReward/std": 0.025877460837364197, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9212500005960464, + "rewards/MveiLLMConsistencyReward/std": 0.10005183294415473, + "step": 470, + "step_time": 66.18105836473406 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 229.95, + "completions/mean_length": 198.90625, + "completions/min_length": 174.05, + "epoch": 0.34532897128316975, + "frac_reward_zero_std": 0.35, + "grad_norm": 1.3500653569538534, + "kl": 0.05451708110049367, + "learning_rate": 7.802160992956419e-07, + "loss": 0.0021812736988067625, + "reward": 2.2687500178813935, + "reward_std": 0.145114778727293, + "rewards/MveiAccuracyReward/mean": 0.84375, + "rewards/MveiAccuracyReward/std": 0.07617851048707962, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9249999955296516, + "rewards/MveiLLMConsistencyReward/std": 0.08570018373429775, + "step": 475, + "step_time": 63.98813854046166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.9, + "completions/mean_length": 195.975, + "completions/min_length": 168.25, + "epoch": 0.3489640130861505, + "frac_reward_zero_std": 0.45, + "grad_norm": 1.1991339535324204, + "kl": 0.0536842804402113, + "learning_rate": 7.75219187895074e-07, + "loss": 0.0021476447582244873, + "reward": 2.1912500083446504, + "reward_std": 0.1769543681293726, + "rewards/MveiAccuracyReward/mean": 0.75625, + "rewards/MveiAccuracyReward/std": 0.12246951609849929, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9349999964237213, + "rewards/MveiLLMConsistencyReward/std": 0.08697443492710591, + "step": 480, + "step_time": 66.7920372530818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.35, + "completions/mean_length": 202.66875, + "completions/min_length": 174.6, + "epoch": 0.35259905488913124, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.9707836906255748, + "kl": 0.05712243542075157, + "learning_rate": 7.701825242128869e-07, + "loss": 0.002285104990005493, + "reward": 2.202500009536743, + "reward_std": 0.11030773371458054, + "rewards/MveiAccuracyReward/mean": 0.80625, + "rewards/MveiAccuracyReward/std": 0.01767766922712326, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.8962499976158143, + "rewards/MveiLLMConsistencyReward/std": 0.11521266996860505, + "step": 485, + "step_time": 69.42208571247757 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 263.3, + "completions/mean_length": 213.5625, + "completions/min_length": 178.05, + "epoch": 0.356234096692112, + "frac_reward_zero_std": 0.1, + "grad_norm": 1.4659303026360888, + "kl": 0.05164956394582987, + "learning_rate": 7.651068357378676e-07, + "loss": 0.0020667314529418945, + "reward": 2.263750010728836, + "reward_std": 0.20939640030264856, + "rewards/MveiAccuracyReward/mean": 0.8375, + "rewards/MveiAccuracyReward/std": 0.13971776217222215, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9262499958276749, + "rewards/MveiLLMConsistencyReward/std": 0.09756103716790676, + "step": 490, + "step_time": 72.6419134542346 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00625, + "completions/max_length": 273.75, + "completions/mean_length": 205.7625, + "completions/min_length": 173.25, + "epoch": 0.35986913849509267, + "frac_reward_zero_std": 0.35, + "grad_norm": 1.053758263654244, + "kl": 0.05370880430564284, + "learning_rate": 7.599928555954906e-07, + "loss": 0.0021483659744262694, + "reward": 2.309375, + "reward_std": 0.09666542373597622, + "rewards/MveiAccuracyReward/mean": 0.88125, + "rewards/MveiAccuracyReward/std": 0.044403792917728425, + "rewards/MveiFormatReward/mean": 0.99375, + "rewards/MveiFormatReward/std": 0.01767766922712326, + "rewards/MveiLLMConsistencyReward/mean": 0.9312499970197677, + "rewards/MveiLLMConsistencyReward/std": 0.08384425416588784, + "step": 495, + "step_time": 65.41909863501787 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.5, + "completions/mean_length": 204.11875, + "completions/min_length": 174.6, + "epoch": 0.3635041802980734, + "frac_reward_zero_std": 0.45, + "grad_norm": 1.2068139413662464, + "kl": 0.05257331561297178, + "learning_rate": 7.548413224420258e-07, + "loss": 0.0021028101444244387, + "reward": 2.3625000059604644, + "reward_std": 0.0931675311177969, + "rewards/MveiAccuracyReward/mean": 0.9375, + "rewards/MveiAccuracyReward/std": 0.03535533845424652, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9249999940395355, + "rewards/MveiLLMConsistencyReward/std": 0.07031436897814274, + "step": 500, + "step_time": 64.24607281126082 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 238.15, + "completions/mean_length": 204.14375, + "completions/min_length": 177.4, + "epoch": 0.36713922210105415, + "frac_reward_zero_std": 0.3, + "grad_norm": 0.8831413852549667, + "kl": 0.05326737640425563, + "learning_rate": 7.49652980357849e-07, + "loss": 0.0021309852600097656, + "reward": 2.428124988079071, + "reward_std": 0.119987802952528, + "rewards/MveiAccuracyReward/mean": 0.9875, + "rewards/MveiAccuracyReward/std": 0.023145502805709837, + "rewards/MveiFormatReward/mean": 0.99375, + "rewards/MveiFormatReward/std": 0.01767766922712326, + "rewards/MveiLLMConsistencyReward/mean": 0.9437500029802323, + "rewards/MveiLLMConsistencyReward/std": 0.09070673920214176, + "step": 505, + "step_time": 65.36522831656039 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.05, + "completions/mean_length": 205.71875, + "completions/min_length": 178.15, + "epoch": 0.3707742639040349, + "frac_reward_zero_std": 0.25, + "grad_norm": 1.135326604621287, + "kl": 0.05665736943483353, + "learning_rate": 7.444285787399669e-07, + "loss": 0.002267000079154968, + "reward": 2.2599999964237214, + "reward_std": 0.10505998097360134, + "rewards/MveiAccuracyReward/mean": 0.83125, + "rewards/MveiAccuracyReward/std": 0.025877460837364197, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9287499994039535, + "rewards/MveiLLMConsistencyReward/std": 0.08056277297437191, + "step": 510, + "step_time": 68.83228598050773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 248.6, + "completions/mean_length": 213.84375, + "completions/min_length": 185.05, + "epoch": 0.37440930570701564, + "frac_reward_zero_std": 0.4, + "grad_norm": 1.1027208729128795, + "kl": 0.052945070713758466, + "learning_rate": 7.391688721937763e-07, + "loss": 0.0021178215742111207, + "reward": 2.197500002384186, + "reward_std": 0.1987775254994631, + "rewards/MveiAccuracyReward/mean": 0.7625, + "rewards/MveiAccuracyReward/std": 0.17149247974157333, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9350000053644181, + "rewards/MveiLLMConsistencyReward/std": 0.05919569656252861, + "step": 515, + "step_time": 60.320371775701645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.45, + "completions/mean_length": 203.0875, + "completions/min_length": 177.1, + "epoch": 0.3780443475099964, + "frac_reward_zero_std": 0.25, + "grad_norm": 1.2880362322197543, + "kl": 0.05523567423224449, + "learning_rate": 7.338746204240687e-07, + "loss": 0.0022095203399658202, + "reward": 2.2018749833106996, + "reward_std": 0.18473673313856126, + "rewards/MveiAccuracyReward/mean": 0.775, + "rewards/MveiAccuracyReward/std": 0.1142658069729805, + "rewards/MveiFormatReward/mean": 0.99375, + "rewards/MveiFormatReward/std": 0.01767766922712326, + "rewards/MveiLLMConsistencyReward/mean": 0.9300000041723251, + "rewards/MveiLLMConsistencyReward/std": 0.07789790332317352, + "step": 520, + "step_time": 62.576503900438546 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 249.25, + "completions/mean_length": 205.99375, + "completions/min_length": 175.75, + "epoch": 0.3816793893129771, + "frac_reward_zero_std": 0.3, + "grad_norm": 1.1342890098182488, + "kl": 0.05601891297847032, + "learning_rate": 7.285465881253008e-07, + "loss": 0.002241194248199463, + "reward": 2.1987499952316285, + "reward_std": 0.2141023449599743, + "rewards/MveiAccuracyReward/mean": 0.775, + "rewards/MveiAccuracyReward/std": 0.1672886312007904, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9237499982118607, + "rewards/MveiLLMConsistencyReward/std": 0.09724985435605049, + "step": 525, + "step_time": 62.094839595258236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 243.65, + "completions/mean_length": 205.05625, + "completions/min_length": 170.15, + "epoch": 0.3853144311159578, + "frac_reward_zero_std": 0.45, + "grad_norm": 1.2172909373527205, + "kl": 0.048401630483567715, + "learning_rate": 7.231855448711426e-07, + "loss": 0.0019360661506652832, + "reward": 2.323749989271164, + "reward_std": 0.09478306397795677, + "rewards/MveiAccuracyReward/mean": 0.875, + "rewards/MveiAccuracyReward/std": 0.043555130064487454, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9487500011920929, + "rewards/MveiLLMConsistencyReward/std": 0.06333404555916786, + "step": 530, + "step_time": 63.70999586209655 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.25, + "completions/mean_length": 200.71875, + "completions/min_length": 172.4, + "epoch": 0.38894947291893855, + "frac_reward_zero_std": 0.35, + "grad_norm": 0.9261509026675876, + "kl": 0.05064096115529537, + "learning_rate": 7.177922650033206e-07, + "loss": 0.002025720477104187, + "reward": 2.3612500011920927, + "reward_std": 0.1679020505398512, + "rewards/MveiAccuracyReward/mean": 0.925, + "rewards/MveiAccuracyReward/std": 0.08984613567590713, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9362499952316284, + "rewards/MveiLLMConsistencyReward/std": 0.09141329899430276, + "step": 535, + "step_time": 61.261403454467654 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.1, + "completions/mean_length": 204.66875, + "completions/min_length": 173.7, + "epoch": 0.3925845147219193, + "frac_reward_zero_std": 0.35, + "grad_norm": 0.7197223117139973, + "kl": 0.05635623093694449, + "learning_rate": 7.123675275197738e-07, + "loss": 0.0022535800933837892, + "reward": 2.308749955892563, + "reward_std": 0.15509552210569383, + "rewards/MveiAccuracyReward/mean": 0.88125, + "rewards/MveiAccuracyReward/std": 0.10290463417768478, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9275000095367432, + "rewards/MveiLLMConsistencyReward/std": 0.06624302230775356, + "step": 540, + "step_time": 57.23934747502208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 223.7, + "completions/mean_length": 193.38125, + "completions/min_length": 168.8, + "epoch": 0.39621955652490004, + "frac_reward_zero_std": 0.35, + "grad_norm": 1.2111291692233497, + "kl": 0.04909519180655479, + "learning_rate": 7.069121159621363e-07, + "loss": 0.0019636735320091247, + "reward": 2.3212499558925628, + "reward_std": 0.0717705175280571, + "rewards/MveiAccuracyReward/mean": 0.85625, + "rewards/MveiAccuracyReward/std": 0.01767766922712326, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9650000125169754, + "rewards/MveiLLMConsistencyReward/std": 0.05409285649657249, + "step": 545, + "step_time": 57.13765530437231 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 246.7, + "completions/mean_length": 201.70625, + "completions/min_length": 168.6, + "epoch": 0.3998545983278808, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.9955385708676731, + "kl": 0.04794792728498578, + "learning_rate": 7.014268183025631e-07, + "loss": 0.0019185051321983337, + "reward": 2.3812500119209288, + "reward_std": 0.1366258893162012, + "rewards/MveiAccuracyReward/mean": 0.925, + "rewards/MveiAccuracyReward/std": 0.09385617971420288, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9562499970197678, + "rewards/MveiLLMConsistencyReward/std": 0.06136959791183472, + "step": 550, + "step_time": 62.35990489758551 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 255.7, + "completions/mean_length": 213.225, + "completions/min_length": 178.2, + "epoch": 0.4034896401308615, + "frac_reward_zero_std": 0.25, + "grad_norm": 1.3799567537012887, + "kl": 0.059047107398509976, + "learning_rate": 6.959124268299166e-07, + "loss": 0.002361568808555603, + "reward": 2.2499999403953552, + "reward_std": 0.12058468423783779, + "rewards/MveiAccuracyReward/mean": 0.81875, + "rewards/MveiAccuracyReward/std": 0.06123279929161072, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9312500178813934, + "rewards/MveiLLMConsistencyReward/std": 0.07831006348133088, + "step": 555, + "step_time": 63.79237565882504 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 249.45, + "completions/mean_length": 209.0125, + "completions/min_length": 178.9, + "epoch": 0.4071246819338422, + "frac_reward_zero_std": 0.2, + "grad_norm": 1.0840077239241812, + "kl": 0.0486101808026433, + "learning_rate": 6.903697380353298e-07, + "loss": 0.0019450187683105469, + "reward": 2.4093749940395357, + "reward_std": 0.11802061796188354, + "rewards/MveiAccuracyReward/mean": 0.9625, + "rewards/MveiAccuracyReward/std": 0.023145502805709837, + "rewards/MveiFormatReward/mean": 0.98125, + "rewards/MveiFormatReward/std": 0.05303300768136978, + "rewards/MveiLLMConsistencyReward/mean": 0.9562500029802322, + "rewards/MveiLLMConsistencyReward/std": 0.07798741534352302, + "step": 560, + "step_time": 60.060036178678274 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 238.1, + "completions/mean_length": 198.9625, + "completions/min_length": 169.45, + "epoch": 0.41075972373682296, + "frac_reward_zero_std": 0.25, + "grad_norm": 1.1811123278584712, + "kl": 0.05753252692520618, + "learning_rate": 6.847995524971619e-07, + "loss": 0.0023005783557891847, + "reward": 2.263750010728836, + "reward_std": 0.13544689528644085, + "rewards/MveiAccuracyReward/mean": 0.80625, + "rewards/MveiAccuracyReward/std": 0.0757490873336792, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9574999928474426, + "rewards/MveiLLMConsistencyReward/std": 0.0672094389796257, + "step": 565, + "step_time": 64.18464987687767 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.2, + "completions/mean_length": 204.38125, + "completions/min_length": 175.4, + "epoch": 0.4143947655398037, + "frac_reward_zero_std": 0.45, + "grad_norm": 0.5912123187155788, + "kl": 0.05996296042576432, + "learning_rate": 6.792026747653643e-07, + "loss": 0.0023984819650650024, + "reward": 2.26499999165535, + "reward_std": 0.050737760961055756, + "rewards/MveiAccuracyReward/mean": 0.8, + "rewards/MveiAccuracyReward/std": 0.0, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9650000005960464, + "rewards/MveiLLMConsistencyReward/std": 0.05073775686323643, + "step": 570, + "step_time": 58.63816139064729 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 229.5, + "completions/mean_length": 196.88125, + "completions/min_length": 166.7, + "epoch": 0.41802980734278444, + "frac_reward_zero_std": 0.5, + "grad_norm": 1.0153067081656428, + "kl": 0.05443270392715931, + "learning_rate": 6.735799132452719e-07, + "loss": 0.0021768227219581605, + "reward": 2.431249976158142, + "reward_std": 0.09589770957827567, + "rewards/MveiAccuracyReward/mean": 0.95625, + "rewards/MveiAccuracyReward/std": 0.06754929572343826, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9750000029802323, + "rewards/MveiLLMConsistencyReward/std": 0.050441878661513326, + "step": 575, + "step_time": 59.038442390412094 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.3, + "completions/mean_length": 205.7625, + "completions/min_length": 180.05, + "epoch": 0.4216648491457652, + "frac_reward_zero_std": 0.15, + "grad_norm": 1.134960103349415, + "kl": 0.05283217802643776, + "learning_rate": 6.679320800808386e-07, + "loss": 0.0021133676171302797, + "reward": 2.2875, + "reward_std": 0.1303585585206747, + "rewards/MveiAccuracyReward/mean": 0.8625, + "rewards/MveiAccuracyReward/std": 0.043555130064487454, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9250000029802322, + "rewards/MveiLLMConsistencyReward/std": 0.09563095308840275, + "step": 580, + "step_time": 63.406537940725684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 237.45, + "completions/mean_length": 200.23125, + "completions/min_length": 173.05, + "epoch": 0.42529989094874593, + "frac_reward_zero_std": 0.3, + "grad_norm": 0.8157496467150972, + "kl": 0.06471217218786478, + "learning_rate": 6.622599910373317e-07, + "loss": 0.002588161826133728, + "reward": 2.372499966621399, + "reward_std": 0.0735352661460638, + "rewards/MveiAccuracyReward/mean": 0.90625, + "rewards/MveiAccuracyReward/std": 0.01767766922712326, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9662500083446502, + "rewards/MveiLLMConsistencyReward/std": 0.06310496106743813, + "step": 585, + "step_time": 65.69516163021326 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.3, + "completions/mean_length": 201.24375, + "completions/min_length": 172.7, + "epoch": 0.42893493275172667, + "frac_reward_zero_std": 0.4, + "grad_norm": 1.237601541269308, + "kl": 0.056501855980604886, + "learning_rate": 6.565644653835053e-07, + "loss": 0.002260762453079224, + "reward": 2.410000002384186, + "reward_std": 0.13696115352213384, + "rewards/MveiAccuracyReward/mean": 0.975, + "rewards/MveiAccuracyReward/std": 0.043555130064487454, + "rewards/MveiFormatReward/mean": 0.9875, + "rewards/MveiFormatReward/std": 0.03535533845424652, + "rewards/MveiLLMConsistencyReward/mean": 0.9412499994039536, + "rewards/MveiLLMConsistencyReward/std": 0.0803681381046772, + "step": 590, + "step_time": 61.110890195518735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 234.9, + "completions/mean_length": 202.1, + "completions/min_length": 175.1, + "epoch": 0.43256997455470736, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.08556000264668923, + "kl": 0.06673226188868284, + "learning_rate": 6.508463257732644e-07, + "loss": 0.0026691555976867676, + "reward": 2.352500003576279, + "reward_std": 0.07930894531309604, + "rewards/MveiAccuracyReward/mean": 0.8875, + "rewards/MveiAccuracyReward/std": 0.03535533845424652, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9649999976158142, + "rewards/MveiLLMConsistencyReward/std": 0.04610480107367039, + "step": 595, + "step_time": 63.12421229444444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.45, + "completions/mean_length": 199.64375, + "completions/min_length": 172.9, + "epoch": 0.4362050163576881, + "frac_reward_zero_std": 0.4, + "grad_norm": 1.0473929863133022, + "kl": 0.050745103228837254, + "learning_rate": 6.451063981268433e-07, + "loss": 0.0020303398370742796, + "reward": 2.468750023841858, + "reward_std": 0.05933799929916859, + "rewards/MveiAccuracyReward/mean": 1.0, + "rewards/MveiAccuracyReward/std": 0.0, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9687499940395355, + "rewards/MveiLLMConsistencyReward/std": 0.05933800302445889, + "step": 600, + "step_time": 66.82194680832326 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.95, + "completions/mean_length": 201.2375, + "completions/min_length": 174.6, + "epoch": 0.43984005816066885, + "frac_reward_zero_std": 0.2, + "grad_norm": 0.9629096188446689, + "kl": 0.059405024908483026, + "learning_rate": 6.393455115115104e-07, + "loss": 0.0023769378662109376, + "reward": 2.2493750154972076, + "reward_std": 0.2034198261797428, + "rewards/MveiAccuracyReward/mean": 0.825, + "rewards/MveiAccuracyReward/std": 0.12878209501504898, + "rewards/MveiFormatReward/mean": 0.99375, + "rewards/MveiFormatReward/std": 0.01767766922712326, + "rewards/MveiLLMConsistencyReward/mean": 0.9274999976158143, + "rewards/MveiLLMConsistencyReward/std": 0.09270746670663357, + "step": 605, + "step_time": 61.957865204662085 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 252.15, + "completions/mean_length": 206.30625, + "completions/min_length": 176.05, + "epoch": 0.4434750999636496, + "frac_reward_zero_std": 0.3, + "grad_norm": 1.1432314216345052, + "kl": 0.059602943155914544, + "learning_rate": 6.335644980218184e-07, + "loss": 0.0023843377828598024, + "reward": 2.223749977350235, + "reward_std": 0.21208821050822735, + "rewards/MveiAccuracyReward/mean": 0.7875, + "rewards/MveiAccuracyReward/std": 0.1483469769358635, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9362499982118606, + "rewards/MveiLLMConsistencyReward/std": 0.10256377197802066, + "step": 610, + "step_time": 61.47646673768759 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 227.55, + "completions/mean_length": 197.175, + "completions/min_length": 167.75, + "epoch": 0.44711014176663033, + "frac_reward_zero_std": 0.2, + "grad_norm": 1.2303025699265895, + "kl": 0.05380919221788645, + "learning_rate": 6.277641926594185e-07, + "loss": 0.0021525368094444273, + "reward": 2.3424999892711638, + "reward_std": 0.11703605018556118, + "rewards/MveiAccuracyReward/mean": 0.9, + "rewards/MveiAccuracyReward/std": 0.03535533845424652, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9425000071525573, + "rewards/MveiLLMConsistencyReward/std": 0.08437751457095147, + "step": 615, + "step_time": 57.86312923617661 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.45, + "completions/mean_length": 198.2, + "completions/min_length": 168.55, + "epoch": 0.4507451835696111, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.6999681934447477, + "kl": 0.05240498147904873, + "learning_rate": 6.219454332124531e-07, + "loss": 0.0020967841148376463, + "reward": 2.39375, + "reward_std": 0.14114195108413696, + "rewards/MveiAccuracyReward/mean": 0.95625, + "rewards/MveiAccuracyReward/std": 0.09932401329278946, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9375, + "rewards/MveiLLMConsistencyReward/std": 0.06857122667133808, + "step": 620, + "step_time": 58.16082239821553 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 234.1, + "completions/mean_length": 202.7625, + "completions/min_length": 176.9, + "epoch": 0.45438022537259176, + "frac_reward_zero_std": 0.5, + "grad_norm": 1.1825385245122997, + "kl": 0.050347439106553794, + "learning_rate": 6.161090601345476e-07, + "loss": 0.0020137086510658266, + "reward": 2.3837499916553497, + "reward_std": 0.09976745806634427, + "rewards/MveiAccuracyReward/mean": 0.9125, + "rewards/MveiAccuracyReward/std": 0.043555130064487454, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9712500005960465, + "rewards/MveiLLMConsistencyReward/std": 0.05759679488837719, + "step": 625, + "step_time": 63.5189865604043 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 238.2, + "completions/mean_length": 199.525, + "completions/min_length": 173.85, + "epoch": 0.4580152671755725, + "frac_reward_zero_std": 0.55, + "grad_norm": 0.6265346042375116, + "kl": 0.04690617267042398, + "learning_rate": 6.102559164234163e-07, + "loss": 0.0018758177757263184, + "reward": 2.4387499928474425, + "reward_std": 0.07619544602930546, + "rewards/MveiAccuracyReward/mean": 0.98125, + "rewards/MveiAccuracyReward/std": 0.0408231720328331, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9575000017881393, + "rewards/MveiLLMConsistencyReward/std": 0.046797850355505946, + "step": 630, + "step_time": 58.02576370835304 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 229.15, + "completions/mean_length": 197.35, + "completions/min_length": 170.8, + "epoch": 0.46165030897855325, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.5489047862993448, + "kl": 0.05437676999717951, + "learning_rate": 6.043868474990996e-07, + "loss": 0.0021751759573817252, + "reward": 2.4262499809265137, + "reward_std": 0.07427813299000263, + "rewards/MveiAccuracyReward/mean": 0.95, + "rewards/MveiAccuracyReward/std": 0.046291005611419675, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9762500077486038, + "rewards/MveiLLMConsistencyReward/std": 0.04104428850114346, + "step": 635, + "step_time": 64.33075705319644 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 237.75, + "completions/mean_length": 200.06875, + "completions/min_length": 168.8, + "epoch": 0.465285350781534, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.6484025378502892, + "kl": 0.060050109215080735, + "learning_rate": 5.985027010818552e-07, + "loss": 0.0024017333984375, + "reward": 2.381249988079071, + "reward_std": 0.06976461969316006, + "rewards/MveiAccuracyReward/mean": 0.925, + "rewards/MveiAccuracyReward/std": 0.026726123690605164, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.95625, + "rewards/MveiLLMConsistencyReward/std": 0.05022066310048103, + "step": 640, + "step_time": 60.223006937652826 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 222.95, + "completions/mean_length": 193.0, + "completions/min_length": 165.75, + "epoch": 0.46892039258451473, + "frac_reward_zero_std": 0.15, + "grad_norm": 1.276040088015636, + "kl": 0.06414886265993118, + "learning_rate": 5.926043270697123e-07, + "loss": 0.002565912902355194, + "reward": 2.211249965429306, + "reward_std": 0.12579258158802986, + "rewards/MveiAccuracyReward/mean": 0.80625, + "rewards/MveiAccuracyReward/std": 0.06396867483854293, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9050000071525574, + "rewards/MveiLLMConsistencyReward/std": 0.09588906839489937, + "step": 645, + "step_time": 67.38408568166196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 231.95, + "completions/mean_length": 196.225, + "completions/min_length": 166.15, + "epoch": 0.4725554343874955, + "frac_reward_zero_std": 0.3, + "grad_norm": 0.633168537470746, + "kl": 0.059809874184429644, + "learning_rate": 5.866925774157148e-07, + "loss": 0.002393084764480591, + "reward": 2.3537499964237214, + "reward_std": 0.13260694704949855, + "rewards/MveiAccuracyReward/mean": 0.9, + "rewards/MveiAccuracyReward/std": 0.0816463440656662, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9537500023841858, + "rewards/MveiLLMConsistencyReward/std": 0.06922099255025387, + "step": 650, + "step_time": 64.30584721565246 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 241.55, + "completions/mean_length": 203.3125, + "completions/min_length": 175.65, + "epoch": 0.47619047619047616, + "frac_reward_zero_std": 0.45, + "grad_norm": 1.0780184778390836, + "kl": 0.05320450058206916, + "learning_rate": 5.807683060048658e-07, + "loss": 0.0021285176277160644, + "reward": 2.3974999845027924, + "reward_std": 0.06946404278278351, + "rewards/MveiAccuracyReward/mean": 0.94375, + "rewards/MveiAccuracyReward/std": 0.01767766922712326, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9537500023841858, + "rewards/MveiLLMConsistencyReward/std": 0.05438485443592071, + "step": 655, + "step_time": 61.625752969831225 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.2, + "completions/mean_length": 195.99375, + "completions/min_length": 168.4, + "epoch": 0.4798255179934569, + "frac_reward_zero_std": 0.6, + "grad_norm": 0.8238856197498328, + "kl": 0.053611588478088376, + "learning_rate": 5.748323685307936e-07, + "loss": 0.002144557237625122, + "reward": 2.4399999916553496, + "reward_std": 0.07517711184918881, + "rewards/MveiAccuracyReward/mean": 0.96875, + "rewards/MveiAccuracyReward/std": 0.044403792917728425, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9712500035762787, + "rewards/MveiLLMConsistencyReward/std": 0.04230758845806122, + "step": 660, + "step_time": 63.852589287608865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 222.2, + "completions/mean_length": 195.25, + "completions/min_length": 169.35, + "epoch": 0.48346055979643765, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.9303723416284634, + "kl": 0.05216963700950146, + "learning_rate": 5.68885622372157e-07, + "loss": 0.0020866133272647858, + "reward": 2.4524999856948853, + "reward_std": 0.07636669464409351, + "rewards/MveiAccuracyReward/mean": 1.0, + "rewards/MveiAccuracyReward/std": 0.0, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9525000035762787, + "rewards/MveiLLMConsistencyReward/std": 0.07636669613420963, + "step": 665, + "step_time": 63.11184250563383 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 224.35, + "completions/mean_length": 194.8625, + "completions/min_length": 169.7, + "epoch": 0.4870956015994184, + "frac_reward_zero_std": 0.25, + "grad_norm": 1.1743371080404714, + "kl": 0.06406292896717787, + "learning_rate": 5.629289264688061e-07, + "loss": 0.002563074231147766, + "reward": 2.348749989271164, + "reward_std": 0.14019065350294113, + "rewards/MveiAccuracyReward/mean": 0.89375, + "rewards/MveiAccuracyReward/std": 0.08795892298221589, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9550000041723251, + "rewards/MveiLLMConsistencyReward/std": 0.06603162772953511, + "step": 670, + "step_time": 180.70229631587864 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.2, + "completions/mean_length": 203.6125, + "completions/min_length": 174.4, + "epoch": 0.49073064340239914, + "frac_reward_zero_std": 0.3, + "grad_norm": 0.8537166656005972, + "kl": 0.05742366947233677, + "learning_rate": 5.569631411977183e-07, + "loss": 0.0022970378398895265, + "reward": 2.3449999690055847, + "reward_std": 0.11332251727581025, + "rewards/MveiAccuracyReward/mean": 0.88125, + "rewards/MveiAccuracyReward/std": 0.06123279929161072, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.963750010728836, + "rewards/MveiLLMConsistencyReward/std": 0.054786515235900876, + "step": 675, + "step_time": 60.93428806103766 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 246.75, + "completions/mean_length": 199.125, + "completions/min_length": 166.9, + "epoch": 0.4943656852053799, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.6826603397734794, + "kl": 0.05021212715655565, + "learning_rate": 5.509891282487275e-07, + "loss": 0.0020084142684936523, + "reward": 2.4324999928474424, + "reward_std": 0.09412019178271294, + "rewards/MveiAccuracyReward/mean": 0.98125, + "rewards/MveiAccuracyReward/std": 0.0408231720328331, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9512500017881393, + "rewards/MveiLLMConsistencyReward/std": 0.06251149512827396, + "step": 680, + "step_time": 59.32948975674808 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 234.2, + "completions/mean_length": 201.675, + "completions/min_length": 172.55, + "epoch": 0.4980007270083606, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.9879158907430384, + "kl": 0.059967555291950705, + "learning_rate": 5.450077505000612e-07, + "loss": 0.002398298680782318, + "reward": 2.2287499845027923, + "reward_std": 0.13407860919833184, + "rewards/MveiAccuracyReward/mean": 0.81875, + "rewards/MveiAccuracyReward/std": 0.06396867483854293, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9100000038743019, + "rewards/MveiLLMConsistencyReward/std": 0.08820019215345383, + "step": 685, + "step_time": 60.08720072694123 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 231.8, + "completions/mean_length": 201.78125, + "completions/min_length": 176.0, + "epoch": 0.5016357688113413, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.834265898422456, + "kl": 0.05460627768188715, + "learning_rate": 5.390198718937091e-07, + "loss": 0.002184931933879852, + "reward": 2.2975000143051147, + "reward_std": 0.19341775104403497, + "rewards/MveiAccuracyReward/mean": 0.8625, + "rewards/MveiAccuracyReward/std": 0.12015288025140762, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9349999994039535, + "rewards/MveiLLMConsistencyReward/std": 0.10299718491733074, + "step": 690, + "step_time": 64.25981214381754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.05, + "completions/mean_length": 198.2875, + "completions/min_length": 170.55, + "epoch": 0.5052708106143221, + "frac_reward_zero_std": 0.35, + "grad_norm": 1.2727083271601063, + "kl": 0.05565820280462504, + "learning_rate": 5.330263573106357e-07, + "loss": 0.0022266149520874024, + "reward": 2.341250002384186, + "reward_std": 0.117528111115098, + "rewards/MveiAccuracyReward/mean": 0.89375, + "rewards/MveiAccuracyReward/std": 0.06943259090185165, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9474999994039536, + "rewards/MveiLLMConsistencyReward/std": 0.07141387052834033, + "step": 695, + "step_time": 180.17640886865556 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.1, + "completions/mean_length": 198.4875, + "completions/min_length": 170.75, + "epoch": 0.5089058524173028, + "frac_reward_zero_std": 0.4, + "grad_norm": 1.1063117753508258, + "kl": 0.05431645596399903, + "learning_rate": 5.270280724458579e-07, + "loss": 0.0021736010909080505, + "reward": 2.364999997615814, + "reward_std": 0.11570962965488434, + "rewards/MveiAccuracyReward/mean": 0.9, + "rewards/MveiAccuracyReward/std": 0.07028125375509262, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9649999976158142, + "rewards/MveiLLMConsistencyReward/std": 0.05579419694840908, + "step": 700, + "step_time": 62.90832507573068 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 227.4, + "completions/mean_length": 201.525, + "completions/min_length": 174.3, + "epoch": 0.5125408942202835, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.9227917459769821, + "kl": 0.050359622575342655, + "learning_rate": 5.210258836834061e-07, + "loss": 0.0020142778754234313, + "reward": 2.4349999904632567, + "reward_std": 0.056512949988245964, + "rewards/MveiAccuracyReward/mean": 0.95625, + "rewards/MveiAccuracyReward/std": 0.01767766922712326, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9787500023841857, + "rewards/MveiLLMConsistencyReward/std": 0.04191595278680325, + "step": 705, + "step_time": 65.42733619399368 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.35, + "completions/mean_length": 201.425, + "completions/min_length": 170.75, + "epoch": 0.5161759360232643, + "frac_reward_zero_std": 0.55, + "grad_norm": 1.1155377821534105, + "kl": 0.050790361035615204, + "learning_rate": 5.150206579711841e-07, + "loss": 0.0020315080881118776, + "reward": 2.3724999964237212, + "reward_std": 0.1378706492483616, + "rewards/MveiAccuracyReward/mean": 0.9125, + "rewards/MveiAccuracyReward/std": 0.12015288025140762, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9599999964237214, + "rewards/MveiLLMConsistencyReward/std": 0.05531466566026211, + "step": 710, + "step_time": 56.96984634622932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 243.2, + "completions/mean_length": 207.6375, + "completions/min_length": 181.4, + "epoch": 0.519810977826245, + "frac_reward_zero_std": 0.25, + "grad_norm": 1.5449226412042787, + "kl": 0.05760031826794147, + "learning_rate": 5.090132626957483e-07, + "loss": 0.002303384244441986, + "reward": 2.3712499737739563, + "reward_std": 0.1599471341818571, + "rewards/MveiAccuracyReward/mean": 0.9125, + "rewards/MveiAccuracyReward/std": 0.10205597132444381, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9587500065565109, + "rewards/MveiLLMConsistencyReward/std": 0.07709407061338425, + "step": 715, + "step_time": 61.04374643303454 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.85, + "completions/mean_length": 197.825, + "completions/min_length": 171.25, + "epoch": 0.5234460196292258, + "frac_reward_zero_std": 0.5, + "grad_norm": 1.2051537027260706, + "kl": 0.04487507436424494, + "learning_rate": 5.030045655570239e-07, + "loss": 0.001795440912246704, + "reward": 2.443750023841858, + "reward_std": 0.07350295260548592, + "rewards/MveiAccuracyReward/mean": 0.99375, + "rewards/MveiAccuracyReward/std": 0.01767766922712326, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9499999940395355, + "rewards/MveiLLMConsistencyReward/std": 0.06459763720631599, + "step": 720, + "step_time": 61.92764515392482 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.05, + "completions/mean_length": 198.3125, + "completions/min_length": 175.1, + "epoch": 0.5270810614322065, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.7604026219628306, + "kl": 0.05456447992473841, + "learning_rate": 4.96995434442976e-07, + "loss": 0.0021826371550559996, + "reward": 2.3399999678134917, + "reward_std": 0.10628600828349591, + "rewards/MveiAccuracyReward/mean": 0.90625, + "rewards/MveiAccuracyReward/std": 0.06754929572343826, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.933750008046627, + "rewards/MveiLLMConsistencyReward/std": 0.06852892078459263, + "step": 725, + "step_time": 56.75429000556469 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.6, + "completions/mean_length": 201.1125, + "completions/min_length": 172.55, + "epoch": 0.5307161032351873, + "frac_reward_zero_std": 0.35, + "grad_norm": 1.0330584260741134, + "kl": 0.05688001066446304, + "learning_rate": 4.909867373042517e-07, + "loss": 0.0022754698991775514, + "reward": 2.2087499797344208, + "reward_std": 0.14862833879888057, + "rewards/MveiAccuracyReward/mean": 0.8, + "rewards/MveiAccuracyReward/std": 0.09258201122283935, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.908750006556511, + "rewards/MveiLLMConsistencyReward/std": 0.07889666929841041, + "step": 730, + "step_time": 64.73999388627708 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.65, + "completions/mean_length": 196.98125, + "completions/min_length": 169.45, + "epoch": 0.5343511450381679, + "frac_reward_zero_std": 0.5, + "grad_norm": 1.1474298194768007, + "kl": 0.05623842403292656, + "learning_rate": 4.84979342028816e-07, + "loss": 0.0022498369216918945, + "reward": 2.4000000238418577, + "reward_std": 0.09364267885684967, + "rewards/MveiAccuracyReward/mean": 0.9375, + "rewards/MveiAccuracyReward/std": 0.03535533845424652, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9624999940395356, + "rewards/MveiLLMConsistencyReward/std": 0.06328011900186539, + "step": 735, + "step_time": 177.2352121386677 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 229.8, + "completions/mean_length": 199.1375, + "completions/min_length": 170.8, + "epoch": 0.5379861868411486, + "frac_reward_zero_std": 0.7, + "grad_norm": 0.6896036376003111, + "kl": 0.051563126314431426, + "learning_rate": 4.789741163165938e-07, + "loss": 0.002062559127807617, + "reward": 2.3737499892711638, + "reward_std": 0.05844749584794044, + "rewards/MveiAccuracyReward/mean": 0.90625, + "rewards/MveiAccuracyReward/std": 0.05260358452796936, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9675000011920929, + "rewards/MveiLLMConsistencyReward/std": 0.036222582682967186, + "step": 740, + "step_time": 70.48028190284967 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 224.35, + "completions/mean_length": 196.5, + "completions/min_length": 166.2, + "epoch": 0.5416212286441294, + "frac_reward_zero_std": 0.5, + "grad_norm": 1.1805001713373837, + "kl": 0.049361081700772046, + "learning_rate": 4.7297192755414196e-07, + "loss": 0.001974627375602722, + "reward": 2.4225000143051147, + "reward_std": 0.1044437676668167, + "rewards/MveiAccuracyReward/mean": 0.9625, + "rewards/MveiAccuracyReward/std": 0.0667006328701973, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9599999964237214, + "rewards/MveiLLMConsistencyReward/std": 0.07520394884049893, + "step": 745, + "step_time": 61.17317264787853 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 223.45, + "completions/mean_length": 195.6, + "completions/min_length": 172.1, + "epoch": 0.5452562704471101, + "frac_reward_zero_std": 0.55, + "grad_norm": 0.6491740465797488, + "kl": 0.056783065758645536, + "learning_rate": 4.669736426893644e-07, + "loss": 0.002271723747253418, + "reward": 2.352499985694885, + "reward_std": 0.0707980003207922, + "rewards/MveiAccuracyReward/mean": 0.8875, + "rewards/MveiAccuracyReward/std": 0.023145502805709837, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9650000035762787, + "rewards/MveiLLMConsistencyReward/std": 0.051695066317915914, + "step": 750, + "step_time": 58.426505132764575 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.5, + "completions/mean_length": 202.5125, + "completions/min_length": 175.3, + "epoch": 0.5488913122500909, + "frac_reward_zero_std": 0.3, + "grad_norm": 1.0801325006021063, + "kl": 0.05887562576681375, + "learning_rate": 4.609801281062909e-07, + "loss": 0.0023553848266601564, + "reward": 2.3574999988079073, + "reward_std": 0.15311881639063357, + "rewards/MveiAccuracyReward/mean": 0.9125, + "rewards/MveiAccuracyReward/std": 0.09616263210773468, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9449999988079071, + "rewards/MveiLLMConsistencyReward/std": 0.07789404280483722, + "step": 755, + "step_time": 59.36040690355003 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.05, + "completions/mean_length": 202.99375, + "completions/min_length": 175.95, + "epoch": 0.5525263540530716, + "frac_reward_zero_std": 0.45, + "grad_norm": 1.110540886692785, + "kl": 0.050079621654003856, + "learning_rate": 4.5499224949993894e-07, + "loss": 0.002003321796655655, + "reward": 2.368750011920929, + "reward_std": 0.07481234222650528, + "rewards/MveiAccuracyReward/mean": 0.93125, + "rewards/MveiAccuracyReward/std": 0.025877460837364197, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9375, + "rewards/MveiLLMConsistencyReward/std": 0.050710825994610784, + "step": 760, + "step_time": 60.89374854378402 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 223.1, + "completions/mean_length": 194.1375, + "completions/min_length": 167.9, + "epoch": 0.5561613958560524, + "frac_reward_zero_std": 0.45, + "grad_norm": 1.1723132750119714, + "kl": 0.05370624531060457, + "learning_rate": 4.490108717512726e-07, + "loss": 0.0021477729082107545, + "reward": 2.396249973773956, + "reward_std": 0.10729667954146863, + "rewards/MveiAccuracyReward/mean": 0.925, + "rewards/MveiAccuracyReward/std": 0.05850084125995636, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.971250006556511, + "rewards/MveiLLMConsistencyReward/std": 0.0538467600941658, + "step": 765, + "step_time": 57.5007011462003 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 238.05, + "completions/mean_length": 200.59375, + "completions/min_length": 171.35, + "epoch": 0.5597964376590331, + "frac_reward_zero_std": 0.4, + "grad_norm": 0.5824919884796961, + "kl": 0.06538908109068871, + "learning_rate": 4.4303685880228156e-07, + "loss": 0.0026160240173339843, + "reward": 2.414999985694885, + "reward_std": 0.05796761065721512, + "rewards/MveiAccuracyReward/mean": 0.95, + "rewards/MveiAccuracyReward/std": 0.0, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9650000035762787, + "rewards/MveiLLMConsistencyReward/std": 0.05796760767698288, + "step": 770, + "step_time": 64.03310044296086 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.05, + "completions/mean_length": 201.54375, + "completions/min_length": 173.15, + "epoch": 0.5634314794620138, + "frac_reward_zero_std": 0.45, + "grad_norm": 0.5927449477272995, + "kl": 0.06196275260299444, + "learning_rate": 4.370710735311941e-07, + "loss": 0.002478864789009094, + "reward": 2.2337499976158144, + "reward_std": 0.1062652364373207, + "rewards/MveiAccuracyReward/mean": 0.7875, + "rewards/MveiAccuracyReward/std": 0.0667006328701973, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9462500035762786, + "rewards/MveiLLMConsistencyReward/std": 0.05085046738386154, + "step": 775, + "step_time": 67.91525550037622 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.75, + "completions/mean_length": 201.84375, + "completions/min_length": 172.1, + "epoch": 0.5670665212649946, + "frac_reward_zero_std": 0.35, + "grad_norm": 1.7612246576704227, + "kl": 0.054537341371178626, + "learning_rate": 4.3111437762784305e-07, + "loss": 0.002182258665561676, + "reward": 2.321249985694885, + "reward_std": 0.11211095787584782, + "rewards/MveiAccuracyReward/mean": 0.86875, + "rewards/MveiAccuracyReward/std": 0.044403792917728425, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9525000005960464, + "rewards/MveiLLMConsistencyReward/std": 0.07240946032106876, + "step": 780, + "step_time": 62.06473238617182 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 255.0, + "completions/mean_length": 212.86875, + "completions/min_length": 181.3, + "epoch": 0.5707015630679753, + "frac_reward_zero_std": 0.3, + "grad_norm": 1.0629382148331044, + "kl": 0.05230696480721235, + "learning_rate": 4.2516763146920646e-07, + "loss": 0.0020924389362335207, + "reward": 2.356249988079071, + "reward_std": 0.17326271906495094, + "rewards/MveiAccuracyReward/mean": 0.9125, + "rewards/MveiAccuracyReward/std": 0.10205597132444381, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.94375, + "rewards/MveiLLMConsistencyReward/std": 0.08507692925632, + "step": 785, + "step_time": 63.057847008854154 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 248.8, + "completions/mean_length": 199.65625, + "completions/min_length": 170.55, + "epoch": 0.5743366048709561, + "frac_reward_zero_std": 0.55, + "grad_norm": 1.1475089053384309, + "kl": 0.04890742050483823, + "learning_rate": 4.1923169399513425e-07, + "loss": 0.001956340670585632, + "reward": 2.4231250047683717, + "reward_std": 0.09638397991657258, + "rewards/MveiAccuracyReward/mean": 0.9625, + "rewards/MveiAccuracyReward/std": 0.043555130064487454, + "rewards/MveiFormatReward/mean": 0.99375, + "rewards/MveiFormatReward/std": 0.01767766922712326, + "rewards/MveiLLMConsistencyReward/mean": 0.9637499988079071, + "rewards/MveiLLMConsistencyReward/std": 0.05671881660819054, + "step": 790, + "step_time": 94.11746585927904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.9, + "completions/mean_length": 203.5875, + "completions/min_length": 173.15, + "epoch": 0.5779716466739367, + "frac_reward_zero_std": 0.45, + "grad_norm": 1.0134052005819736, + "kl": 0.056848237104713914, + "learning_rate": 4.133074225842851e-07, + "loss": 0.0022740095853805544, + "reward": 2.3612500011920927, + "reward_std": 0.12292048744857312, + "rewards/MveiAccuracyReward/mean": 0.925, + "rewards/MveiAccuracyReward/std": 0.043555130064487454, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9362500011920929, + "rewards/MveiLLMConsistencyReward/std": 0.09131217785179616, + "step": 795, + "step_time": 65.19886676594615 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 231.8, + "completions/mean_length": 200.49375, + "completions/min_length": 175.05, + "epoch": 0.5816066884769174, + "frac_reward_zero_std": 0.35, + "grad_norm": 0.9991836210947987, + "kl": 0.055218469258397816, + "learning_rate": 4.0739567293028764e-07, + "loss": 0.002208811044692993, + "reward": 2.383749985694885, + "reward_std": 0.10879868492484093, + "rewards/MveiAccuracyReward/mean": 0.925, + "rewards/MveiAccuracyReward/std": 0.043555130064487454, + "rewards/MveiFormatReward/mean": 1.0, + "rewards/MveiFormatReward/std": 0.0, + "rewards/MveiLLMConsistencyReward/mean": 0.9587500035762787, + "rewards/MveiLLMConsistencyReward/std": 0.0751047309488058, + "step": 800, + "step_time": 59.56771714054048 + } + ], + "logging_steps": 5, + "max_steps": 1376, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 200, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +}