| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 2.992, |
| "eval_steps": 10, |
| "global_step": 279, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 231.22917556762695, |
| "epoch": 0.010666666666666666, |
| "grad_norm": 0.1495383381843567, |
| "kl": 0.0, |
| "learning_rate": 1.0714285714285714e-07, |
| "loss": -0.0555, |
| "reward": 0.1458333358168602, |
| "reward_std": 0.2350771240890026, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/format_reward": 0.06250000186264515, |
| "step": 1 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 271.77084732055664, |
| "epoch": 0.021333333333333333, |
| "grad_norm": 0.11679606884717941, |
| "kl": 0.0, |
| "learning_rate": 2.1428571428571428e-07, |
| "loss": -0.0391, |
| "reward": 0.1458333358168602, |
| "reward_std": 0.27258946746587753, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/format_reward": 0.0833333358168602, |
| "step": 2 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 331.87500953674316, |
| "epoch": 0.032, |
| "grad_norm": 0.16674447059631348, |
| "kl": 0.00020113587379455566, |
| "learning_rate": 3.214285714285714e-07, |
| "loss": -0.0214, |
| "reward": 0.14583333767950535, |
| "reward_std": 0.28219256922602654, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/format_reward": 0.0416666679084301, |
| "step": 3 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 305.97917556762695, |
| "epoch": 0.042666666666666665, |
| "grad_norm": 0.16403762996196747, |
| "kl": 9.453296661376953e-05, |
| "learning_rate": 4.2857142857142857e-07, |
| "loss": 0.0042, |
| "reward": 0.1666666716337204, |
| "reward_std": 0.3332235999405384, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/format_reward": 0.1250000037252903, |
| "step": 4 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 292.6458463668823, |
| "epoch": 0.05333333333333334, |
| "grad_norm": 0.026218686252832413, |
| "kl": 0.00013911724090576172, |
| "learning_rate": 5.357142857142858e-07, |
| "loss": -0.0034, |
| "reward": 0.02083333395421505, |
| "reward_std": 0.05103103443980217, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/format_reward": 0.0, |
| "step": 5 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 235.87500858306885, |
| "epoch": 0.064, |
| "grad_norm": 0.17623676359653473, |
| "kl": 0.0001900792121887207, |
| "learning_rate": 6.428571428571428e-07, |
| "loss": -0.082, |
| "reward": 0.1875000037252903, |
| "reward_std": 0.2996268533170223, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/format_reward": 0.0833333358168602, |
| "step": 6 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 266.6666774749756, |
| "epoch": 0.07466666666666667, |
| "grad_norm": 0.13698340952396393, |
| "kl": 0.00019168853759765625, |
| "learning_rate": 7.5e-07, |
| "loss": 0.1021, |
| "reward": 0.2083333395421505, |
| "reward_std": 0.43528567254543304, |
| "rewards/accuracy_reward": 0.1250000037252903, |
| "rewards/format_reward": 0.0833333358168602, |
| "step": 7 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 221.31250762939453, |
| "epoch": 0.08533333333333333, |
| "grad_norm": 0.20153407752513885, |
| "kl": 0.0002624690532684326, |
| "learning_rate": 8.571428571428571e-07, |
| "loss": -0.0188, |
| "reward": 0.35416667349636555, |
| "reward_std": 0.48367293924093246, |
| "rewards/accuracy_reward": 0.1666666716337204, |
| "rewards/format_reward": 0.1875000037252903, |
| "step": 8 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 256.7291736602783, |
| "epoch": 0.096, |
| "grad_norm": 0.14511053264141083, |
| "kl": 0.0002912282943725586, |
| "learning_rate": 9.642857142857145e-07, |
| "loss": 0.0278, |
| "reward": 0.27083334140479565, |
| "reward_std": 0.4177170880138874, |
| "rewards/accuracy_reward": 0.14583333767950535, |
| "rewards/format_reward": 0.1250000037252903, |
| "step": 9 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 275.1458396911621, |
| "epoch": 0.10666666666666667, |
| "grad_norm": 0.10830721259117126, |
| "kl": 0.0003374814987182617, |
| "learning_rate": 1.0714285714285716e-06, |
| "loss": -0.0591, |
| "reward": 0.16666666977107525, |
| "reward_std": 0.18404608592391014, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/format_reward": 0.02083333395421505, |
| "step": 10 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 203.83333778381348, |
| "epoch": 0.11733333333333333, |
| "grad_norm": 0.16608387231826782, |
| "kl": 0.0005745887756347656, |
| "learning_rate": 1.1785714285714285e-06, |
| "loss": 0.0155, |
| "reward": 0.22916667349636555, |
| "reward_std": 0.41129202395677567, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/format_reward": 0.1250000037252903, |
| "step": 11 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 275.6458396911621, |
| "epoch": 0.128, |
| "grad_norm": 0.16276516020298004, |
| "kl": 0.0009386539459228516, |
| "learning_rate": 1.2857142857142856e-06, |
| "loss": -0.1055, |
| "reward": 0.3333333395421505, |
| "reward_std": 0.46875908970832825, |
| "rewards/accuracy_reward": 0.1666666716337204, |
| "rewards/format_reward": 0.16666666977107525, |
| "step": 12 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 209.54167366027832, |
| "epoch": 0.13866666666666666, |
| "grad_norm": 0.20444652438163757, |
| "kl": 0.0017161369323730469, |
| "learning_rate": 1.392857142857143e-06, |
| "loss": 0.0046, |
| "reward": 0.27083334140479565, |
| "reward_std": 0.4217669703066349, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/format_reward": 0.1666666679084301, |
| "step": 13 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 211.5000057220459, |
| "epoch": 0.14933333333333335, |
| "grad_norm": 0.1791943609714508, |
| "kl": 0.0035800933837890625, |
| "learning_rate": 1.5e-06, |
| "loss": -0.0985, |
| "reward": 0.3750000074505806, |
| "reward_std": 0.49719163402915, |
| "rewards/accuracy_reward": 0.16666666977107525, |
| "rewards/format_reward": 0.2083333395421505, |
| "step": 14 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 292.8750057220459, |
| "epoch": 0.16, |
| "grad_norm": 0.1786985695362091, |
| "kl": 0.0047359466552734375, |
| "learning_rate": 1.6071428571428572e-06, |
| "loss": -0.0043, |
| "reward": 0.37500000931322575, |
| "reward_std": 0.44172170385718346, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/format_reward": 0.3125000074505806, |
| "step": 15 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 273.43750762939453, |
| "epoch": 0.17066666666666666, |
| "grad_norm": 0.17925456166267395, |
| "kl": 0.010840415954589844, |
| "learning_rate": 1.7142857142857143e-06, |
| "loss": 0.0074, |
| "reward": 0.4583333432674408, |
| "reward_std": 0.38552645966410637, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/format_reward": 0.3541666716337204, |
| "step": 16 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 264.7916793823242, |
| "epoch": 0.18133333333333335, |
| "grad_norm": 0.1461903601884842, |
| "kl": 0.02383136749267578, |
| "learning_rate": 1.8214285714285714e-06, |
| "loss": -0.0197, |
| "reward": 0.6250000037252903, |
| "reward_std": 0.46026117354631424, |
| "rewards/accuracy_reward": 0.2083333395421505, |
| "rewards/format_reward": 0.4166666753590107, |
| "step": 17 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 319.12500762939453, |
| "epoch": 0.192, |
| "grad_norm": 0.17338670790195465, |
| "kl": 0.036746978759765625, |
| "learning_rate": 1.928571428571429e-06, |
| "loss": 0.0239, |
| "reward": 0.6458333395421505, |
| "reward_std": 0.5965815186500549, |
| "rewards/accuracy_reward": 0.1875000037252903, |
| "rewards/format_reward": 0.45833334513008595, |
| "step": 18 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 218.85417366027832, |
| "epoch": 0.20266666666666666, |
| "grad_norm": 0.15455670654773712, |
| "kl": 0.028100967407226562, |
| "learning_rate": 2.035714285714286e-06, |
| "loss": 0.0165, |
| "reward": 0.9791666939854622, |
| "reward_std": 0.6326877325773239, |
| "rewards/accuracy_reward": 0.2291666716337204, |
| "rewards/format_reward": 0.7500000298023224, |
| "step": 19 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 328.1875114440918, |
| "epoch": 0.21333333333333335, |
| "grad_norm": 0.3660297095775604, |
| "kl": 0.2696857452392578, |
| "learning_rate": 2.142857142857143e-06, |
| "loss": 0.0074, |
| "reward": 0.5416666828095913, |
| "reward_std": 0.6121251657605171, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/format_reward": 0.4791666753590107, |
| "step": 20 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 185.58334159851074, |
| "epoch": 0.224, |
| "grad_norm": 0.18908099830150604, |
| "kl": 0.034381866455078125, |
| "learning_rate": 2.25e-06, |
| "loss": 0.0302, |
| "reward": 0.9166666846722364, |
| "reward_std": 0.46722716465592384, |
| "rewards/accuracy_reward": 0.2708333432674408, |
| "rewards/format_reward": 0.6458333488553762, |
| "step": 21 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 269.4583435058594, |
| "epoch": 0.23466666666666666, |
| "grad_norm": 0.23737290501594543, |
| "kl": 0.07430791854858398, |
| "learning_rate": 2.357142857142857e-06, |
| "loss": 0.0205, |
| "reward": 0.6875000186264515, |
| "reward_std": 0.41912321373820305, |
| "rewards/accuracy_reward": 0.02083333395421505, |
| "rewards/format_reward": 0.6666666846722364, |
| "step": 22 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 289.9791774749756, |
| "epoch": 0.24533333333333332, |
| "grad_norm": 0.11483895033597946, |
| "kl": 0.026653289794921875, |
| "learning_rate": 2.4642857142857143e-06, |
| "loss": 0.0206, |
| "reward": 0.7708333469927311, |
| "reward_std": 0.34674229472875595, |
| "rewards/accuracy_reward": 0.06250000186264515, |
| "rewards/format_reward": 0.7083333395421505, |
| "step": 23 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 256.7083377838135, |
| "epoch": 0.256, |
| "grad_norm": 0.10966558754444122, |
| "kl": 0.009334564208984375, |
| "learning_rate": 2.571428571428571e-06, |
| "loss": 0.0168, |
| "reward": 0.9583333730697632, |
| "reward_std": 0.2861081585288048, |
| "rewards/accuracy_reward": 0.1875000074505806, |
| "rewards/format_reward": 0.7708333395421505, |
| "step": 24 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 255.9583396911621, |
| "epoch": 0.26666666666666666, |
| "grad_norm": 0.19421663880348206, |
| "kl": 0.01947784423828125, |
| "learning_rate": 2.678571428571429e-06, |
| "loss": 0.0648, |
| "reward": 0.8750000335276127, |
| "reward_std": 0.46787626668810844, |
| "rewards/accuracy_reward": 0.1250000037252903, |
| "rewards/format_reward": 0.7500000186264515, |
| "step": 25 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 270.50000762939453, |
| "epoch": 0.2773333333333333, |
| "grad_norm": 0.13444656133651733, |
| "kl": 0.013866424560546875, |
| "learning_rate": 2.785714285714286e-06, |
| "loss": 0.132, |
| "reward": 0.833333358168602, |
| "reward_std": 0.43123577907681465, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/format_reward": 0.7500000074505806, |
| "step": 26 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 292.6458377838135, |
| "epoch": 0.288, |
| "grad_norm": 0.10114189982414246, |
| "kl": 0.01815032958984375, |
| "learning_rate": 2.892857142857143e-06, |
| "loss": 0.0158, |
| "reward": 0.9166666939854622, |
| "reward_std": 0.37223926186561584, |
| "rewards/accuracy_reward": 0.20833333767950535, |
| "rewards/format_reward": 0.7083333432674408, |
| "step": 27 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 263.29167079925537, |
| "epoch": 0.2986666666666667, |
| "grad_norm": 0.14505182206630707, |
| "kl": 0.022125244140625, |
| "learning_rate": 3e-06, |
| "loss": 0.082, |
| "reward": 0.9166667126119137, |
| "reward_std": 0.41380149498581886, |
| "rewards/accuracy_reward": 0.20833333767950535, |
| "rewards/format_reward": 0.7083333507180214, |
| "step": 28 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 256.2500057220459, |
| "epoch": 0.30933333333333335, |
| "grad_norm": 0.07133425027132034, |
| "kl": 0.026397705078125, |
| "learning_rate": 2.999882508108334e-06, |
| "loss": 0.0633, |
| "reward": 0.8958333432674408, |
| "reward_std": 0.1801304928958416, |
| "rewards/accuracy_reward": 0.0, |
| "rewards/format_reward": 0.8958333432674408, |
| "step": 29 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 241.0000057220459, |
| "epoch": 0.32, |
| "grad_norm": 0.1608133763074875, |
| "kl": 0.03473663330078125, |
| "learning_rate": 2.9995300508391286e-06, |
| "loss": 0.0251, |
| "reward": 1.0416666939854622, |
| "reward_std": 0.4235428161919117, |
| "rewards/accuracy_reward": 0.1875000074505806, |
| "rewards/format_reward": 0.854166679084301, |
| "step": 30 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 242.22917366027832, |
| "epoch": 0.33066666666666666, |
| "grad_norm": 0.15004193782806396, |
| "kl": 0.02217864990234375, |
| "learning_rate": 2.998942683406879e-06, |
| "loss": 0.0159, |
| "reward": 1.0000000223517418, |
| "reward_std": 0.5234047770500183, |
| "rewards/accuracy_reward": 0.1875000037252903, |
| "rewards/format_reward": 0.8125000298023224, |
| "step": 31 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 195.08334064483643, |
| "epoch": 0.3413333333333333, |
| "grad_norm": 0.1392156332731247, |
| "kl": 0.03990936279296875, |
| "learning_rate": 2.998120497826133e-06, |
| "loss": 0.048, |
| "reward": 0.9791666939854622, |
| "reward_std": 0.29669977352023125, |
| "rewards/accuracy_reward": 0.1458333358168602, |
| "rewards/format_reward": 0.8333333469927311, |
| "step": 32 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 229.68751049041748, |
| "epoch": 0.352, |
| "grad_norm": 0.15871571004390717, |
| "kl": 0.030364990234375, |
| "learning_rate": 2.9970636228970757e-06, |
| "loss": 0.0226, |
| "reward": 1.0416666939854622, |
| "reward_std": 0.2957112565636635, |
| "rewards/accuracy_reward": 0.10416666977107525, |
| "rewards/format_reward": 0.9375000074505806, |
| "step": 33 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 158.83333778381348, |
| "epoch": 0.3626666666666667, |
| "grad_norm": 0.16293902695178986, |
| "kl": 0.02352142333984375, |
| "learning_rate": 2.995772224185354e-06, |
| "loss": 0.0433, |
| "reward": 1.2708333432674408, |
| "reward_std": 0.37061261385679245, |
| "rewards/accuracy_reward": 0.2916666716337204, |
| "rewards/format_reward": 0.9791666716337204, |
| "step": 34 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 249.41667556762695, |
| "epoch": 0.37333333333333335, |
| "grad_norm": 0.1561712920665741, |
| "kl": 0.02712249755859375, |
| "learning_rate": 2.9942465039961373e-06, |
| "loss": 0.0097, |
| "reward": 1.1041666716337204, |
| "reward_std": 0.34607556089758873, |
| "rewards/accuracy_reward": 0.1875000037252903, |
| "rewards/format_reward": 0.9166666865348816, |
| "step": 35 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 269.75000762939453, |
| "epoch": 0.384, |
| "grad_norm": 0.12326981872320175, |
| "kl": 0.0439910888671875, |
| "learning_rate": 2.992486701342427e-06, |
| "loss": 0.0882, |
| "reward": 1.0000000298023224, |
| "reward_std": 0.39079636335372925, |
| "rewards/accuracy_reward": 0.1250000037252903, |
| "rewards/format_reward": 0.8750000223517418, |
| "step": 36 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 195.60417461395264, |
| "epoch": 0.39466666666666667, |
| "grad_norm": 0.14010730385780334, |
| "kl": 0.0265045166015625, |
| "learning_rate": 2.9904930919076145e-06, |
| "loss": 0.0183, |
| "reward": 1.1875000298023224, |
| "reward_std": 0.2996268570423126, |
| "rewards/accuracy_reward": 0.2291666716337204, |
| "rewards/format_reward": 0.9583333432674408, |
| "step": 37 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 264.81250953674316, |
| "epoch": 0.4053333333333333, |
| "grad_norm": 0.16210472583770752, |
| "kl": 0.0369720458984375, |
| "learning_rate": 2.9882659880022905e-06, |
| "loss": 0.0318, |
| "reward": 1.1250000521540642, |
| "reward_std": 0.5272140763700008, |
| "rewards/accuracy_reward": 0.3333333432674408, |
| "rewards/format_reward": 0.7916666865348816, |
| "step": 38 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 180.5833396911621, |
| "epoch": 0.416, |
| "grad_norm": 0.13184523582458496, |
| "kl": 0.0410919189453125, |
| "learning_rate": 2.985805738515323e-06, |
| "loss": -0.0404, |
| "reward": 1.2291666865348816, |
| "reward_std": 0.3411714509129524, |
| "rewards/accuracy_reward": 0.25000000558793545, |
| "rewards/format_reward": 0.9791666716337204, |
| "step": 39 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 206.54166793823242, |
| "epoch": 0.4266666666666667, |
| "grad_norm": 0.13690926134586334, |
| "kl": 0.039154052734375, |
| "learning_rate": 2.9831127288592004e-06, |
| "loss": 0.0823, |
| "reward": 1.062500037252903, |
| "reward_std": 0.40168892219662666, |
| "rewards/accuracy_reward": 0.1666666716337204, |
| "rewards/format_reward": 0.8958333507180214, |
| "step": 40 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 258.16667556762695, |
| "epoch": 0.43733333333333335, |
| "grad_norm": 0.17689989507198334, |
| "kl": 0.03472900390625, |
| "learning_rate": 2.9801873809096543e-06, |
| "loss": 0.0241, |
| "reward": 1.145833346992731, |
| "reward_std": 0.46066781133413315, |
| "rewards/accuracy_reward": 0.3125000074505806, |
| "rewards/format_reward": 0.8333333414047956, |
| "step": 41 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 201.5000057220459, |
| "epoch": 0.448, |
| "grad_norm": 0.15305694937705994, |
| "kl": 0.08642578125, |
| "learning_rate": 2.97703015293957e-06, |
| "loss": -0.0321, |
| "reward": 1.1875000298023224, |
| "reward_std": 0.4862990714609623, |
| "rewards/accuracy_reward": 0.2916666753590107, |
| "rewards/format_reward": 0.8958333507180214, |
| "step": 42 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 164.06250286102295, |
| "epoch": 0.45866666666666667, |
| "grad_norm": 0.14044952392578125, |
| "kl": 0.0565032958984375, |
| "learning_rate": 2.9736415395471975e-06, |
| "loss": 0.0723, |
| "reward": 1.333333358168602, |
| "reward_std": 0.4847872518002987, |
| "rewards/accuracy_reward": 0.45833334513008595, |
| "rewards/format_reward": 0.8750000149011612, |
| "step": 43 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 190.35417366027832, |
| "epoch": 0.4693333333333333, |
| "grad_norm": 0.16754870116710663, |
| "kl": 0.05095672607421875, |
| "learning_rate": 2.970022071578666e-06, |
| "loss": 0.0547, |
| "reward": 1.4166667014360428, |
| "reward_std": 0.43123577907681465, |
| "rewards/accuracy_reward": 0.5416666846722364, |
| "rewards/format_reward": 0.8750000074505806, |
| "step": 44 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 257.3541736602783, |
| "epoch": 0.48, |
| "grad_norm": 0.11928242444992065, |
| "kl": 0.05057525634765625, |
| "learning_rate": 2.9661723160448267e-06, |
| "loss": -0.0214, |
| "reward": 1.2083333656191826, |
| "reward_std": 0.3922024853527546, |
| "rewards/accuracy_reward": 0.29166667349636555, |
| "rewards/format_reward": 0.916666679084301, |
| "step": 45 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 248.25000953674316, |
| "epoch": 0.49066666666666664, |
| "grad_norm": 0.406486839056015, |
| "kl": 0.0576629638671875, |
| "learning_rate": 2.962092876032427e-06, |
| "loss": 0.0367, |
| "reward": 1.2708333656191826, |
| "reward_std": 0.36417657881975174, |
| "rewards/accuracy_reward": 0.3750000111758709, |
| "rewards/format_reward": 0.8958333432674408, |
| "step": 46 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 240.0833387374878, |
| "epoch": 0.5013333333333333, |
| "grad_norm": 0.10968135297298431, |
| "kl": 0.059417724609375, |
| "learning_rate": 2.957784390609632e-06, |
| "loss": 0.0364, |
| "reward": 1.3958333656191826, |
| "reward_std": 0.34674229472875595, |
| "rewards/accuracy_reward": 0.4791666753590107, |
| "rewards/format_reward": 0.916666679084301, |
| "step": 47 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 255.45833778381348, |
| "epoch": 0.512, |
| "grad_norm": 0.09772133082151413, |
| "kl": 0.0493011474609375, |
| "learning_rate": 2.9532475347259113e-06, |
| "loss": 0.0263, |
| "reward": 1.5416667014360428, |
| "reward_std": 0.3881702236831188, |
| "rewards/accuracy_reward": 0.625000013038516, |
| "rewards/format_reward": 0.916666679084301, |
| "step": 48 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 230.87500762939453, |
| "epoch": 0.5226666666666666, |
| "grad_norm": 0.1263599842786789, |
| "kl": 0.0523681640625, |
| "learning_rate": 2.9484830191063057e-06, |
| "loss": 0.0451, |
| "reward": 1.3541666939854622, |
| "reward_std": 0.5295314528048038, |
| "rewards/accuracy_reward": 0.4791666753590107, |
| "rewards/format_reward": 0.8750000149011612, |
| "step": 49 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 312.2708377838135, |
| "epoch": 0.5333333333333333, |
| "grad_norm": 0.08942104130983353, |
| "kl": 0.0318603515625, |
| "learning_rate": 2.9434915901400857e-06, |
| "loss": 0.06, |
| "reward": 1.3125000298023224, |
| "reward_std": 0.3947119526565075, |
| "rewards/accuracy_reward": 0.41666667349636555, |
| "rewards/format_reward": 0.8958333432674408, |
| "step": 50 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 233.83334159851074, |
| "epoch": 0.544, |
| "grad_norm": 0.15756255388259888, |
| "kl": 0.071136474609375, |
| "learning_rate": 2.938274029763826e-06, |
| "loss": 0.054, |
| "reward": 1.250000037252903, |
| "reward_std": 0.45534609258174896, |
| "rewards/accuracy_reward": 0.37500000931322575, |
| "rewards/format_reward": 0.8750000223517418, |
| "step": 51 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 258.87500762939453, |
| "epoch": 0.5546666666666666, |
| "grad_norm": 0.11138404905796051, |
| "kl": 0.033782958984375, |
| "learning_rate": 2.9328311553389116e-06, |
| "loss": 0.0751, |
| "reward": 1.354166716337204, |
| "reward_std": 0.5238114185631275, |
| "rewards/accuracy_reward": 0.4583333432674408, |
| "rewards/format_reward": 0.8958333507180214, |
| "step": 52 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 256.56250953674316, |
| "epoch": 0.5653333333333334, |
| "grad_norm": 0.10758252441883087, |
| "kl": 0.0422515869140625, |
| "learning_rate": 2.927163819523492e-06, |
| "loss": 0.0381, |
| "reward": 1.437500037252903, |
| "reward_std": 0.3842546418309212, |
| "rewards/accuracy_reward": 0.5208333507180214, |
| "rewards/format_reward": 0.916666679084301, |
| "step": 53 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 309.83333587646484, |
| "epoch": 0.576, |
| "grad_norm": 0.10542929172515869, |
| "kl": 0.0524139404296875, |
| "learning_rate": 2.9212729101389084e-06, |
| "loss": 0.0156, |
| "reward": 1.5416666939854622, |
| "reward_std": 0.5237057246267796, |
| "rewards/accuracy_reward": 0.6875000149011612, |
| "rewards/format_reward": 0.8541666939854622, |
| "step": 54 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 369.87500762939453, |
| "epoch": 0.5866666666666667, |
| "grad_norm": 0.07796894013881683, |
| "kl": 0.04473876953125, |
| "learning_rate": 2.9151593500306103e-06, |
| "loss": 0.0122, |
| "reward": 1.2500000298023224, |
| "reward_std": 0.3506578877568245, |
| "rewards/accuracy_reward": 0.29166666977107525, |
| "rewards/format_reward": 0.9583333432674408, |
| "step": 55 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 243.08334159851074, |
| "epoch": 0.5973333333333334, |
| "grad_norm": 0.12414506822824478, |
| "kl": 0.0529937744140625, |
| "learning_rate": 2.9088240969235863e-06, |
| "loss": 0.0219, |
| "reward": 1.4583333879709244, |
| "reward_std": 0.5347039774060249, |
| "rewards/accuracy_reward": 0.5000000149011612, |
| "rewards/format_reward": 0.9583333432674408, |
| "step": 56 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 176.43750476837158, |
| "epoch": 0.608, |
| "grad_norm": 0.12592697143554688, |
| "kl": 0.0508575439453125, |
| "learning_rate": 2.902268143272333e-06, |
| "loss": -0.0349, |
| "reward": 1.8125000298023224, |
| "reward_std": 0.33713918924331665, |
| "rewards/accuracy_reward": 0.8125000223517418, |
| "rewards/format_reward": 1.0, |
| "step": 57 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 284.16666984558105, |
| "epoch": 0.6186666666666667, |
| "grad_norm": 0.08683699369430542, |
| "kl": 0.0404052734375, |
| "learning_rate": 2.895492516105378e-06, |
| "loss": 0.0729, |
| "reward": 1.4791666865348816, |
| "reward_std": 0.37575671821832657, |
| "rewards/accuracy_reward": 0.5208333507180214, |
| "rewards/format_reward": 0.9583333432674408, |
| "step": 58 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 305.87500381469727, |
| "epoch": 0.6293333333333333, |
| "grad_norm": 0.09318725019693375, |
| "kl": 0.03826904296875, |
| "learning_rate": 2.888498276864393e-06, |
| "loss": 0.0262, |
| "reward": 1.2708333656191826, |
| "reward_std": 0.36417657881975174, |
| "rewards/accuracy_reward": 0.33333333767950535, |
| "rewards/format_reward": 0.9375000074505806, |
| "step": 59 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 337.0416736602783, |
| "epoch": 0.64, |
| "grad_norm": 0.09196493029594421, |
| "kl": 0.045013427734375, |
| "learning_rate": 2.8812865212379087e-06, |
| "loss": 0.0465, |
| "reward": 1.4791667014360428, |
| "reward_std": 0.33713918924331665, |
| "rewards/accuracy_reward": 0.500000013038516, |
| "rewards/format_reward": 0.9791666716337204, |
| "step": 60 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 285.41667556762695, |
| "epoch": 0.6506666666666666, |
| "grad_norm": 0.12164270132780075, |
| "kl": 0.0420989990234375, |
| "learning_rate": 2.8738583789896743e-06, |
| "loss": 0.0701, |
| "reward": 1.5208333879709244, |
| "reward_std": 0.5107103250920773, |
| "rewards/accuracy_reward": 0.562500013038516, |
| "rewards/format_reward": 0.9583333432674408, |
| "step": 61 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 346.9166793823242, |
| "epoch": 0.6613333333333333, |
| "grad_norm": 0.08939547091722488, |
| "kl": 0.0489654541015625, |
| "learning_rate": 2.866215013781668e-06, |
| "loss": -0.0066, |
| "reward": 1.3750000447034836, |
| "reward_std": 0.4743013270199299, |
| "rewards/accuracy_reward": 0.45833334140479565, |
| "rewards/format_reward": 0.916666679084301, |
| "step": 62 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 301.2708435058594, |
| "epoch": 0.672, |
| "grad_norm": 0.10984641313552856, |
| "kl": 0.0488128662109375, |
| "learning_rate": 2.858357622991806e-06, |
| "loss": -0.0121, |
| "reward": 1.541666716337204, |
| "reward_std": 0.46232306212186813, |
| "rewards/accuracy_reward": 0.6458333563059568, |
| "rewards/format_reward": 0.8958333507180214, |
| "step": 63 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 269.2500066757202, |
| "epoch": 0.6826666666666666, |
| "grad_norm": 0.11388546973466873, |
| "kl": 0.067291259765625, |
| "learning_rate": 2.8502874375263646e-06, |
| "loss": 0.0396, |
| "reward": 1.2916666977107525, |
| "reward_std": 0.3332235999405384, |
| "rewards/accuracy_reward": 0.43750001303851604, |
| "rewards/format_reward": 0.8541666753590107, |
| "step": 64 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 227.08334159851074, |
| "epoch": 0.6933333333333334, |
| "grad_norm": 0.13996504247188568, |
| "kl": 0.078887939453125, |
| "learning_rate": 2.8420057216271533e-06, |
| "loss": 0.0677, |
| "reward": 1.4166667014360428, |
| "reward_std": 0.3977733291685581, |
| "rewards/accuracy_reward": 0.5208333432674408, |
| "rewards/format_reward": 0.8958333358168602, |
| "step": 65 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 279.1458396911621, |
| "epoch": 0.704, |
| "grad_norm": 0.061006851494312286, |
| "kl": 0.0504913330078125, |
| "learning_rate": 2.8335137726734607e-06, |
| "loss": -0.0082, |
| "reward": 1.5000000149011612, |
| "reward_std": 0.18404608592391014, |
| "rewards/accuracy_reward": 0.5208333358168602, |
| "rewards/format_reward": 0.9791666716337204, |
| "step": 66 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 227.02083587646484, |
| "epoch": 0.7146666666666667, |
| "grad_norm": 0.12499740719795227, |
| "kl": 0.0751800537109375, |
| "learning_rate": 2.8248129209788164e-06, |
| "loss": -0.0358, |
| "reward": 1.479166716337204, |
| "reward_std": 0.47038573771715164, |
| "rewards/accuracy_reward": 0.5625000186264515, |
| "rewards/format_reward": 0.916666679084301, |
| "step": 67 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 279.9791774749756, |
| "epoch": 0.7253333333333334, |
| "grad_norm": 0.08705603331327438, |
| "kl": 0.0396881103515625, |
| "learning_rate": 2.8159045295825857e-06, |
| "loss": -0.0075, |
| "reward": 1.4583333730697632, |
| "reward_std": 0.45827317237854004, |
| "rewards/accuracy_reward": 0.500000013038516, |
| "rewards/format_reward": 0.9583333432674408, |
| "step": 68 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 240.45834159851074, |
| "epoch": 0.736, |
| "grad_norm": 0.1296224296092987, |
| "kl": 0.04766845703125, |
| "learning_rate": 2.806789994036445e-06, |
| "loss": 0.0998, |
| "reward": 1.4583333879709244, |
| "reward_std": 0.5228229016065598, |
| "rewards/accuracy_reward": 0.47916668094694614, |
| "rewards/format_reward": 0.9791666716337204, |
| "step": 69 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 301.2916793823242, |
| "epoch": 0.7466666666666667, |
| "grad_norm": 0.10123682767152786, |
| "kl": 0.06207275390625, |
| "learning_rate": 2.7974707421857584e-06, |
| "loss": 0.0305, |
| "reward": 1.520833358168602, |
| "reward_std": 0.2982207275927067, |
| "rewards/accuracy_reward": 0.645833345130086, |
| "rewards/format_reward": 0.8750000074505806, |
| "step": 70 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 292.62500762939453, |
| "epoch": 0.7573333333333333, |
| "grad_norm": 0.10017960518598557, |
| "kl": 0.0426025390625, |
| "learning_rate": 2.7879482339458973e-06, |
| "loss": 0.0018, |
| "reward": 1.458333358168602, |
| "reward_std": 0.2861081585288048, |
| "rewards/accuracy_reward": 0.479166679084301, |
| "rewards/format_reward": 0.9791666716337204, |
| "step": 71 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 300.12500953674316, |
| "epoch": 0.768, |
| "grad_norm": 0.12141850590705872, |
| "kl": 0.0420684814453125, |
| "learning_rate": 2.778223961073537e-06, |
| "loss": 0.0561, |
| "reward": 1.4375000298023224, |
| "reward_std": 0.41619613766670227, |
| "rewards/accuracy_reward": 0.5000000186264515, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 72 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 232.16666984558105, |
| "epoch": 0.7786666666666666, |
| "grad_norm": 0.12889333069324493, |
| "kl": 0.057708740234375, |
| "learning_rate": 2.7682994469329646e-06, |
| "loss": -0.04, |
| "reward": 1.5625000596046448, |
| "reward_std": 0.5363415889441967, |
| "rewards/accuracy_reward": 0.6250000223517418, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 73 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 307.2291793823242, |
| "epoch": 0.7893333333333333, |
| "grad_norm": 0.13781766593456268, |
| "kl": 0.0377960205078125, |
| "learning_rate": 2.7581762462574334e-06, |
| "loss": 0.014, |
| "reward": 1.3958333730697632, |
| "reward_std": 0.4997011013329029, |
| "rewards/accuracy_reward": 0.47916668467223644, |
| "rewards/format_reward": 0.9166666716337204, |
| "step": 74 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 397.3958511352539, |
| "epoch": 0.8, |
| "grad_norm": 0.08684942871332169, |
| "kl": 0.0388031005859375, |
| "learning_rate": 2.747855944905606e-06, |
| "loss": 0.0606, |
| "reward": 1.229166716337204, |
| "reward_std": 0.5326506160199642, |
| "rewards/accuracy_reward": 0.35416666977107525, |
| "rewards/format_reward": 0.8750000149011612, |
| "step": 75 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 299.77084159851074, |
| "epoch": 0.8106666666666666, |
| "grad_norm": 0.10988432168960571, |
| "kl": 0.0503997802734375, |
| "learning_rate": 2.73734015961312e-06, |
| "loss": -0.0247, |
| "reward": 1.3333333730697632, |
| "reward_std": 0.5198958218097687, |
| "rewards/accuracy_reward": 0.5208333469927311, |
| "rewards/format_reward": 0.8125000149011612, |
| "step": 76 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 324.7916774749756, |
| "epoch": 0.8213333333333334, |
| "grad_norm": 0.13095711171627045, |
| "kl": 0.070037841796875, |
| "learning_rate": 2.7266305377393183e-06, |
| "loss": 0.0494, |
| "reward": 1.5833333432674408, |
| "reward_std": 0.400033675134182, |
| "rewards/accuracy_reward": 0.6666666734963655, |
| "rewards/format_reward": 0.9166666865348816, |
| "step": 77 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 356.4583435058594, |
| "epoch": 0.832, |
| "grad_norm": 0.10965587943792343, |
| "kl": 0.04058837890625, |
| "learning_rate": 2.7157287570091765e-06, |
| "loss": 0.036, |
| "reward": 1.354166716337204, |
| "reward_std": 0.6204284392297268, |
| "rewards/accuracy_reward": 0.47916668094694614, |
| "rewards/format_reward": 0.8750000223517418, |
| "step": 78 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 352.4166831970215, |
| "epoch": 0.8426666666666667, |
| "grad_norm": 0.11178199201822281, |
| "kl": 0.048248291015625, |
| "learning_rate": 2.7046365252504835e-06, |
| "loss": 0.019, |
| "reward": 1.5000000298023224, |
| "reward_std": 0.5423886179924011, |
| "rewards/accuracy_reward": 0.6250000186264515, |
| "rewards/format_reward": 0.8750000223517418, |
| "step": 79 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 318.5000171661377, |
| "epoch": 0.8533333333333334, |
| "grad_norm": 0.11200235784053802, |
| "kl": 0.04592132568359375, |
| "learning_rate": 2.693355580126295e-06, |
| "loss": 0.0629, |
| "reward": 1.416666716337204, |
| "reward_std": 0.5645233504474163, |
| "rewards/accuracy_reward": 0.4791666828095913, |
| "rewards/format_reward": 0.9375000074505806, |
| "step": 80 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 349.708345413208, |
| "epoch": 0.864, |
| "grad_norm": 0.09885663539171219, |
| "kl": 0.0634307861328125, |
| "learning_rate": 2.6818876888627203e-06, |
| "loss": 0.0439, |
| "reward": 1.5416666865348816, |
| "reward_std": 0.49666832759976387, |
| "rewards/accuracy_reward": 0.6875000074505806, |
| "rewards/format_reward": 0.8541666865348816, |
| "step": 81 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 292.77084159851074, |
| "epoch": 0.8746666666666667, |
| "grad_norm": 0.09445224702358246, |
| "kl": 0.0454254150390625, |
| "learning_rate": 2.6702346479720777e-06, |
| "loss": 0.0829, |
| "reward": 1.4791667088866234, |
| "reward_std": 0.48782002553343773, |
| "rewards/accuracy_reward": 0.6041666865348816, |
| "rewards/format_reward": 0.8750000223517418, |
| "step": 82 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 373.3125057220459, |
| "epoch": 0.8853333333333333, |
| "grad_norm": 0.6235383152961731, |
| "kl": 0.069488525390625, |
| "learning_rate": 2.6583982829714576e-06, |
| "loss": 0.04, |
| "reward": 1.3958333730697632, |
| "reward_std": 0.4621887616813183, |
| "rewards/accuracy_reward": 0.520833345130086, |
| "rewards/format_reward": 0.8750000149011612, |
| "step": 83 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 298.9583396911621, |
| "epoch": 0.896, |
| "grad_norm": 0.10517609119415283, |
| "kl": 0.0439910888671875, |
| "learning_rate": 2.6463804480967447e-06, |
| "loss": 0.0363, |
| "reward": 1.5000000298023224, |
| "reward_std": 0.4338619112968445, |
| "rewards/accuracy_reward": 0.5833333469927311, |
| "rewards/format_reward": 0.916666679084301, |
| "step": 84 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 347.3750057220459, |
| "epoch": 0.9066666666666666, |
| "grad_norm": 0.10065872222185135, |
| "kl": 0.0599365234375, |
| "learning_rate": 2.6341830260121445e-06, |
| "loss": 0.1186, |
| "reward": 1.5000000447034836, |
| "reward_std": 0.5534840673208237, |
| "rewards/accuracy_reward": 0.625000013038516, |
| "rewards/format_reward": 0.8750000223517418, |
| "step": 85 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 387.0833396911621, |
| "epoch": 0.9173333333333333, |
| "grad_norm": 0.12794648110866547, |
| "kl": 0.090423583984375, |
| "learning_rate": 2.6218079275152485e-06, |
| "loss": 0.045, |
| "reward": 1.2500000447034836, |
| "reward_std": 0.4701542444527149, |
| "rewards/accuracy_reward": 0.4583333469927311, |
| "rewards/format_reward": 0.7916666828095913, |
| "step": 86 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 391.37501525878906, |
| "epoch": 0.928, |
| "grad_norm": 0.09694866091012955, |
| "kl": 0.0460968017578125, |
| "learning_rate": 2.609257091237699e-06, |
| "loss": 0.1368, |
| "reward": 1.2916667014360428, |
| "reward_std": 0.57484245672822, |
| "rewards/accuracy_reward": 0.41666667722165585, |
| "rewards/format_reward": 0.8750000149011612, |
| "step": 87 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 362.1875104904175, |
| "epoch": 0.9386666666666666, |
| "grad_norm": 0.08955412358045578, |
| "kl": 0.0487823486328125, |
| "learning_rate": 2.5965324833414926e-06, |
| "loss": 0.0046, |
| "reward": 1.5416667014360428, |
| "reward_std": 0.39372343569993973, |
| "rewards/accuracy_reward": 0.625000013038516, |
| "rewards/format_reward": 0.9166666865348816, |
| "step": 88 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 412.12501525878906, |
| "epoch": 0.9493333333333334, |
| "grad_norm": 0.5077324509620667, |
| "kl": 0.1267852783203125, |
| "learning_rate": 2.5836360972109642e-06, |
| "loss": 0.0258, |
| "reward": 1.2500000223517418, |
| "reward_std": 0.46246128901839256, |
| "rewards/accuracy_reward": 0.4583333395421505, |
| "rewards/format_reward": 0.7916666865348816, |
| "step": 89 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 304.1666717529297, |
| "epoch": 0.96, |
| "grad_norm": 0.11206508427858353, |
| "kl": 0.064788818359375, |
| "learning_rate": 2.5705699531405174e-06, |
| "loss": 0.0635, |
| "reward": 1.708333358168602, |
| "reward_std": 0.4028032459318638, |
| "rewards/accuracy_reward": 0.7916666753590107, |
| "rewards/format_reward": 0.9166666865348816, |
| "step": 90 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 345.958345413208, |
| "epoch": 0.9706666666666667, |
| "grad_norm": 0.09778965264558792, |
| "kl": 0.050811767578125, |
| "learning_rate": 2.55733609801813e-06, |
| "loss": -0.0167, |
| "reward": 1.5416667014360428, |
| "reward_std": 0.3776952736079693, |
| "rewards/accuracy_reward": 0.6458333469927311, |
| "rewards/format_reward": 0.8958333358168602, |
| "step": 91 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 296.50000381469727, |
| "epoch": 0.9813333333333333, |
| "grad_norm": 0.12521831691265106, |
| "kl": 0.0517120361328125, |
| "learning_rate": 2.5439366050046985e-06, |
| "loss": 0.0045, |
| "reward": 1.5416666865348816, |
| "reward_std": 0.42428741604089737, |
| "rewards/accuracy_reward": 0.6041666753590107, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 92 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 335.70834159851074, |
| "epoch": 0.992, |
| "grad_norm": 0.10977984219789505, |
| "kl": 0.0595550537109375, |
| "learning_rate": 2.5303735732092655e-06, |
| "loss": 0.0105, |
| "reward": 1.4375000149011612, |
| "reward_std": 0.47319112718105316, |
| "rewards/accuracy_reward": 0.6458333488553762, |
| "rewards/format_reward": 0.791666679084301, |
| "step": 93 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 298.2500057220459, |
| "epoch": 1.0106666666666666, |
| "grad_norm": 0.11219625920057297, |
| "kl": 0.0492706298828125, |
| "learning_rate": 2.51664912736018e-06, |
| "loss": -0.0016, |
| "reward": 1.5625000447034836, |
| "reward_std": 0.48367293924093246, |
| "rewards/accuracy_reward": 0.5625000167638063, |
| "rewards/format_reward": 1.0, |
| "step": 94 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 291.3125114440918, |
| "epoch": 1.0213333333333334, |
| "grad_norm": 0.097702756524086, |
| "kl": 0.0426025390625, |
| "learning_rate": 2.5027654174722495e-06, |
| "loss": 0.0599, |
| "reward": 1.5000000298023224, |
| "reward_std": 0.45132480561733246, |
| "rewards/accuracy_reward": 0.6041666828095913, |
| "rewards/format_reward": 0.8958333507180214, |
| "step": 95 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 289.00000953674316, |
| "epoch": 1.032, |
| "grad_norm": 0.12104532122612, |
| "kl": 0.0511474609375, |
| "learning_rate": 2.4887246185099237e-06, |
| "loss": 0.0013, |
| "reward": 1.6875000298023224, |
| "reward_std": 0.4161961302161217, |
| "rewards/accuracy_reward": 0.7500000204890966, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 96 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 218.04167366027832, |
| "epoch": 1.0426666666666666, |
| "grad_norm": 0.148694708943367, |
| "kl": 0.09942626953125, |
| "learning_rate": 2.474528930046578e-06, |
| "loss": -0.0069, |
| "reward": 1.6041666865348816, |
| "reward_std": 0.33713919296860695, |
| "rewards/accuracy_reward": 0.6458333395421505, |
| "rewards/format_reward": 0.9583333432674408, |
| "step": 97 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 246.31250858306885, |
| "epoch": 1.0533333333333332, |
| "grad_norm": 0.08420921862125397, |
| "kl": 0.0697021484375, |
| "learning_rate": 2.4601805759199317e-06, |
| "loss": -0.0038, |
| "reward": 1.645833358168602, |
| "reward_std": 0.26070838421583176, |
| "rewards/accuracy_reward": 0.666666679084301, |
| "rewards/format_reward": 0.9791666716337204, |
| "step": 98 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 337.3333396911621, |
| "epoch": 1.064, |
| "grad_norm": 0.12480373680591583, |
| "kl": 0.054962158203125, |
| "learning_rate": 2.445681803883678e-06, |
| "loss": 0.0435, |
| "reward": 1.4791667014360428, |
| "reward_std": 0.5356924869120121, |
| "rewards/accuracy_reward": 0.5625000111758709, |
| "rewards/format_reward": 0.9166666865348816, |
| "step": 99 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 257.0625057220459, |
| "epoch": 1.0746666666666667, |
| "grad_norm": 0.1519816517829895, |
| "kl": 0.0865478515625, |
| "learning_rate": 2.4310348852553554e-06, |
| "loss": -0.0236, |
| "reward": 1.520833358168602, |
| "reward_std": 0.5126873664557934, |
| "rewards/accuracy_reward": 0.6250000186264515, |
| "rewards/format_reward": 0.895833358168602, |
| "step": 100 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 279.02084732055664, |
| "epoch": 1.0853333333333333, |
| "grad_norm": 0.10319739580154419, |
| "kl": 0.0516510009765625, |
| "learning_rate": 2.4162421145605308e-06, |
| "loss": 0.0704, |
| "reward": 1.5416666865348816, |
| "reward_std": 0.37979811057448387, |
| "rewards/accuracy_reward": 0.6458333507180214, |
| "rewards/format_reward": 0.8958333432674408, |
| "step": 101 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 294.45834159851074, |
| "epoch": 1.096, |
| "grad_norm": 0.13318555057048798, |
| "kl": 0.069122314453125, |
| "learning_rate": 2.4013058091733546e-06, |
| "loss": 0.0196, |
| "reward": 1.3958333730697632, |
| "reward_std": 0.4621887616813183, |
| "rewards/accuracy_reward": 0.4583333469927311, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 102 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 257.93750953674316, |
| "epoch": 1.1066666666666667, |
| "grad_norm": 0.49007534980773926, |
| "kl": 0.1214599609375, |
| "learning_rate": 2.3862283089535265e-06, |
| "loss": 0.0464, |
| "reward": 1.6041666865348816, |
| "reward_std": 0.4967849962413311, |
| "rewards/accuracy_reward": 0.750000013038516, |
| "rewards/format_reward": 0.8541666865348816, |
| "step": 103 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 215.58334159851074, |
| "epoch": 1.1173333333333333, |
| "grad_norm": 0.15618140995502472, |
| "kl": 0.0696868896484375, |
| "learning_rate": 2.371011975879742e-06, |
| "loss": -0.0101, |
| "reward": 1.5000000149011612, |
| "reward_std": 0.43767181411385536, |
| "rewards/accuracy_reward": 0.6875000111758709, |
| "rewards/format_reward": 0.8125000149011612, |
| "step": 104 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 220.60417366027832, |
| "epoch": 1.1280000000000001, |
| "grad_norm": 0.34512364864349365, |
| "kl": 0.144317626953125, |
| "learning_rate": 2.3556591936796777e-06, |
| "loss": 0.0326, |
| "reward": 1.5208333730697632, |
| "reward_std": 0.44323352724313736, |
| "rewards/accuracy_reward": 0.687500013038516, |
| "rewards/format_reward": 0.8333333507180214, |
| "step": 105 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 238.3750057220459, |
| "epoch": 1.1386666666666667, |
| "grad_norm": 0.1124984472990036, |
| "kl": 0.0436859130859375, |
| "learning_rate": 2.340172367456564e-06, |
| "loss": -0.0044, |
| "reward": 1.6041667014360428, |
| "reward_std": 0.5307883769273758, |
| "rewards/accuracy_reward": 0.729166679084301, |
| "rewards/format_reward": 0.8750000149011612, |
| "step": 106 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 235.91667366027832, |
| "epoch": 1.1493333333333333, |
| "grad_norm": 0.09784164279699326, |
| "kl": 0.053192138671875, |
| "learning_rate": 2.3245539233124133e-06, |
| "loss": -0.0207, |
| "reward": 1.5000000298023224, |
| "reward_std": 0.3506578914821148, |
| "rewards/accuracy_reward": 0.5625000093132257, |
| "rewards/format_reward": 0.9375000074505806, |
| "step": 107 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 222.02083778381348, |
| "epoch": 1.16, |
| "grad_norm": 0.11026975512504578, |
| "kl": 0.0535888671875, |
| "learning_rate": 2.308806307967955e-06, |
| "loss": 0.0335, |
| "reward": 1.645833358168602, |
| "reward_std": 0.36124950647354126, |
| "rewards/accuracy_reward": 0.7500000223517418, |
| "rewards/format_reward": 0.8958333358168602, |
| "step": 108 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 280.7291793823242, |
| "epoch": 1.1706666666666667, |
| "grad_norm": 0.13585244119167328, |
| "kl": 0.0595703125, |
| "learning_rate": 2.2929319883793442e-06, |
| "loss": 0.026, |
| "reward": 1.479166716337204, |
| "reward_std": 0.5093041993677616, |
| "rewards/accuracy_reward": 0.666666679084301, |
| "rewards/format_reward": 0.8125000149011612, |
| "step": 109 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 319.4583396911621, |
| "epoch": 1.1813333333333333, |
| "grad_norm": 0.11203665286302567, |
| "kl": 0.056427001953125, |
| "learning_rate": 2.276933451351696e-06, |
| "loss": 0.0592, |
| "reward": 1.2708333805203438, |
| "reward_std": 0.5797660648822784, |
| "rewards/accuracy_reward": 0.500000013038516, |
| "rewards/format_reward": 0.7708333432674408, |
| "step": 110 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 298.00000381469727, |
| "epoch": 1.192, |
| "grad_norm": 0.14347125589847565, |
| "kl": 0.056915283203125, |
| "learning_rate": 2.2608132031495184e-06, |
| "loss": 0.0683, |
| "reward": 1.4583333730697632, |
| "reward_std": 0.6156517639756203, |
| "rewards/accuracy_reward": 0.6875000149011612, |
| "rewards/format_reward": 0.7708333507180214, |
| "step": 111 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 287.2916774749756, |
| "epoch": 1.2026666666666666, |
| "grad_norm": 0.11557626724243164, |
| "kl": 0.0446014404296875, |
| "learning_rate": 2.244573769104084e-06, |
| "loss": 0.0335, |
| "reward": 1.479166716337204, |
| "reward_std": 0.5093042030930519, |
| "rewards/accuracy_reward": 0.6666666716337204, |
| "rewards/format_reward": 0.8125000186264515, |
| "step": 112 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 312.93750762939453, |
| "epoch": 1.2133333333333334, |
| "grad_norm": 0.10706419497728348, |
| "kl": 0.045074462890625, |
| "learning_rate": 2.228217693217826e-06, |
| "loss": 0.0568, |
| "reward": 1.354166716337204, |
| "reward_std": 0.6839981600642204, |
| "rewards/accuracy_reward": 0.6041666828095913, |
| "rewards/format_reward": 0.7500000298023224, |
| "step": 113 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 352.7083435058594, |
| "epoch": 1.224, |
| "grad_norm": 0.10810855031013489, |
| "kl": 0.050567626953125, |
| "learning_rate": 2.211747537765808e-06, |
| "loss": 0.0611, |
| "reward": 1.2708333805203438, |
| "reward_std": 0.7179597914218903, |
| "rewards/accuracy_reward": 0.5625000149011612, |
| "rewards/format_reward": 0.7083333507180214, |
| "step": 114 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 265.6250057220459, |
| "epoch": 1.2346666666666666, |
| "grad_norm": 0.13173377513885498, |
| "kl": 0.217529296875, |
| "learning_rate": 2.195165882894322e-06, |
| "loss": -0.0123, |
| "reward": 1.4583333879709244, |
| "reward_std": 0.6409131288528442, |
| "rewards/accuracy_reward": 0.6875000111758709, |
| "rewards/format_reward": 0.770833358168602, |
| "step": 115 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 386.06250762939453, |
| "epoch": 1.2453333333333334, |
| "grad_norm": 0.09368361532688141, |
| "kl": 0.0453033447265625, |
| "learning_rate": 2.1784753262166986e-06, |
| "loss": 0.0297, |
| "reward": 1.2083333805203438, |
| "reward_std": 0.46787626668810844, |
| "rewards/accuracy_reward": 0.35416667349636555, |
| "rewards/format_reward": 0.854166679084301, |
| "step": 116 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 281.27084159851074, |
| "epoch": 1.256, |
| "grad_norm": 0.10779143124818802, |
| "kl": 0.054718017578125, |
| "learning_rate": 2.161678482406374e-06, |
| "loss": 0.0179, |
| "reward": 1.5000000447034836, |
| "reward_std": 0.4393179304897785, |
| "rewards/accuracy_reward": 0.6666666753590107, |
| "rewards/format_reward": 0.833333358168602, |
| "step": 117 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 350.04167079925537, |
| "epoch": 1.2666666666666666, |
| "grad_norm": 0.08757103979587555, |
| "kl": 0.053070068359375, |
| "learning_rate": 2.144777982787286e-06, |
| "loss": 0.0239, |
| "reward": 1.4166666865348816, |
| "reward_std": 0.5397448539733887, |
| "rewards/accuracy_reward": 0.604166679084301, |
| "rewards/format_reward": 0.8125000223517418, |
| "step": 118 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 248.54167556762695, |
| "epoch": 1.2773333333333334, |
| "grad_norm": 0.0667010247707367, |
| "kl": 0.058319091796875, |
| "learning_rate": 2.127776474921661e-06, |
| "loss": 0.004, |
| "reward": 1.8125000298023224, |
| "reward_std": 0.26070839166641235, |
| "rewards/accuracy_reward": 0.8541666753590107, |
| "rewards/format_reward": 0.9583333432674408, |
| "step": 119 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 340.87500953674316, |
| "epoch": 1.288, |
| "grad_norm": 0.1155412346124649, |
| "kl": 0.0574951171875, |
| "learning_rate": 2.1106766221952607e-06, |
| "loss": 0.0287, |
| "reward": 1.500000037252903, |
| "reward_std": 0.5172696895897388, |
| "rewards/accuracy_reward": 0.6041666828095913, |
| "rewards/format_reward": 0.8958333507180214, |
| "step": 120 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 300.62500953674316, |
| "epoch": 1.2986666666666666, |
| "grad_norm": 0.10186120867729187, |
| "kl": 0.053436279296875, |
| "learning_rate": 2.0934811034001427e-06, |
| "loss": 0.055, |
| "reward": 1.5000000149011612, |
| "reward_std": 0.400033675134182, |
| "rewards/accuracy_reward": 0.6250000111758709, |
| "rewards/format_reward": 0.8750000074505806, |
| "step": 121 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 292.72917556762695, |
| "epoch": 1.3093333333333335, |
| "grad_norm": 0.1855934113264084, |
| "kl": 0.0568084716796875, |
| "learning_rate": 2.0761926123150166e-06, |
| "loss": 0.0365, |
| "reward": 1.5625000298023224, |
| "reward_std": 0.2446802221238613, |
| "rewards/accuracy_reward": 0.5833333395421505, |
| "rewards/format_reward": 0.9791666716337204, |
| "step": 122 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 348.6666793823242, |
| "epoch": 1.32, |
| "grad_norm": 0.10667165368795395, |
| "kl": 0.076904296875, |
| "learning_rate": 2.058813857283244e-06, |
| "loss": 0.0521, |
| "reward": 1.5208333656191826, |
| "reward_std": 0.4552404023706913, |
| "rewards/accuracy_reward": 0.6458333544433117, |
| "rewards/format_reward": 0.8750000074505806, |
| "step": 123 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 328.22918224334717, |
| "epoch": 1.3306666666666667, |
| "grad_norm": 27.496614456176758, |
| "kl": 1.2711334228515625, |
| "learning_rate": 2.041347560788562e-06, |
| "loss": 0.1228, |
| "reward": 1.6041667014360428, |
| "reward_std": 0.4337562173604965, |
| "rewards/accuracy_reward": 0.6666666809469461, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 124 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 326.8333435058594, |
| "epoch": 1.3413333333333333, |
| "grad_norm": 0.13310205936431885, |
| "kl": 0.0502166748046875, |
| "learning_rate": 2.0237964590285924e-06, |
| "loss": 0.0235, |
| "reward": 1.5625000447034836, |
| "reward_std": 0.5452955961227417, |
| "rewards/accuracy_reward": 0.6458333488553762, |
| "rewards/format_reward": 0.9166666865348816, |
| "step": 125 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 217.8958396911621, |
| "epoch": 1.3519999999999999, |
| "grad_norm": 0.1421211063861847, |
| "kl": 0.07293701171875, |
| "learning_rate": 2.0061633014861975e-06, |
| "loss": 0.0336, |
| "reward": 1.5208333879709244, |
| "reward_std": 0.4217669740319252, |
| "rewards/accuracy_reward": 0.5625000186264515, |
| "rewards/format_reward": 0.9583333432674408, |
| "step": 126 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 287.79168128967285, |
| "epoch": 1.3626666666666667, |
| "grad_norm": 0.11143215000629425, |
| "kl": 0.078277587890625, |
| "learning_rate": 1.988450850498758e-06, |
| "loss": 0.0335, |
| "reward": 1.5416667014360428, |
| "reward_std": 0.3881702274084091, |
| "rewards/accuracy_reward": 0.6458333432674408, |
| "rewards/format_reward": 0.8958333507180214, |
| "step": 127 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 357.0833511352539, |
| "epoch": 1.3733333333333333, |
| "grad_norm": 0.08851885795593262, |
| "kl": 0.05126953125, |
| "learning_rate": 1.970661880825437e-06, |
| "loss": 0.0769, |
| "reward": 1.3125000223517418, |
| "reward_std": 0.4403064474463463, |
| "rewards/accuracy_reward": 0.4375000074505806, |
| "rewards/format_reward": 0.8750000223517418, |
| "step": 128 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 245.2708396911621, |
| "epoch": 1.384, |
| "grad_norm": 0.11802337318658829, |
| "kl": 0.0472564697265625, |
| "learning_rate": 1.952799179212498e-06, |
| "loss": 0.0265, |
| "reward": 1.583333358168602, |
| "reward_std": 0.40823065117001534, |
| "rewards/accuracy_reward": 0.625000013038516, |
| "rewards/format_reward": 0.9583333432674408, |
| "step": 129 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 185.97917079925537, |
| "epoch": 1.3946666666666667, |
| "grad_norm": 0.15199995040893555, |
| "kl": 0.080413818359375, |
| "learning_rate": 1.934865543956745e-06, |
| "loss": 0.0216, |
| "reward": 1.7083333730697632, |
| "reward_std": 0.4326419048011303, |
| "rewards/accuracy_reward": 0.7083333507180214, |
| "rewards/format_reward": 1.0, |
| "step": 130 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 291.2916793823242, |
| "epoch": 1.4053333333333333, |
| "grad_norm": 0.11653240770101547, |
| "kl": 0.0476531982421875, |
| "learning_rate": 1.916863784467152e-06, |
| "loss": 0.0277, |
| "reward": 1.5416667088866234, |
| "reward_std": 0.4870755970478058, |
| "rewards/accuracy_reward": 0.6250000149011612, |
| "rewards/format_reward": 0.9166666716337204, |
| "step": 131 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 310.37501335144043, |
| "epoch": 1.416, |
| "grad_norm": 0.11371561139822006, |
| "kl": 0.049407958984375, |
| "learning_rate": 1.8987967208247542e-06, |
| "loss": 0.0587, |
| "reward": 1.5833333730697632, |
| "reward_std": 0.39372343569993973, |
| "rewards/accuracy_reward": 0.6250000111758709, |
| "rewards/format_reward": 0.9583333432674408, |
| "step": 132 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 217.06250762939453, |
| "epoch": 1.4266666666666667, |
| "grad_norm": 0.05566725507378578, |
| "kl": 0.06402587890625, |
| "learning_rate": 1.8806671833408633e-06, |
| "loss": 0.0115, |
| "reward": 1.75, |
| "reward_std": 0.17304783314466476, |
| "rewards/accuracy_reward": 0.8333333358168602, |
| "rewards/format_reward": 0.9166666716337204, |
| "step": 133 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 233.18750762939453, |
| "epoch": 1.4373333333333334, |
| "grad_norm": 0.09656023234128952, |
| "kl": 0.0499114990234375, |
| "learning_rate": 1.8624780121136837e-06, |
| "loss": 0.0088, |
| "reward": 1.833333358168602, |
| "reward_std": 0.2861081622540951, |
| "rewards/accuracy_reward": 0.854166679084301, |
| "rewards/format_reward": 0.9791666716337204, |
| "step": 134 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 265.62500953674316, |
| "epoch": 1.448, |
| "grad_norm": 0.12826627492904663, |
| "kl": 0.062347412109375, |
| "learning_rate": 1.8442320565833972e-06, |
| "loss": 0.0249, |
| "reward": 1.5000000447034836, |
| "reward_std": 0.5420222245156765, |
| "rewards/accuracy_reward": 0.5416666809469461, |
| "rewards/format_reward": 0.9583333358168602, |
| "step": 135 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 239.37500953674316, |
| "epoch": 1.4586666666666668, |
| "grad_norm": 0.10006551444530487, |
| "kl": 0.0582275390625, |
| "learning_rate": 1.8259321750857772e-06, |
| "loss": -0.0374, |
| "reward": 1.7291667014360428, |
| "reward_std": 0.4090071730315685, |
| "rewards/accuracy_reward": 0.7708333507180214, |
| "rewards/format_reward": 0.9583333358168602, |
| "step": 136 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 259.45834159851074, |
| "epoch": 1.4693333333333334, |
| "grad_norm": 0.1015496775507927, |
| "kl": 0.0584716796875, |
| "learning_rate": 1.8075812344044163e-06, |
| "loss": 0.0308, |
| "reward": 1.7083333730697632, |
| "reward_std": 0.39372343569993973, |
| "rewards/accuracy_reward": 0.7708333469927311, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 137 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 241.56250381469727, |
| "epoch": 1.48, |
| "grad_norm": 0.0967569649219513, |
| "kl": 0.085906982421875, |
| "learning_rate": 1.7891821093216275e-06, |
| "loss": 0.0161, |
| "reward": 1.6875000447034836, |
| "reward_std": 0.41129202023148537, |
| "rewards/accuracy_reward": 0.7916666716337204, |
| "rewards/format_reward": 0.8958333507180214, |
| "step": 138 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 341.37500953674316, |
| "epoch": 1.4906666666666666, |
| "grad_norm": 0.9276979565620422, |
| "kl": 0.562530517578125, |
| "learning_rate": 1.770737682168091e-06, |
| "loss": 0.1031, |
| "reward": 1.5833333730697632, |
| "reward_std": 0.4673529714345932, |
| "rewards/accuracy_reward": 0.6458333544433117, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 139 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 204.06250381469727, |
| "epoch": 1.5013333333333332, |
| "grad_norm": 0.09736520051956177, |
| "kl": 0.105438232421875, |
| "learning_rate": 1.7522508423713243e-06, |
| "loss": 0.0251, |
| "reward": 1.8541667014360428, |
| "reward_std": 0.2446802221238613, |
| "rewards/accuracy_reward": 0.854166679084301, |
| "rewards/format_reward": 1.0, |
| "step": 140 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 347.8541679382324, |
| "epoch": 1.512, |
| "grad_norm": 0.12234628945589066, |
| "kl": 0.0500640869140625, |
| "learning_rate": 1.7337244860030312e-06, |
| "loss": 0.0863, |
| "reward": 1.3333333656191826, |
| "reward_std": 0.47278037667274475, |
| "rewards/accuracy_reward": 0.47916667722165585, |
| "rewards/format_reward": 0.854166679084301, |
| "step": 141 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 237.06250858306885, |
| "epoch": 1.5226666666666666, |
| "grad_norm": 0.15673072636127472, |
| "kl": 0.0669403076171875, |
| "learning_rate": 1.7151615153254192e-06, |
| "loss": 0.0404, |
| "reward": 1.4791667014360428, |
| "reward_std": 0.5729752071201801, |
| "rewards/accuracy_reward": 0.5833333469927311, |
| "rewards/format_reward": 0.8958333395421505, |
| "step": 142 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 256.10417652130127, |
| "epoch": 1.5333333333333332, |
| "grad_norm": 0.16718435287475586, |
| "kl": 0.0815582275390625, |
| "learning_rate": 1.6965648383365416e-06, |
| "loss": 0.1272, |
| "reward": 1.4583333730697632, |
| "reward_std": 0.5867235325276852, |
| "rewards/accuracy_reward": 0.5625000186264515, |
| "rewards/format_reward": 0.895833358168602, |
| "step": 143 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 282.1458377838135, |
| "epoch": 1.544, |
| "grad_norm": 0.11697231978178024, |
| "kl": 0.060821533203125, |
| "learning_rate": 1.6779373683147423e-06, |
| "loss": 0.1063, |
| "reward": 1.3541667014360428, |
| "reward_std": 0.5518574230372906, |
| "rewards/accuracy_reward": 0.47916668094694614, |
| "rewards/format_reward": 0.8750000223517418, |
| "step": 144 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 275.29167556762695, |
| "epoch": 1.5546666666666666, |
| "grad_norm": 0.13972164690494537, |
| "kl": 0.0799560546875, |
| "learning_rate": 1.6592820233622743e-06, |
| "loss": -0.0019, |
| "reward": 1.2500000298023224, |
| "reward_std": 0.5609171241521835, |
| "rewards/accuracy_reward": 0.43750001303851604, |
| "rewards/format_reward": 0.8125000298023224, |
| "step": 145 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 235.35417461395264, |
| "epoch": 1.5653333333333332, |
| "grad_norm": 0.1573922336101532, |
| "kl": 0.084381103515625, |
| "learning_rate": 1.6406017259481624e-06, |
| "loss": 0.1167, |
| "reward": 1.3125000298023224, |
| "reward_std": 0.46066780760884285, |
| "rewards/accuracy_reward": 0.41666666977107525, |
| "rewards/format_reward": 0.8958333358168602, |
| "step": 146 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 315.25001335144043, |
| "epoch": 1.576, |
| "grad_norm": 0.15172944962978363, |
| "kl": 0.080963134765625, |
| "learning_rate": 1.621899402450379e-06, |
| "loss": 0.1219, |
| "reward": 1.2708333656191826, |
| "reward_std": 0.6084391921758652, |
| "rewards/accuracy_reward": 0.4583333358168602, |
| "rewards/format_reward": 0.8125000298023224, |
| "step": 147 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 263.4375066757202, |
| "epoch": 1.5866666666666667, |
| "grad_norm": 0.12377157062292099, |
| "kl": 0.07403564453125, |
| "learning_rate": 1.6031779826974138e-06, |
| "loss": 0.0255, |
| "reward": 1.395833358168602, |
| "reward_std": 0.4362565539777279, |
| "rewards/accuracy_reward": 0.5416666716337204, |
| "rewards/format_reward": 0.8541666865348816, |
| "step": 148 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 272.3750114440918, |
| "epoch": 1.5973333333333333, |
| "grad_norm": 0.46126094460487366, |
| "kl": 0.209686279296875, |
| "learning_rate": 1.5844403995092944e-06, |
| "loss": 0.0734, |
| "reward": 1.145833358168602, |
| "reward_std": 0.5279995314776897, |
| "rewards/accuracy_reward": 0.31250000931322575, |
| "rewards/format_reward": 0.8333333507180214, |
| "step": 149 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 219.7291717529297, |
| "epoch": 1.608, |
| "grad_norm": 0.17344419658184052, |
| "kl": 0.077972412109375, |
| "learning_rate": 1.5656895882381465e-06, |
| "loss": 0.0446, |
| "reward": 1.104166716337204, |
| "reward_std": 0.5293822586536407, |
| "rewards/accuracy_reward": 0.2500000037252903, |
| "rewards/format_reward": 0.8541666939854622, |
| "step": 150 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 295.9166717529297, |
| "epoch": 1.6186666666666667, |
| "grad_norm": 0.13225041329860687, |
| "kl": 0.078277587890625, |
| "learning_rate": 1.5469284863083523e-06, |
| "loss": 0.1267, |
| "reward": 1.2083333805203438, |
| "reward_std": 0.44083888083696365, |
| "rewards/accuracy_reward": 0.33333334140479565, |
| "rewards/format_reward": 0.8750000223517418, |
| "step": 151 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 253.854172706604, |
| "epoch": 1.6293333333333333, |
| "grad_norm": 0.14670467376708984, |
| "kl": 0.06512451171875, |
| "learning_rate": 1.5281600327563853e-06, |
| "loss": 0.0528, |
| "reward": 1.2916667014360428, |
| "reward_std": 0.46722717583179474, |
| "rewards/accuracy_reward": 0.354166679084301, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 152 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 305.4166774749756, |
| "epoch": 1.6400000000000001, |
| "grad_norm": 0.14602994918823242, |
| "kl": 0.1068115234375, |
| "learning_rate": 1.5093871677703947e-06, |
| "loss": 0.0707, |
| "reward": 1.1875000149011612, |
| "reward_std": 0.6286378428339958, |
| "rewards/accuracy_reward": 0.41666667349636555, |
| "rewards/format_reward": 0.7708333507180214, |
| "step": 153 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 284.2083396911621, |
| "epoch": 1.6506666666666665, |
| "grad_norm": 0.15345576405525208, |
| "kl": 0.06597900390625, |
| "learning_rate": 1.4906128322296056e-06, |
| "loss": 0.0552, |
| "reward": 1.2291666865348816, |
| "reward_std": 0.5342973358929157, |
| "rewards/accuracy_reward": 0.31250000558793545, |
| "rewards/format_reward": 0.9166666865348816, |
| "step": 154 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 309.1666736602783, |
| "epoch": 1.6613333333333333, |
| "grad_norm": 0.17641578614711761, |
| "kl": 0.1270904541015625, |
| "learning_rate": 1.471839967243615e-06, |
| "loss": 0.0082, |
| "reward": 1.1041667088866234, |
| "reward_std": 0.5523697510361671, |
| "rewards/accuracy_reward": 0.3125000111758709, |
| "rewards/format_reward": 0.791666679084301, |
| "step": 155 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 322.75000762939453, |
| "epoch": 1.6720000000000002, |
| "grad_norm": 0.15972626209259033, |
| "kl": 0.0926513671875, |
| "learning_rate": 1.453071513691648e-06, |
| "loss": 0.1647, |
| "reward": 1.2291667014360428, |
| "reward_std": 0.5694059133529663, |
| "rewards/accuracy_reward": 0.3958333469927311, |
| "rewards/format_reward": 0.833333358168602, |
| "step": 156 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 258.6875114440918, |
| "epoch": 1.6826666666666665, |
| "grad_norm": 0.1186479851603508, |
| "kl": 0.081817626953125, |
| "learning_rate": 1.4343104117618536e-06, |
| "loss": 0.0049, |
| "reward": 1.2500000223517418, |
| "reward_std": 0.4595342054963112, |
| "rewards/accuracy_reward": 0.3750000074505806, |
| "rewards/format_reward": 0.8750000149011612, |
| "step": 157 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 193.104172706604, |
| "epoch": 1.6933333333333334, |
| "grad_norm": 0.08673811703920364, |
| "kl": 0.0623321533203125, |
| "learning_rate": 1.415559600490706e-06, |
| "loss": 0.0657, |
| "reward": 1.6041666716337204, |
| "reward_std": 0.18796167895197868, |
| "rewards/accuracy_reward": 0.6458333358168602, |
| "rewards/format_reward": 0.9583333432674408, |
| "step": 158 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 218.0416717529297, |
| "epoch": 1.704, |
| "grad_norm": 0.1484222710132599, |
| "kl": 0.088836669921875, |
| "learning_rate": 1.3968220173025865e-06, |
| "loss": 0.0912, |
| "reward": 1.5208333879709244, |
| "reward_std": 0.572458777576685, |
| "rewards/accuracy_reward": 0.5833333544433117, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 159 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 303.95834159851074, |
| "epoch": 1.7146666666666666, |
| "grad_norm": 0.2447938323020935, |
| "kl": 0.1371002197265625, |
| "learning_rate": 1.3781005975496212e-06, |
| "loss": 0.0618, |
| "reward": 1.2083333730697632, |
| "reward_std": 0.34925176575779915, |
| "rewards/accuracy_reward": 0.31250001303851604, |
| "rewards/format_reward": 0.8958333507180214, |
| "step": 160 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 263.3125057220459, |
| "epoch": 1.7253333333333334, |
| "grad_norm": 0.14777325093746185, |
| "kl": 0.159271240234375, |
| "learning_rate": 1.359398274051838e-06, |
| "loss": 0.0329, |
| "reward": 1.3125000149011612, |
| "reward_std": 0.5235714204609394, |
| "rewards/accuracy_reward": 0.4375000037252903, |
| "rewards/format_reward": 0.8750000149011612, |
| "step": 161 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 279.91667556762695, |
| "epoch": 1.736, |
| "grad_norm": 0.11606959253549576, |
| "kl": 0.12255859375, |
| "learning_rate": 1.340717976637726e-06, |
| "loss": 0.0324, |
| "reward": 1.4375000298023224, |
| "reward_std": 0.45774073526263237, |
| "rewards/accuracy_reward": 0.5416666753590107, |
| "rewards/format_reward": 0.8958333507180214, |
| "step": 162 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 252.3125057220459, |
| "epoch": 1.7466666666666666, |
| "grad_norm": 0.1397114396095276, |
| "kl": 0.0972442626953125, |
| "learning_rate": 1.3220626316852584e-06, |
| "loss": 0.0687, |
| "reward": 1.4791666865348816, |
| "reward_std": 0.4081249535083771, |
| "rewards/accuracy_reward": 0.6666666772216558, |
| "rewards/format_reward": 0.8125000223517418, |
| "step": 163 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 315.2708435058594, |
| "epoch": 1.7573333333333334, |
| "grad_norm": 0.1347351223230362, |
| "kl": 0.113433837890625, |
| "learning_rate": 1.303435161663459e-06, |
| "loss": 0.1106, |
| "reward": 1.1666667088866234, |
| "reward_std": 0.45271995291113853, |
| "rewards/accuracy_reward": 0.29166666977107525, |
| "rewards/format_reward": 0.8750000223517418, |
| "step": 164 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 216.47917556762695, |
| "epoch": 1.768, |
| "grad_norm": 0.15313582122325897, |
| "kl": 0.0865936279296875, |
| "learning_rate": 1.2848384846745813e-06, |
| "loss": 0.0198, |
| "reward": 1.5416667014360428, |
| "reward_std": 0.41228053718805313, |
| "rewards/accuracy_reward": 0.604166679084301, |
| "rewards/format_reward": 0.9375000074505806, |
| "step": 165 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 250.6875114440918, |
| "epoch": 1.7786666666666666, |
| "grad_norm": 0.1456070989370346, |
| "kl": 0.09686279296875, |
| "learning_rate": 1.2662755139969693e-06, |
| "loss": -0.0239, |
| "reward": 1.333333358168602, |
| "reward_std": 0.5820958688855171, |
| "rewards/accuracy_reward": 0.4166666753590107, |
| "rewards/format_reward": 0.9166666716337204, |
| "step": 166 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 318.395845413208, |
| "epoch": 1.7893333333333334, |
| "grad_norm": 0.1404711902141571, |
| "kl": 0.091278076171875, |
| "learning_rate": 1.2477491576286764e-06, |
| "loss": 0.0999, |
| "reward": 1.3125000149011612, |
| "reward_std": 0.4405704736709595, |
| "rewards/accuracy_reward": 0.41666667722165585, |
| "rewards/format_reward": 0.8958333507180214, |
| "step": 167 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 231.89583778381348, |
| "epoch": 1.8, |
| "grad_norm": 0.14985869824886322, |
| "kl": 0.104583740234375, |
| "learning_rate": 1.2292623178319094e-06, |
| "loss": 0.0291, |
| "reward": 1.333333358168602, |
| "reward_std": 0.5864746198058128, |
| "rewards/accuracy_reward": 0.5000000204890966, |
| "rewards/format_reward": 0.8333333432674408, |
| "step": 168 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 268.00000762939453, |
| "epoch": 1.8106666666666666, |
| "grad_norm": 0.16692346334457397, |
| "kl": 0.071533203125, |
| "learning_rate": 1.2108178906783732e-06, |
| "loss": 0.0409, |
| "reward": 1.354166716337204, |
| "reward_std": 0.5293822661042213, |
| "rewards/accuracy_reward": 0.45833334140479565, |
| "rewards/format_reward": 0.895833358168602, |
| "step": 169 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 246.33333826065063, |
| "epoch": 1.8213333333333335, |
| "grad_norm": 0.17591020464897156, |
| "kl": 0.09210205078125, |
| "learning_rate": 1.1924187655955838e-06, |
| "loss": 0.0802, |
| "reward": 1.4583333805203438, |
| "reward_std": 0.44083888456225395, |
| "rewards/accuracy_reward": 0.5416666772216558, |
| "rewards/format_reward": 0.916666679084301, |
| "step": 170 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 283.4166774749756, |
| "epoch": 1.8319999999999999, |
| "grad_norm": 0.1486297994852066, |
| "kl": 0.1509857177734375, |
| "learning_rate": 1.1740678249142229e-06, |
| "loss": 0.0796, |
| "reward": 1.3958333730697632, |
| "reward_std": 0.532146617770195, |
| "rewards/accuracy_reward": 0.5000000149011612, |
| "rewards/format_reward": 0.8958333507180214, |
| "step": 171 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 198.7083396911621, |
| "epoch": 1.8426666666666667, |
| "grad_norm": 0.23767401278018951, |
| "kl": 0.145599365234375, |
| "learning_rate": 1.1557679434166027e-06, |
| "loss": 0.0035, |
| "reward": 1.4791666865348816, |
| "reward_std": 0.47406983748078346, |
| "rewards/accuracy_reward": 0.541666679084301, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 172 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 224.75000476837158, |
| "epoch": 1.8533333333333335, |
| "grad_norm": 0.13122382760047913, |
| "kl": 0.11077880859375, |
| "learning_rate": 1.1375219878863162e-06, |
| "loss": 0.0779, |
| "reward": 1.5416667014360428, |
| "reward_std": 0.38677507638931274, |
| "rewards/accuracy_reward": 0.6041666809469461, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 173 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 246.08333587646484, |
| "epoch": 1.8639999999999999, |
| "grad_norm": 0.13901762664318085, |
| "kl": 0.099822998046875, |
| "learning_rate": 1.1193328166591372e-06, |
| "loss": 0.0029, |
| "reward": 1.458333358168602, |
| "reward_std": 0.45446737110614777, |
| "rewards/accuracy_reward": 0.5625000055879354, |
| "rewards/format_reward": 0.8958333432674408, |
| "step": 174 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 279.9166736602783, |
| "epoch": 1.8746666666666667, |
| "grad_norm": 0.304503470659256, |
| "kl": 0.228118896484375, |
| "learning_rate": 1.1012032791752457e-06, |
| "loss": 0.0385, |
| "reward": 1.2291667088866234, |
| "reward_std": 0.4801270626485348, |
| "rewards/accuracy_reward": 0.3750000074505806, |
| "rewards/format_reward": 0.8541666865348816, |
| "step": 175 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 244.0208396911621, |
| "epoch": 1.8853333333333333, |
| "grad_norm": 0.2561553120613098, |
| "kl": 0.191009521484375, |
| "learning_rate": 1.0831362155328477e-06, |
| "loss": 0.0531, |
| "reward": 1.6875000149011612, |
| "reward_std": 0.4051978886127472, |
| "rewards/accuracy_reward": 0.750000013038516, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 176 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 344.43750953674316, |
| "epoch": 1.896, |
| "grad_norm": 0.24141816794872284, |
| "kl": 0.21771240234375, |
| "learning_rate": 1.0651344560432548e-06, |
| "loss": 0.1034, |
| "reward": 1.4375000223517418, |
| "reward_std": 0.5041032508015633, |
| "rewards/accuracy_reward": 0.6041666734963655, |
| "rewards/format_reward": 0.8333333469927311, |
| "step": 177 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 237.37500762939453, |
| "epoch": 1.9066666666666667, |
| "grad_norm": 1.193719744682312, |
| "kl": 0.32879638671875, |
| "learning_rate": 1.0472008207875023e-06, |
| "loss": 0.1244, |
| "reward": 1.479166716337204, |
| "reward_std": 0.5614495500922203, |
| "rewards/accuracy_reward": 0.6250000186264515, |
| "rewards/format_reward": 0.854166679084301, |
| "step": 178 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 261.7083435058594, |
| "epoch": 1.9173333333333333, |
| "grad_norm": 0.46435582637786865, |
| "kl": 0.276611328125, |
| "learning_rate": 1.0293381191745637e-06, |
| "loss": 0.0264, |
| "reward": 1.4375000447034836, |
| "reward_std": 0.4688647873699665, |
| "rewards/accuracy_reward": 0.5000000204890966, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 179 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 292.27084255218506, |
| "epoch": 1.928, |
| "grad_norm": 0.19408948719501495, |
| "kl": 0.187713623046875, |
| "learning_rate": 1.0115491495012424e-06, |
| "loss": -0.002, |
| "reward": 1.5000000149011612, |
| "reward_std": 0.20815639942884445, |
| "rewards/accuracy_reward": 0.5208333376795053, |
| "rewards/format_reward": 0.9791666716337204, |
| "step": 180 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 326.85417079925537, |
| "epoch": 1.9386666666666668, |
| "grad_norm": 0.34896135330200195, |
| "kl": 0.253265380859375, |
| "learning_rate": 9.938366985138028e-07, |
| "loss": 0.1119, |
| "reward": 1.3750000186264515, |
| "reward_std": 0.4847872592508793, |
| "rewards/accuracy_reward": 0.5625000093132257, |
| "rewards/format_reward": 0.8125000111758709, |
| "step": 181 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 246.3333396911621, |
| "epoch": 1.9493333333333334, |
| "grad_norm": 0.242198184132576, |
| "kl": 0.17230224609375, |
| "learning_rate": 9.762035409714077e-07, |
| "loss": 0.0225, |
| "reward": 1.2500000298023224, |
| "reward_std": 0.45622891560196877, |
| "rewards/accuracy_reward": 0.37500000931322575, |
| "rewards/format_reward": 0.8750000149011612, |
| "step": 182 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 252.4375057220459, |
| "epoch": 1.96, |
| "grad_norm": 1.2867988348007202, |
| "kl": 0.237274169921875, |
| "learning_rate": 9.586524392114385e-07, |
| "loss": 0.1312, |
| "reward": 1.4166667014360428, |
| "reward_std": 0.5692892409861088, |
| "rewards/accuracy_reward": 0.5416666753590107, |
| "rewards/format_reward": 0.8750000223517418, |
| "step": 183 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 252.2500114440918, |
| "epoch": 1.9706666666666668, |
| "grad_norm": 0.4133116602897644, |
| "kl": 0.166107177734375, |
| "learning_rate": 9.411861427167567e-07, |
| "loss": 0.1611, |
| "reward": 1.5208333507180214, |
| "reward_std": 0.508791871368885, |
| "rewards/accuracy_reward": 0.6250000093132257, |
| "rewards/format_reward": 0.8958333432674408, |
| "step": 184 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 297.16666984558105, |
| "epoch": 1.9813333333333332, |
| "grad_norm": 2.6324448585510254, |
| "kl": 0.236083984375, |
| "learning_rate": 9.238073876849837e-07, |
| "loss": 0.0799, |
| "reward": 1.4166666865348816, |
| "reward_std": 0.6997495777904987, |
| "rewards/accuracy_reward": 0.5833333525806665, |
| "rewards/format_reward": 0.8333333507180214, |
| "step": 185 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 350.0833435058594, |
| "epoch": 1.992, |
| "grad_norm": 1.6513264179229736, |
| "kl": 0.191741943359375, |
| "learning_rate": 9.065188965998574e-07, |
| "loss": 0.1732, |
| "reward": 1.4375000447034836, |
| "reward_std": 0.6652876362204552, |
| "rewards/accuracy_reward": 0.6041666828095913, |
| "rewards/format_reward": 0.8333333656191826, |
| "step": 186 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 263.8333387374878, |
| "epoch": 2.010666666666667, |
| "grad_norm": 0.46523988246917725, |
| "kl": 0.34124755859375, |
| "learning_rate": 8.893233778047394e-07, |
| "loss": 0.0944, |
| "reward": 1.4166666865348816, |
| "reward_std": 0.4923461005091667, |
| "rewards/accuracy_reward": 0.5833333469927311, |
| "rewards/format_reward": 0.8333333544433117, |
| "step": 187 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 241.3750057220459, |
| "epoch": 2.021333333333333, |
| "grad_norm": 5.950314521789551, |
| "kl": 0.441497802734375, |
| "learning_rate": 8.722235250783389e-07, |
| "loss": 0.0085, |
| "reward": 1.5625000298023224, |
| "reward_std": 0.5480775870382786, |
| "rewards/accuracy_reward": 0.6875000167638063, |
| "rewards/format_reward": 0.8750000223517418, |
| "step": 188 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 239.95834255218506, |
| "epoch": 2.032, |
| "grad_norm": 2.313253879547119, |
| "kl": 0.481903076171875, |
| "learning_rate": 8.552220172127148e-07, |
| "loss": 0.1195, |
| "reward": 1.6250000223517418, |
| "reward_std": 0.4617997705936432, |
| "rewards/accuracy_reward": 0.750000013038516, |
| "rewards/format_reward": 0.8750000149011612, |
| "step": 189 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 333.2291774749756, |
| "epoch": 2.042666666666667, |
| "grad_norm": 393.3891296386719, |
| "kl": 54.324493408203125, |
| "learning_rate": 8.383215175936263e-07, |
| "loss": 3.1915, |
| "reward": 1.3750000223517418, |
| "reward_std": 0.5343292653560638, |
| "rewards/accuracy_reward": 0.5625000037252903, |
| "rewards/format_reward": 0.8125000149011612, |
| "step": 190 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 288.06250381469727, |
| "epoch": 2.0533333333333332, |
| "grad_norm": 17.025423049926758, |
| "kl": 5.16571044921875, |
| "learning_rate": 8.215246737833017e-07, |
| "loss": 0.3571, |
| "reward": 1.3333333730697632, |
| "reward_std": 0.5607051253318787, |
| "rewards/accuracy_reward": 0.4791666753590107, |
| "rewards/format_reward": 0.8541666865348816, |
| "step": 191 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 242.31250762939453, |
| "epoch": 2.064, |
| "grad_norm": 1.5437623262405396, |
| "kl": 1.153076171875, |
| "learning_rate": 8.048341171056786e-07, |
| "loss": 0.0747, |
| "reward": 1.5625000447034836, |
| "reward_std": 0.49742312356829643, |
| "rewards/accuracy_reward": 0.666666679084301, |
| "rewards/format_reward": 0.8958333432674408, |
| "step": 192 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 250.291672706604, |
| "epoch": 2.074666666666667, |
| "grad_norm": 2.553690195083618, |
| "kl": 0.81292724609375, |
| "learning_rate": 7.882524622341923e-07, |
| "loss": 0.1897, |
| "reward": 1.5208333656191826, |
| "reward_std": 0.6309319958090782, |
| "rewards/accuracy_reward": 0.6666666939854622, |
| "rewards/format_reward": 0.8541666865348816, |
| "step": 193 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 276.79167556762695, |
| "epoch": 2.0853333333333333, |
| "grad_norm": 0.9575795531272888, |
| "kl": 0.219451904296875, |
| "learning_rate": 7.717823067821738e-07, |
| "loss": 0.0771, |
| "reward": 1.4166667014360428, |
| "reward_std": 0.3477308116853237, |
| "rewards/accuracy_reward": 0.5000000111758709, |
| "rewards/format_reward": 0.9166666865348816, |
| "step": 194 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 236.22917366027832, |
| "epoch": 2.096, |
| "grad_norm": 1.3622400760650635, |
| "kl": 0.867919921875, |
| "learning_rate": 7.554262308959165e-07, |
| "loss": 0.0914, |
| "reward": 1.1458333805203438, |
| "reward_std": 0.5349354632198811, |
| "rewards/accuracy_reward": 0.29166667349636555, |
| "rewards/format_reward": 0.854166679084301, |
| "step": 195 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 277.3958396911621, |
| "epoch": 2.1066666666666665, |
| "grad_norm": 4.180922031402588, |
| "kl": 0.69647216796875, |
| "learning_rate": 7.39186796850482e-07, |
| "loss": 0.1392, |
| "reward": 1.416666679084301, |
| "reward_std": 0.5008263885974884, |
| "rewards/accuracy_reward": 0.5416666734963655, |
| "rewards/format_reward": 0.8750000149011612, |
| "step": 196 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 233.37500190734863, |
| "epoch": 2.1173333333333333, |
| "grad_norm": 0.28783828020095825, |
| "kl": 0.64825439453125, |
| "learning_rate": 7.23066548648304e-07, |
| "loss": 0.0323, |
| "reward": 1.458333358168602, |
| "reward_std": 0.5146066322922707, |
| "rewards/accuracy_reward": 0.5833333432674408, |
| "rewards/format_reward": 0.8750000223517418, |
| "step": 197 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 400.9791831970215, |
| "epoch": 2.128, |
| "grad_norm": 10.193907737731934, |
| "kl": 5.00872802734375, |
| "learning_rate": 7.070680116206564e-07, |
| "loss": 0.3473, |
| "reward": 1.0625000335276127, |
| "reward_std": 0.5005942918360233, |
| "rewards/accuracy_reward": 0.3125000037252903, |
| "rewards/format_reward": 0.7500000111758709, |
| "step": 198 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 292.7916736602783, |
| "epoch": 2.1386666666666665, |
| "grad_norm": 4.077642440795898, |
| "kl": 2.90966796875, |
| "learning_rate": 6.911936920320457e-07, |
| "loss": 0.1194, |
| "reward": 1.4166667014360428, |
| "reward_std": 0.5656285360455513, |
| "rewards/accuracy_reward": 0.5000000186264515, |
| "rewards/format_reward": 0.916666679084301, |
| "step": 199 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 225.8958396911621, |
| "epoch": 2.1493333333333333, |
| "grad_norm": 0.7050092816352844, |
| "kl": 0.34698486328125, |
| "learning_rate": 6.754460766875871e-07, |
| "loss": 0.0508, |
| "reward": 1.5208333730697632, |
| "reward_std": 0.40168892219662666, |
| "rewards/accuracy_reward": 0.583333345130086, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 200 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 173.95833778381348, |
| "epoch": 2.16, |
| "grad_norm": 0.23802195489406586, |
| "kl": 0.095611572265625, |
| "learning_rate": 6.598276325434357e-07, |
| "loss": 0.0267, |
| "reward": 1.7916666865348816, |
| "reward_std": 0.3410547971725464, |
| "rewards/accuracy_reward": 0.8333333432674408, |
| "rewards/format_reward": 0.9583333432674408, |
| "step": 201 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 266.3541736602783, |
| "epoch": 2.1706666666666665, |
| "grad_norm": 0.40206822752952576, |
| "kl": 0.666473388671875, |
| "learning_rate": 6.443408063203226e-07, |
| "loss": 0.0029, |
| "reward": 1.4375000447034836, |
| "reward_std": 0.5253323614597321, |
| "rewards/accuracy_reward": 0.5625000018626451, |
| "rewards/format_reward": 0.8750000223517418, |
| "step": 202 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 228.9583396911621, |
| "epoch": 2.1813333333333333, |
| "grad_norm": 1.0147886276245117, |
| "kl": 0.338043212890625, |
| "learning_rate": 6.289880241202581e-07, |
| "loss": 0.043, |
| "reward": 1.2708333730697632, |
| "reward_std": 0.5363415889441967, |
| "rewards/accuracy_reward": 0.3541666753590107, |
| "rewards/format_reward": 0.916666679084301, |
| "step": 203 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 221.7916774749756, |
| "epoch": 2.192, |
| "grad_norm": 0.427720844745636, |
| "kl": 0.18035888671875, |
| "learning_rate": 6.137716910464735e-07, |
| "loss": 0.0187, |
| "reward": 1.5625000447034836, |
| "reward_std": 0.5079090520739555, |
| "rewards/accuracy_reward": 0.6250000111758709, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 204 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 235.62500762939453, |
| "epoch": 2.2026666666666666, |
| "grad_norm": 0.5082700848579407, |
| "kl": 1.269378662109375, |
| "learning_rate": 5.986941908266454e-07, |
| "loss": 0.1469, |
| "reward": 1.4375000447034836, |
| "reward_std": 0.5468165427446365, |
| "rewards/accuracy_reward": 0.5833333488553762, |
| "rewards/format_reward": 0.854166679084301, |
| "step": 205 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 228.1875057220459, |
| "epoch": 2.2133333333333334, |
| "grad_norm": 7.290508270263672, |
| "kl": 4.46014404296875, |
| "learning_rate": 5.837578854394692e-07, |
| "loss": 0.2142, |
| "reward": 1.3125000298023224, |
| "reward_std": 0.44334835186600685, |
| "rewards/accuracy_reward": 0.45833333767950535, |
| "rewards/format_reward": 0.8541666865348816, |
| "step": 206 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 248.1875057220459, |
| "epoch": 2.224, |
| "grad_norm": 5.754692077636719, |
| "kl": 6.70330810546875, |
| "learning_rate": 5.689651147446448e-07, |
| "loss": 0.4078, |
| "reward": 1.2500000223517418, |
| "reward_std": 0.8125958703458309, |
| "rewards/accuracy_reward": 0.520833345130086, |
| "rewards/format_reward": 0.7291666865348816, |
| "step": 207 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 276.5000057220459, |
| "epoch": 2.2346666666666666, |
| "grad_norm": 0.4427819550037384, |
| "kl": 0.77191162109375, |
| "learning_rate": 5.543181961163219e-07, |
| "loss": 0.1116, |
| "reward": 1.3958333730697632, |
| "reward_std": 0.4217669740319252, |
| "rewards/accuracy_reward": 0.479166679084301, |
| "rewards/format_reward": 0.9166666865348816, |
| "step": 208 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 191.64583778381348, |
| "epoch": 2.2453333333333334, |
| "grad_norm": 0.782500684261322, |
| "kl": 1.3341064453125, |
| "learning_rate": 5.398194240800684e-07, |
| "loss": 0.0145, |
| "reward": 1.5833333730697632, |
| "reward_std": 0.5104185156524181, |
| "rewards/accuracy_reward": 0.6875000149011612, |
| "rewards/format_reward": 0.895833358168602, |
| "step": 209 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 267.18750762939453, |
| "epoch": 2.2560000000000002, |
| "grad_norm": 1.0296471118927002, |
| "kl": 1.434326171875, |
| "learning_rate": 5.254710699534226e-07, |
| "loss": 0.1032, |
| "reward": 1.3750000298023224, |
| "reward_std": 0.4010416902601719, |
| "rewards/accuracy_reward": 0.5416666697710752, |
| "rewards/format_reward": 0.8333333432674408, |
| "step": 210 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 308.89584159851074, |
| "epoch": 2.2666666666666666, |
| "grad_norm": 2.316876173019409, |
| "kl": 1.29791259765625, |
| "learning_rate": 5.112753814900761e-07, |
| "loss": 0.1656, |
| "reward": 1.312500037252903, |
| "reward_std": 0.5606077648699284, |
| "rewards/accuracy_reward": 0.4583333395421505, |
| "rewards/format_reward": 0.8541666865348816, |
| "step": 211 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 198.3125057220459, |
| "epoch": 2.2773333333333334, |
| "grad_norm": 0.34542033076286316, |
| "kl": 1.111846923828125, |
| "learning_rate": 4.97234582527751e-07, |
| "loss": 0.018, |
| "reward": 1.3333333805203438, |
| "reward_std": 0.6683094389736652, |
| "rewards/accuracy_reward": 0.5416666846722364, |
| "rewards/format_reward": 0.7916666828095913, |
| "step": 212 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 316.75001335144043, |
| "epoch": 2.288, |
| "grad_norm": 1.203397274017334, |
| "kl": 1.38311767578125, |
| "learning_rate": 4.8335087263982e-07, |
| "loss": 0.2258, |
| "reward": 1.3333333730697632, |
| "reward_std": 0.607686098664999, |
| "rewards/accuracy_reward": 0.5000000055879354, |
| "rewards/format_reward": 0.833333358168602, |
| "step": 213 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 296.8125114440918, |
| "epoch": 2.2986666666666666, |
| "grad_norm": 0.3318883180618286, |
| "kl": 0.48272705078125, |
| "learning_rate": 4.6962642679073465e-07, |
| "loss": 0.034, |
| "reward": 1.2083333730697632, |
| "reward_std": 0.5359239801764488, |
| "rewards/accuracy_reward": 0.31250000931322575, |
| "rewards/format_reward": 0.8958333507180214, |
| "step": 214 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 286.9166736602783, |
| "epoch": 2.3093333333333335, |
| "grad_norm": 0.5079717636108398, |
| "kl": 0.31884765625, |
| "learning_rate": 4.560633949953017e-07, |
| "loss": 0.051, |
| "reward": 1.333333358168602, |
| "reward_std": 0.5560254342854023, |
| "rewards/accuracy_reward": 0.5000000149011612, |
| "rewards/format_reward": 0.8333333507180214, |
| "step": 215 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 211.1250057220459, |
| "epoch": 2.32, |
| "grad_norm": 0.673991858959198, |
| "kl": 0.325927734375, |
| "learning_rate": 4.426639019818704e-07, |
| "loss": -0.0239, |
| "reward": 1.4375000298023224, |
| "reward_std": 0.5759567655622959, |
| "rewards/accuracy_reward": 0.5625000149011612, |
| "rewards/format_reward": 0.8750000223517418, |
| "step": 216 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 249.66667556762695, |
| "epoch": 2.3306666666666667, |
| "grad_norm": 0.620853841304779, |
| "kl": 0.3583984375, |
| "learning_rate": 4.2943004685948286e-07, |
| "loss": 0.0766, |
| "reward": 1.2708333432674408, |
| "reward_std": 0.5561920776963234, |
| "rewards/accuracy_reward": 0.4166666753590107, |
| "rewards/format_reward": 0.854166679084301, |
| "step": 217 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 291.3333396911621, |
| "epoch": 2.3413333333333335, |
| "grad_norm": 10.582584381103516, |
| "kl": 2.179534912109375, |
| "learning_rate": 4.163639027890363e-07, |
| "loss": 0.1387, |
| "reward": 1.3125000409781933, |
| "reward_std": 0.5307883769273758, |
| "rewards/accuracy_reward": 0.4375000111758709, |
| "rewards/format_reward": 0.8750000111758709, |
| "step": 218 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 249.1250057220459, |
| "epoch": 2.352, |
| "grad_norm": 1.4043989181518555, |
| "kl": 0.674163818359375, |
| "learning_rate": 4.034675166585079e-07, |
| "loss": 0.1281, |
| "reward": 1.4166667312383652, |
| "reward_std": 0.5429009459912777, |
| "rewards/accuracy_reward": 0.5000000186264515, |
| "rewards/format_reward": 0.9166666865348816, |
| "step": 219 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 297.16667461395264, |
| "epoch": 2.3626666666666667, |
| "grad_norm": 3.6459124088287354, |
| "kl": 2.717041015625, |
| "learning_rate": 3.907429087623007e-07, |
| "loss": 0.1155, |
| "reward": 1.2291666865348816, |
| "reward_std": 0.41912320628762245, |
| "rewards/accuracy_reward": 0.3958333432674408, |
| "rewards/format_reward": 0.8333333432674408, |
| "step": 220 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 152.5416717529297, |
| "epoch": 2.3733333333333335, |
| "grad_norm": 0.8172355890274048, |
| "kl": 0.51953125, |
| "learning_rate": 3.7819207248475174e-07, |
| "loss": 0.0473, |
| "reward": 1.6666666865348816, |
| "reward_std": 0.40420936793088913, |
| "rewards/accuracy_reward": 0.7500000111758709, |
| "rewards/format_reward": 0.916666679084301, |
| "step": 221 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 234.00000953674316, |
| "epoch": 2.384, |
| "grad_norm": 0.4453510046005249, |
| "kl": 0.760650634765625, |
| "learning_rate": 3.6581697398785615e-07, |
| "loss": -0.0164, |
| "reward": 1.4791666865348816, |
| "reward_std": 0.3892955183982849, |
| "rewards/accuracy_reward": 0.5208333414047956, |
| "rewards/format_reward": 0.9583333432674408, |
| "step": 222 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 223.3958396911621, |
| "epoch": 2.3946666666666667, |
| "grad_norm": 0.7755252122879028, |
| "kl": 1.18450927734375, |
| "learning_rate": 3.5361955190325563e-07, |
| "loss": 0.0217, |
| "reward": 1.5833333730697632, |
| "reward_std": 0.5788923390209675, |
| "rewards/accuracy_reward": 0.7291666846722364, |
| "rewards/format_reward": 0.8541666939854622, |
| "step": 223 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 200.2916717529297, |
| "epoch": 2.405333333333333, |
| "grad_norm": 0.29831400513648987, |
| "kl": 0.40399169921875, |
| "learning_rate": 3.4160171702854286e-07, |
| "loss": -0.0345, |
| "reward": 1.4583333730697632, |
| "reward_std": 0.5722163133323193, |
| "rewards/accuracy_reward": 0.6041666753590107, |
| "rewards/format_reward": 0.8541666865348816, |
| "step": 224 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 232.39583778381348, |
| "epoch": 2.416, |
| "grad_norm": 0.2781633138656616, |
| "kl": 0.917236328125, |
| "learning_rate": 3.297653520279226e-07, |
| "loss": 0.0222, |
| "reward": 1.3958333656191826, |
| "reward_std": 0.6155543886125088, |
| "rewards/accuracy_reward": 0.5625000186264515, |
| "rewards/format_reward": 0.833333358168602, |
| "step": 225 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 236.77083778381348, |
| "epoch": 2.4266666666666667, |
| "grad_norm": 0.24500270187854767, |
| "kl": 0.32452392578125, |
| "learning_rate": 3.1811231113727974e-07, |
| "loss": 0.043, |
| "reward": 1.5208333730697632, |
| "reward_std": 0.5257022529840469, |
| "rewards/accuracy_reward": 0.6250000186264515, |
| "rewards/format_reward": 0.8958333507180214, |
| "step": 226 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 210.14583778381348, |
| "epoch": 2.437333333333333, |
| "grad_norm": 0.28082796931266785, |
| "kl": 0.30218505859375, |
| "learning_rate": 3.066444198737053e-07, |
| "loss": 0.0462, |
| "reward": 1.5208333730697632, |
| "reward_std": 0.5423685237765312, |
| "rewards/accuracy_reward": 0.6250000186264515, |
| "rewards/format_reward": 0.895833358168602, |
| "step": 227 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 236.43750762939453, |
| "epoch": 2.448, |
| "grad_norm": 0.32010531425476074, |
| "kl": 0.2791748046875, |
| "learning_rate": 2.9536347474951645e-07, |
| "loss": 0.0316, |
| "reward": 1.7291667014360428, |
| "reward_std": 0.4177280552685261, |
| "rewards/accuracy_reward": 0.791666679084301, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 228 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 243.35417366027832, |
| "epoch": 2.458666666666667, |
| "grad_norm": 0.4656471014022827, |
| "kl": 0.6329345703125, |
| "learning_rate": 2.842712429908237e-07, |
| "loss": 0.0578, |
| "reward": 1.187500037252903, |
| "reward_std": 0.566876970231533, |
| "rewards/accuracy_reward": 0.3125000074505806, |
| "rewards/format_reward": 0.8750000149011612, |
| "step": 229 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 266.68750381469727, |
| "epoch": 2.469333333333333, |
| "grad_norm": 5.0469970703125, |
| "kl": 2.81072998046875, |
| "learning_rate": 2.733694622606824e-07, |
| "loss": 0.109, |
| "reward": 1.2708333730697632, |
| "reward_std": 0.7010384388267994, |
| "rewards/accuracy_reward": 0.4583333432674408, |
| "rewards/format_reward": 0.8125000149011612, |
| "step": 230 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 239.02083683013916, |
| "epoch": 2.48, |
| "grad_norm": 0.692191481590271, |
| "kl": 0.800323486328125, |
| "learning_rate": 2.626598403868801e-07, |
| "loss": 0.0852, |
| "reward": 1.5000000298023224, |
| "reward_std": 0.5464099086821079, |
| "rewards/accuracy_reward": 0.6041666828095913, |
| "rewards/format_reward": 0.8958333507180214, |
| "step": 231 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 204.97917556762695, |
| "epoch": 2.490666666666667, |
| "grad_norm": 0.18402956426143646, |
| "kl": 0.36541748046875, |
| "learning_rate": 2.521440550943943e-07, |
| "loss": 0.0105, |
| "reward": 1.7500000149011612, |
| "reward_std": 0.3694790154695511, |
| "rewards/accuracy_reward": 0.8333333507180214, |
| "rewards/format_reward": 0.9166666716337204, |
| "step": 232 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 261.1666774749756, |
| "epoch": 2.501333333333333, |
| "grad_norm": 0.18328428268432617, |
| "kl": 0.095855712890625, |
| "learning_rate": 2.418237537425666e-07, |
| "loss": 0.0137, |
| "reward": 1.7291667014360428, |
| "reward_std": 0.40168891847133636, |
| "rewards/accuracy_reward": 0.7708333432674408, |
| "rewards/format_reward": 0.9583333432674408, |
| "step": 233 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 215.5416717529297, |
| "epoch": 2.512, |
| "grad_norm": 0.19842679798603058, |
| "kl": 0.108673095703125, |
| "learning_rate": 2.3170055306703512e-07, |
| "loss": 0.028, |
| "reward": 1.7500000447034836, |
| "reward_std": 0.45132480934262276, |
| "rewards/accuracy_reward": 0.770833358168602, |
| "rewards/format_reward": 0.9791666716337204, |
| "step": 234 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 258.0208396911621, |
| "epoch": 2.522666666666667, |
| "grad_norm": 0.9317381381988525, |
| "kl": 0.91571044921875, |
| "learning_rate": 2.2177603892646277e-07, |
| "loss": 0.0325, |
| "reward": 1.4791667014360428, |
| "reward_std": 0.49830594658851624, |
| "rewards/accuracy_reward": 0.604166679084301, |
| "rewards/format_reward": 0.8750000149011612, |
| "step": 235 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 216.5208396911621, |
| "epoch": 2.533333333333333, |
| "grad_norm": 2.2101762294769287, |
| "kl": 1.422576904296875, |
| "learning_rate": 2.1205176605410259e-07, |
| "loss": 0.0724, |
| "reward": 1.6875000447034836, |
| "reward_std": 0.3842546343803406, |
| "rewards/accuracy_reward": 0.7708333414047956, |
| "rewards/format_reward": 0.916666679084301, |
| "step": 236 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 243.12500667572021, |
| "epoch": 2.544, |
| "grad_norm": 1.1581038236618042, |
| "kl": 1.95135498046875, |
| "learning_rate": 2.0252925781424153e-07, |
| "loss": 0.1607, |
| "reward": 1.270833358168602, |
| "reward_std": 0.6249947585165501, |
| "rewards/accuracy_reward": 0.47916667722165585, |
| "rewards/format_reward": 0.7916666939854622, |
| "step": 237 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 242.5208444595337, |
| "epoch": 2.554666666666667, |
| "grad_norm": 0.6918954849243164, |
| "kl": 1.07720947265625, |
| "learning_rate": 1.932100059635551e-07, |
| "loss": 0.119, |
| "reward": 1.3541667014360428, |
| "reward_std": 0.51335409283638, |
| "rewards/accuracy_reward": 0.4791666865348816, |
| "rewards/format_reward": 0.8750000149011612, |
| "step": 238 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 281.1458377838135, |
| "epoch": 2.5653333333333332, |
| "grad_norm": 0.5743387341499329, |
| "kl": 0.401519775390625, |
| "learning_rate": 1.8409547041741436e-07, |
| "loss": 0.0128, |
| "reward": 1.3125000298023224, |
| "reward_std": 0.4759799800813198, |
| "rewards/accuracy_reward": 0.4375000111758709, |
| "rewards/format_reward": 0.8750000074505806, |
| "step": 239 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 338.0000057220459, |
| "epoch": 2.576, |
| "grad_norm": 0.6467074155807495, |
| "kl": 1.7817535400390625, |
| "learning_rate": 1.751870790211838e-07, |
| "loss": 0.1999, |
| "reward": 1.3750000447034836, |
| "reward_std": 0.5198958218097687, |
| "rewards/accuracy_reward": 0.479166679084301, |
| "rewards/format_reward": 0.8958333507180214, |
| "step": 240 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 275.14584732055664, |
| "epoch": 2.586666666666667, |
| "grad_norm": 0.6359933614730835, |
| "kl": 0.8695068359375, |
| "learning_rate": 1.6648622732653946e-07, |
| "loss": 0.0497, |
| "reward": 1.4375000521540642, |
| "reward_std": 0.547709733247757, |
| "rewards/accuracy_reward": 0.5416666753590107, |
| "rewards/format_reward": 0.8958333432674408, |
| "step": 241 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 241.33334159851074, |
| "epoch": 2.5973333333333333, |
| "grad_norm": 0.8483588695526123, |
| "kl": 0.48760986328125, |
| "learning_rate": 1.5799427837284686e-07, |
| "loss": 0.1047, |
| "reward": 1.6666667014360428, |
| "reward_std": 0.37073594331741333, |
| "rewards/accuracy_reward": 0.7500000149011612, |
| "rewards/format_reward": 0.9166666865348816, |
| "step": 242 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 235.08333587646484, |
| "epoch": 2.608, |
| "grad_norm": 0.25336307287216187, |
| "kl": 0.22003173828125, |
| "learning_rate": 1.4971256247363534e-07, |
| "loss": 0.0624, |
| "reward": 1.5833333730697632, |
| "reward_std": 0.5708101950585842, |
| "rewards/accuracy_reward": 0.6875000279396772, |
| "rewards/format_reward": 0.8958333432674408, |
| "step": 243 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 291.1041793823242, |
| "epoch": 2.618666666666667, |
| "grad_norm": 0.47303256392478943, |
| "kl": 0.993865966796875, |
| "learning_rate": 1.416423770081941e-07, |
| "loss": 0.1299, |
| "reward": 1.4583333544433117, |
| "reward_std": 0.5008263885974884, |
| "rewards/accuracy_reward": 0.6666666828095913, |
| "rewards/format_reward": 0.7916666902601719, |
| "step": 244 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 282.0416774749756, |
| "epoch": 2.6293333333333333, |
| "grad_norm": 0.39533504843711853, |
| "kl": 0.520782470703125, |
| "learning_rate": 1.3378498621833236e-07, |
| "loss": -0.0644, |
| "reward": 1.3958333730697632, |
| "reward_std": 0.6059102527797222, |
| "rewards/accuracy_reward": 0.5833333507180214, |
| "rewards/format_reward": 0.8125000223517418, |
| "step": 245 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 315.14584732055664, |
| "epoch": 2.64, |
| "grad_norm": 0.33665895462036133, |
| "kl": 0.689300537109375, |
| "learning_rate": 1.2614162101032601e-07, |
| "loss": 0.0385, |
| "reward": 1.4166666939854622, |
| "reward_std": 0.4743013381958008, |
| "rewards/accuracy_reward": 0.5625000093132257, |
| "rewards/format_reward": 0.8541666716337204, |
| "step": 246 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 213.68750762939453, |
| "epoch": 2.6506666666666665, |
| "grad_norm": 0.25907960534095764, |
| "kl": 0.36181640625, |
| "learning_rate": 1.1871347876209121e-07, |
| "loss": 0.0062, |
| "reward": 1.520833358168602, |
| "reward_std": 0.5440386645495892, |
| "rewards/accuracy_reward": 0.6458333525806665, |
| "rewards/format_reward": 0.8750000149011612, |
| "step": 247 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 305.4166793823242, |
| "epoch": 2.6613333333333333, |
| "grad_norm": 0.44137367606163025, |
| "kl": 0.75616455078125, |
| "learning_rate": 1.1150172313560741e-07, |
| "loss": 0.0573, |
| "reward": 1.3750000223517418, |
| "reward_std": 0.5896422937512398, |
| "rewards/accuracy_reward": 0.5208333544433117, |
| "rewards/format_reward": 0.8541666865348816, |
| "step": 248 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 230.31250667572021, |
| "epoch": 2.672, |
| "grad_norm": 0.3653818666934967, |
| "kl": 0.227935791015625, |
| "learning_rate": 1.0450748389462184e-07, |
| "loss": 0.0375, |
| "reward": 1.3958333805203438, |
| "reward_std": 0.48087166249752045, |
| "rewards/accuracy_reward": 0.5000000149011612, |
| "rewards/format_reward": 0.8958333432674408, |
| "step": 249 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 240.56250762939453, |
| "epoch": 2.6826666666666665, |
| "grad_norm": 0.27294793725013733, |
| "kl": 0.223388671875, |
| "learning_rate": 9.773185672766705e-08, |
| "loss": -0.0183, |
| "reward": 1.6041666865348816, |
| "reward_std": 0.4456373043358326, |
| "rewards/accuracy_reward": 0.666666679084301, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 250 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 414.2916774749756, |
| "epoch": 2.6933333333333334, |
| "grad_norm": 0.2690182328224182, |
| "kl": 0.7999267578125, |
| "learning_rate": 9.117590307641361e-08, |
| "loss": 0.1233, |
| "reward": 1.000000026077032, |
| "reward_std": 0.6379684209823608, |
| "rewards/accuracy_reward": 0.2708333358168602, |
| "rewards/format_reward": 0.7291666902601719, |
| "step": 251 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 290.645845413208, |
| "epoch": 2.7039999999999997, |
| "grad_norm": 0.19759991765022278, |
| "kl": 0.17645263671875, |
| "learning_rate": 8.484064996939023e-08, |
| "loss": 0.0121, |
| "reward": 1.437500037252903, |
| "reward_std": 0.5738539323210716, |
| "rewards/accuracy_reward": 0.5833333469927311, |
| "rewards/format_reward": 0.8541666828095913, |
| "step": 252 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 327.3541736602783, |
| "epoch": 2.7146666666666666, |
| "grad_norm": 0.2967400848865509, |
| "kl": 0.84234619140625, |
| "learning_rate": 7.872708986109183e-08, |
| "loss": 0.112, |
| "reward": 1.2916667014360428, |
| "reward_std": 0.519895825535059, |
| "rewards/accuracy_reward": 0.4791666716337204, |
| "rewards/format_reward": 0.8125000149011612, |
| "step": 253 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 320.6458396911621, |
| "epoch": 2.7253333333333334, |
| "grad_norm": 1.6768146753311157, |
| "kl": 1.749481201171875, |
| "learning_rate": 7.283618047650808e-08, |
| "loss": 0.1938, |
| "reward": 1.2291666865348816, |
| "reward_std": 0.5054984018206596, |
| "rewards/accuracy_reward": 0.41666667349636555, |
| "rewards/format_reward": 0.8125000149011612, |
| "step": 254 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 190.7916717529297, |
| "epoch": 2.7359999999999998, |
| "grad_norm": 8.16866397857666, |
| "kl": 2.960968017578125, |
| "learning_rate": 6.716884466108869e-08, |
| "loss": 0.283, |
| "reward": 1.6875000298023224, |
| "reward_std": 0.3987618423998356, |
| "rewards/accuracy_reward": 0.750000013038516, |
| "rewards/format_reward": 0.9375000149011612, |
| "step": 255 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 282.4166736602783, |
| "epoch": 2.7466666666666666, |
| "grad_norm": 0.19895876944065094, |
| "kl": 0.157257080078125, |
| "learning_rate": 6.17259702361741e-08, |
| "loss": 0.0076, |
| "reward": 1.645833358168602, |
| "reward_std": 0.4148009791970253, |
| "rewards/accuracy_reward": 0.6875000167638063, |
| "rewards/format_reward": 0.9583333432674408, |
| "step": 256 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 289.1875057220459, |
| "epoch": 2.7573333333333334, |
| "grad_norm": 0.549140214920044, |
| "kl": 0.666778564453125, |
| "learning_rate": 5.65084098599144e-08, |
| "loss": 0.0692, |
| "reward": 1.2500000447034836, |
| "reward_std": 0.6597342528402805, |
| "rewards/accuracy_reward": 0.4583333507180214, |
| "rewards/format_reward": 0.7916666865348816, |
| "step": 257 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 229.06250381469727, |
| "epoch": 2.768, |
| "grad_norm": 0.37792423367500305, |
| "kl": 0.859893798828125, |
| "learning_rate": 5.151698089369433e-08, |
| "loss": 0.0291, |
| "reward": 1.4791667088866234, |
| "reward_std": 0.5829372256994247, |
| "rewards/accuracy_reward": 0.6875000055879354, |
| "rewards/format_reward": 0.791666679084301, |
| "step": 258 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 240.12500762939453, |
| "epoch": 2.7786666666666666, |
| "grad_norm": 0.340668261051178, |
| "kl": 0.816009521484375, |
| "learning_rate": 4.675246527408872e-08, |
| "loss": 0.009, |
| "reward": 1.5000000447034836, |
| "reward_std": 0.6165339834988117, |
| "rewards/accuracy_reward": 0.7083333395421505, |
| "rewards/format_reward": 0.7916666865348816, |
| "step": 259 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 246.7708396911621, |
| "epoch": 2.7893333333333334, |
| "grad_norm": 0.18053409457206726, |
| "kl": 0.27801513671875, |
| "learning_rate": 4.221560939036828e-08, |
| "loss": -0.0014, |
| "reward": 1.3125000298023224, |
| "reward_std": 0.5489303506910801, |
| "rewards/accuracy_reward": 0.4166666753590107, |
| "rewards/format_reward": 0.895833358168602, |
| "step": 260 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 306.04167556762695, |
| "epoch": 2.8, |
| "grad_norm": 0.19975829124450684, |
| "kl": 0.544586181640625, |
| "learning_rate": 3.790712396757317e-08, |
| "loss": 0.0101, |
| "reward": 1.416666716337204, |
| "reward_std": 0.604806911200285, |
| "rewards/accuracy_reward": 0.5208333488553762, |
| "rewards/format_reward": 0.8958333358168602, |
| "step": 261 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 227.0000057220459, |
| "epoch": 2.8106666666666666, |
| "grad_norm": 0.27892568707466125, |
| "kl": 0.60784912109375, |
| "learning_rate": 3.382768395517355e-08, |
| "loss": 0.0042, |
| "reward": 1.3750000298023224, |
| "reward_std": 0.5251008756458759, |
| "rewards/accuracy_reward": 0.479166679084301, |
| "rewards/format_reward": 0.8958333432674408, |
| "step": 262 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 317.79167652130127, |
| "epoch": 2.8213333333333335, |
| "grad_norm": 0.2454572319984436, |
| "kl": 0.6148834228515625, |
| "learning_rate": 2.997792842133445e-08, |
| "loss": 0.0885, |
| "reward": 1.5208333507180214, |
| "reward_std": 0.3212082237005234, |
| "rewards/accuracy_reward": 0.6250000018626451, |
| "rewards/format_reward": 0.8958333432674408, |
| "step": 263 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 240.93750762939453, |
| "epoch": 2.832, |
| "grad_norm": 3.458434820175171, |
| "kl": 1.1170654296875, |
| "learning_rate": 2.6358460452802768e-08, |
| "loss": 0.1461, |
| "reward": 1.458333358168602, |
| "reward_std": 0.5896422863006592, |
| "rewards/accuracy_reward": 0.5833333469927311, |
| "rewards/format_reward": 0.8750000223517418, |
| "step": 264 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 245.12500762939453, |
| "epoch": 2.8426666666666667, |
| "grad_norm": 0.4039785861968994, |
| "kl": 1.179351806640625, |
| "learning_rate": 2.296984706042993e-08, |
| "loss": 0.0867, |
| "reward": 1.5625000298023224, |
| "reward_std": 0.47457385063171387, |
| "rewards/accuracy_reward": 0.6875000074505806, |
| "rewards/format_reward": 0.8750000149011612, |
| "step": 265 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 293.22916984558105, |
| "epoch": 2.8533333333333335, |
| "grad_norm": 0.689044713973999, |
| "kl": 0.975433349609375, |
| "learning_rate": 1.9812619090345984e-08, |
| "loss": 0.1793, |
| "reward": 1.6250000298023224, |
| "reward_std": 0.4975505545735359, |
| "rewards/accuracy_reward": 0.7083333432674408, |
| "rewards/format_reward": 0.916666679084301, |
| "step": 266 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 348.16668128967285, |
| "epoch": 2.864, |
| "grad_norm": 0.7113028764724731, |
| "kl": 2.130126953125, |
| "learning_rate": 1.688727114079963e-08, |
| "loss": 0.1079, |
| "reward": 0.9583333507180214, |
| "reward_std": 0.6428273692727089, |
| "rewards/accuracy_reward": 0.33333334513008595, |
| "rewards/format_reward": 0.6250000037252903, |
| "step": 267 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 289.5208435058594, |
| "epoch": 2.8746666666666667, |
| "grad_norm": 0.8639241456985474, |
| "kl": 1.56182861328125, |
| "learning_rate": 1.4194261484677007e-08, |
| "loss": 0.1785, |
| "reward": 1.5000000298023224, |
| "reward_std": 0.514606636017561, |
| "rewards/accuracy_reward": 0.6458333488553762, |
| "rewards/format_reward": 0.8541666865348816, |
| "step": 268 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 196.2291717529297, |
| "epoch": 2.8853333333333335, |
| "grad_norm": 0.28236812353134155, |
| "kl": 0.495849609375, |
| "learning_rate": 1.1734011997709626e-08, |
| "loss": 0.0362, |
| "reward": 1.6041667014360428, |
| "reward_std": 0.5063771232962608, |
| "rewards/accuracy_reward": 0.7291666809469461, |
| "rewards/format_reward": 0.8750000223517418, |
| "step": 269 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 254.4791717529297, |
| "epoch": 2.896, |
| "grad_norm": 0.4359928071498871, |
| "kl": 0.602508544921875, |
| "learning_rate": 9.50690809238558e-09, |
| "loss": 0.0818, |
| "reward": 1.6458333730697632, |
| "reward_std": 0.5747367590665817, |
| "rewards/accuracy_reward": 0.7291666865348816, |
| "rewards/format_reward": 0.9166666865348816, |
| "step": 270 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 300.5208492279053, |
| "epoch": 2.9066666666666667, |
| "grad_norm": 0.33899879455566406, |
| "kl": 0.670623779296875, |
| "learning_rate": 7.513298657572831e-09, |
| "loss": 0.0838, |
| "reward": 1.3958333432674408, |
| "reward_std": 0.36908068880438805, |
| "rewards/accuracy_reward": 0.47916666977107525, |
| "rewards/format_reward": 0.9166666865348816, |
| "step": 271 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 225.29167079925537, |
| "epoch": 2.9173333333333336, |
| "grad_norm": 0.22239045798778534, |
| "kl": 0.837432861328125, |
| "learning_rate": 5.753496003862935e-09, |
| "loss": 0.0877, |
| "reward": 1.416666690260172, |
| "reward_std": 0.46875908970832825, |
| "rewards/accuracy_reward": 0.6041666772216558, |
| "rewards/format_reward": 0.8125000037252903, |
| "step": 272 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 289.27084159851074, |
| "epoch": 2.928, |
| "grad_norm": 0.4770262837409973, |
| "kl": 0.46295166015625, |
| "learning_rate": 4.227775814646173e-09, |
| "loss": 0.0406, |
| "reward": 1.1458333805203438, |
| "reward_std": 0.4217669777572155, |
| "rewards/accuracy_reward": 0.25000000931322575, |
| "rewards/format_reward": 0.8958333432674408, |
| "step": 273 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 249.45834159851074, |
| "epoch": 2.9386666666666668, |
| "grad_norm": 0.523398220539093, |
| "kl": 0.4814453125, |
| "learning_rate": 2.9363771029242705e-09, |
| "loss": 0.0887, |
| "reward": 1.5625000596046448, |
| "reward_std": 0.5939319953322411, |
| "rewards/accuracy_reward": 0.7083333544433117, |
| "rewards/format_reward": 0.8541667014360428, |
| "step": 274 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 246.04167366027832, |
| "epoch": 2.9493333333333336, |
| "grad_norm": 0.3436894416809082, |
| "kl": 0.813873291015625, |
| "learning_rate": 1.8795021738670093e-09, |
| "loss": 0.0461, |
| "reward": 1.5208333507180214, |
| "reward_std": 0.5120971687138081, |
| "rewards/accuracy_reward": 0.6666666828095913, |
| "rewards/format_reward": 0.854166679084301, |
| "step": 275 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 274.6666774749756, |
| "epoch": 2.96, |
| "grad_norm": 0.20680680871009827, |
| "kl": 0.593109130859375, |
| "learning_rate": 1.0573165931209162e-09, |
| "loss": 0.0129, |
| "reward": 1.5416667014360428, |
| "reward_std": 0.43123577535152435, |
| "rewards/accuracy_reward": 0.6666666846722364, |
| "rewards/format_reward": 0.8750000111758709, |
| "step": 276 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 222.81250762939453, |
| "epoch": 2.970666666666667, |
| "grad_norm": 0.2603120803833008, |
| "kl": 0.629364013671875, |
| "learning_rate": 4.699491608713413e-10, |
| "loss": -0.0279, |
| "reward": 1.479166716337204, |
| "reward_std": 0.5843288898468018, |
| "rewards/accuracy_reward": 0.604166692122817, |
| "rewards/format_reward": 0.8750000223517418, |
| "step": 277 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 321.8958435058594, |
| "epoch": 2.981333333333333, |
| "grad_norm": 1.4922990798950195, |
| "kl": 1.21771240234375, |
| "learning_rate": 1.1749189166609943e-10, |
| "loss": 0.1116, |
| "reward": 1.2708333656191826, |
| "reward_std": 0.677177045494318, |
| "rewards/accuracy_reward": 0.45833333767950535, |
| "rewards/format_reward": 0.8125000298023224, |
| "step": 278 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 225.45833778381348, |
| "epoch": 2.992, |
| "grad_norm": 0.13138599693775177, |
| "kl": 0.090789794921875, |
| "learning_rate": 0.0, |
| "loss": -0.0398, |
| "reward": 1.770833358168602, |
| "reward_std": 0.26070838794112206, |
| "rewards/accuracy_reward": 0.791666679084301, |
| "rewards/format_reward": 0.9791666716337204, |
| "step": 279 |
| }, |
| { |
| "epoch": 2.992, |
| "step": 279, |
| "total_flos": 0.0, |
| "train_loss": 0.061770447607879864, |
| "train_runtime": 25801.9425, |
| "train_samples_per_second": 0.087, |
| "train_steps_per_second": 0.011 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 279, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 3, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
