{ "best_metric": null, "best_model_checkpoint": null, "epoch": 0.5179558011049724, "eval_steps": 500, "global_step": 1500, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "Total_reward": 3.7678651665373764e-07, "advantage_std": 1.3013015985488892, "completion_length": 544.9285888671875, "epoch": 0.0003453038674033149, "grad_norm": 3.820777610972956, "kl": 0.0, "learning_rate": 4.999998529000244e-07, "loss": -0.0, "rewards_raw/CoT ICLG": 0.2053774744272232, "rewards_raw/accuracy_reward": 0.5391521453857422, "rewards_raw/clip_reward_wrapper": 0.2592490315437317, "rewards_raw/format_reward": 5.0, "step": 1 }, { "Total_reward": 2.3416111716301202e-08, "advantage_std": 1.5877742767333984, "completion_length": 548.5357666015625, "epoch": 0.0006906077348066298, "grad_norm": 5.0533640327817375, "kl": 0.00049591064453125, "learning_rate": 4.999994116002706e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.1567649245262146, "rewards_raw/accuracy_reward": 0.5490314960479736, "rewards_raw/clip_reward_wrapper": 0.2823711633682251, "rewards_raw/format_reward": 5.0, "step": 2 }, { "Total_reward": -4.257474728319721e-08, "advantage_std": 1.6971460580825806, "completion_length": 536.75, "epoch": 0.0010359116022099447, "grad_norm": 8.52958796532309, "kl": 0.00054931640625, "learning_rate": 4.99998676101258e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.5830060243606567, "rewards_raw/accuracy_reward": 0.5456159114837646, "rewards_raw/clip_reward_wrapper": 0.26301491260528564, "rewards_raw/format_reward": 5.0, "step": 3 }, { "Total_reward": 4.363911543237009e-08, "advantage_std": 1.5525906085968018, "completion_length": 530.3214721679688, "epoch": 0.0013812154696132596, "grad_norm": 3.2851160397686865, "kl": 0.0004215240478515625, "learning_rate": 4.999976464038522e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.13023559749126434, "rewards_raw/accuracy_reward": 0.5426667928695679, "rewards_raw/clip_reward_wrapper": 0.2651683986186981, "rewards_raw/format_reward": 5.0, "step": 4 }, { "Total_reward": 5.10896995820076e-08, "advantage_std": 1.6370394229888916, "completion_length": 553.6607666015625, "epoch": 0.0017265193370165745, "grad_norm": 7.168393885906233, "kl": 0.0005645751953125, "learning_rate": 4.999963225092648e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.0029400205239653587, "rewards_raw/accuracy_reward": 0.5666831731796265, "rewards_raw/clip_reward_wrapper": 0.26911136507987976, "rewards_raw/format_reward": 5.0, "step": 5 }, { "Total_reward": 9.792191946189632e-08, "advantage_std": 1.3893508911132812, "completion_length": 548.857177734375, "epoch": 0.0020718232044198894, "grad_norm": 3.1681764903755383, "kl": 0.0004825592041015625, "learning_rate": 4.999947044190538e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.00489798653870821, "rewards_raw/accuracy_reward": 0.5392516255378723, "rewards_raw/clip_reward_wrapper": 0.26430901885032654, "rewards_raw/format_reward": 5.0, "step": 6 }, { "Total_reward": -5.747590989813034e-07, "advantage_std": 1.540329933166504, "completion_length": 516.4107666015625, "epoch": 0.0024171270718232043, "grad_norm": 5.356843131621023, "kl": 0.0005340576171875, "learning_rate": 4.999927921351234e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.13179372251033783, "rewards_raw/accuracy_reward": 0.5478073954582214, "rewards_raw/clip_reward_wrapper": 0.25724899768829346, "rewards_raw/format_reward": 5.0, "step": 7 }, { "Total_reward": -6.173338817916374e-08, "advantage_std": 1.8101165294647217, "completion_length": 545.0, "epoch": 0.0027624309392265192, "grad_norm": 8.835716575423502, "kl": 0.000537872314453125, "learning_rate": 4.99990585659724e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.25604355335235596, "rewards_raw/accuracy_reward": 0.5468096137046814, "rewards_raw/clip_reward_wrapper": 0.2707265317440033, "rewards_raw/format_reward": 5.0, "step": 8 }, { "Total_reward": 4.645969511329895e-07, "advantage_std": 1.6038609743118286, "completion_length": 548.25, "epoch": 0.003107734806629834, "grad_norm": 4.931037451440273, "kl": 0.000675201416015625, "learning_rate": 4.999880849954522e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.13242053985595703, "rewards_raw/accuracy_reward": 0.5505883097648621, "rewards_raw/clip_reward_wrapper": 0.26960790157318115, "rewards_raw/format_reward": 5.0, "step": 9 }, { "Total_reward": -7.237706967089252e-08, "advantage_std": 1.4872801303863525, "completion_length": 539.25, "epoch": 0.003453038674033149, "grad_norm": 4.3071400372745705, "kl": 0.000579833984375, "learning_rate": 4.999852901452507e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.7705937623977661, "rewards_raw/accuracy_reward": 0.5542618036270142, "rewards_raw/clip_reward_wrapper": 0.26355570554733276, "rewards_raw/format_reward": 5.0, "step": 10 }, { "Total_reward": 2.660921722963394e-07, "advantage_std": 1.5245392322540283, "completion_length": 532.982177734375, "epoch": 0.003798342541436464, "grad_norm": 4.892174579881816, "kl": 0.000637054443359375, "learning_rate": 4.999822011124084e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.12882253527641296, "rewards_raw/accuracy_reward": 0.5502899289131165, "rewards_raw/clip_reward_wrapper": 0.2625480890274048, "rewards_raw/format_reward": 5.0, "step": 11 }, { "Total_reward": 1.2069940567016602e-06, "advantage_std": 1.3909637928009033, "completion_length": 521.482177734375, "epoch": 0.004143646408839779, "grad_norm": 3.6900702863259283, "kl": 0.00054168701171875, "learning_rate": 4.999788179005607e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.08975247293710709, "rewards_raw/accuracy_reward": 0.5593005418777466, "rewards_raw/clip_reward_wrapper": 0.26997071504592896, "rewards_raw/format_reward": 5.0, "step": 12 }, { "Total_reward": -2.575772271029564e-07, "advantage_std": 1.5189039707183838, "completion_length": 554.25, "epoch": 0.004488950276243094, "grad_norm": 3.508857254815982, "kl": 0.000640869140625, "learning_rate": 4.999751405136889e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.168446883559227, "rewards_raw/accuracy_reward": 0.5342169404029846, "rewards_raw/clip_reward_wrapper": 0.2550903856754303, "rewards_raw/format_reward": 5.0, "step": 13 }, { "Total_reward": 4.470348713425665e-08, "advantage_std": 1.5492525100708008, "completion_length": 540.857177734375, "epoch": 0.004834254143646409, "grad_norm": 5.162262542237812, "kl": 0.000640869140625, "learning_rate": 4.999711689561203e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.045171741396188736, "rewards_raw/accuracy_reward": 0.5622472763061523, "rewards_raw/clip_reward_wrapper": 0.26976853609085083, "rewards_raw/format_reward": 5.0, "step": 14 }, { "Total_reward": 3.7891524584665603e-07, "advantage_std": 1.6218661069869995, "completion_length": 529.6964721679688, "epoch": 0.0051795580110497235, "grad_norm": 3.43032784045246, "kl": 0.000568389892578125, "learning_rate": 4.99966903232529e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.01099963765591383, "rewards_raw/accuracy_reward": 0.5659701228141785, "rewards_raw/clip_reward_wrapper": 0.27172842621803284, "rewards_raw/format_reward": 5.0, "step": 15 }, { "Total_reward": -1.9158636632710113e-07, "advantage_std": 1.4006550312042236, "completion_length": 539.5357666015625, "epoch": 0.0055248618784530384, "grad_norm": 5.815245163620592, "kl": 0.000579833984375, "learning_rate": 4.999623433479346e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.22928695380687714, "rewards_raw/accuracy_reward": 0.5572542548179626, "rewards_raw/clip_reward_wrapper": 0.2580520808696747, "rewards_raw/format_reward": 5.0, "step": 16 }, { "Total_reward": 3.916876778475853e-07, "advantage_std": 1.462172508239746, "completion_length": 531.2678833007812, "epoch": 0.005870165745856353, "grad_norm": 7.282604458165541, "kl": 0.000637054443359375, "learning_rate": 4.999574893077033e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.07184437662363052, "rewards_raw/accuracy_reward": 0.5495157837867737, "rewards_raw/clip_reward_wrapper": 0.2808060348033905, "rewards_raw/format_reward": 5.0, "step": 17 }, { "Total_reward": 1.9797258232756576e-07, "advantage_std": 1.5979071855545044, "completion_length": 527.1785888671875, "epoch": 0.006215469613259668, "grad_norm": 6.193892248382822, "kl": 0.00061798095703125, "learning_rate": 4.999523411175473e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.154600590467453, "rewards_raw/accuracy_reward": 0.5461306571960449, "rewards_raw/clip_reward_wrapper": 0.2588827908039093, "rewards_raw/format_reward": 5.0, "step": 18 }, { "Total_reward": -5.800809503853088e-07, "advantage_std": 1.381419062614441, "completion_length": 530.0, "epoch": 0.006560773480662983, "grad_norm": 5.328347015193286, "kl": 0.000690460205078125, "learning_rate": 4.999468987835252e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.02479776367545128, "rewards_raw/accuracy_reward": 0.5618874430656433, "rewards_raw/clip_reward_wrapper": 0.2700769901275635, "rewards_raw/format_reward": 5.0, "step": 19 }, { "Total_reward": -8.791685672804306e-07, "advantage_std": 1.49555242061615, "completion_length": 553.5, "epoch": 0.006906077348066298, "grad_norm": 3.7014603641341917, "kl": 0.00109100341796875, "learning_rate": 4.999411623120412e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.40345117449760437, "rewards_raw/accuracy_reward": 0.524497389793396, "rewards_raw/clip_reward_wrapper": 0.2833903431892395, "rewards_raw/format_reward": 5.0, "step": 20 }, { "Total_reward": -1.6817025993987045e-07, "advantage_std": 1.4033880233764648, "completion_length": 539.7142944335938, "epoch": 0.007251381215469613, "grad_norm": 2.828570129622724, "kl": 0.0007171630859375, "learning_rate": 4.999351317098463e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.018932510167360306, "rewards_raw/accuracy_reward": 0.5370692014694214, "rewards_raw/clip_reward_wrapper": 0.26074516773223877, "rewards_raw/format_reward": 5.0, "step": 21 }, { "Total_reward": 4.5342108023760375e-07, "advantage_std": 1.6576473712921143, "completion_length": 548.4107666015625, "epoch": 0.007596685082872928, "grad_norm": 13.35859852260371, "kl": 0.00067901611328125, "learning_rate": 4.99928806984037e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.06995975226163864, "rewards_raw/accuracy_reward": 0.5420945882797241, "rewards_raw/clip_reward_wrapper": 0.2833397090435028, "rewards_raw/format_reward": 5.0, "step": 22 }, { "Total_reward": -1.319817215517105e-07, "advantage_std": 1.4263255596160889, "completion_length": 526.5178833007812, "epoch": 0.007941988950276244, "grad_norm": 3.7793025442736528, "kl": 0.0006866455078125, "learning_rate": 4.999221881420563e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.037917934358119965, "rewards_raw/accuracy_reward": 0.5455657243728638, "rewards_raw/clip_reward_wrapper": 0.2594003975391388, "rewards_raw/format_reward": 5.0, "step": 23 }, { "Total_reward": -4.002026230409683e-07, "advantage_std": 1.2331974506378174, "completion_length": 529.732177734375, "epoch": 0.008287292817679558, "grad_norm": 3.051674317263212, "kl": 0.00077056884765625, "learning_rate": 4.999152751916935e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.42752644419670105, "rewards_raw/accuracy_reward": 0.5502229928970337, "rewards_raw/clip_reward_wrapper": 0.2607329487800598, "rewards_raw/format_reward": 5.0, "step": 24 }, { "Total_reward": -3.139887780889694e-07, "advantage_std": 1.418794870376587, "completion_length": 528.9642944335938, "epoch": 0.008632596685082873, "grad_norm": 3.900645200255051, "kl": 0.000732421875, "learning_rate": 4.999080681410835e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.02365497313439846, "rewards_raw/accuracy_reward": 0.565673828125, "rewards_raw/clip_reward_wrapper": 0.27329570055007935, "rewards_raw/format_reward": 5.0, "step": 25 }, { "Total_reward": 7.535730333074753e-07, "advantage_std": 1.5951372385025024, "completion_length": 527.0892944335938, "epoch": 0.008977900552486187, "grad_norm": 4.500315761978984, "kl": 0.000732421875, "learning_rate": 4.999005669987078e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.03977833315730095, "rewards_raw/accuracy_reward": 0.5455085635185242, "rewards_raw/clip_reward_wrapper": 0.2774393856525421, "rewards_raw/format_reward": 5.0, "step": 26 }, { "Total_reward": 1.4901161193847656e-08, "advantage_std": 1.527603268623352, "completion_length": 524.9464721679688, "epoch": 0.009323204419889503, "grad_norm": 4.275311376134378, "kl": 0.00069427490234375, "learning_rate": 4.998927717733934e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.284778356552124, "rewards_raw/accuracy_reward": 0.5641474723815918, "rewards_raw/clip_reward_wrapper": 0.2588571012020111, "rewards_raw/format_reward": 5.0, "step": 27 }, { "Total_reward": -5.960464477539062e-07, "advantage_std": 1.416927695274353, "completion_length": 540.232177734375, "epoch": 0.009668508287292817, "grad_norm": 8.795198290257902, "kl": 0.000797271728515625, "learning_rate": 4.99884682474314e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.17683598399162292, "rewards_raw/accuracy_reward": 0.5529056191444397, "rewards_raw/clip_reward_wrapper": 0.260739803314209, "rewards_raw/format_reward": 5.0, "step": 28 }, { "Total_reward": -2.7247838829680404e-07, "advantage_std": 1.6397868394851685, "completion_length": 544.232177734375, "epoch": 0.010013812154696133, "grad_norm": 6.197843735406051, "kl": 0.0008392333984375, "learning_rate": 4.998762991109889e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.027827152982354164, "rewards_raw/accuracy_reward": 0.5569967031478882, "rewards_raw/clip_reward_wrapper": 0.2746717929840088, "rewards_raw/format_reward": 5.0, "step": 29 }, { "Total_reward": -1.5114035534224968e-07, "advantage_std": 1.4956527948379517, "completion_length": 528.8214721679688, "epoch": 0.010359116022099447, "grad_norm": 3.0952870570769426, "kl": 0.00086212158203125, "learning_rate": 4.998676216932839e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.16699431836605072, "rewards_raw/accuracy_reward": 0.5593418478965759, "rewards_raw/clip_reward_wrapper": 0.2633999288082123, "rewards_raw/format_reward": 5.0, "step": 30 }, { "Total_reward": 6.737453759342316e-07, "advantage_std": 1.4399821758270264, "completion_length": 522.125, "epoch": 0.010704419889502763, "grad_norm": 6.5362477880363805, "kl": 0.00089263916015625, "learning_rate": 4.998586502314104e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.0362359881401062, "rewards_raw/accuracy_reward": 0.5512585043907166, "rewards_raw/clip_reward_wrapper": 0.2727758586406708, "rewards_raw/format_reward": 5.0, "step": 31 }, { "Total_reward": 5.3218434103996515e-09, "advantage_std": 1.4898673295974731, "completion_length": 542.4642944335938, "epoch": 0.011049723756906077, "grad_norm": 3.028278449572225, "kl": 0.0009307861328125, "learning_rate": 4.998493847359259e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.22690129280090332, "rewards_raw/accuracy_reward": 0.5464760065078735, "rewards_raw/clip_reward_wrapper": 0.27615436911582947, "rewards_raw/format_reward": 5.0, "step": 32 }, { "Total_reward": 3.618853483544626e-08, "advantage_std": 1.3925362825393677, "completion_length": 511.64288330078125, "epoch": 0.011395027624309393, "grad_norm": 3.854172875518781, "kl": 0.0008697509765625, "learning_rate": 4.998398252177343e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.10484276711940765, "rewards_raw/accuracy_reward": 0.5374171733856201, "rewards_raw/clip_reward_wrapper": 0.2627907395362854, "rewards_raw/format_reward": 5.0, "step": 33 }, { "Total_reward": -2.7460711748972244e-07, "advantage_std": 1.592063069343567, "completion_length": 532.8928833007812, "epoch": 0.011740331491712707, "grad_norm": 3.8699760178505764, "kl": 0.000949859619140625, "learning_rate": 4.99829971688085e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.12083924561738968, "rewards_raw/accuracy_reward": 0.5541420578956604, "rewards_raw/clip_reward_wrapper": 0.265118807554245, "rewards_raw/format_reward": 5.0, "step": 34 }, { "Total_reward": 1.4262541014886665e-07, "advantage_std": 1.5054094791412354, "completion_length": 526.625, "epoch": 0.012085635359116022, "grad_norm": 10.005628249926396, "kl": 0.000942230224609375, "learning_rate": 4.998198241585739e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 1.230852484703064, "rewards_raw/accuracy_reward": 0.5284959077835083, "rewards_raw/clip_reward_wrapper": 0.26017701625823975, "rewards_raw/format_reward": 5.0, "step": 35 }, { "Total_reward": -5.790165573671402e-07, "advantage_std": 1.5012624263763428, "completion_length": 534.7678833007812, "epoch": 0.012430939226519336, "grad_norm": 7.705749629698544, "kl": 0.000949859619140625, "learning_rate": 4.998093826411422e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.08841613680124283, "rewards_raw/accuracy_reward": 0.5507487654685974, "rewards_raw/clip_reward_wrapper": 0.26477134227752686, "rewards_raw/format_reward": 5.0, "step": 36 }, { "Total_reward": -4.789659335813212e-09, "advantage_std": 1.5348552465438843, "completion_length": 543.9107666015625, "epoch": 0.012776243093922652, "grad_norm": 3.4827345568855823, "kl": 0.0010528564453125, "learning_rate": 4.997986471480779e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.028610214591026306, "rewards_raw/accuracy_reward": 0.5680136680603027, "rewards_raw/clip_reward_wrapper": 0.275068998336792, "rewards_raw/format_reward": 5.0, "step": 37 }, { "Total_reward": 3.991382584445091e-07, "advantage_std": 1.4227604866027832, "completion_length": 531.3392944335938, "epoch": 0.013121546961325966, "grad_norm": 5.086774276387851, "kl": 0.00104522705078125, "learning_rate": 4.997876176920143e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.05556211993098259, "rewards_raw/accuracy_reward": 0.5515257120132446, "rewards_raw/clip_reward_wrapper": 0.27020615339279175, "rewards_raw/format_reward": 5.0, "step": 38 }, { "Total_reward": -3.4059798537100505e-08, "advantage_std": 1.4755719900131226, "completion_length": 541.1607666015625, "epoch": 0.013466850828729282, "grad_norm": 4.954930728574558, "kl": 0.00107574462890625, "learning_rate": 4.99776294285931e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.21244117617607117, "rewards_raw/accuracy_reward": 0.5501272678375244, "rewards_raw/clip_reward_wrapper": 0.2861342430114746, "rewards_raw/format_reward": 5.0, "step": 39 }, { "Total_reward": -6.631017299696396e-07, "advantage_std": 1.3219239711761475, "completion_length": 554.8035888671875, "epoch": 0.013812154696132596, "grad_norm": 13.816749311048396, "kl": 0.001068115234375, "learning_rate": 4.997646769431531e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.10124067217111588, "rewards_raw/accuracy_reward": 0.5623647570610046, "rewards_raw/clip_reward_wrapper": 0.2624461054801941, "rewards_raw/format_reward": 5.0, "step": 40 }, { "Total_reward": 2.55448497910038e-07, "advantage_std": 1.5635424852371216, "completion_length": 534.625, "epoch": 0.014157458563535912, "grad_norm": 4.955056934317048, "kl": 0.00110626220703125, "learning_rate": 4.997527656773523e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.14737151563167572, "rewards_raw/accuracy_reward": 0.5329960584640503, "rewards_raw/clip_reward_wrapper": 0.2665625810623169, "rewards_raw/format_reward": 5.0, "step": 41 }, { "Total_reward": -4.2574748704282683e-07, "advantage_std": 1.70897376537323, "completion_length": 530.5535888671875, "epoch": 0.014502762430939226, "grad_norm": 5.0771638160970936, "kl": 0.0011444091796875, "learning_rate": 4.997405605025455e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.16018721461296082, "rewards_raw/accuracy_reward": 0.5455630421638489, "rewards_raw/clip_reward_wrapper": 0.27412906289100647, "rewards_raw/format_reward": 5.0, "step": 42 }, { "Total_reward": 1.4901161193847656e-08, "advantage_std": 1.4481912851333618, "completion_length": 547.357177734375, "epoch": 0.014848066298342542, "grad_norm": 5.323692978819396, "kl": 0.0011749267578125, "learning_rate": 4.997280614330958e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.1494537889957428, "rewards_raw/accuracy_reward": 0.5487316250801086, "rewards_raw/clip_reward_wrapper": 0.273599773645401, "rewards_raw/format_reward": 5.0, "step": 43 }, { "Total_reward": 2.7247838829680404e-07, "advantage_std": 1.3923403024673462, "completion_length": 534.5535888671875, "epoch": 0.015193370165745856, "grad_norm": 2.6783040064954884, "kl": 0.00112152099609375, "learning_rate": 4.99715268483712e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.0031592294108122587, "rewards_raw/accuracy_reward": 0.5316653251647949, "rewards_raw/clip_reward_wrapper": 0.2691628634929657, "rewards_raw/format_reward": 5.0, "step": 44 }, { "Total_reward": 2.852508202977333e-07, "advantage_std": 1.499343991279602, "completion_length": 537.107177734375, "epoch": 0.015538674033149171, "grad_norm": 4.335028739714115, "kl": 0.001129150390625, "learning_rate": 4.99702181669449e-07, "loss": 0.0, "rewards_raw/CoT ICLG": -0.15742892026901245, "rewards_raw/accuracy_reward": 0.5567408204078674, "rewards_raw/clip_reward_wrapper": 0.2634657323360443, "rewards_raw/format_reward": 5.0, "step": 45 }, { "Total_reward": -3.83172746865057e-08, "advantage_std": 1.4470787048339844, "completion_length": 547.625, "epoch": 0.015883977900552487, "grad_norm": 4.119615251043658, "kl": 0.001251220703125, "learning_rate": 4.996888010057074e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": 0.0712594985961914, "rewards_raw/accuracy_reward": 0.5534456372261047, "rewards_raw/clip_reward_wrapper": 0.25799593329429626, "rewards_raw/format_reward": 5.0, "step": 46 }, { "Total_reward": -1.0643687176070671e-07, "advantage_std": 1.6197601556777954, "completion_length": 538.357177734375, "epoch": 0.0162292817679558, "grad_norm": 4.562940070875313, "kl": 0.00128173828125, "learning_rate": 4.996751265082334e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.05841388553380966, "rewards_raw/accuracy_reward": 0.5449908971786499, "rewards_raw/clip_reward_wrapper": 0.2700492739677429, "rewards_raw/format_reward": 5.0, "step": 47 }, { "Total_reward": 6.003039629831619e-07, "advantage_std": 1.5313037633895874, "completion_length": 535.8392944335938, "epoch": 0.016574585635359115, "grad_norm": 8.391467753405983, "kl": 0.00115203857421875, "learning_rate": 4.996611581931193e-07, "loss": 0.0, "rewards_raw/CoT ICLG": 0.26393580436706543, "rewards_raw/accuracy_reward": 0.5492407083511353, "rewards_raw/clip_reward_wrapper": 0.26276934146881104, "rewards_raw/format_reward": 5.0, "step": 48 }, { "Total_reward": -2.937657654911163e-07, "advantage_std": 1.6209067106246948, "completion_length": 556.1607666015625, "epoch": 0.01691988950276243, "grad_norm": 6.432403573887328, "kl": 0.0012664794921875, "learning_rate": 4.996468960768027e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.0022906577214598656, "rewards_raw/accuracy_reward": 0.5363568067550659, "rewards_raw/clip_reward_wrapper": 0.28627046942710876, "rewards_raw/format_reward": 5.0, "step": 49 }, { "Total_reward": -1.7562084053679428e-07, "advantage_std": 1.4604822397232056, "completion_length": 521.3035888671875, "epoch": 0.017265193370165747, "grad_norm": 3.19432074902889, "kl": 0.00136566162109375, "learning_rate": 4.996323401760677e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.08484985679388046, "rewards_raw/accuracy_reward": 0.5458639860153198, "rewards_raw/clip_reward_wrapper": 0.2580966055393219, "rewards_raw/format_reward": 5.0, "step": 50 }, { "Total_reward": 7.876328567135715e-08, "advantage_std": 1.394093632698059, "completion_length": 537.7857666015625, "epoch": 0.01761049723756906, "grad_norm": 8.383219493084745, "kl": 0.00141143798828125, "learning_rate": 4.996174905080433e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.04570167884230614, "rewards_raw/accuracy_reward": 0.5725643038749695, "rewards_raw/clip_reward_wrapper": 0.28410252928733826, "rewards_raw/format_reward": 5.0, "step": 51 }, { "Total_reward": -2.55448497910038e-08, "advantage_std": 1.7771891355514526, "completion_length": 528.357177734375, "epoch": 0.017955801104972375, "grad_norm": 4.7661829947875525, "kl": 0.0012664794921875, "learning_rate": 4.99602347090205e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": 0.8049647808074951, "rewards_raw/accuracy_reward": 0.5399656295776367, "rewards_raw/clip_reward_wrapper": 0.2555745244026184, "rewards_raw/format_reward": 5.0, "step": 52 }, { "Total_reward": 3.8530146184712066e-07, "advantage_std": 1.4607365131378174, "completion_length": 540.1964721679688, "epoch": 0.01830110497237569, "grad_norm": 3.9544578142243574, "kl": 0.00131988525390625, "learning_rate": 4.995869099403731e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.142437145113945, "rewards_raw/accuracy_reward": 0.5110816955566406, "rewards_raw/clip_reward_wrapper": 0.2706608772277832, "rewards_raw/format_reward": 5.0, "step": 53 }, { "Total_reward": -5.683729114025482e-07, "advantage_std": 1.589028239250183, "completion_length": 539.0535888671875, "epoch": 0.018646408839779006, "grad_norm": 4.441439992398222, "kl": 0.00148773193359375, "learning_rate": 4.995711790767143e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.08036550879478455, "rewards_raw/accuracy_reward": 0.5660966634750366, "rewards_raw/clip_reward_wrapper": 0.27623388171195984, "rewards_raw/format_reward": 5.0, "step": 54 }, { "Total_reward": 1.0005065860241302e-06, "advantage_std": 1.62366783618927, "completion_length": 547.8035888671875, "epoch": 0.01899171270718232, "grad_norm": 6.010170790228934, "kl": 0.00148773193359375, "learning_rate": 4.995551545177406e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": 0.4828476905822754, "rewards_raw/accuracy_reward": 0.5435479283332825, "rewards_raw/clip_reward_wrapper": 0.27734375, "rewards_raw/format_reward": 5.0, "step": 55 }, { "Total_reward": -1.1069434435739822e-07, "advantage_std": 1.681904911994934, "completion_length": 551.875, "epoch": 0.019337016574585635, "grad_norm": 3.8529423881893443, "kl": 0.0016021728515625, "learning_rate": 4.995388362823098e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.05221758782863617, "rewards_raw/accuracy_reward": 0.5520492792129517, "rewards_raw/clip_reward_wrapper": 0.2594911754131317, "rewards_raw/format_reward": 5.0, "step": 56 }, { "Total_reward": -6.737453759342316e-07, "advantage_std": 1.3590770959854126, "completion_length": 536.2678833007812, "epoch": 0.01968232044198895, "grad_norm": 3.2069550525516863, "kl": 0.0015716552734375, "learning_rate": 4.995222243896252e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": 0.06733390688896179, "rewards_raw/accuracy_reward": 0.5559384822845459, "rewards_raw/clip_reward_wrapper": 0.2733931243419647, "rewards_raw/format_reward": 5.0, "step": 57 }, { "Total_reward": -3.640140846528084e-07, "advantage_std": 1.4492419958114624, "completion_length": 536.607177734375, "epoch": 0.020027624309392266, "grad_norm": 6.571093726112222, "kl": 0.001678466796875, "learning_rate": 4.995053188592355e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.18158669769763947, "rewards_raw/accuracy_reward": 0.534403920173645, "rewards_raw/clip_reward_wrapper": 0.2735101878643036, "rewards_raw/format_reward": 5.0, "step": 58 }, { "Total_reward": -3.4059798537100505e-08, "advantage_std": 1.4412157535552979, "completion_length": 542.8392944335938, "epoch": 0.02037292817679558, "grad_norm": 4.0204948997683605, "kl": 0.00164031982421875, "learning_rate": 4.994881197110352e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.13755694031715393, "rewards_raw/accuracy_reward": 0.545576274394989, "rewards_raw/clip_reward_wrapper": 0.27181172370910645, "rewards_raw/format_reward": 5.0, "step": 59 }, { "Total_reward": 1.2133803295455436e-07, "advantage_std": 1.5062249898910522, "completion_length": 537.857177734375, "epoch": 0.020718232044198894, "grad_norm": 6.96539599915367, "kl": 0.00168609619140625, "learning_rate": 4.994706269652644e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.012379472143948078, "rewards_raw/accuracy_reward": 0.5545734167098999, "rewards_raw/clip_reward_wrapper": 0.2525492310523987, "rewards_raw/format_reward": 5.0, "step": 60 }, { "Total_reward": 5.556004794016189e-07, "advantage_std": 1.3153735399246216, "completion_length": 560.5892944335938, "epoch": 0.02106353591160221, "grad_norm": 3.229768935852245, "kl": 0.00164794921875, "learning_rate": 4.994528406425083e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": 0.0657782182097435, "rewards_raw/accuracy_reward": 0.555535078048706, "rewards_raw/clip_reward_wrapper": 0.27611052989959717, "rewards_raw/format_reward": 5.0, "step": 61 }, { "Total_reward": 5.3218435880353354e-08, "advantage_std": 1.578580379486084, "completion_length": 538.8035888671875, "epoch": 0.021408839779005526, "grad_norm": 4.37068727124411, "kl": 0.00177001953125, "learning_rate": 4.994347607636981e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": 0.4029756784439087, "rewards_raw/accuracy_reward": 0.5279491543769836, "rewards_raw/clip_reward_wrapper": 0.25686556100845337, "rewards_raw/format_reward": 5.0, "step": 62 }, { "Total_reward": 5.662441253662109e-07, "advantage_std": 1.681679368019104, "completion_length": 538.7678833007812, "epoch": 0.02175414364640884, "grad_norm": 6.08301306766795, "kl": 0.00188446044921875, "learning_rate": 4.9941638735011e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.23750388622283936, "rewards_raw/accuracy_reward": 0.5716294050216675, "rewards_raw/clip_reward_wrapper": 0.266650527715683, "rewards_raw/format_reward": 5.0, "step": 63 }, { "Total_reward": 9.196145356327179e-07, "advantage_std": 1.3781418800354004, "completion_length": 539.2678833007812, "epoch": 0.022099447513812154, "grad_norm": 3.5282641615767547, "kl": 0.00167083740234375, "learning_rate": 4.993977204233659e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.1997544914484024, "rewards_raw/accuracy_reward": 0.5532044768333435, "rewards_raw/clip_reward_wrapper": 0.2707071006298065, "rewards_raw/format_reward": 5.0, "step": 64 }, { "Total_reward": 2.0223005492425727e-07, "advantage_std": 1.4838593006134033, "completion_length": 540.875, "epoch": 0.02244475138121547, "grad_norm": 36.55200168002825, "kl": 0.0019989013671875, "learning_rate": 4.993787600054332e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.030470725148916245, "rewards_raw/accuracy_reward": 0.5403492450714111, "rewards_raw/clip_reward_wrapper": 0.2567340135574341, "rewards_raw/format_reward": 5.0, "step": 65 }, { "Total_reward": 7.333500207096222e-07, "advantage_std": 1.488895058631897, "completion_length": 542.857177734375, "epoch": 0.022790055248618785, "grad_norm": 4.358059463067183, "kl": 0.001983642578125, "learning_rate": 4.993595061186241e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.1277601420879364, "rewards_raw/accuracy_reward": 0.544142484664917, "rewards_raw/clip_reward_wrapper": 0.2736986577510834, "rewards_raw/format_reward": 5.0, "step": 66 }, { "Total_reward": -7.66345493730114e-08, "advantage_std": 1.4726346731185913, "completion_length": 540.1428833007812, "epoch": 0.0231353591160221, "grad_norm": 4.4317553997652945, "kl": 0.002105712890625, "learning_rate": 4.993399587855969e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.2728242874145508, "rewards_raw/accuracy_reward": 0.5565522909164429, "rewards_raw/clip_reward_wrapper": 0.28400561213493347, "rewards_raw/format_reward": 5.0, "step": 67 }, { "Total_reward": 9.206789286508865e-07, "advantage_std": 1.5442893505096436, "completion_length": 544.7142944335938, "epoch": 0.023480662983425413, "grad_norm": 3.3159554603232255, "kl": 0.0018768310546875, "learning_rate": 4.993201180293549e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.07945090532302856, "rewards_raw/accuracy_reward": 0.5449894070625305, "rewards_raw/clip_reward_wrapper": 0.2736750543117523, "rewards_raw/format_reward": 5.0, "step": 68 }, { "Total_reward": -5.27926886206842e-07, "advantage_std": 1.708790898323059, "completion_length": 527.0, "epoch": 0.02382596685082873, "grad_norm": 4.330565736331735, "kl": 0.00201416015625, "learning_rate": 4.992999838732467e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": 0.5248808860778809, "rewards_raw/accuracy_reward": 0.5392948985099792, "rewards_raw/clip_reward_wrapper": 0.2581956386566162, "rewards_raw/format_reward": 5.0, "step": 69 }, { "Total_reward": 6.066901505619171e-07, "advantage_std": 1.4175939559936523, "completion_length": 535.0, "epoch": 0.024171270718232045, "grad_norm": 3.699130854096596, "kl": 0.00225830078125, "learning_rate": 4.99279556340966e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.24624647200107574, "rewards_raw/accuracy_reward": 0.5361307263374329, "rewards_raw/clip_reward_wrapper": 0.26364895701408386, "rewards_raw/format_reward": 5.0, "step": 70 }, { "Total_reward": 1.128230792346585e-06, "advantage_std": 1.7603671550750732, "completion_length": 551.0, "epoch": 0.02451657458563536, "grad_norm": 13.10658051277705, "kl": 0.002105712890625, "learning_rate": 4.99258835456552e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.03606429323554039, "rewards_raw/accuracy_reward": 0.5472319722175598, "rewards_raw/clip_reward_wrapper": 0.2755392789840698, "rewards_raw/format_reward": 5.0, "step": 71 }, { "Total_reward": -4.76837158203125e-07, "advantage_std": 1.6161551475524902, "completion_length": 542.732177734375, "epoch": 0.024861878453038673, "grad_norm": 11.080452231092535, "kl": 0.0023193359375, "learning_rate": 4.99237821244389e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.03310293331742287, "rewards_raw/accuracy_reward": 0.5531694889068604, "rewards_raw/clip_reward_wrapper": 0.2868482172489166, "rewards_raw/format_reward": 5.0, "step": 72 }, { "Total_reward": -1.6178404393940582e-07, "advantage_std": 1.5717582702636719, "completion_length": 540.5178833007812, "epoch": 0.02520718232044199, "grad_norm": 2.9887887197749006, "kl": 0.002227783203125, "learning_rate": 4.992165137292067e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": 0.2318764179944992, "rewards_raw/accuracy_reward": 0.5462968349456787, "rewards_raw/clip_reward_wrapper": 0.2776066064834595, "rewards_raw/format_reward": 5.0, "step": 73 }, { "Total_reward": 1.7029898913278885e-07, "advantage_std": 1.6714789867401123, "completion_length": 547.625, "epoch": 0.025552486187845305, "grad_norm": 14.022226873021259, "kl": 0.0024566650390625, "learning_rate": 4.991949129360795e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": 0.049976374953985214, "rewards_raw/accuracy_reward": 0.5467252731323242, "rewards_raw/clip_reward_wrapper": 0.2700766921043396, "rewards_raw/format_reward": 5.0, "step": 74 }, { "Total_reward": 5.939177185609879e-07, "advantage_std": 1.450186848640442, "completion_length": 521.0714721679688, "epoch": 0.02589779005524862, "grad_norm": 3.141635095442637, "kl": 0.002410888671875, "learning_rate": 4.991730188904275e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": 0.14063866436481476, "rewards_raw/accuracy_reward": 0.5543889403343201, "rewards_raw/clip_reward_wrapper": 0.27688437700271606, "rewards_raw/format_reward": 5.0, "step": 75 }, { "Total_reward": 8.089202196970291e-08, "advantage_std": 1.5466766357421875, "completion_length": 552.625, "epoch": 0.026243093922651933, "grad_norm": 4.851212691578106, "kl": 0.0024261474609375, "learning_rate": 4.991508316180153e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.07256332784891129, "rewards_raw/accuracy_reward": 0.5688727498054504, "rewards_raw/clip_reward_wrapper": 0.26008743047714233, "rewards_raw/format_reward": 5.0, "step": 76 }, { "Total_reward": 1.0005065576024208e-07, "advantage_std": 1.5232013463974, "completion_length": 552.982177734375, "epoch": 0.02658839779005525, "grad_norm": 3.755548084626777, "kl": 0.0025177001953125, "learning_rate": 4.991283511449532e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.10310900956392288, "rewards_raw/accuracy_reward": 0.5463109016418457, "rewards_raw/clip_reward_wrapper": 0.2712312638759613, "rewards_raw/format_reward": 5.0, "step": 77 }, { "Total_reward": -4.470348642371391e-07, "advantage_std": 1.509238600730896, "completion_length": 564.2857666015625, "epoch": 0.026933701657458564, "grad_norm": 4.603381275882107, "kl": 0.0025787353515625, "learning_rate": 4.99105577497696e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.06541189551353455, "rewards_raw/accuracy_reward": 0.5475320219993591, "rewards_raw/clip_reward_wrapper": 0.2665116488933563, "rewards_raw/format_reward": 5.0, "step": 78 }, { "Total_reward": 1.021793991640152e-07, "advantage_std": 1.5462901592254639, "completion_length": 541.25, "epoch": 0.027279005524861878, "grad_norm": 10.637490678581266, "kl": 0.002655029296875, "learning_rate": 4.990825107030438e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.29512640833854675, "rewards_raw/accuracy_reward": 0.5774394273757935, "rewards_raw/clip_reward_wrapper": 0.27652066946029663, "rewards_raw/format_reward": 5.0, "step": 79 }, { "Total_reward": -2.7673587510435027e-07, "advantage_std": 1.5617196559906006, "completion_length": 540.0892944335938, "epoch": 0.027624309392265192, "grad_norm": 3.7828148521879807, "kl": 0.00274658203125, "learning_rate": 4.990591507881416e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": 0.023654701188206673, "rewards_raw/accuracy_reward": 0.5523390173912048, "rewards_raw/clip_reward_wrapper": 0.275909423828125, "rewards_raw/format_reward": 5.0, "step": 80 }, { "Total_reward": 1.2665987014770508e-07, "advantage_std": 1.431259274482727, "completion_length": 530.5357666015625, "epoch": 0.02796961325966851, "grad_norm": 3.437693026340378, "kl": 0.0028839111328125, "learning_rate": 4.990354977804792e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": 0.28551632165908813, "rewards_raw/accuracy_reward": 0.5711768269538879, "rewards_raw/clip_reward_wrapper": 0.27442002296447754, "rewards_raw/format_reward": 5.0, "step": 81 }, { "Total_reward": -1.6604151653609733e-07, "advantage_std": 1.3799837827682495, "completion_length": 543.9107666015625, "epoch": 0.028314917127071824, "grad_norm": 3.096630684544752, "kl": 0.002655029296875, "learning_rate": 4.990115517078917e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.11071396619081497, "rewards_raw/accuracy_reward": 0.5533071756362915, "rewards_raw/clip_reward_wrapper": 0.2741154134273529, "rewards_raw/format_reward": 5.0, "step": 82 }, { "Total_reward": -4.225543648317398e-07, "advantage_std": 1.3226157426834106, "completion_length": 555.482177734375, "epoch": 0.028660220994475138, "grad_norm": 4.8542169381629385, "kl": 0.002899169921875, "learning_rate": 4.989873125985586e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": 0.2519896626472473, "rewards_raw/accuracy_reward": 0.5496246814727783, "rewards_raw/clip_reward_wrapper": 0.26967334747314453, "rewards_raw/format_reward": 5.0, "step": 83 }, { "Total_reward": -2.2777490471526107e-07, "advantage_std": 1.5050489902496338, "completion_length": 533.232177734375, "epoch": 0.029005524861878452, "grad_norm": 3.984061187242299, "kl": 0.0033416748046875, "learning_rate": 4.989627804810047e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.19753830134868622, "rewards_raw/accuracy_reward": 0.5535197257995605, "rewards_raw/clip_reward_wrapper": 0.2681303918361664, "rewards_raw/format_reward": 5.0, "step": 84 }, { "Total_reward": 3.448554650731239e-07, "advantage_std": 1.598309874534607, "completion_length": 547.857177734375, "epoch": 0.02935082872928177, "grad_norm": 4.850514883023093, "kl": 0.00323486328125, "learning_rate": 4.989379553840992e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.042796265333890915, "rewards_raw/accuracy_reward": 0.547249436378479, "rewards_raw/clip_reward_wrapper": 0.27979880571365356, "rewards_raw/format_reward": 5.0, "step": 85 }, { "Total_reward": -1.1920928955078125e-07, "advantage_std": 1.5344841480255127, "completion_length": 532.9107666015625, "epoch": 0.029696132596685083, "grad_norm": 4.053330659847069, "kl": 0.0032958984375, "learning_rate": 4.989128373370563e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.15033647418022156, "rewards_raw/accuracy_reward": 0.5532473921775818, "rewards_raw/clip_reward_wrapper": 0.2751409411430359, "rewards_raw/format_reward": 5.0, "step": 86 }, { "Total_reward": 3.08666926684964e-07, "advantage_std": 1.3224024772644043, "completion_length": 535.5714721679688, "epoch": 0.030041436464088397, "grad_norm": 2.9149381461689248, "kl": 0.003173828125, "learning_rate": 4.988874263694351e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": 0.14702759683132172, "rewards_raw/accuracy_reward": 0.5553412437438965, "rewards_raw/clip_reward_wrapper": 0.27594193816185, "rewards_raw/format_reward": 5.0, "step": 87 }, { "Total_reward": 8.578811616644089e-07, "advantage_std": 1.4901316165924072, "completion_length": 550.482177734375, "epoch": 0.03038674033149171, "grad_norm": 3.642029875681976, "kl": 0.003448486328125, "learning_rate": 4.988617225111391e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": -0.01895509473979473, "rewards_raw/accuracy_reward": 0.5575993657112122, "rewards_raw/clip_reward_wrapper": 0.28677380084991455, "rewards_raw/format_reward": 5.0, "step": 88 }, { "Total_reward": 1.0707549336075317e-06, "advantage_std": 1.4228986501693726, "completion_length": 547.5892944335938, "epoch": 0.03073204419889503, "grad_norm": 4.992907854742592, "kl": 0.00341796875, "learning_rate": 4.988357257924165e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": 0.24341440200805664, "rewards_raw/accuracy_reward": 0.5624955892562866, "rewards_raw/clip_reward_wrapper": 0.27124232053756714, "rewards_raw/format_reward": 5.0, "step": 89 }, { "Total_reward": -5.843384087711456e-07, "advantage_std": 1.5542560815811157, "completion_length": 530.0, "epoch": 0.031077348066298343, "grad_norm": 4.3197804281038605, "kl": 0.003662109375, "learning_rate": 4.988094362438604e-07, "loss": 0.0001, "rewards_raw/CoT ICLG": 0.1774117648601532, "rewards_raw/accuracy_reward": 0.5503580570220947, "rewards_raw/clip_reward_wrapper": 0.26699572801589966, "rewards_raw/format_reward": 5.0, "step": 90 }, { "Total_reward": -1.4901161193847656e-07, "advantage_std": 1.3654721975326538, "completion_length": 524.482177734375, "epoch": 0.03142265193370166, "grad_norm": 3.4499673161634528, "kl": 0.00390625, "learning_rate": 4.987828538964081e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": 0.14534983038902283, "rewards_raw/accuracy_reward": 0.5591360926628113, "rewards_raw/clip_reward_wrapper": 0.2734087407588959, "rewards_raw/format_reward": 5.0, "step": 91 }, { "Total_reward": 1.0643687176070671e-07, "advantage_std": 1.486676812171936, "completion_length": 528.6785888671875, "epoch": 0.031767955801104975, "grad_norm": 5.69409161399665, "kl": 0.003814697265625, "learning_rate": 4.98755978781342e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": 0.1226472556591034, "rewards_raw/accuracy_reward": 0.5469090342521667, "rewards_raw/clip_reward_wrapper": 0.27164116501808167, "rewards_raw/format_reward": 5.0, "step": 92 }, { "Total_reward": 1.0750123635716591e-07, "advantage_std": 1.4619635343551636, "completion_length": 527.8392944335938, "epoch": 0.03211325966850829, "grad_norm": 5.07077683322132, "kl": 0.004119873046875, "learning_rate": 4.987288109302885e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": -0.1795245260000229, "rewards_raw/accuracy_reward": 0.568863570690155, "rewards_raw/clip_reward_wrapper": 0.2749401926994324, "rewards_raw/format_reward": 5.0, "step": 93 }, { "Total_reward": 4.193612710423622e-07, "advantage_std": 1.4533272981643677, "completion_length": 535.4464721679688, "epoch": 0.0324585635359116, "grad_norm": 2.9551864268078503, "kl": 0.0037841796875, "learning_rate": 4.987013503752189e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": -0.12452757358551025, "rewards_raw/accuracy_reward": 0.5681349039077759, "rewards_raw/clip_reward_wrapper": 0.275720477104187, "rewards_raw/format_reward": 5.0, "step": 94 }, { "Total_reward": -5.151544542059128e-07, "advantage_std": 1.4500651359558105, "completion_length": 531.8392944335938, "epoch": 0.03280386740331492, "grad_norm": 5.970817021005036, "kl": 0.004150390625, "learning_rate": 4.986735971484486e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": -0.05569273233413696, "rewards_raw/accuracy_reward": 0.5596421957015991, "rewards_raw/clip_reward_wrapper": 0.2655959725379944, "rewards_raw/format_reward": 5.0, "step": 95 }, { "Total_reward": 5.662441253662109e-07, "advantage_std": 1.493851900100708, "completion_length": 540.2142944335938, "epoch": 0.03314917127071823, "grad_norm": 4.862492719447053, "kl": 0.004180908203125, "learning_rate": 4.986455512826377e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": -0.03637164458632469, "rewards_raw/accuracy_reward": 0.5573148727416992, "rewards_raw/clip_reward_wrapper": 0.25513479113578796, "rewards_raw/format_reward": 5.0, "step": 96 }, { "Total_reward": -7.557017625003937e-07, "advantage_std": 1.5171374082565308, "completion_length": 527.0357666015625, "epoch": 0.033494475138121545, "grad_norm": 4.019344957600761, "kl": 0.00433349609375, "learning_rate": 4.986172128107905e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": 0.10719563066959381, "rewards_raw/accuracy_reward": 0.5603432655334473, "rewards_raw/clip_reward_wrapper": 0.2652685344219208, "rewards_raw/format_reward": 5.0, "step": 97 }, { "Total_reward": -1.9158636632710113e-07, "advantage_std": 1.4693865776062012, "completion_length": 521.732177734375, "epoch": 0.03383977900552486, "grad_norm": 3.861966814334728, "kl": 0.00439453125, "learning_rate": 4.985885817662557e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": 0.027044394984841347, "rewards_raw/accuracy_reward": 0.553304135799408, "rewards_raw/clip_reward_wrapper": 0.28632527589797974, "rewards_raw/format_reward": 5.0, "step": 98 }, { "Total_reward": 0.0, "advantage_std": 1.5183796882629395, "completion_length": 525.2142944335938, "epoch": 0.03418508287292818, "grad_norm": 4.500398899936733, "kl": 0.0045166015625, "learning_rate": 4.985596581827265e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": 0.018529480323195457, "rewards_raw/accuracy_reward": 0.5634405016899109, "rewards_raw/clip_reward_wrapper": 0.271349161863327, "rewards_raw/format_reward": 5.0, "step": 99 }, { "Total_reward": -2.55448497910038e-07, "advantage_std": 1.4871606826782227, "completion_length": 525.982177734375, "epoch": 0.034530386740331494, "grad_norm": 5.288944981796944, "kl": 0.004425048828125, "learning_rate": 4.985304420942399e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": 0.28080740571022034, "rewards_raw/accuracy_reward": 0.5571067929267883, "rewards_raw/clip_reward_wrapper": 0.2719470262527466, "rewards_raw/format_reward": 5.0, "step": 100 }, { "Total_reward": -3.0228071068449935e-07, "advantage_std": 1.556200385093689, "completion_length": 506.8214416503906, "epoch": 0.03487569060773481, "grad_norm": 5.096670689072091, "kl": 0.004913330078125, "learning_rate": 4.985009335351776e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": -0.09699644148349762, "rewards_raw/accuracy_reward": 0.5545779466629028, "rewards_raw/clip_reward_wrapper": 0.27555975317955017, "rewards_raw/format_reward": 5.0, "step": 101 }, { "Total_reward": -4.896095973094816e-08, "advantage_std": 1.6057206392288208, "completion_length": 542.1785888671875, "epoch": 0.03522099447513812, "grad_norm": 5.179921032482085, "kl": 0.004730224609375, "learning_rate": 4.984711325402651e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": -0.019612960517406464, "rewards_raw/accuracy_reward": 0.5672138333320618, "rewards_raw/clip_reward_wrapper": 0.26362112164497375, "rewards_raw/format_reward": 5.0, "step": 102 }, { "Total_reward": -5.10896995820076e-08, "advantage_std": 1.5900546312332153, "completion_length": 522.482177734375, "epoch": 0.035566298342541436, "grad_norm": 3.169206689362739, "kl": 0.0047607421875, "learning_rate": 4.984410391445724e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": 0.27151840925216675, "rewards_raw/accuracy_reward": 0.5116754174232483, "rewards_raw/clip_reward_wrapper": 0.28932368755340576, "rewards_raw/format_reward": 4.9375, "step": 103 }, { "Total_reward": -2.55448497910038e-07, "advantage_std": 1.2117950916290283, "completion_length": 532.732177734375, "epoch": 0.03591160220994475, "grad_norm": 7.108786265543804, "kl": 0.00506591796875, "learning_rate": 4.984106533835132e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": -0.0033349567092955112, "rewards_raw/accuracy_reward": 0.555770754814148, "rewards_raw/clip_reward_wrapper": 0.29113849997520447, "rewards_raw/format_reward": 5.0, "step": 104 }, { "Total_reward": 5.066394805908203e-07, "advantage_std": 1.5032185316085815, "completion_length": 536.3035888671875, "epoch": 0.036256906077348064, "grad_norm": 5.361139264841673, "kl": 0.00555419921875, "learning_rate": 4.983799752928455e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": 0.089925616979599, "rewards_raw/accuracy_reward": 0.5641834735870361, "rewards_raw/clip_reward_wrapper": 0.2760183811187744, "rewards_raw/format_reward": 5.0, "step": 105 }, { "Total_reward": 1.548656456407116e-07, "advantage_std": 1.497544527053833, "completion_length": 515.0892944335938, "epoch": 0.03660220994475138, "grad_norm": 12.790660571470823, "kl": 0.00531005859375, "learning_rate": 4.983490049086714e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": -0.0821966677904129, "rewards_raw/accuracy_reward": 0.5424985885620117, "rewards_raw/clip_reward_wrapper": 0.29660165309906006, "rewards_raw/format_reward": 5.0, "step": 106 }, { "Total_reward": -4.6406475462390517e-07, "advantage_std": 1.330548882484436, "completion_length": 512.0892944335938, "epoch": 0.0369475138121547, "grad_norm": 6.234721093292275, "kl": 0.0054931640625, "learning_rate": 4.983177422674367e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": 0.2810308337211609, "rewards_raw/accuracy_reward": 0.5575412511825562, "rewards_raw/clip_reward_wrapper": 0.28687775135040283, "rewards_raw/format_reward": 5.0, "step": 107 }, { "Total_reward": -8.727823086474018e-08, "advantage_std": 1.3989101648330688, "completion_length": 525.8214721679688, "epoch": 0.03729281767955801, "grad_norm": 4.197587287009315, "kl": 0.005767822265625, "learning_rate": 4.982861874059314e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": 0.28203609585762024, "rewards_raw/accuracy_reward": 0.5391165614128113, "rewards_raw/clip_reward_wrapper": 0.2827070951461792, "rewards_raw/format_reward": 5.0, "step": 108 }, { "Total_reward": -1.0537250005882015e-07, "advantage_std": 1.4285361766815186, "completion_length": 504.2500305175781, "epoch": 0.03763812154696133, "grad_norm": 3.8176296075390153, "kl": 0.005889892578125, "learning_rate": 4.982543403612891e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": 0.04469750076532364, "rewards_raw/accuracy_reward": 0.5633372664451599, "rewards_raw/clip_reward_wrapper": 0.2779842019081116, "rewards_raw/format_reward": 5.0, "step": 109 }, { "Total_reward": 2.4906228190957336e-07, "advantage_std": 1.2634960412979126, "completion_length": 528.4642944335938, "epoch": 0.03798342541436464, "grad_norm": 3.158646837856019, "kl": 0.006011962890625, "learning_rate": 4.982222011709875e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": 0.011542031541466713, "rewards_raw/accuracy_reward": 0.5901047587394714, "rewards_raw/clip_reward_wrapper": 0.27277126908302307, "rewards_raw/format_reward": 5.0, "step": 110 }, { "Total_reward": -8.94069742685133e-08, "advantage_std": 1.6588972806930542, "completion_length": 519.4642944335938, "epoch": 0.038328729281767955, "grad_norm": 16.628413977230775, "kl": 0.005828857421875, "learning_rate": 4.98189769872848e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": 0.062463048845529556, "rewards_raw/accuracy_reward": 0.5619646310806274, "rewards_raw/clip_reward_wrapper": 0.2963189482688904, "rewards_raw/format_reward": 5.0, "step": 111 }, { "Total_reward": 1.0388238251834991e-06, "advantage_std": 1.4417369365692139, "completion_length": 531.9642944335938, "epoch": 0.03867403314917127, "grad_norm": 3.290464468758497, "kl": 0.0059814453125, "learning_rate": 4.981570465050357e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": -0.06176147982478142, "rewards_raw/accuracy_reward": 0.5574870705604553, "rewards_raw/clip_reward_wrapper": 0.27208608388900757, "rewards_raw/format_reward": 5.0, "step": 112 }, { "Total_reward": -6.811959565311554e-07, "advantage_std": 1.5938453674316406, "completion_length": 525.482177734375, "epoch": 0.03901933701657458, "grad_norm": 4.102104410163163, "kl": 0.00616455078125, "learning_rate": 4.981240311060594e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": -0.050940629094839096, "rewards_raw/accuracy_reward": 0.5439350605010986, "rewards_raw/clip_reward_wrapper": 0.2877698838710785, "rewards_raw/format_reward": 5.0, "step": 113 }, { "Total_reward": -1.7029899268550253e-08, "advantage_std": 1.3366965055465698, "completion_length": 510.0000305175781, "epoch": 0.0393646408839779, "grad_norm": 3.4391742569188346, "kl": 0.0064697265625, "learning_rate": 4.980907237147718e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": -0.2708454728126526, "rewards_raw/accuracy_reward": 0.5598379373550415, "rewards_raw/clip_reward_wrapper": 0.28361448645591736, "rewards_raw/format_reward": 5.0, "step": 114 }, { "Total_reward": 6.407499881788681e-07, "advantage_std": 1.6627007722854614, "completion_length": 521.0357666015625, "epoch": 0.03970994475138122, "grad_norm": 3.306917040828258, "kl": 0.006805419921875, "learning_rate": 4.980571243703688e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": 0.0024803453125059605, "rewards_raw/accuracy_reward": 0.5553105473518372, "rewards_raw/clip_reward_wrapper": 0.2869442403316498, "rewards_raw/format_reward": 5.0, "step": 115 }, { "Total_reward": -3.469841942660423e-07, "advantage_std": 1.3182111978530884, "completion_length": 513.9285888671875, "epoch": 0.04005524861878453, "grad_norm": 9.472603548186225, "kl": 0.006591796875, "learning_rate": 4.980232331123903e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": -0.023304743692278862, "rewards_raw/accuracy_reward": 0.5547733902931213, "rewards_raw/clip_reward_wrapper": 0.2768118679523468, "rewards_raw/format_reward": 5.0, "step": 116 }, { "Total_reward": 6.088188797548355e-07, "advantage_std": 1.4380186796188354, "completion_length": 521.8035888671875, "epoch": 0.040400552486187846, "grad_norm": 4.2723358325161795, "kl": 0.006622314453125, "learning_rate": 4.979890499807194e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": 0.16441947221755981, "rewards_raw/accuracy_reward": 0.5575407147407532, "rewards_raw/clip_reward_wrapper": 0.2923314571380615, "rewards_raw/format_reward": 5.0, "step": 117 }, { "Total_reward": 6.215913117557648e-07, "advantage_std": 1.5080684423446655, "completion_length": 528.482177734375, "epoch": 0.04074585635359116, "grad_norm": 11.973457991909747, "kl": 0.00640869140625, "learning_rate": 4.979545750155829e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": -0.12270514667034149, "rewards_raw/accuracy_reward": 0.5644799470901489, "rewards_raw/clip_reward_wrapper": 0.2830794155597687, "rewards_raw/format_reward": 5.0, "step": 118 }, { "Total_reward": 3.08666926684964e-07, "advantage_std": 1.7452293634414673, "completion_length": 532.5178833007812, "epoch": 0.041091160220994474, "grad_norm": 4.354566824017738, "kl": 0.00616455078125, "learning_rate": 4.979198082575508e-07, "loss": 0.0002, "rewards_raw/CoT ICLG": 0.01133409608155489, "rewards_raw/accuracy_reward": 0.5550068616867065, "rewards_raw/clip_reward_wrapper": 0.28957927227020264, "rewards_raw/format_reward": 5.0, "step": 119 }, { "Total_reward": 8.514949456639442e-08, "advantage_std": 1.698884129524231, "completion_length": 525.3928833007812, "epoch": 0.04143646408839779, "grad_norm": 4.524469905476979, "kl": 0.006866455078125, "learning_rate": 4.978847497475368e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": -0.015224401839077473, "rewards_raw/accuracy_reward": 0.5511736869812012, "rewards_raw/clip_reward_wrapper": 0.29158198833465576, "rewards_raw/format_reward": 5.0, "step": 120 }, { "Total_reward": -5.534717217869911e-08, "advantage_std": 1.3667527437210083, "completion_length": 503.8214416503906, "epoch": 0.0417817679558011, "grad_norm": 3.294013736043703, "kl": 0.006744384765625, "learning_rate": 4.978493995267976e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": 0.2838781476020813, "rewards_raw/accuracy_reward": 0.5711387395858765, "rewards_raw/clip_reward_wrapper": 0.28888440132141113, "rewards_raw/format_reward": 5.0, "step": 121 }, { "Total_reward": 7.216419817268616e-07, "advantage_std": 1.6466139554977417, "completion_length": 506.607177734375, "epoch": 0.04212707182320442, "grad_norm": 3.7763681958694044, "kl": 0.006805419921875, "learning_rate": 4.978137576369334e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": -0.029733335599303246, "rewards_raw/accuracy_reward": 0.553954005241394, "rewards_raw/clip_reward_wrapper": 0.27836933732032776, "rewards_raw/format_reward": 5.0, "step": 122 }, { "Total_reward": -1.1282308065574398e-07, "advantage_std": 1.6448744535446167, "completion_length": 514.9285888671875, "epoch": 0.04247237569060774, "grad_norm": 4.520190899846128, "kl": 0.006805419921875, "learning_rate": 4.977778241198874e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": -0.25298595428466797, "rewards_raw/accuracy_reward": 0.5747519731521606, "rewards_raw/clip_reward_wrapper": 0.283920019865036, "rewards_raw/format_reward": 5.0, "step": 123 }, { "Total_reward": -6.482005687757919e-07, "advantage_std": 1.5241632461547852, "completion_length": 529.0535888671875, "epoch": 0.04281767955801105, "grad_norm": 3.9025510197178916, "kl": 0.006988525390625, "learning_rate": 4.977415990179463e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": -0.14653651416301727, "rewards_raw/accuracy_reward": 0.5753042697906494, "rewards_raw/clip_reward_wrapper": 0.274982750415802, "rewards_raw/format_reward": 5.0, "step": 124 }, { "Total_reward": 3.08666926684964e-07, "advantage_std": 1.4232532978057861, "completion_length": 517.4464721679688, "epoch": 0.043162983425414365, "grad_norm": 4.558047435288674, "kl": 0.0068359375, "learning_rate": 4.9770508237374e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": 0.006379102356731892, "rewards_raw/accuracy_reward": 0.5462497472763062, "rewards_raw/clip_reward_wrapper": 0.27715879678726196, "rewards_raw/format_reward": 5.0, "step": 125 }, { "Total_reward": 3.2569681707173004e-07, "advantage_std": 1.5555254220962524, "completion_length": 535.3214721679688, "epoch": 0.04350828729281768, "grad_norm": 3.929021066640046, "kl": 0.00726318359375, "learning_rate": 4.976682742302409e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": 0.14482660591602325, "rewards_raw/accuracy_reward": 0.5474557876586914, "rewards_raw/clip_reward_wrapper": 0.2801978290081024, "rewards_raw/format_reward": 5.0, "step": 126 }, { "Total_reward": 2.2351743211856956e-07, "advantage_std": 1.7467254400253296, "completion_length": 514.0535888671875, "epoch": 0.043853591160220994, "grad_norm": 3.7740099608854583, "kl": 0.00732421875, "learning_rate": 4.97631174630765e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": 0.004415588919073343, "rewards_raw/accuracy_reward": 0.569057047367096, "rewards_raw/clip_reward_wrapper": 0.29046544432640076, "rewards_raw/format_reward": 5.0, "step": 127 }, { "Total_reward": -4.108463258489792e-07, "advantage_std": 1.3878499269485474, "completion_length": 527.5892944335938, "epoch": 0.04419889502762431, "grad_norm": 10.888379803011471, "kl": 0.007598876953125, "learning_rate": 4.975937836189711e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": -0.05652032420039177, "rewards_raw/accuracy_reward": 0.5593966245651245, "rewards_raw/clip_reward_wrapper": 0.27564069628715515, "rewards_raw/format_reward": 5.0, "step": 128 }, { "Total_reward": -1.1069434435739822e-07, "advantage_std": 1.6290395259857178, "completion_length": 518.75, "epoch": 0.04454419889502762, "grad_norm": 5.6434459093423115, "kl": 0.00787353515625, "learning_rate": 4.97556101238861e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": -0.08490315824747086, "rewards_raw/accuracy_reward": 0.5722637176513672, "rewards_raw/clip_reward_wrapper": 0.2831626534461975, "rewards_raw/format_reward": 5.0, "step": 129 }, { "Total_reward": 2.341611207157257e-07, "advantage_std": 1.3949594497680664, "completion_length": 520.75, "epoch": 0.04488950276243094, "grad_norm": 3.1863281438568167, "kl": 0.007080078125, "learning_rate": 4.975181275347791e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": -0.14004868268966675, "rewards_raw/accuracy_reward": 0.5396770238876343, "rewards_raw/clip_reward_wrapper": 0.2871198058128357, "rewards_raw/format_reward": 5.0, "step": 130 }, { "Total_reward": 5.460211696117767e-07, "advantage_std": 1.4947121143341064, "completion_length": 517.232177734375, "epoch": 0.04523480662983426, "grad_norm": 3.3349332346092697, "kl": 0.00732421875, "learning_rate": 4.974798625514132e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": 0.060957469046115875, "rewards_raw/accuracy_reward": 0.5819981694221497, "rewards_raw/clip_reward_wrapper": 0.26614370942115784, "rewards_raw/format_reward": 5.0, "step": 131 }, { "Total_reward": -6.279775561779388e-07, "advantage_std": 1.6704750061035156, "completion_length": 511.58929443359375, "epoch": 0.04558011049723757, "grad_norm": 4.44238579934027, "kl": 0.00787353515625, "learning_rate": 4.974413063337931e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": 0.3109298646450043, "rewards_raw/accuracy_reward": 0.5235889554023743, "rewards_raw/clip_reward_wrapper": 0.2815716862678528, "rewards_raw/format_reward": 5.0, "step": 132 }, { "Total_reward": -1.021793991640152e-07, "advantage_std": 1.574791669845581, "completion_length": 502.1607360839844, "epoch": 0.045925414364640885, "grad_norm": 6.010242733822932, "kl": 0.00830078125, "learning_rate": 4.974024589272921e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": -0.14844532310962677, "rewards_raw/accuracy_reward": 0.5655229091644287, "rewards_raw/clip_reward_wrapper": 0.2618003487586975, "rewards_raw/format_reward": 5.0, "step": 133 }, { "Total_reward": 2.3416111716301202e-08, "advantage_std": 1.436211347579956, "completion_length": 516.8392944335938, "epoch": 0.0462707182320442, "grad_norm": 3.8705359142884332, "kl": 0.007598876953125, "learning_rate": 4.973633203776256e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": -0.08949045091867447, "rewards_raw/accuracy_reward": 0.5664572715759277, "rewards_raw/clip_reward_wrapper": 0.294656902551651, "rewards_raw/format_reward": 5.0, "step": 134 }, { "Total_reward": -9.366444686520481e-08, "advantage_std": 1.7136293649673462, "completion_length": 527.1428833007812, "epoch": 0.04661602209944751, "grad_norm": 4.146517021426908, "kl": 0.007781982421875, "learning_rate": 4.973238907308519e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": -0.15505468845367432, "rewards_raw/accuracy_reward": 0.5536661744117737, "rewards_raw/clip_reward_wrapper": 0.3008856475353241, "rewards_raw/format_reward": 5.0, "step": 135 }, { "Total_reward": -4.5874290321989974e-07, "advantage_std": 1.298261046409607, "completion_length": 535.75, "epoch": 0.04696132596685083, "grad_norm": 3.4262770998074554, "kl": 0.0086669921875, "learning_rate": 4.972841700333719e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": 0.09361543506383896, "rewards_raw/accuracy_reward": 0.5485143065452576, "rewards_raw/clip_reward_wrapper": 0.2883041501045227, "rewards_raw/format_reward": 5.0, "step": 136 }, { "Total_reward": -2.3416111716301202e-08, "advantage_std": 1.767530918121338, "completion_length": 525.2678833007812, "epoch": 0.04730662983425414, "grad_norm": 7.2575269585674596, "kl": 0.007598876953125, "learning_rate": 4.972441583319288e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": 0.09905204176902771, "rewards_raw/accuracy_reward": 0.5440556406974792, "rewards_raw/clip_reward_wrapper": 0.30747920274734497, "rewards_raw/format_reward": 5.0, "step": 137 }, { "Total_reward": -6.992902399360901e-07, "advantage_std": 1.4200681447982788, "completion_length": 523.8214721679688, "epoch": 0.04765193370165746, "grad_norm": 5.490150017589277, "kl": 0.0086669921875, "learning_rate": 4.972038556736084e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": 0.08590802550315857, "rewards_raw/accuracy_reward": 0.5509113669395447, "rewards_raw/clip_reward_wrapper": 0.28057020902633667, "rewards_raw/format_reward": 5.0, "step": 138 }, { "Total_reward": 7.237706967089252e-08, "advantage_std": 1.607130527496338, "completion_length": 510.6785888671875, "epoch": 0.047997237569060776, "grad_norm": 6.43479423712289, "kl": 0.0079345703125, "learning_rate": 4.971632621058388e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": -0.059060461819171906, "rewards_raw/accuracy_reward": 0.5720551013946533, "rewards_raw/clip_reward_wrapper": 0.2963559031486511, "rewards_raw/format_reward": 5.0, "step": 139 }, { "Total_reward": 7.152557941481064e-07, "advantage_std": 1.5315024852752686, "completion_length": 501.76788330078125, "epoch": 0.04834254143646409, "grad_norm": 4.156110218467648, "kl": 0.00946044921875, "learning_rate": 4.971223776763907e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": 0.13351985812187195, "rewards_raw/accuracy_reward": 0.5619311332702637, "rewards_raw/clip_reward_wrapper": 0.2670201063156128, "rewards_raw/format_reward": 5.0, "step": 140 }, { "Total_reward": -1.0856560805905247e-07, "advantage_std": 1.5070858001708984, "completion_length": 517.607177734375, "epoch": 0.048687845303867404, "grad_norm": 4.078183845884685, "kl": 0.008544921875, "learning_rate": 4.970812024333765e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": 0.02232750505208969, "rewards_raw/accuracy_reward": 0.5530190467834473, "rewards_raw/clip_reward_wrapper": 0.28469812870025635, "rewards_raw/format_reward": 5.0, "step": 141 }, { "Total_reward": -3.4059798537100505e-08, "advantage_std": 1.4376537799835205, "completion_length": 501.8750305175781, "epoch": 0.04903314917127072, "grad_norm": 5.761111051489745, "kl": 0.0078125, "learning_rate": 4.970397364252516e-07, "loss": 0.0003, "rewards_raw/CoT ICLG": 0.5554707646369934, "rewards_raw/accuracy_reward": 0.551210880279541, "rewards_raw/clip_reward_wrapper": 0.28278088569641113, "rewards_raw/format_reward": 5.0, "step": 142 }, { "Total_reward": -5.534717217869911e-08, "advantage_std": 1.4895880222320557, "completion_length": 528.4464721679688, "epoch": 0.04937845303867403, "grad_norm": 4.151351065932804, "kl": 0.009033203125, "learning_rate": 4.969979797008132e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": 0.026737414300441742, "rewards_raw/accuracy_reward": 0.5261349081993103, "rewards_raw/clip_reward_wrapper": 0.27502885460853577, "rewards_raw/format_reward": 5.0, "step": 143 }, { "Total_reward": 7.557017767112484e-08, "advantage_std": 1.4485719203948975, "completion_length": 525.5892944335938, "epoch": 0.049723756906077346, "grad_norm": 4.005873037959841, "kl": 0.0093994140625, "learning_rate": 4.969559323092003e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": 0.0378478467464447, "rewards_raw/accuracy_reward": 0.5650350451469421, "rewards_raw/clip_reward_wrapper": 0.27778488397598267, "rewards_raw/format_reward": 5.0, "step": 144 }, { "Total_reward": -5.832740725963959e-07, "advantage_std": 1.6272189617156982, "completion_length": 526.7142944335938, "epoch": 0.05006906077348066, "grad_norm": 3.697944777715662, "kl": 0.009521484375, "learning_rate": 4.969135942998945e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": -0.08300291746854782, "rewards_raw/accuracy_reward": 0.542961597442627, "rewards_raw/clip_reward_wrapper": 0.285362184047699, "rewards_raw/format_reward": 5.0, "step": 145 }, { "Total_reward": 1.6071967934294662e-07, "advantage_std": 1.4456908702850342, "completion_length": 527.732177734375, "epoch": 0.05041436464088398, "grad_norm": 4.324153580265714, "kl": 0.00921630859375, "learning_rate": 4.968709657227189e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": -0.02490800805389881, "rewards_raw/accuracy_reward": 0.566914439201355, "rewards_raw/clip_reward_wrapper": 0.2816513180732727, "rewards_raw/format_reward": 5.0, "step": 146 }, { "Total_reward": 2.575772271029564e-07, "advantage_std": 1.6975507736206055, "completion_length": 512.7857666015625, "epoch": 0.050759668508287295, "grad_norm": 17.671003555792208, "kl": 0.010009765625, "learning_rate": 4.968280466278391e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": -0.13559190928936005, "rewards_raw/accuracy_reward": 0.54875248670578, "rewards_raw/clip_reward_wrapper": 0.2737201154232025, "rewards_raw/format_reward": 5.0, "step": 147 }, { "Total_reward": 2.628990785069618e-07, "advantage_std": 1.628423810005188, "completion_length": 517.7678833007812, "epoch": 0.05110497237569061, "grad_norm": 3.9640047042059434, "kl": 0.00970458984375, "learning_rate": 4.967848370657622e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": -0.11161623895168304, "rewards_raw/accuracy_reward": 0.55605149269104, "rewards_raw/clip_reward_wrapper": 0.2834872901439667, "rewards_raw/format_reward": 5.0, "step": 148 }, { "Total_reward": 5.641153961732925e-07, "advantage_std": 1.4231380224227905, "completion_length": 512.3928833007812, "epoch": 0.05145027624309392, "grad_norm": 5.527583677761199, "kl": 0.00958251953125, "learning_rate": 4.967413370873371e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": -0.07730408012866974, "rewards_raw/accuracy_reward": 0.5604280233383179, "rewards_raw/clip_reward_wrapper": 0.28690576553344727, "rewards_raw/format_reward": 5.0, "step": 149 }, { "Total_reward": 3.129243850708008e-07, "advantage_std": 1.382338285446167, "completion_length": 519.3214721679688, "epoch": 0.05179558011049724, "grad_norm": 3.6625257514412684, "kl": 0.0091552734375, "learning_rate": 4.966975467437547e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": -0.06825140118598938, "rewards_raw/accuracy_reward": 0.5721582770347595, "rewards_raw/clip_reward_wrapper": 0.27330413460731506, "rewards_raw/format_reward": 5.0, "step": 150 }, { "Total_reward": 7.66345493730114e-08, "advantage_std": 1.64508056640625, "completion_length": 527.0178833007812, "epoch": 0.05214088397790055, "grad_norm": 3.471294591678261, "kl": 0.00982666015625, "learning_rate": 4.966534660865473e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": -0.2040381282567978, "rewards_raw/accuracy_reward": 0.5508667826652527, "rewards_raw/clip_reward_wrapper": 0.27360039949417114, "rewards_raw/format_reward": 4.901785850524902, "step": 151 }, { "Total_reward": -2.4267606590910873e-07, "advantage_std": 1.5280027389526367, "completion_length": 513.6785888671875, "epoch": 0.052486187845303865, "grad_norm": 3.5740777688104663, "kl": 0.01080322265625, "learning_rate": 4.966090951675892e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": 0.0751904770731926, "rewards_raw/accuracy_reward": 0.5695961713790894, "rewards_raw/clip_reward_wrapper": 0.27914708852767944, "rewards_raw/format_reward": 5.0, "step": 152 }, { "Total_reward": 5.3218435880353354e-08, "advantage_std": 1.2631573677062988, "completion_length": 530.75, "epoch": 0.05283149171270718, "grad_norm": 2.8560179134514283, "kl": 0.01043701171875, "learning_rate": 4.96564434039096e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": 0.08189540356397629, "rewards_raw/accuracy_reward": 0.5577921867370605, "rewards_raw/clip_reward_wrapper": 0.28954052925109863, "rewards_raw/format_reward": 5.0, "step": 153 }, { "Total_reward": -1.2155090871601715e-06, "advantage_std": 1.4194211959838867, "completion_length": 500.7857360839844, "epoch": 0.0531767955801105, "grad_norm": 5.713861117756891, "kl": 0.01055908203125, "learning_rate": 4.965194827536249e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": -0.2575479745864868, "rewards_raw/accuracy_reward": 0.5310795307159424, "rewards_raw/clip_reward_wrapper": 0.27704185247421265, "rewards_raw/format_reward": 5.0, "step": 154 }, { "Total_reward": 1.575265713427143e-07, "advantage_std": 1.5396692752838135, "completion_length": 507.5535888671875, "epoch": 0.053522099447513814, "grad_norm": 3.409341582894292, "kl": 0.00933837890625, "learning_rate": 4.964742413640745e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": -0.22218090295791626, "rewards_raw/accuracy_reward": 0.5746353268623352, "rewards_raw/clip_reward_wrapper": 0.29156652092933655, "rewards_raw/format_reward": 5.0, "step": 155 }, { "Total_reward": -3.698681325658981e-07, "advantage_std": 1.5075180530548096, "completion_length": 504.01788330078125, "epoch": 0.05386740331491713, "grad_norm": 4.122226676148195, "kl": 0.01025390625, "learning_rate": 4.964287099236851e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": 0.20818476378917694, "rewards_raw/accuracy_reward": 0.565675675868988, "rewards_raw/clip_reward_wrapper": 0.2721414864063263, "rewards_raw/format_reward": 5.0, "step": 156 }, { "Total_reward": 3.2569681707173004e-07, "advantage_std": 1.4233213663101196, "completion_length": 514.6428833007812, "epoch": 0.05421270718232044, "grad_norm": 3.9420420277131614, "kl": 0.00994873046875, "learning_rate": 4.963828884860377e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": 0.053985390812158585, "rewards_raw/accuracy_reward": 0.5499867796897888, "rewards_raw/clip_reward_wrapper": 0.27744436264038086, "rewards_raw/format_reward": 5.0, "step": 157 }, { "Total_reward": -4.15103784234816e-07, "advantage_std": 1.503002405166626, "completion_length": 494.14288330078125, "epoch": 0.054558011049723756, "grad_norm": 4.256675455422387, "kl": 0.010009765625, "learning_rate": 4.963367771050553e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": 0.3670518100261688, "rewards_raw/accuracy_reward": 0.5510427355766296, "rewards_raw/clip_reward_wrapper": 0.2864399552345276, "rewards_raw/format_reward": 5.0, "step": 158 }, { "Total_reward": 1.6178404393940582e-07, "advantage_std": 1.3469398021697998, "completion_length": 506.4107360839844, "epoch": 0.05490331491712707, "grad_norm": 2.9961900658397997, "kl": 0.01068115234375, "learning_rate": 4.962903758350016e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": 0.3939068913459778, "rewards_raw/accuracy_reward": 0.5542668104171753, "rewards_raw/clip_reward_wrapper": 0.2854616045951843, "rewards_raw/format_reward": 5.0, "step": 159 }, { "Total_reward": 4.5554980943052215e-07, "advantage_std": 1.6594568490982056, "completion_length": 520.8928833007812, "epoch": 0.055248618784530384, "grad_norm": 3.329108812450529, "kl": 0.01190185546875, "learning_rate": 4.962436847304817e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": -0.12299103289842606, "rewards_raw/accuracy_reward": 0.5411401391029358, "rewards_raw/clip_reward_wrapper": 0.2857493758201599, "rewards_raw/format_reward": 5.0, "step": 160 }, { "Total_reward": 1.532690987460228e-07, "advantage_std": 1.5635669231414795, "completion_length": 521.9642944335938, "epoch": 0.0555939226519337, "grad_norm": 5.220183240579194, "kl": 0.009521484375, "learning_rate": 4.961967038464416e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": 0.04646393284201622, "rewards_raw/accuracy_reward": 0.5482580661773682, "rewards_raw/clip_reward_wrapper": 0.2874627709388733, "rewards_raw/format_reward": 5.0, "step": 161 }, { "Total_reward": -4.3000494542866363e-07, "advantage_std": 1.6508915424346924, "completion_length": 505.4285888671875, "epoch": 0.05593922651933702, "grad_norm": 6.98251780667839, "kl": 0.01055908203125, "learning_rate": 4.961494332381685e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": 0.35364028811454773, "rewards_raw/accuracy_reward": 0.542191207408905, "rewards_raw/clip_reward_wrapper": 0.29294776916503906, "rewards_raw/format_reward": 5.0, "step": 162 }, { "Total_reward": 1.788139485370266e-07, "advantage_std": 1.170664668083191, "completion_length": 510.2500305175781, "epoch": 0.056284530386740333, "grad_norm": 3.1011436666268586, "kl": 0.01165771484375, "learning_rate": 4.961018729612903e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.3278099596500397, "rewards_raw/accuracy_reward": 0.565270185470581, "rewards_raw/clip_reward_wrapper": 0.2735218405723572, "rewards_raw/format_reward": 5.0, "step": 163 }, { "Total_reward": -2.3416111716301202e-08, "advantage_std": 1.6241581439971924, "completion_length": 532.982177734375, "epoch": 0.05662983425414365, "grad_norm": 3.8621948895819327, "kl": 0.01116943359375, "learning_rate": 4.960540230717761e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": 0.24381965398788452, "rewards_raw/accuracy_reward": 0.5597196221351624, "rewards_raw/clip_reward_wrapper": 0.2773340344429016, "rewards_raw/format_reward": 5.0, "step": 164 }, { "Total_reward": 1.415610427102365e-06, "advantage_std": 1.4106059074401855, "completion_length": 523.857177734375, "epoch": 0.05697513812154696, "grad_norm": 3.4839040072135745, "kl": 0.01123046875, "learning_rate": 4.960058836259353e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": -0.23893745243549347, "rewards_raw/accuracy_reward": 0.5547304749488831, "rewards_raw/clip_reward_wrapper": 0.2689056694507599, "rewards_raw/format_reward": 5.0, "step": 165 }, { "Total_reward": -2.4693355271665496e-07, "advantage_std": 1.4492673873901367, "completion_length": 520.5178833007812, "epoch": 0.057320441988950276, "grad_norm": 3.299876378282781, "kl": 0.011962890625, "learning_rate": 4.959574546804187e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.19716228544712067, "rewards_raw/accuracy_reward": 0.5403835773468018, "rewards_raw/clip_reward_wrapper": 0.2753221392631531, "rewards_raw/format_reward": 5.0, "step": 166 }, { "Total_reward": -5.428280474006897e-07, "advantage_std": 1.4289418458938599, "completion_length": 522.1607666015625, "epoch": 0.05766574585635359, "grad_norm": 3.7990287365439537, "kl": 0.0107421875, "learning_rate": 4.959087362922175e-07, "loss": 0.0004, "rewards_raw/CoT ICLG": 0.15593068301677704, "rewards_raw/accuracy_reward": 0.5521643161773682, "rewards_raw/clip_reward_wrapper": 0.27485743165016174, "rewards_raw/format_reward": 5.0, "step": 167 }, { "Total_reward": 1.7668520513325348e-07, "advantage_std": 1.6180325746536255, "completion_length": 517.9464721679688, "epoch": 0.058011049723756904, "grad_norm": 3.7535116441384146, "kl": 0.01129150390625, "learning_rate": 4.958597285186634e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.23871847987174988, "rewards_raw/accuracy_reward": 0.5617122054100037, "rewards_raw/clip_reward_wrapper": 0.30343925952911377, "rewards_raw/format_reward": 5.0, "step": 168 }, { "Total_reward": -5.534717217869911e-08, "advantage_std": 1.3394261598587036, "completion_length": 500.96429443359375, "epoch": 0.05835635359116022, "grad_norm": 13.94286783348791, "kl": 0.01153564453125, "learning_rate": 4.958104314174287e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.6720725893974304, "rewards_raw/accuracy_reward": 0.5588024258613586, "rewards_raw/clip_reward_wrapper": 0.29596269130706787, "rewards_raw/format_reward": 5.0, "step": 169 }, { "Total_reward": 9.302582384407287e-07, "advantage_std": 1.603474497795105, "completion_length": 529.3392944335938, "epoch": 0.05870165745856354, "grad_norm": 4.359638404900999, "kl": 0.01190185546875, "learning_rate": 4.957608450465262e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.050970658659935, "rewards_raw/accuracy_reward": 0.5536025762557983, "rewards_raw/clip_reward_wrapper": 0.2904462516307831, "rewards_raw/format_reward": 5.0, "step": 170 }, { "Total_reward": 7.258994401126984e-07, "advantage_std": 1.4940024614334106, "completion_length": 518.5, "epoch": 0.05904696132596685, "grad_norm": 6.959037619930989, "kl": 0.0120849609375, "learning_rate": 4.957109694643092e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": -0.07871396839618683, "rewards_raw/accuracy_reward": 0.5694586634635925, "rewards_raw/clip_reward_wrapper": 0.29095011949539185, "rewards_raw/format_reward": 5.0, "step": 171 }, { "Total_reward": 1.1920928955078125e-07, "advantage_std": 1.646762728691101, "completion_length": 502.51788330078125, "epoch": 0.05939226519337017, "grad_norm": 9.7149302834341, "kl": 0.0120849609375, "learning_rate": 4.956608047294712e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.0586007721722126, "rewards_raw/accuracy_reward": 0.5667936205863953, "rewards_raw/clip_reward_wrapper": 0.2849215567111969, "rewards_raw/format_reward": 5.0, "step": 172 }, { "Total_reward": 1.0111502746212864e-07, "advantage_std": 1.6033051013946533, "completion_length": 510.6964416503906, "epoch": 0.05973756906077348, "grad_norm": 4.751815642101673, "kl": 0.0118408203125, "learning_rate": 4.956103509010461e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": -0.018963303416967392, "rewards_raw/accuracy_reward": 0.5808303356170654, "rewards_raw/clip_reward_wrapper": 0.28626739978790283, "rewards_raw/format_reward": 5.0, "step": 173 }, { "Total_reward": 2.086162709247219e-07, "advantage_std": 1.6692168712615967, "completion_length": 517.607177734375, "epoch": 0.060082872928176795, "grad_norm": 8.105624992181953, "kl": 0.01202392578125, "learning_rate": 4.955596080384081e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.1378893256187439, "rewards_raw/accuracy_reward": 0.566072940826416, "rewards_raw/clip_reward_wrapper": 0.267341673374176, "rewards_raw/format_reward": 5.0, "step": 174 }, { "Total_reward": 2.0010131152048416e-07, "advantage_std": 1.638774037361145, "completion_length": 509.9464416503906, "epoch": 0.06042817679558011, "grad_norm": 4.881618376529889, "kl": 0.01214599609375, "learning_rate": 4.955085762012712e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": -0.02389630861580372, "rewards_raw/accuracy_reward": 0.5773419737815857, "rewards_raw/clip_reward_wrapper": 0.279889851808548, "rewards_raw/format_reward": 5.0, "step": 175 }, { "Total_reward": 9.366444686520481e-08, "advantage_std": 1.3771945238113403, "completion_length": 501.0714416503906, "epoch": 0.06077348066298342, "grad_norm": 3.811501396209456, "kl": 0.01239013671875, "learning_rate": 4.954572554496896e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": -0.2895646095275879, "rewards_raw/accuracy_reward": 0.5719171762466431, "rewards_raw/clip_reward_wrapper": 0.28920072317123413, "rewards_raw/format_reward": 5.0, "step": 176 }, { "Total_reward": 5.172831833988312e-07, "advantage_std": 1.474685788154602, "completion_length": 515.982177734375, "epoch": 0.06111878453038674, "grad_norm": 3.762165770666687, "kl": 0.01336669921875, "learning_rate": 4.954056458440577e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.11086411774158478, "rewards_raw/accuracy_reward": 0.5642876029014587, "rewards_raw/clip_reward_wrapper": 0.28981825709342957, "rewards_raw/format_reward": 5.0, "step": 177 }, { "Total_reward": -4.864165248363861e-07, "advantage_std": 1.6425962448120117, "completion_length": 517.5357666015625, "epoch": 0.06146408839779006, "grad_norm": 4.9321898523295316, "kl": 0.011962890625, "learning_rate": 4.953537474451098e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.0773528516292572, "rewards_raw/accuracy_reward": 0.5573384761810303, "rewards_raw/clip_reward_wrapper": 0.2984551787376404, "rewards_raw/format_reward": 5.0, "step": 178 }, { "Total_reward": 4.2468312244636763e-07, "advantage_std": 1.5192439556121826, "completion_length": 495.3750305175781, "epoch": 0.06180939226519337, "grad_norm": 3.3876988665058567, "kl": 0.0123291015625, "learning_rate": 4.953015603139195e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": -0.11431697756052017, "rewards_raw/accuracy_reward": 0.5509729385375977, "rewards_raw/clip_reward_wrapper": 0.2969561517238617, "rewards_raw/format_reward": 5.0, "step": 179 }, { "Total_reward": -1.852001503266365e-07, "advantage_std": 1.2956804037094116, "completion_length": 497.64288330078125, "epoch": 0.062154696132596686, "grad_norm": 6.556268418908927, "kl": 0.01287841796875, "learning_rate": 4.95249084511901e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.17161186039447784, "rewards_raw/accuracy_reward": 0.5504719614982605, "rewards_raw/clip_reward_wrapper": 0.2882999777793884, "rewards_raw/format_reward": 5.0, "step": 180 }, { "Total_reward": 6.034970283508301e-07, "advantage_std": 1.5378026962280273, "completion_length": 501.2500305175781, "epoch": 0.0625, "grad_norm": 7.6056267508375575, "kl": 0.0123291015625, "learning_rate": 4.951963201008075e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": -0.0284075066447258, "rewards_raw/accuracy_reward": 0.5699392557144165, "rewards_raw/clip_reward_wrapper": 0.28145310282707214, "rewards_raw/format_reward": 4.964285850524902, "step": 181 }, { "Total_reward": -2.937657654911163e-07, "advantage_std": 1.574235439300537, "completion_length": 529.7678833007812, "epoch": 0.06284530386740332, "grad_norm": 8.746127315256338, "kl": 0.012451171875, "learning_rate": 4.951432671427325e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.053850941359996796, "rewards_raw/accuracy_reward": 0.5686709880828857, "rewards_raw/clip_reward_wrapper": 0.2769218683242798, "rewards_raw/format_reward": 5.0, "step": 182 }, { "Total_reward": 3.5549913945942535e-07, "advantage_std": 1.4462498426437378, "completion_length": 529.7678833007812, "epoch": 0.06319060773480663, "grad_norm": 5.367015319205287, "kl": 0.0128173828125, "learning_rate": 4.950899257001085e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.050106409937143326, "rewards_raw/accuracy_reward": 0.5441453456878662, "rewards_raw/clip_reward_wrapper": 0.2855337858200073, "rewards_raw/format_reward": 5.0, "step": 183 }, { "Total_reward": 6.811959707420101e-08, "advantage_std": 1.6445728540420532, "completion_length": 503.9285888671875, "epoch": 0.06353591160220995, "grad_norm": 4.241590068677655, "kl": 0.013427734375, "learning_rate": 4.950362958357078e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.03109976276755333, "rewards_raw/accuracy_reward": 0.5872928500175476, "rewards_raw/clip_reward_wrapper": 0.2874682545661926, "rewards_raw/format_reward": 5.0, "step": 184 }, { "Total_reward": -1.0452100696056732e-06, "advantage_std": 1.6726597547531128, "completion_length": 497.732177734375, "epoch": 0.06388121546961326, "grad_norm": 5.037082097447055, "kl": 0.01251220703125, "learning_rate": 4.949823776126418e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.01848173700273037, "rewards_raw/accuracy_reward": 0.5454745292663574, "rewards_raw/clip_reward_wrapper": 0.27307456731796265, "rewards_raw/format_reward": 5.0, "step": 185 }, { "Total_reward": -2.043587983280304e-07, "advantage_std": 1.5187867879867554, "completion_length": 514.875, "epoch": 0.06422651933701658, "grad_norm": 3.4030898666761296, "kl": 0.01324462890625, "learning_rate": 4.949281710943617e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": -0.16471393406391144, "rewards_raw/accuracy_reward": 0.556415319442749, "rewards_raw/clip_reward_wrapper": 0.27119237184524536, "rewards_raw/format_reward": 5.0, "step": 186 }, { "Total_reward": 1.2133803295455436e-07, "advantage_std": 1.4243242740631104, "completion_length": 502.3214416503906, "epoch": 0.06457182320441988, "grad_norm": 5.09980483086766, "kl": 0.012939453125, "learning_rate": 4.948736763446576e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": -0.31164437532424927, "rewards_raw/accuracy_reward": 0.5464898347854614, "rewards_raw/clip_reward_wrapper": 0.2914060354232788, "rewards_raw/format_reward": 5.0, "step": 187 }, { "Total_reward": -7.024833337254677e-08, "advantage_std": 1.5553253889083862, "completion_length": 512.0357666015625, "epoch": 0.0649171270718232, "grad_norm": 5.432225191038284, "kl": 0.01361083984375, "learning_rate": 4.948188934276589e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.18689264357089996, "rewards_raw/accuracy_reward": 0.5795437097549438, "rewards_raw/clip_reward_wrapper": 0.28361886739730835, "rewards_raw/format_reward": 5.0, "step": 188 }, { "Total_reward": 5.27926886206842e-07, "advantage_std": 1.516660451889038, "completion_length": 504.732177734375, "epoch": 0.06526243093922653, "grad_norm": 6.223824772467652, "kl": 0.0135498046875, "learning_rate": 4.947638224078343e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.13446775078773499, "rewards_raw/accuracy_reward": 0.5717757344245911, "rewards_raw/clip_reward_wrapper": 0.2816329896450043, "rewards_raw/format_reward": 5.0, "step": 189 }, { "Total_reward": 1.830714211337181e-07, "advantage_std": 1.7298113107681274, "completion_length": 510.8214416503906, "epoch": 0.06560773480662983, "grad_norm": 5.546987228593812, "kl": 0.01336669921875, "learning_rate": 4.94708463349991e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": -0.1532973349094391, "rewards_raw/accuracy_reward": 0.5697504281997681, "rewards_raw/clip_reward_wrapper": 0.29129573702812195, "rewards_raw/format_reward": 5.0, "step": 190 }, { "Total_reward": -3.895589486546669e-07, "advantage_std": 1.4455777406692505, "completion_length": 514.9464721679688, "epoch": 0.06595303867403315, "grad_norm": 5.08751291116219, "kl": 0.01373291015625, "learning_rate": 4.94652816319276e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.31587615609169006, "rewards_raw/accuracy_reward": 0.5622051954269409, "rewards_raw/clip_reward_wrapper": 0.2871090769767761, "rewards_raw/format_reward": 5.0, "step": 191 }, { "Total_reward": 6.173338817916374e-08, "advantage_std": 1.4759451150894165, "completion_length": 506.5000305175781, "epoch": 0.06629834254143646, "grad_norm": 5.860231184937193, "kl": 0.012451171875, "learning_rate": 4.945968813811743e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.08401873707771301, "rewards_raw/accuracy_reward": 0.5420577526092529, "rewards_raw/clip_reward_wrapper": 0.2772436738014221, "rewards_raw/format_reward": 5.0, "step": 192 }, { "Total_reward": -4.5767853862344055e-07, "advantage_std": 1.5504201650619507, "completion_length": 518.8214721679688, "epoch": 0.06664364640883978, "grad_norm": 3.541374092879815, "kl": 0.01275634765625, "learning_rate": 4.945406586015104e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.04351310804486275, "rewards_raw/accuracy_reward": 0.5476694703102112, "rewards_raw/clip_reward_wrapper": 0.2936071455478668, "rewards_raw/format_reward": 5.0, "step": 193 }, { "Total_reward": 1.852001503266365e-07, "advantage_std": 1.7862707376480103, "completion_length": 516.3928833007812, "epoch": 0.06698895027624309, "grad_norm": 5.399682510404869, "kl": 0.0133056640625, "learning_rate": 4.94484148046447e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": -0.10633828490972519, "rewards_raw/accuracy_reward": 0.5636649131774902, "rewards_raw/clip_reward_wrapper": 0.2891509234905243, "rewards_raw/format_reward": 5.0, "step": 194 }, { "Total_reward": -1.2559550555124588e-07, "advantage_std": 1.558029294013977, "completion_length": 503.2857360839844, "epoch": 0.06733425414364641, "grad_norm": 4.703523182132355, "kl": 0.01385498046875, "learning_rate": 4.944273497824859e-07, "loss": 0.0006, "rewards_raw/CoT ICLG": -0.07945737987756729, "rewards_raw/accuracy_reward": 0.5425638556480408, "rewards_raw/clip_reward_wrapper": 0.3140902519226074, "rewards_raw/format_reward": 5.0, "step": 195 }, { "Total_reward": 3.469841942660423e-07, "advantage_std": 1.4069358110427856, "completion_length": 505.6607360839844, "epoch": 0.06767955801104972, "grad_norm": 4.483721446760816, "kl": 0.0128173828125, "learning_rate": 4.943702638764673e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.04721741005778313, "rewards_raw/accuracy_reward": 0.572493851184845, "rewards_raw/clip_reward_wrapper": 0.2833981513977051, "rewards_raw/format_reward": 5.0, "step": 196 }, { "Total_reward": 2.650278076998802e-07, "advantage_std": 1.2679930925369263, "completion_length": 513.1785888671875, "epoch": 0.06802486187845304, "grad_norm": 3.124139804404352, "kl": 0.013916015625, "learning_rate": 4.943128903955698e-07, "loss": 0.0006, "rewards_raw/CoT ICLG": -0.17242877185344696, "rewards_raw/accuracy_reward": 0.5593235492706299, "rewards_raw/clip_reward_wrapper": 0.2768571674823761, "rewards_raw/format_reward": 5.0, "step": 197 }, { "Total_reward": -2.1287373641598606e-08, "advantage_std": 1.6205300092697144, "completion_length": 489.0714416503906, "epoch": 0.06837016574585636, "grad_norm": 4.63314525590045, "kl": 0.01458740234375, "learning_rate": 4.942552294073106e-07, "loss": 0.0006, "rewards_raw/CoT ICLG": -0.07675658166408539, "rewards_raw/accuracy_reward": 0.5447031259536743, "rewards_raw/clip_reward_wrapper": 0.29330193996429443, "rewards_raw/format_reward": 5.0, "step": 198 }, { "Total_reward": 1.0409526112198364e-06, "advantage_std": 1.5475388765335083, "completion_length": 508.76788330078125, "epoch": 0.06871546961325967, "grad_norm": 4.776908278331817, "kl": 0.0145263671875, "learning_rate": 4.94197280979545e-07, "loss": 0.0006, "rewards_raw/CoT ICLG": 0.3233896791934967, "rewards_raw/accuracy_reward": 0.5738892555236816, "rewards_raw/clip_reward_wrapper": 0.2749997079372406, "rewards_raw/format_reward": 5.0, "step": 199 }, { "Total_reward": 1.1239733339607483e-06, "advantage_std": 1.418381929397583, "completion_length": 508.51788330078125, "epoch": 0.06906077348066299, "grad_norm": 6.076078802760186, "kl": 0.01361083984375, "learning_rate": 4.941390451804668e-07, "loss": 0.0005, "rewards_raw/CoT ICLG": 0.5181885957717896, "rewards_raw/accuracy_reward": 0.5389074683189392, "rewards_raw/clip_reward_wrapper": 0.28233450651168823, "rewards_raw/format_reward": 5.0, "step": 200 }, { "Total_reward": 2.2777490471526107e-07, "advantage_std": 1.555677890777588, "completion_length": 506.107177734375, "epoch": 0.0694060773480663, "grad_norm": 3.792391216983554, "kl": 0.0162353515625, "learning_rate": 4.940805220786078e-07, "loss": 0.0006, "rewards_raw/CoT ICLG": 0.08688168972730637, "rewards_raw/accuracy_reward": 0.5860782861709595, "rewards_raw/clip_reward_wrapper": 0.29488858580589294, "rewards_raw/format_reward": 5.0, "step": 201 }, { "Total_reward": -1.4688288274555816e-07, "advantage_std": 1.5135338306427002, "completion_length": 491.8214416503906, "epoch": 0.06975138121546962, "grad_norm": 3.6656968573773114, "kl": 0.01483154296875, "learning_rate": 4.94021711742838e-07, "loss": 0.0006, "rewards_raw/CoT ICLG": -0.1686454862356186, "rewards_raw/accuracy_reward": 0.550775408744812, "rewards_raw/clip_reward_wrapper": 0.28613317012786865, "rewards_raw/format_reward": 4.919642925262451, "step": 202 }, { "Total_reward": -2.448047986547408e-08, "advantage_std": 1.4554182291030884, "completion_length": 499.3035888671875, "epoch": 0.07009668508287292, "grad_norm": 3.3021723028351238, "kl": 0.01397705078125, "learning_rate": 4.939626142423654e-07, "loss": 0.0006, "rewards_raw/CoT ICLG": 0.1387176215648651, "rewards_raw/accuracy_reward": 0.5710609555244446, "rewards_raw/clip_reward_wrapper": 0.28921496868133545, "rewards_raw/format_reward": 5.0, "step": 203 }, { "Total_reward": -5.002532930120651e-07, "advantage_std": 1.540892481803894, "completion_length": 504.2857360839844, "epoch": 0.07044198895027624, "grad_norm": 3.7935450964026987, "kl": 0.0150146484375, "learning_rate": 4.93903229646736e-07, "loss": 0.0006, "rewards_raw/CoT ICLG": -0.016731075942516327, "rewards_raw/accuracy_reward": 0.5480803847312927, "rewards_raw/clip_reward_wrapper": 0.2896498441696167, "rewards_raw/format_reward": 5.0, "step": 204 }, { "Total_reward": -7.791178973093338e-07, "advantage_std": 1.5665221214294434, "completion_length": 497.58929443359375, "epoch": 0.07078729281767956, "grad_norm": 4.310848675217961, "kl": 0.01513671875, "learning_rate": 4.938435580258335e-07, "loss": 0.0006, "rewards_raw/CoT ICLG": -0.06406281888484955, "rewards_raw/accuracy_reward": 0.5607409477233887, "rewards_raw/clip_reward_wrapper": 0.29194962978363037, "rewards_raw/format_reward": 5.0, "step": 205 }, { "Total_reward": 4.417130128331337e-07, "advantage_std": 1.5936275720596313, "completion_length": 507.1607360839844, "epoch": 0.07113259668508287, "grad_norm": 7.836461407557938, "kl": 0.01531982421875, "learning_rate": 4.937835994498795e-07, "loss": 0.0006, "rewards_raw/CoT ICLG": 0.2645582854747772, "rewards_raw/accuracy_reward": 0.5444351434707642, "rewards_raw/clip_reward_wrapper": 0.2814715504646301, "rewards_raw/format_reward": 5.0, "step": 206 }, { "Total_reward": -1.3027872682869202e-06, "advantage_std": 1.3925861120224, "completion_length": 506.4464416503906, "epoch": 0.07147790055248619, "grad_norm": 2.897097496995919, "kl": 0.017822265625, "learning_rate": 4.937233539894331e-07, "loss": 0.0007, "rewards_raw/CoT ICLG": -0.24193145334720612, "rewards_raw/accuracy_reward": 0.5598382949829102, "rewards_raw/clip_reward_wrapper": 0.27770888805389404, "rewards_raw/format_reward": 5.0, "step": 207 }, { "Total_reward": 2.2564616131148796e-07, "advantage_std": 1.4578649997711182, "completion_length": 504.9107360839844, "epoch": 0.0718232044198895, "grad_norm": 3.685719262701774, "kl": 0.0164794921875, "learning_rate": 4.936628217153914e-07, "loss": 0.0007, "rewards_raw/CoT ICLG": -0.020081818103790283, "rewards_raw/accuracy_reward": 0.5734533071517944, "rewards_raw/clip_reward_wrapper": 0.2817386984825134, "rewards_raw/format_reward": 5.0, "step": 208 }, { "Total_reward": -9.494168580204132e-07, "advantage_std": 1.756820559501648, "completion_length": 503.357177734375, "epoch": 0.07216850828729282, "grad_norm": 3.7773146443573387, "kl": 0.015380859375, "learning_rate": 4.936020026989885e-07, "loss": 0.0006, "rewards_raw/CoT ICLG": 0.1973571628332138, "rewards_raw/accuracy_reward": 0.5617734789848328, "rewards_raw/clip_reward_wrapper": 0.29060542583465576, "rewards_raw/format_reward": 5.0, "step": 209 }, { "Total_reward": 2.0223005492425727e-07, "advantage_std": 1.7778713703155518, "completion_length": 505.8750305175781, "epoch": 0.07251381215469613, "grad_norm": 4.895479095456174, "kl": 0.01611328125, "learning_rate": 4.935408970117965e-07, "loss": 0.0006, "rewards_raw/CoT ICLG": 0.2186170369386673, "rewards_raw/accuracy_reward": 0.5827112197875977, "rewards_raw/clip_reward_wrapper": 0.2920427918434143, "rewards_raw/format_reward": 5.0, "step": 210 }, { "Total_reward": 9.792191946189632e-08, "advantage_std": 1.532712697982788, "completion_length": 489.857177734375, "epoch": 0.07285911602209945, "grad_norm": 8.094243890079436, "kl": 0.0166015625, "learning_rate": 4.934795047257242e-07, "loss": 0.0007, "rewards_raw/CoT ICLG": -0.25082069635391235, "rewards_raw/accuracy_reward": 0.5631587505340576, "rewards_raw/clip_reward_wrapper": 0.283652663230896, "rewards_raw/format_reward": 5.0, "step": 211 }, { "Total_reward": -2.4267606590910873e-07, "advantage_std": 1.409688115119934, "completion_length": 487.01788330078125, "epoch": 0.07320441988950276, "grad_norm": 3.8389477588906855, "kl": 0.0189208984375, "learning_rate": 4.934178259130183e-07, "loss": 0.0008, "rewards_raw/CoT ICLG": 0.3114447295665741, "rewards_raw/accuracy_reward": 0.5576962828636169, "rewards_raw/clip_reward_wrapper": 0.2784334421157837, "rewards_raw/format_reward": 5.0, "step": 212 }, { "Total_reward": -4.2361875784990843e-07, "advantage_std": 1.3376318216323853, "completion_length": 505.8214416503906, "epoch": 0.07354972375690608, "grad_norm": 17.304845998378987, "kl": 0.017822265625, "learning_rate": 4.933558606462623e-07, "loss": 0.0007, "rewards_raw/CoT ICLG": 0.07990580052137375, "rewards_raw/accuracy_reward": 0.5825483202934265, "rewards_raw/clip_reward_wrapper": 0.2799948751926422, "rewards_raw/format_reward": 5.0, "step": 213 }, { "Total_reward": -1.7029898913278885e-07, "advantage_std": 1.4121431112289429, "completion_length": 507.8750305175781, "epoch": 0.0738950276243094, "grad_norm": 4.421286346221083, "kl": 0.0167236328125, "learning_rate": 4.93293608998377e-07, "loss": 0.0007, "rewards_raw/CoT ICLG": -0.2808973491191864, "rewards_raw/accuracy_reward": 0.5697479844093323, "rewards_raw/clip_reward_wrapper": 0.28699010610580444, "rewards_raw/format_reward": 5.0, "step": 214 }, { "Total_reward": -8.514949634275126e-09, "advantage_std": 1.5634936094284058, "completion_length": 490.96429443359375, "epoch": 0.0742403314917127, "grad_norm": 3.775813909208265, "kl": 0.0164794921875, "learning_rate": 4.932310710426201e-07, "loss": 0.0007, "rewards_raw/CoT ICLG": -0.08251398056745529, "rewards_raw/accuracy_reward": 0.5547329187393188, "rewards_raw/clip_reward_wrapper": 0.2888900935649872, "rewards_raw/format_reward": 5.0, "step": 215 }, { "Total_reward": -8.238213808908768e-07, "advantage_std": 1.4879287481307983, "completion_length": 503.857177734375, "epoch": 0.07458563535911603, "grad_norm": 4.767540870226309, "kl": 0.01806640625, "learning_rate": 4.931682468525862e-07, "loss": 0.0007, "rewards_raw/CoT ICLG": 0.08497200906276703, "rewards_raw/accuracy_reward": 0.5552334189414978, "rewards_raw/clip_reward_wrapper": 0.2736639380455017, "rewards_raw/format_reward": 5.0, "step": 216 }, { "Total_reward": 2.2990363390817947e-07, "advantage_std": 1.4041056632995605, "completion_length": 508.607177734375, "epoch": 0.07493093922651933, "grad_norm": 4.061054067314394, "kl": 0.0169677734375, "learning_rate": 4.931051365022068e-07, "loss": 0.0007, "rewards_raw/CoT ICLG": 0.21189472079277039, "rewards_raw/accuracy_reward": 0.5480632781982422, "rewards_raw/clip_reward_wrapper": 0.2880631685256958, "rewards_raw/format_reward": 5.0, "step": 217 }, { "Total_reward": 4.3426243223620986e-07, "advantage_std": 1.401737093925476, "completion_length": 500.0357360839844, "epoch": 0.07527624309392265, "grad_norm": 3.689530213300563, "kl": 0.0181884765625, "learning_rate": 4.930417400657502e-07, "loss": 0.0007, "rewards_raw/CoT ICLG": 0.3166710138320923, "rewards_raw/accuracy_reward": 0.553358256816864, "rewards_raw/clip_reward_wrapper": 0.2852186858654022, "rewards_raw/format_reward": 5.0, "step": 218 }, { "Total_reward": 2.1500247271433182e-07, "advantage_std": 1.5425214767456055, "completion_length": 486.6250305175781, "epoch": 0.07562154696132596, "grad_norm": 9.004726801862775, "kl": 0.018798828125, "learning_rate": 4.929780576178213e-07, "loss": 0.0008, "rewards_raw/CoT ICLG": 0.2751413583755493, "rewards_raw/accuracy_reward": 0.5715619325637817, "rewards_raw/clip_reward_wrapper": 0.2900683879852295, "rewards_raw/format_reward": 5.0, "step": 219 }, { "Total_reward": 4.385199190437561e-07, "advantage_std": 1.5962166786193848, "completion_length": 481.982177734375, "epoch": 0.07596685082872928, "grad_norm": 17.18643105966378, "kl": 0.01953125, "learning_rate": 4.929140892333615e-07, "loss": 0.0008, "rewards_raw/CoT ICLG": 0.2194872349500656, "rewards_raw/accuracy_reward": 0.5554612278938293, "rewards_raw/clip_reward_wrapper": 0.29020437598228455, "rewards_raw/format_reward": 5.0, "step": 220 }, { "Total_reward": -4.864165248363861e-07, "advantage_std": 1.2759031057357788, "completion_length": 506.96429443359375, "epoch": 0.0763121546961326, "grad_norm": 3.3888993874038533, "kl": 0.01806640625, "learning_rate": 4.92849834987649e-07, "loss": 0.0007, "rewards_raw/CoT ICLG": -0.038895588368177414, "rewards_raw/accuracy_reward": 0.5465409159660339, "rewards_raw/clip_reward_wrapper": 0.2869451642036438, "rewards_raw/format_reward": 5.0, "step": 221 }, { "Total_reward": 1.5965531119377374e-08, "advantage_std": 1.6731700897216797, "completion_length": 504.08929443359375, "epoch": 0.07665745856353591, "grad_norm": 4.79274208224429, "kl": 0.0181884765625, "learning_rate": 4.92785294956298e-07, "loss": 0.0007, "rewards_raw/CoT ICLG": 0.40108349919319153, "rewards_raw/accuracy_reward": 0.5461398959159851, "rewards_raw/clip_reward_wrapper": 0.30528566241264343, "rewards_raw/format_reward": 5.0, "step": 222 }, { "Total_reward": 1.1708055858150601e-08, "advantage_std": 1.545292615890503, "completion_length": 465.2500305175781, "epoch": 0.07700276243093923, "grad_norm": 3.601238807872157, "kl": 0.0181884765625, "learning_rate": 4.927204692152591e-07, "loss": 0.0007, "rewards_raw/CoT ICLG": 0.27434054017066956, "rewards_raw/accuracy_reward": 0.56340092420578, "rewards_raw/clip_reward_wrapper": 0.2833518981933594, "rewards_raw/format_reward": 5.0, "step": 223 }, { "Total_reward": 9.260007516331825e-08, "advantage_std": 1.4071803092956543, "completion_length": 479.4285888671875, "epoch": 0.07734806629834254, "grad_norm": 5.716580955397403, "kl": 0.017822265625, "learning_rate": 4.926553578408195e-07, "loss": 0.0007, "rewards_raw/CoT ICLG": 0.6115318536758423, "rewards_raw/accuracy_reward": 0.551483154296875, "rewards_raw/clip_reward_wrapper": 0.2930271029472351, "rewards_raw/format_reward": 5.0, "step": 224 }, { "Total_reward": -3.8104400346128386e-07, "advantage_std": 1.4216307401657104, "completion_length": 487.4107360839844, "epoch": 0.07769337016574586, "grad_norm": 2.897035151202171, "kl": 0.0189208984375, "learning_rate": 4.925899609096022e-07, "loss": 0.0008, "rewards_raw/CoT ICLG": 0.12099312245845795, "rewards_raw/accuracy_reward": 0.564297616481781, "rewards_raw/clip_reward_wrapper": 0.2916184365749359, "rewards_raw/format_reward": 5.0, "step": 225 }, { "Total_reward": -3.3314739766865387e-07, "advantage_std": 1.4773719310760498, "completion_length": 482.1785888671875, "epoch": 0.07803867403314917, "grad_norm": 6.34864350617746, "kl": 0.02001953125, "learning_rate": 4.925242784985661e-07, "loss": 0.0008, "rewards_raw/CoT ICLG": 0.3721778094768524, "rewards_raw/accuracy_reward": 0.5638289451599121, "rewards_raw/clip_reward_wrapper": 0.2982778251171112, "rewards_raw/format_reward": 5.0, "step": 226 }, { "Total_reward": -1.915863734325285e-08, "advantage_std": 1.5492805242538452, "completion_length": 497.46429443359375, "epoch": 0.07838397790055249, "grad_norm": 4.097463177601697, "kl": 0.0191650390625, "learning_rate": 4.924583106850064e-07, "loss": 0.0008, "rewards_raw/CoT ICLG": 0.08945746719837189, "rewards_raw/accuracy_reward": 0.5715360641479492, "rewards_raw/clip_reward_wrapper": 0.29637670516967773, "rewards_raw/format_reward": 5.0, "step": 227 }, { "Total_reward": -9.089708896681259e-07, "advantage_std": 1.3572466373443604, "completion_length": 471.4107360839844, "epoch": 0.0787292817679558, "grad_norm": 4.4498288968816135, "kl": 0.0206298828125, "learning_rate": 4.923920575465539e-07, "loss": 0.0008, "rewards_raw/CoT ICLG": 0.007627168204635382, "rewards_raw/accuracy_reward": 0.5875865817070007, "rewards_raw/clip_reward_wrapper": 0.29940176010131836, "rewards_raw/format_reward": 5.0, "step": 228 }, { "Total_reward": -4.257474817137563e-09, "advantage_std": 1.6454626321792603, "completion_length": 473.7500305175781, "epoch": 0.07907458563535912, "grad_norm": 4.045765564162108, "kl": 0.01953125, "learning_rate": 4.923255191611753e-07, "loss": 0.0008, "rewards_raw/CoT ICLG": 0.03572952374815941, "rewards_raw/accuracy_reward": 0.5705129504203796, "rewards_raw/clip_reward_wrapper": 0.2811674475669861, "rewards_raw/format_reward": 5.0, "step": 229 }, { "Total_reward": -4.6619348381682357e-07, "advantage_std": 1.5327098369598389, "completion_length": 497.0000305175781, "epoch": 0.07941988950276244, "grad_norm": 5.276643162326877, "kl": 0.01904296875, "learning_rate": 4.922586956071731e-07, "loss": 0.0008, "rewards_raw/CoT ICLG": 0.0523027740418911, "rewards_raw/accuracy_reward": 0.572675347328186, "rewards_raw/clip_reward_wrapper": 0.3035823702812195, "rewards_raw/format_reward": 5.0, "step": 230 }, { "Total_reward": 3.395336136691185e-07, "advantage_std": 1.4359383583068848, "completion_length": 490.5714416503906, "epoch": 0.07976519337016574, "grad_norm": 3.2180993837042746, "kl": 0.0198974609375, "learning_rate": 4.921915869631852e-07, "loss": 0.0008, "rewards_raw/CoT ICLG": 0.17936387658119202, "rewards_raw/accuracy_reward": 0.5599880218505859, "rewards_raw/clip_reward_wrapper": 0.2829244136810303, "rewards_raw/format_reward": 5.0, "step": 231 }, { "Total_reward": -2.043587983280304e-07, "advantage_std": 1.3515844345092773, "completion_length": 486.7857360839844, "epoch": 0.08011049723756906, "grad_norm": 4.662661715490669, "kl": 0.0223388671875, "learning_rate": 4.921241933081849e-07, "loss": 0.0009, "rewards_raw/CoT ICLG": 0.11898837983608246, "rewards_raw/accuracy_reward": 0.5557588338851929, "rewards_raw/clip_reward_wrapper": 0.2901642322540283, "rewards_raw/format_reward": 5.0, "step": 232 }, { "Total_reward": 2.2990363390817947e-07, "advantage_std": 1.5682915449142456, "completion_length": 491.3214416503906, "epoch": 0.08045580110497237, "grad_norm": 4.012237843448517, "kl": 0.0181884765625, "learning_rate": 4.920565147214812e-07, "loss": 0.0007, "rewards_raw/CoT ICLG": 0.7751486301422119, "rewards_raw/accuracy_reward": 0.5611006021499634, "rewards_raw/clip_reward_wrapper": 0.2920724153518677, "rewards_raw/format_reward": 5.0, "step": 233 }, { "Total_reward": -6.737453759342316e-07, "advantage_std": 1.4105229377746582, "completion_length": 483.71429443359375, "epoch": 0.08080110497237569, "grad_norm": 3.749192813695744, "kl": 0.019775390625, "learning_rate": 4.919885512827182e-07, "loss": 0.0008, "rewards_raw/CoT ICLG": 0.008989471010863781, "rewards_raw/accuracy_reward": 0.5650462508201599, "rewards_raw/clip_reward_wrapper": 0.29224973917007446, "rewards_raw/format_reward": 5.0, "step": 234 }, { "Total_reward": 2.2351743211856956e-07, "advantage_std": 1.3978214263916016, "completion_length": 504.4107360839844, "epoch": 0.081146408839779, "grad_norm": 3.719007359428609, "kl": 0.0201416015625, "learning_rate": 4.919203030718752e-07, "loss": 0.0008, "rewards_raw/CoT ICLG": -0.15718504786491394, "rewards_raw/accuracy_reward": 0.5639793872833252, "rewards_raw/clip_reward_wrapper": 0.29076290130615234, "rewards_raw/format_reward": 5.0, "step": 235 }, { "Total_reward": 2.7673587510435027e-07, "advantage_std": 1.6776580810546875, "completion_length": 488.8214416503906, "epoch": 0.08149171270718232, "grad_norm": 3.980350832720385, "kl": 0.0211181640625, "learning_rate": 4.918517701692667e-07, "loss": 0.0008, "rewards_raw/CoT ICLG": 0.6026254892349243, "rewards_raw/accuracy_reward": 0.5525531768798828, "rewards_raw/clip_reward_wrapper": 0.2885596454143524, "rewards_raw/format_reward": 5.0, "step": 236 }, { "Total_reward": 1.6178404393940582e-07, "advantage_std": 1.4165712594985962, "completion_length": 490.58929443359375, "epoch": 0.08183701657458564, "grad_norm": 3.5729969435105513, "kl": 0.0208740234375, "learning_rate": 4.917829526555423e-07, "loss": 0.0008, "rewards_raw/CoT ICLG": 0.15305356681346893, "rewards_raw/accuracy_reward": 0.5688011050224304, "rewards_raw/clip_reward_wrapper": 0.29362741112709045, "rewards_raw/format_reward": 5.0, "step": 237 }, { "Total_reward": 2.3203237731195259e-07, "advantage_std": 1.5956885814666748, "completion_length": 508.2857360839844, "epoch": 0.08218232044198895, "grad_norm": 4.064993306843834, "kl": 0.0198974609375, "learning_rate": 4.917138506116863e-07, "loss": 0.0008, "rewards_raw/CoT ICLG": 0.11754552274942398, "rewards_raw/accuracy_reward": 0.5508336424827576, "rewards_raw/clip_reward_wrapper": 0.31070175766944885, "rewards_raw/format_reward": 5.0, "step": 238 }, { "Total_reward": -1.0430813546236095e-07, "advantage_std": 1.5778899192810059, "completion_length": 492.0000305175781, "epoch": 0.08252762430939227, "grad_norm": 5.428046909964664, "kl": 0.0201416015625, "learning_rate": 4.916444641190181e-07, "loss": 0.0008, "rewards_raw/CoT ICLG": 0.2514152526855469, "rewards_raw/accuracy_reward": 0.5390273928642273, "rewards_raw/clip_reward_wrapper": 0.2918488681316376, "rewards_raw/format_reward": 5.0, "step": 239 }, { "Total_reward": -2.3203237731195259e-07, "advantage_std": 1.3156840801239014, "completion_length": 484.3214416503906, "epoch": 0.08287292817679558, "grad_norm": 3.5019808542493736, "kl": 0.0220947265625, "learning_rate": 4.915747932591916e-07, "loss": 0.0009, "rewards_raw/CoT ICLG": -0.0364522710442543, "rewards_raw/accuracy_reward": 0.5716397166252136, "rewards_raw/clip_reward_wrapper": 0.29614993929862976, "rewards_raw/format_reward": 5.0, "step": 240 }, { "Total_reward": -4.108463258489792e-07, "advantage_std": 1.5803987979888916, "completion_length": 498.8214416503906, "epoch": 0.0832182320441989, "grad_norm": 4.885790374892769, "kl": 0.0223388671875, "learning_rate": 4.915048381141954e-07, "loss": 0.0009, "rewards_raw/CoT ICLG": 0.002065224340185523, "rewards_raw/accuracy_reward": 0.5649864673614502, "rewards_raw/clip_reward_wrapper": 0.29083457589149475, "rewards_raw/format_reward": 5.0, "step": 241 }, { "Total_reward": 4.2787621623574523e-07, "advantage_std": 1.6102813482284546, "completion_length": 475.14288330078125, "epoch": 0.0835635359116022, "grad_norm": 4.957821992190591, "kl": 0.0205078125, "learning_rate": 4.914345987663531e-07, "loss": 0.0008, "rewards_raw/CoT ICLG": 0.041946038603782654, "rewards_raw/accuracy_reward": 0.5477690696716309, "rewards_raw/clip_reward_wrapper": 0.29744720458984375, "rewards_raw/format_reward": 5.0, "step": 242 }, { "Total_reward": -2.384185791015625e-07, "advantage_std": 1.3232988119125366, "completion_length": 490.3035888671875, "epoch": 0.08390883977900553, "grad_norm": 6.716409322783883, "kl": 0.02490234375, "learning_rate": 4.913640752983218e-07, "loss": 0.001, "rewards_raw/CoT ICLG": 0.11542578041553497, "rewards_raw/accuracy_reward": 0.5567033886909485, "rewards_raw/clip_reward_wrapper": 0.28643566370010376, "rewards_raw/format_reward": 5.0, "step": 243 }, { "Total_reward": 1.1069434435739822e-07, "advantage_std": 1.2177863121032715, "completion_length": 490.8214416503906, "epoch": 0.08425414364640885, "grad_norm": 3.01959653732289, "kl": 0.021240234375, "learning_rate": 4.912932677930938e-07, "loss": 0.0009, "rewards_raw/CoT ICLG": 0.07262596487998962, "rewards_raw/accuracy_reward": 0.5633370876312256, "rewards_raw/clip_reward_wrapper": 0.3008340299129486, "rewards_raw/format_reward": 4.973214626312256, "step": 244 }, { "Total_reward": -1.1920928955078125e-07, "advantage_std": 1.4958244562149048, "completion_length": 486.9107360839844, "epoch": 0.08459944751381215, "grad_norm": 3.6600331990777577, "kl": 0.02197265625, "learning_rate": 4.912221763339954e-07, "loss": 0.0009, "rewards_raw/CoT ICLG": 0.04260852187871933, "rewards_raw/accuracy_reward": 0.5555170178413391, "rewards_raw/clip_reward_wrapper": 0.2894883155822754, "rewards_raw/format_reward": 5.0, "step": 245 }, { "Total_reward": -2.575772271029564e-07, "advantage_std": 1.2540935277938843, "completion_length": 477.76788330078125, "epoch": 0.08494475138121547, "grad_norm": 2.773953904709332, "kl": 0.0225830078125, "learning_rate": 4.911508010046868e-07, "loss": 0.0009, "rewards_raw/CoT ICLG": 0.8960698246955872, "rewards_raw/accuracy_reward": 0.5654431581497192, "rewards_raw/clip_reward_wrapper": 0.2879599928855896, "rewards_raw/format_reward": 5.0, "step": 246 }, { "Total_reward": 4.7577282202837523e-07, "advantage_std": 1.2953518629074097, "completion_length": 494.5000305175781, "epoch": 0.08529005524861878, "grad_norm": 3.22635965425088, "kl": 0.0223388671875, "learning_rate": 4.910791418891628e-07, "loss": 0.0009, "rewards_raw/CoT ICLG": 0.25722822546958923, "rewards_raw/accuracy_reward": 0.5513840913772583, "rewards_raw/clip_reward_wrapper": 0.284451425075531, "rewards_raw/format_reward": 5.0, "step": 247 }, { "Total_reward": -6.301062853708572e-07, "advantage_std": 1.6999200582504272, "completion_length": 484.5357360839844, "epoch": 0.0856353591160221, "grad_norm": 7.069591251093891, "kl": 0.0224609375, "learning_rate": 4.910071990717514e-07, "loss": 0.0009, "rewards_raw/CoT ICLG": 0.1606362909078598, "rewards_raw/accuracy_reward": 0.5634171366691589, "rewards_raw/clip_reward_wrapper": 0.29726970195770264, "rewards_raw/format_reward": 5.0, "step": 248 }, { "Total_reward": -4.3639116142912826e-07, "advantage_std": 1.4854180812835693, "completion_length": 490.9285888671875, "epoch": 0.08598066298342541, "grad_norm": 4.431386886772037, "kl": 0.0230712890625, "learning_rate": 4.909349726371154e-07, "loss": 0.0009, "rewards_raw/CoT ICLG": -0.21518905460834503, "rewards_raw/accuracy_reward": 0.55348140001297, "rewards_raw/clip_reward_wrapper": 0.2940467596054077, "rewards_raw/format_reward": 5.0, "step": 249 }, { "Total_reward": 2.7460711748972244e-07, "advantage_std": 1.531346321105957, "completion_length": 495.58929443359375, "epoch": 0.08632596685082873, "grad_norm": 6.489844180880644, "kl": 0.0218505859375, "learning_rate": 4.908624626702504e-07, "loss": 0.0009, "rewards_raw/CoT ICLG": 0.2981480360031128, "rewards_raw/accuracy_reward": 0.5615089535713196, "rewards_raw/clip_reward_wrapper": 0.29276975989341736, "rewards_raw/format_reward": 5.0, "step": 250 }, { "Total_reward": 4.853521318182175e-07, "advantage_std": 1.5678112506866455, "completion_length": 492.6785888671875, "epoch": 0.08667127071823204, "grad_norm": 4.549391781453348, "kl": 0.0223388671875, "learning_rate": 4.907896692564864e-07, "loss": 0.0009, "rewards_raw/CoT ICLG": 0.1088840663433075, "rewards_raw/accuracy_reward": 0.5486457943916321, "rewards_raw/clip_reward_wrapper": 0.28672897815704346, "rewards_raw/format_reward": 5.0, "step": 251 }, { "Total_reward": -1.6391278734317893e-07, "advantage_std": 1.464723825454712, "completion_length": 477.5535888671875, "epoch": 0.08701657458563536, "grad_norm": 3.6141217647588584, "kl": 0.0230712890625, "learning_rate": 4.907165924814865e-07, "loss": 0.0009, "rewards_raw/CoT ICLG": 0.23386557400226593, "rewards_raw/accuracy_reward": 0.5922436714172363, "rewards_raw/clip_reward_wrapper": 0.2996317446231842, "rewards_raw/format_reward": 5.0, "step": 252 }, { "Total_reward": -2.07551892117408e-07, "advantage_std": 1.5158897638320923, "completion_length": 480.482177734375, "epoch": 0.08736187845303868, "grad_norm": 4.110984477735708, "kl": 0.0250244140625, "learning_rate": 4.906432324312476e-07, "loss": 0.001, "rewards_raw/CoT ICLG": 0.1029849424958229, "rewards_raw/accuracy_reward": 0.5555786490440369, "rewards_raw/clip_reward_wrapper": 0.2829669117927551, "rewards_raw/format_reward": 5.0, "step": 253 }, { "Total_reward": -2.2564616131148796e-07, "advantage_std": 1.5141600370407104, "completion_length": 478.982177734375, "epoch": 0.08770718232044199, "grad_norm": 4.853348594198094, "kl": 0.0230712890625, "learning_rate": 4.905695891920997e-07, "loss": 0.0009, "rewards_raw/CoT ICLG": 0.02693568542599678, "rewards_raw/accuracy_reward": 0.5576996207237244, "rewards_raw/clip_reward_wrapper": 0.29937735199928284, "rewards_raw/format_reward": 5.0, "step": 254 }, { "Total_reward": -7.791178973093338e-07, "advantage_std": 1.543488621711731, "completion_length": 474.51788330078125, "epoch": 0.08805248618784531, "grad_norm": 4.520667752259219, "kl": 0.026611328125, "learning_rate": 4.90495662850706e-07, "loss": 0.0011, "rewards_raw/CoT ICLG": 0.3264912962913513, "rewards_raw/accuracy_reward": 0.5632156133651733, "rewards_raw/clip_reward_wrapper": 0.29384300112724304, "rewards_raw/format_reward": 5.0, "step": 255 }, { "Total_reward": -7.66345493730114e-08, "advantage_std": 1.3003735542297363, "completion_length": 483.107177734375, "epoch": 0.08839779005524862, "grad_norm": 2.832349614343257, "kl": 0.0233154296875, "learning_rate": 4.904214534940633e-07, "loss": 0.0009, "rewards_raw/CoT ICLG": 0.01299280859529972, "rewards_raw/accuracy_reward": 0.5607337355613708, "rewards_raw/clip_reward_wrapper": 0.29483461380004883, "rewards_raw/format_reward": 5.0, "step": 256 }, { "Total_reward": -1.154840063577467e-07, "advantage_std": 1.336219310760498, "completion_length": 481.83929443359375, "epoch": 0.08874309392265194, "grad_norm": 2.9726029856661835, "kl": 0.02490234375, "learning_rate": 4.90346961209501e-07, "loss": 0.001, "rewards_raw/CoT ICLG": -0.10798069089651108, "rewards_raw/accuracy_reward": 0.572564959526062, "rewards_raw/clip_reward_wrapper": 0.28609296679496765, "rewards_raw/format_reward": 5.0, "step": 257 }, { "Total_reward": 6.045614213689987e-07, "advantage_std": 1.4405900239944458, "completion_length": 478.96429443359375, "epoch": 0.08908839779005524, "grad_norm": 3.7740044047666004, "kl": 0.0247802734375, "learning_rate": 4.902721860846816e-07, "loss": 0.001, "rewards_raw/CoT ICLG": 0.054205212742090225, "rewards_raw/accuracy_reward": 0.5458335280418396, "rewards_raw/clip_reward_wrapper": 0.28517499566078186, "rewards_raw/format_reward": 5.0, "step": 258 }, { "Total_reward": 2.7034965910388564e-07, "advantage_std": 1.3862894773483276, "completion_length": 484.8035888671875, "epoch": 0.08943370165745856, "grad_norm": 6.8760626220183045, "kl": 0.025634765625, "learning_rate": 4.901971282076005e-07, "loss": 0.001, "rewards_raw/CoT ICLG": 0.12093314528465271, "rewards_raw/accuracy_reward": 0.5572730898857117, "rewards_raw/clip_reward_wrapper": 0.2852571904659271, "rewards_raw/format_reward": 5.0, "step": 259 }, { "Total_reward": -3.491129234589607e-07, "advantage_std": 1.5480287075042725, "completion_length": 482.9285888671875, "epoch": 0.08977900552486189, "grad_norm": 3.8147753258682275, "kl": 0.0255126953125, "learning_rate": 4.901217876665857e-07, "loss": 0.001, "rewards_raw/CoT ICLG": 0.11653050035238266, "rewards_raw/accuracy_reward": 0.5741154551506042, "rewards_raw/clip_reward_wrapper": 0.29048436880111694, "rewards_raw/format_reward": 5.0, "step": 260 }, { "Total_reward": 8.685248644724197e-07, "advantage_std": 1.4946304559707642, "completion_length": 482.01788330078125, "epoch": 0.09012430939226519, "grad_norm": 5.018268627018665, "kl": 0.0252685546875, "learning_rate": 4.900461645502981e-07, "loss": 0.001, "rewards_raw/CoT ICLG": -0.16013240814208984, "rewards_raw/accuracy_reward": 0.561482310295105, "rewards_raw/clip_reward_wrapper": 0.2998617887496948, "rewards_raw/format_reward": 5.0, "step": 261 }, { "Total_reward": -3.916876778475853e-07, "advantage_std": 1.644363522529602, "completion_length": 488.8750305175781, "epoch": 0.09046961325966851, "grad_norm": 4.456421146960366, "kl": 0.026611328125, "learning_rate": 4.899702589477308e-07, "loss": 0.0011, "rewards_raw/CoT ICLG": 0.11140438914299011, "rewards_raw/accuracy_reward": 0.5819857716560364, "rewards_raw/clip_reward_wrapper": 0.2868296802043915, "rewards_raw/format_reward": 5.0, "step": 262 }, { "Total_reward": 1.4682966593682067e-06, "advantage_std": 1.7349853515625, "completion_length": 475.21429443359375, "epoch": 0.09081491712707182, "grad_norm": 4.25339468106343, "kl": 0.0274658203125, "learning_rate": 4.898940709482096e-07, "loss": 0.0011, "rewards_raw/CoT ICLG": 0.3507837653160095, "rewards_raw/accuracy_reward": 0.5543650984764099, "rewards_raw/clip_reward_wrapper": 0.28295403718948364, "rewards_raw/format_reward": 5.0, "step": 263 }, { "Total_reward": -5.577292085945373e-07, "advantage_std": 1.408698320388794, "completion_length": 456.232177734375, "epoch": 0.09116022099447514, "grad_norm": 8.17715317838858, "kl": 0.0260009765625, "learning_rate": 4.898176006413925e-07, "loss": 0.001, "rewards_raw/CoT ICLG": 0.08705244958400726, "rewards_raw/accuracy_reward": 0.5782259106636047, "rewards_raw/clip_reward_wrapper": 0.30272752046585083, "rewards_raw/format_reward": 5.0, "step": 264 }, { "Total_reward": 2.852508202977333e-07, "advantage_std": 1.313490390777588, "completion_length": 468.5714416503906, "epoch": 0.09150552486187845, "grad_norm": 3.168269868603701, "kl": 0.02783203125, "learning_rate": 4.897408481172697e-07, "loss": 0.0011, "rewards_raw/CoT ICLG": 0.33343759179115295, "rewards_raw/accuracy_reward": 0.5579774975776672, "rewards_raw/clip_reward_wrapper": 0.29780837893486023, "rewards_raw/format_reward": 5.0, "step": 265 }, { "Total_reward": -1.5114035534224968e-07, "advantage_std": 1.5371440649032593, "completion_length": 458.0000305175781, "epoch": 0.09185082872928177, "grad_norm": 4.7956099613953285, "kl": 0.029296875, "learning_rate": 4.896638134661636e-07, "loss": 0.0012, "rewards_raw/CoT ICLG": 0.3186013102531433, "rewards_raw/accuracy_reward": 0.5668014287948608, "rewards_raw/clip_reward_wrapper": 0.31144899129867554, "rewards_raw/format_reward": 5.0, "step": 266 }, { "Total_reward": -2.6928529450742644e-07, "advantage_std": 1.2564846277236938, "completion_length": 463.7857360839844, "epoch": 0.09219613259668508, "grad_norm": 4.443340690521326, "kl": 0.027099609375, "learning_rate": 4.895864967787285e-07, "loss": 0.0011, "rewards_raw/CoT ICLG": -0.020427338778972626, "rewards_raw/accuracy_reward": 0.5796898603439331, "rewards_raw/clip_reward_wrapper": 0.2818624675273895, "rewards_raw/format_reward": 5.0, "step": 267 }, { "Total_reward": -2.6609217940176677e-08, "advantage_std": 1.5700284242630005, "completion_length": 468.7500305175781, "epoch": 0.0925414364640884, "grad_norm": 4.9377294163798675, "kl": 0.0281982421875, "learning_rate": 4.895088981459508e-07, "loss": 0.0011, "rewards_raw/CoT ICLG": 0.253263384103775, "rewards_raw/accuracy_reward": 0.5755363702774048, "rewards_raw/clip_reward_wrapper": 0.3020794987678528, "rewards_raw/format_reward": 5.0, "step": 268 }, { "Total_reward": -2.1287374085687816e-09, "advantage_std": 1.6621402502059937, "completion_length": 474.5714416503906, "epoch": 0.09288674033149172, "grad_norm": 3.3180282688982685, "kl": 0.0279541015625, "learning_rate": 4.894310176591484e-07, "loss": 0.0011, "rewards_raw/CoT ICLG": 0.14533312618732452, "rewards_raw/accuracy_reward": 0.5536447167396545, "rewards_raw/clip_reward_wrapper": 0.28954964876174927, "rewards_raw/format_reward": 5.0, "step": 269 }, { "Total_reward": -6.684235245302261e-07, "advantage_std": 1.233413577079773, "completion_length": 451.14288330078125, "epoch": 0.09323204419889503, "grad_norm": 3.885178347379265, "kl": 0.0299072265625, "learning_rate": 4.893528554099712e-07, "loss": 0.0012, "rewards_raw/CoT ICLG": 0.176591694355011, "rewards_raw/accuracy_reward": 0.5415955781936646, "rewards_raw/clip_reward_wrapper": 0.29656118154525757, "rewards_raw/format_reward": 5.0, "step": 270 }, { "Total_reward": -1.2559550555124588e-07, "advantage_std": 1.3828256130218506, "completion_length": 453.4464416503906, "epoch": 0.09357734806629835, "grad_norm": 4.7304715274336715, "kl": 0.0281982421875, "learning_rate": 4.892744114904004e-07, "loss": 0.0011, "rewards_raw/CoT ICLG": 0.5024570822715759, "rewards_raw/accuracy_reward": 0.5878998041152954, "rewards_raw/clip_reward_wrapper": 0.2985329031944275, "rewards_raw/format_reward": 5.0, "step": 271 }, { "Total_reward": 2.7034965910388564e-07, "advantage_std": 1.496446132659912, "completion_length": 468.8035888671875, "epoch": 0.09392265193370165, "grad_norm": 4.560448995282341, "kl": 0.0277099609375, "learning_rate": 4.891956859927488e-07, "loss": 0.0011, "rewards_raw/CoT ICLG": 0.24462850391864777, "rewards_raw/accuracy_reward": 0.5696033239364624, "rewards_raw/clip_reward_wrapper": 0.2906263768672943, "rewards_raw/format_reward": 5.0, "step": 272 }, { "Total_reward": 2.1500247271433182e-07, "advantage_std": 1.5572667121887207, "completion_length": 481.9285888671875, "epoch": 0.09426795580110497, "grad_norm": 3.90739576538479, "kl": 0.0284423828125, "learning_rate": 4.891166790096606e-07, "loss": 0.0011, "rewards_raw/CoT ICLG": 0.11270648241043091, "rewards_raw/accuracy_reward": 0.5586921572685242, "rewards_raw/clip_reward_wrapper": 0.3035790026187897, "rewards_raw/format_reward": 5.0, "step": 273 }, { "Total_reward": 2.7034965910388564e-07, "advantage_std": 1.6028977632522583, "completion_length": 463.76788330078125, "epoch": 0.09461325966850828, "grad_norm": 4.7185178901599025, "kl": 0.029541015625, "learning_rate": 4.890373906341111e-07, "loss": 0.0012, "rewards_raw/CoT ICLG": 0.6356164216995239, "rewards_raw/accuracy_reward": 0.552966833114624, "rewards_raw/clip_reward_wrapper": 0.29041749238967896, "rewards_raw/format_reward": 5.0, "step": 274 }, { "Total_reward": -1.7029898913278885e-07, "advantage_std": 1.5389026403427124, "completion_length": 450.33929443359375, "epoch": 0.0949585635359116, "grad_norm": 8.099508523864515, "kl": 0.0291748046875, "learning_rate": 4.88957820959407e-07, "loss": 0.0012, "rewards_raw/CoT ICLG": 0.4999101758003235, "rewards_raw/accuracy_reward": 0.5636404752731323, "rewards_raw/clip_reward_wrapper": 0.30940482020378113, "rewards_raw/format_reward": 5.0, "step": 275 }, { "Total_reward": 1.1920928955078125e-07, "advantage_std": 1.539300560951233, "completion_length": 468.982177734375, "epoch": 0.09530386740331492, "grad_norm": 5.9018479835687145, "kl": 0.0284423828125, "learning_rate": 4.888779700791857e-07, "loss": 0.0011, "rewards_raw/CoT ICLG": 0.11960560083389282, "rewards_raw/accuracy_reward": 0.5448002219200134, "rewards_raw/clip_reward_wrapper": 0.29755592346191406, "rewards_raw/format_reward": 5.0, "step": 276 }, { "Total_reward": -2.096806355211811e-07, "advantage_std": 1.6945888996124268, "completion_length": 468.7857360839844, "epoch": 0.09564917127071823, "grad_norm": 5.151800241321184, "kl": 0.0281982421875, "learning_rate": 4.887978380874159e-07, "loss": 0.0011, "rewards_raw/CoT ICLG": 0.5894109606742859, "rewards_raw/accuracy_reward": 0.5354099273681641, "rewards_raw/clip_reward_wrapper": 0.30307736992836, "rewards_raw/format_reward": 5.0, "step": 277 }, { "Total_reward": -5.10896995820076e-08, "advantage_std": 1.634291172027588, "completion_length": 465.8035888671875, "epoch": 0.09599447513812155, "grad_norm": 8.36029192425941, "kl": 0.0289306640625, "learning_rate": 4.887174250783968e-07, "loss": 0.0012, "rewards_raw/CoT ICLG": 0.27099597454071045, "rewards_raw/accuracy_reward": 0.5580532550811768, "rewards_raw/clip_reward_wrapper": 0.2999543845653534, "rewards_raw/format_reward": 5.0, "step": 278 }, { "Total_reward": 3.2356808787881164e-07, "advantage_std": 1.5176494121551514, "completion_length": 450.0535888671875, "epoch": 0.09633977900552486, "grad_norm": 4.11690976771767, "kl": 0.0291748046875, "learning_rate": 4.886367311467584e-07, "loss": 0.0012, "rewards_raw/CoT ICLG": -0.11950356513261795, "rewards_raw/accuracy_reward": 0.5590049028396606, "rewards_raw/clip_reward_wrapper": 0.30766868591308594, "rewards_raw/format_reward": 5.0, "step": 279 }, { "Total_reward": 8.98327186860115e-07, "advantage_std": 1.3466875553131104, "completion_length": 467.96429443359375, "epoch": 0.09668508287292818, "grad_norm": 3.2410892921278167, "kl": 0.028076171875, "learning_rate": 4.885557563874613e-07, "loss": 0.0011, "rewards_raw/CoT ICLG": 0.4437797963619232, "rewards_raw/accuracy_reward": 0.5744026899337769, "rewards_raw/clip_reward_wrapper": 0.2918454110622406, "rewards_raw/format_reward": 5.0, "step": 280 }, { "Total_reward": -1.1415354350674534e-07, "advantage_std": 1.5862510204315186, "completion_length": 465.7857360839844, "epoch": 0.09703038674033149, "grad_norm": 4.054272594392461, "kl": 0.032470703125, "learning_rate": 4.884745008957966e-07, "loss": 0.0013, "rewards_raw/CoT ICLG": 0.2093389630317688, "rewards_raw/accuracy_reward": 0.5740703344345093, "rewards_raw/clip_reward_wrapper": 0.30589038133621216, "rewards_raw/format_reward": 5.0, "step": 281 }, { "Total_reward": -7.557017767112484e-08, "advantage_std": 1.6382842063903809, "completion_length": 475.96429443359375, "epoch": 0.09737569060773481, "grad_norm": 14.404383704020361, "kl": 0.03076171875, "learning_rate": 4.883929647673858e-07, "loss": 0.0012, "rewards_raw/CoT ICLG": 0.17976048588752747, "rewards_raw/accuracy_reward": 0.5777351260185242, "rewards_raw/clip_reward_wrapper": 0.2910054326057434, "rewards_raw/format_reward": 5.0, "step": 282 }, { "Total_reward": -2.7886460429726867e-07, "advantage_std": 1.5986943244934082, "completion_length": 466.58929443359375, "epoch": 0.09772099447513811, "grad_norm": 9.680386861148378, "kl": 0.029052734375, "learning_rate": 4.883111480981805e-07, "loss": 0.0012, "rewards_raw/CoT ICLG": 0.20874670147895813, "rewards_raw/accuracy_reward": 0.5452158451080322, "rewards_raw/clip_reward_wrapper": 0.2875877618789673, "rewards_raw/format_reward": 5.0, "step": 283 }, { "Total_reward": 8.089202196970291e-08, "advantage_std": 1.5076708793640137, "completion_length": 464.9285888671875, "epoch": 0.09806629834254144, "grad_norm": 11.520001400664475, "kl": 0.031494140625, "learning_rate": 4.882290509844626e-07, "loss": 0.0013, "rewards_raw/CoT ICLG": 0.14046378433704376, "rewards_raw/accuracy_reward": 0.5601390600204468, "rewards_raw/clip_reward_wrapper": 0.30581116676330566, "rewards_raw/format_reward": 5.0, "step": 284 }, { "Total_reward": 2.1287373641598606e-08, "advantage_std": 1.6557791233062744, "completion_length": 452.9464416503906, "epoch": 0.09841160220994476, "grad_norm": 3.8657429689200864, "kl": 0.033203125, "learning_rate": 4.88146673522844e-07, "loss": 0.0013, "rewards_raw/CoT ICLG": 0.3062775731086731, "rewards_raw/accuracy_reward": 0.5738981366157532, "rewards_raw/clip_reward_wrapper": 0.27946484088897705, "rewards_raw/format_reward": 5.0, "step": 285 }, { "Total_reward": 2.384185791015625e-07, "advantage_std": 1.5901210308074951, "completion_length": 448.89288330078125, "epoch": 0.09875690607734806, "grad_norm": 4.74775267384986, "kl": 0.033447265625, "learning_rate": 4.880640158102665e-07, "loss": 0.0013, "rewards_raw/CoT ICLG": 0.0621517188847065, "rewards_raw/accuracy_reward": 0.5727666020393372, "rewards_raw/clip_reward_wrapper": 0.31205111742019653, "rewards_raw/format_reward": 5.0, "step": 286 }, { "Total_reward": -8.089202196970291e-08, "advantage_std": 1.5970672369003296, "completion_length": 468.6607360839844, "epoch": 0.09910220994475138, "grad_norm": 3.8501092151784984, "kl": 0.03076171875, "learning_rate": 4.879810779440016e-07, "loss": 0.0012, "rewards_raw/CoT ICLG": 0.07076209038496017, "rewards_raw/accuracy_reward": 0.530329167842865, "rewards_raw/clip_reward_wrapper": 0.29780521988868713, "rewards_raw/format_reward": 5.0, "step": 287 }, { "Total_reward": -2.575772271029564e-07, "advantage_std": 1.465728521347046, "completion_length": 461.5714416503906, "epoch": 0.09944751381215469, "grad_norm": 3.4179073885506672, "kl": 0.03271484375, "learning_rate": 4.878978600216505e-07, "loss": 0.0013, "rewards_raw/CoT ICLG": 0.4322281777858734, "rewards_raw/accuracy_reward": 0.5566182732582092, "rewards_raw/clip_reward_wrapper": 0.2898072600364685, "rewards_raw/format_reward": 5.0, "step": 288 }, { "Total_reward": 1.319817215517105e-07, "advantage_std": 1.4657063484191895, "completion_length": 463.6785888671875, "epoch": 0.09979281767955801, "grad_norm": 4.04830072442153, "kl": 0.03271484375, "learning_rate": 4.878143621411441e-07, "loss": 0.0013, "rewards_raw/CoT ICLG": 0.198085218667984, "rewards_raw/accuracy_reward": 0.5647444725036621, "rewards_raw/clip_reward_wrapper": 0.3093120753765106, "rewards_raw/format_reward": 5.0, "step": 289 }, { "Total_reward": 5.641153961732925e-07, "advantage_std": 1.6184216737747192, "completion_length": 471.732177734375, "epoch": 0.10013812154696132, "grad_norm": 6.292439279692176, "kl": 0.031494140625, "learning_rate": 4.877305844007428e-07, "loss": 0.0013, "rewards_raw/CoT ICLG": 0.03938606381416321, "rewards_raw/accuracy_reward": 0.5452355742454529, "rewards_raw/clip_reward_wrapper": 0.2939234972000122, "rewards_raw/format_reward": 5.0, "step": 290 }, { "Total_reward": 1.319817215517105e-07, "advantage_std": 1.3540023565292358, "completion_length": 441.33929443359375, "epoch": 0.10048342541436464, "grad_norm": 4.467079321219167, "kl": 0.034912109375, "learning_rate": 4.876465268990361e-07, "loss": 0.0014, "rewards_raw/CoT ICLG": 0.8775718212127686, "rewards_raw/accuracy_reward": 0.547944188117981, "rewards_raw/clip_reward_wrapper": 0.2994372844696045, "rewards_raw/format_reward": 5.0, "step": 291 }, { "Total_reward": 5.428280474006897e-07, "advantage_std": 1.5948532819747925, "completion_length": 457.14288330078125, "epoch": 0.10082872928176796, "grad_norm": 4.195282590598284, "kl": 0.033935546875, "learning_rate": 4.875621897349429e-07, "loss": 0.0014, "rewards_raw/CoT ICLG": 0.1830456405878067, "rewards_raw/accuracy_reward": 0.5549792051315308, "rewards_raw/clip_reward_wrapper": 0.3139953017234802, "rewards_raw/format_reward": 5.0, "step": 292 }, { "Total_reward": -2.6928529450742644e-07, "advantage_std": 1.4241403341293335, "completion_length": 466.3035888671875, "epoch": 0.10117403314917127, "grad_norm": 5.476456304855026, "kl": 0.03466796875, "learning_rate": 4.874775730077111e-07, "loss": 0.0014, "rewards_raw/CoT ICLG": 0.19790229201316833, "rewards_raw/accuracy_reward": 0.5758206844329834, "rewards_raw/clip_reward_wrapper": 0.3003704249858856, "rewards_raw/format_reward": 5.0, "step": 293 }, { "Total_reward": -5.449567765936081e-07, "advantage_std": 1.2400246858596802, "completion_length": 450.9107360839844, "epoch": 0.10151933701657459, "grad_norm": 6.962422943863259, "kl": 0.034423828125, "learning_rate": 4.873926768169177e-07, "loss": 0.0014, "rewards_raw/CoT ICLG": 0.25939854979515076, "rewards_raw/accuracy_reward": 0.5550327301025391, "rewards_raw/clip_reward_wrapper": 0.3029191195964813, "rewards_raw/format_reward": 5.0, "step": 294 }, { "Total_reward": -2.586415916994156e-07, "advantage_std": 1.731166124343872, "completion_length": 426.4285888671875, "epoch": 0.1018646408839779, "grad_norm": 4.902909118744395, "kl": 0.033203125, "learning_rate": 4.873075012624687e-07, "loss": 0.0013, "rewards_raw/CoT ICLG": 0.48771679401397705, "rewards_raw/accuracy_reward": 0.5662639141082764, "rewards_raw/clip_reward_wrapper": 0.3001817464828491, "rewards_raw/format_reward": 5.0, "step": 295 }, { "Total_reward": -1.6497715193963813e-07, "advantage_std": 1.3643666505813599, "completion_length": 471.4464416503906, "epoch": 0.10220994475138122, "grad_norm": 4.089090090612342, "kl": 0.034423828125, "learning_rate": 4.872220464445983e-07, "loss": 0.0014, "rewards_raw/CoT ICLG": 0.19761259853839874, "rewards_raw/accuracy_reward": 0.5560311675071716, "rewards_raw/clip_reward_wrapper": 0.3007221817970276, "rewards_raw/format_reward": 5.0, "step": 296 }, { "Total_reward": 1.0750123635716591e-07, "advantage_std": 1.4241597652435303, "completion_length": 467.4107360839844, "epoch": 0.10255524861878453, "grad_norm": 3.3575969803452494, "kl": 0.03466796875, "learning_rate": 4.871363124638699e-07, "loss": 0.0014, "rewards_raw/CoT ICLG": -0.03525393083691597, "rewards_raw/accuracy_reward": 0.5419220924377441, "rewards_raw/clip_reward_wrapper": 0.3057096004486084, "rewards_raw/format_reward": 5.0, "step": 297 }, { "Total_reward": 1.575265713427143e-07, "advantage_std": 1.566164255142212, "completion_length": 458.3035888671875, "epoch": 0.10290055248618785, "grad_norm": 7.182059259349727, "kl": 0.0322265625, "learning_rate": 4.870502994211754e-07, "loss": 0.0013, "rewards_raw/CoT ICLG": 0.03298744186758995, "rewards_raw/accuracy_reward": 0.5821257829666138, "rewards_raw/clip_reward_wrapper": 0.29636698961257935, "rewards_raw/format_reward": 5.0, "step": 298 }, { "Total_reward": 2.980232238769531e-07, "advantage_std": 1.526167392730713, "completion_length": 458.46429443359375, "epoch": 0.10324585635359115, "grad_norm": 3.377182062708586, "kl": 0.034912109375, "learning_rate": 4.869640074177347e-07, "loss": 0.0014, "rewards_raw/CoT ICLG": 0.256696879863739, "rewards_raw/accuracy_reward": 0.5721027851104736, "rewards_raw/clip_reward_wrapper": 0.28780245780944824, "rewards_raw/format_reward": 5.0, "step": 299 }, { "Total_reward": 1.4475413934178505e-07, "advantage_std": 1.5119874477386475, "completion_length": 464.5714416503906, "epoch": 0.10359116022099447, "grad_norm": 3.735328440341552, "kl": 0.03759765625, "learning_rate": 4.868774365550962e-07, "loss": 0.0015, "rewards_raw/CoT ICLG": 0.3723963499069214, "rewards_raw/accuracy_reward": 0.5588865876197815, "rewards_raw/clip_reward_wrapper": 0.3022269010543823, "rewards_raw/format_reward": 5.0, "step": 300 }, { "Total_reward": -4.928027124151413e-07, "advantage_std": 1.4080032110214233, "completion_length": 452.0357360839844, "epoch": 0.1039364640883978, "grad_norm": 3.108230792518154, "kl": 0.036376953125, "learning_rate": 4.867905869351366e-07, "loss": 0.0015, "rewards_raw/CoT ICLG": 0.010252241976559162, "rewards_raw/accuracy_reward": 0.5369073152542114, "rewards_raw/clip_reward_wrapper": 0.307487428188324, "rewards_raw/format_reward": 5.0, "step": 301 }, { "Total_reward": 2.043587983280304e-07, "advantage_std": 1.4283641576766968, "completion_length": 456.0357360839844, "epoch": 0.1042817679558011, "grad_norm": 3.657971736136571, "kl": 0.035888671875, "learning_rate": 4.867034586600605e-07, "loss": 0.0014, "rewards_raw/CoT ICLG": 0.11935491859912872, "rewards_raw/accuracy_reward": 0.5552424192428589, "rewards_raw/clip_reward_wrapper": 0.3182999789714813, "rewards_raw/format_reward": 5.0, "step": 302 }, { "Total_reward": -9.366444828629028e-07, "advantage_std": 1.5113599300384521, "completion_length": 447.0000305175781, "epoch": 0.10462707182320442, "grad_norm": 5.2148053292503596, "kl": 0.0361328125, "learning_rate": 4.866160518324004e-07, "loss": 0.0014, "rewards_raw/CoT ICLG": 0.45687323808670044, "rewards_raw/accuracy_reward": 0.5602638721466064, "rewards_raw/clip_reward_wrapper": 0.30662184953689575, "rewards_raw/format_reward": 5.0, "step": 303 }, { "Total_reward": 2.4480479510202713e-07, "advantage_std": 1.4961776733398438, "completion_length": 445.4285888671875, "epoch": 0.10497237569060773, "grad_norm": 6.074289301258813, "kl": 0.0380859375, "learning_rate": 4.865283665550166e-07, "loss": 0.0015, "rewards_raw/CoT ICLG": 0.3051232695579529, "rewards_raw/accuracy_reward": 0.5809676647186279, "rewards_raw/clip_reward_wrapper": 0.30943745374679565, "rewards_raw/format_reward": 5.0, "step": 304 }, { "Total_reward": 1.4368977474532585e-07, "advantage_std": 1.4985854625701904, "completion_length": 446.1607360839844, "epoch": 0.10531767955801105, "grad_norm": 4.892620677079969, "kl": 0.0380859375, "learning_rate": 4.864404029310972e-07, "loss": 0.0015, "rewards_raw/CoT ICLG": 0.2842741310596466, "rewards_raw/accuracy_reward": 0.557245671749115, "rewards_raw/clip_reward_wrapper": 0.2897734045982361, "rewards_raw/format_reward": 5.0, "step": 305 }, { "Total_reward": -3.8530146184712066e-07, "advantage_std": 1.6134917736053467, "completion_length": 451.5000305175781, "epoch": 0.10566298342541436, "grad_norm": 12.447224722415234, "kl": 0.038818359375, "learning_rate": 4.863521610641578e-07, "loss": 0.0016, "rewards_raw/CoT ICLG": 0.24927660822868347, "rewards_raw/accuracy_reward": 0.5721766352653503, "rewards_raw/clip_reward_wrapper": 0.3071416914463043, "rewards_raw/format_reward": 5.0, "step": 306 }, { "Total_reward": 7.588948847114807e-07, "advantage_std": 1.4512407779693604, "completion_length": 437.6607360839844, "epoch": 0.10600828729281768, "grad_norm": 4.244659581860043, "kl": 0.0390625, "learning_rate": 4.862636410580413e-07, "loss": 0.0016, "rewards_raw/CoT ICLG": 0.1403765231370926, "rewards_raw/accuracy_reward": 0.5559712648391724, "rewards_raw/clip_reward_wrapper": 0.2978423833847046, "rewards_raw/format_reward": 5.0, "step": 307 }, { "Total_reward": -7.620880069225677e-07, "advantage_std": 1.6582695245742798, "completion_length": 456.83929443359375, "epoch": 0.106353591160221, "grad_norm": 4.609485964853714, "kl": 0.03662109375, "learning_rate": 4.861748430169181e-07, "loss": 0.0015, "rewards_raw/CoT ICLG": 0.02069578506052494, "rewards_raw/accuracy_reward": 0.5523171424865723, "rewards_raw/clip_reward_wrapper": 0.3042837083339691, "rewards_raw/format_reward": 5.0, "step": 308 }, { "Total_reward": -7.365431429207092e-07, "advantage_std": 1.4252678155899048, "completion_length": 459.4107360839844, "epoch": 0.10669889502762431, "grad_norm": 12.366611716907784, "kl": 0.037841796875, "learning_rate": 4.860857670452857e-07, "loss": 0.0015, "rewards_raw/CoT ICLG": 0.41293206810951233, "rewards_raw/accuracy_reward": 0.5683772563934326, "rewards_raw/clip_reward_wrapper": 0.3045097589492798, "rewards_raw/format_reward": 5.0, "step": 309 }, { "Total_reward": 4.002026230409683e-07, "advantage_std": 1.689396858215332, "completion_length": 460.39288330078125, "epoch": 0.10704419889502763, "grad_norm": 6.588488451108881, "kl": 0.0400390625, "learning_rate": 4.859964132479687e-07, "loss": 0.0016, "rewards_raw/CoT ICLG": 0.17647430300712585, "rewards_raw/accuracy_reward": 0.5577895641326904, "rewards_raw/clip_reward_wrapper": 0.30305781960487366, "rewards_raw/format_reward": 5.0, "step": 310 }, { "Total_reward": -1.7455646172948036e-07, "advantage_std": 1.4166980981826782, "completion_length": 453.51788330078125, "epoch": 0.10738950276243094, "grad_norm": 6.538857400964025, "kl": 0.038330078125, "learning_rate": 4.859067817301185e-07, "loss": 0.0015, "rewards_raw/CoT ICLG": 0.6233722567558289, "rewards_raw/accuracy_reward": 0.5613826513290405, "rewards_raw/clip_reward_wrapper": 0.29160675406455994, "rewards_raw/format_reward": 5.0, "step": 311 }, { "Total_reward": -1.788139485370266e-07, "advantage_std": 1.3727686405181885, "completion_length": 472.3214416503906, "epoch": 0.10773480662983426, "grad_norm": 3.8599066017348806, "kl": 0.040771484375, "learning_rate": 4.858168725972137e-07, "loss": 0.0016, "rewards_raw/CoT ICLG": 0.31156960129737854, "rewards_raw/accuracy_reward": 0.569201648235321, "rewards_raw/clip_reward_wrapper": 0.3059675395488739, "rewards_raw/format_reward": 5.0, "step": 312 }, { "Total_reward": -7.929546654850128e-07, "advantage_std": 1.57257878780365, "completion_length": 448.6785888671875, "epoch": 0.10808011049723756, "grad_norm": 3.9232464265280518, "kl": 0.036865234375, "learning_rate": 4.857266859550591e-07, "loss": 0.0015, "rewards_raw/CoT ICLG": -0.018837571144104004, "rewards_raw/accuracy_reward": 0.5682346224784851, "rewards_raw/clip_reward_wrapper": 0.307077020406723, "rewards_raw/format_reward": 5.0, "step": 313 }, { "Total_reward": 6.120120019659225e-07, "advantage_std": 1.6285192966461182, "completion_length": 440.51788330078125, "epoch": 0.10842541436464088, "grad_norm": 4.917879145869914, "kl": 0.040771484375, "learning_rate": 4.856362219097865e-07, "loss": 0.0016, "rewards_raw/CoT ICLG": 0.15943777561187744, "rewards_raw/accuracy_reward": 0.567042887210846, "rewards_raw/clip_reward_wrapper": 0.2927122116088867, "rewards_raw/format_reward": 5.0, "step": 314 }, { "Total_reward": 4.853521318182175e-07, "advantage_std": 1.4878710508346558, "completion_length": 463.8750305175781, "epoch": 0.1087707182320442, "grad_norm": 5.787814190928487, "kl": 0.03759765625, "learning_rate": 4.855454805678539e-07, "loss": 0.0015, "rewards_raw/CoT ICLG": 0.11641007661819458, "rewards_raw/accuracy_reward": 0.5561627149581909, "rewards_raw/clip_reward_wrapper": 0.3034154772758484, "rewards_raw/format_reward": 5.0, "step": 315 }, { "Total_reward": -1.1495181695408974e-07, "advantage_std": 1.38889479637146, "completion_length": 452.39288330078125, "epoch": 0.10911602209944751, "grad_norm": 3.21584357483393, "kl": 0.041748046875, "learning_rate": 4.854544620360457e-07, "loss": 0.0017, "rewards_raw/CoT ICLG": -0.008881432935595512, "rewards_raw/accuracy_reward": 0.5774938464164734, "rewards_raw/clip_reward_wrapper": 0.30404168367385864, "rewards_raw/format_reward": 5.0, "step": 316 }, { "Total_reward": 4.385199190437561e-07, "advantage_std": 1.474366307258606, "completion_length": 454.51788330078125, "epoch": 0.10946132596685083, "grad_norm": 3.632935998725406, "kl": 0.039794921875, "learning_rate": 4.853631664214725e-07, "loss": 0.0016, "rewards_raw/CoT ICLG": -0.19163838028907776, "rewards_raw/accuracy_reward": 0.559237003326416, "rewards_raw/clip_reward_wrapper": 0.30837178230285645, "rewards_raw/format_reward": 5.0, "step": 317 }, { "Total_reward": -5.768878281742218e-07, "advantage_std": 1.28582763671875, "completion_length": 449.39288330078125, "epoch": 0.10980662983425414, "grad_norm": 6.996540981484504, "kl": 0.043212890625, "learning_rate": 4.85271593831571e-07, "loss": 0.0017, "rewards_raw/CoT ICLG": 0.12446750700473785, "rewards_raw/accuracy_reward": 0.5624809265136719, "rewards_raw/clip_reward_wrapper": 0.29790744185447693, "rewards_raw/format_reward": 5.0, "step": 318 }, { "Total_reward": -6.066901647727718e-08, "advantage_std": 1.46066153049469, "completion_length": 447.89288330078125, "epoch": 0.11015193370165746, "grad_norm": 3.483656282904529, "kl": 0.04052734375, "learning_rate": 4.851797443741037e-07, "loss": 0.0016, "rewards_raw/CoT ICLG": 0.09998579323291779, "rewards_raw/accuracy_reward": 0.5551614761352539, "rewards_raw/clip_reward_wrapper": 0.31850144267082214, "rewards_raw/format_reward": 5.0, "step": 319 }, { "Total_reward": 8.514949634275126e-09, "advantage_std": 1.4816179275512695, "completion_length": 453.64288330078125, "epoch": 0.11049723756906077, "grad_norm": 5.074763969452544, "kl": 0.04150390625, "learning_rate": 4.850876181571591e-07, "loss": 0.0017, "rewards_raw/CoT ICLG": 0.33230429887771606, "rewards_raw/accuracy_reward": 0.546619713306427, "rewards_raw/clip_reward_wrapper": 0.3152877688407898, "rewards_raw/format_reward": 5.0, "step": 320 }, { "Total_reward": -3.352761552832817e-07, "advantage_std": 1.5820622444152832, "completion_length": 467.89288330078125, "epoch": 0.11084254143646409, "grad_norm": 7.281055042766929, "kl": 0.039794921875, "learning_rate": 4.849952152891513e-07, "loss": 0.0016, "rewards_raw/CoT ICLG": 0.3467289209365845, "rewards_raw/accuracy_reward": 0.5677636861801147, "rewards_raw/clip_reward_wrapper": 0.3020974099636078, "rewards_raw/format_reward": 5.0, "step": 321 }, { "Total_reward": 8.514949634275126e-09, "advantage_std": 1.4087581634521484, "completion_length": 449.1785888671875, "epoch": 0.1111878453038674, "grad_norm": 4.262989031071595, "kl": 0.041259765625, "learning_rate": 4.8490253587882e-07, "loss": 0.0016, "rewards_raw/CoT ICLG": 0.30217286944389343, "rewards_raw/accuracy_reward": 0.5729286670684814, "rewards_raw/clip_reward_wrapper": 0.30680590867996216, "rewards_raw/format_reward": 5.0, "step": 322 }, { "Total_reward": -2.4267606590910873e-07, "advantage_std": 1.504157543182373, "completion_length": 459.6964416503906, "epoch": 0.11153314917127072, "grad_norm": 20.988545254894095, "kl": 0.04052734375, "learning_rate": 4.848095800352303e-07, "loss": 0.0016, "rewards_raw/CoT ICLG": 0.2767605483531952, "rewards_raw/accuracy_reward": 0.5673569440841675, "rewards_raw/clip_reward_wrapper": 0.2985445559024811, "rewards_raw/format_reward": 5.0, "step": 323 }, { "Total_reward": 8.514949456639442e-08, "advantage_std": 1.5886749029159546, "completion_length": 450.5714416503906, "epoch": 0.11187845303867404, "grad_norm": 4.886639626573501, "kl": 0.041015625, "learning_rate": 4.847163478677725e-07, "loss": 0.0016, "rewards_raw/CoT ICLG": 0.15854834020137787, "rewards_raw/accuracy_reward": 0.5732141733169556, "rewards_raw/clip_reward_wrapper": 0.30168378353118896, "rewards_raw/format_reward": 5.0, "step": 324 }, { "Total_reward": 4.5342108023760375e-07, "advantage_std": 1.4740697145462036, "completion_length": 449.01788330078125, "epoch": 0.11222375690607735, "grad_norm": 3.9934600592708405, "kl": 0.042724609375, "learning_rate": 4.846228394861624e-07, "loss": 0.0017, "rewards_raw/CoT ICLG": 0.13972026109695435, "rewards_raw/accuracy_reward": 0.5631281137466431, "rewards_raw/clip_reward_wrapper": 0.3014352321624756, "rewards_raw/format_reward": 5.0, "step": 325 }, { "Total_reward": -6.024326921760803e-07, "advantage_std": 1.3863353729248047, "completion_length": 445.21429443359375, "epoch": 0.11256906077348067, "grad_norm": 6.222941779118397, "kl": 0.043701171875, "learning_rate": 4.845290550004406e-07, "loss": 0.0017, "rewards_raw/CoT ICLG": 0.24727103114128113, "rewards_raw/accuracy_reward": 0.5572870969772339, "rewards_raw/clip_reward_wrapper": 0.3185631036758423, "rewards_raw/format_reward": 5.0, "step": 326 }, { "Total_reward": -9.036490382641205e-07, "advantage_std": 1.4192373752593994, "completion_length": 456.9107360839844, "epoch": 0.11291436464088397, "grad_norm": 5.38462204274006, "kl": 0.04150390625, "learning_rate": 4.844349945209724e-07, "loss": 0.0017, "rewards_raw/CoT ICLG": 0.15155859291553497, "rewards_raw/accuracy_reward": 0.5663836002349854, "rewards_raw/clip_reward_wrapper": 0.30261823534965515, "rewards_raw/format_reward": 5.0, "step": 327 }, { "Total_reward": -9.685755486543712e-08, "advantage_std": 1.550427794456482, "completion_length": 444.857177734375, "epoch": 0.1132596685082873, "grad_norm": 5.2887093487759165, "kl": 0.0419921875, "learning_rate": 4.843406581584486e-07, "loss": 0.0017, "rewards_raw/CoT ICLG": 0.3566831052303314, "rewards_raw/accuracy_reward": 0.5808975100517273, "rewards_raw/clip_reward_wrapper": 0.2928756773471832, "rewards_raw/format_reward": 5.0, "step": 328 }, { "Total_reward": 4.5980726781635894e-07, "advantage_std": 1.5495305061340332, "completion_length": 467.482177734375, "epoch": 0.1136049723756906, "grad_norm": 10.74568517541753, "kl": 0.041259765625, "learning_rate": 4.842460460238839e-07, "loss": 0.0016, "rewards_raw/CoT ICLG": 0.3037559390068054, "rewards_raw/accuracy_reward": 0.5634889602661133, "rewards_raw/clip_reward_wrapper": 0.3101934492588043, "rewards_raw/format_reward": 5.0, "step": 329 }, { "Total_reward": 2.2990363390817947e-07, "advantage_std": 1.5059876441955566, "completion_length": 449.3214416503906, "epoch": 0.11395027624309392, "grad_norm": 5.620728459967688, "kl": 0.0458984375, "learning_rate": 4.841511582286179e-07, "loss": 0.0018, "rewards_raw/CoT ICLG": 0.741118848323822, "rewards_raw/accuracy_reward": 0.5512088537216187, "rewards_raw/clip_reward_wrapper": 0.3121003806591034, "rewards_raw/format_reward": 5.0, "step": 330 }, { "Total_reward": -2.895082786835701e-07, "advantage_std": 1.3206757307052612, "completion_length": 452.9464416503906, "epoch": 0.11429558011049724, "grad_norm": 3.811130250741277, "kl": 0.04052734375, "learning_rate": 4.840559948843146e-07, "loss": 0.0016, "rewards_raw/CoT ICLG": 0.011149300262331963, "rewards_raw/accuracy_reward": 0.5601871013641357, "rewards_raw/clip_reward_wrapper": 0.308531254529953, "rewards_raw/format_reward": 5.0, "step": 331 }, { "Total_reward": 8.706535936653381e-07, "advantage_std": 1.581316351890564, "completion_length": 466.8035888671875, "epoch": 0.11464088397790055, "grad_norm": 5.688453222496396, "kl": 0.045654296875, "learning_rate": 4.839605561029622e-07, "loss": 0.0018, "rewards_raw/CoT ICLG": 0.23512201011180878, "rewards_raw/accuracy_reward": 0.5450880527496338, "rewards_raw/clip_reward_wrapper": 0.3171390891075134, "rewards_raw/format_reward": 5.0, "step": 332 }, { "Total_reward": 8.759754450693436e-07, "advantage_std": 1.5513261556625366, "completion_length": 451.5714416503906, "epoch": 0.11498618784530387, "grad_norm": 4.255854747128638, "kl": 0.04541015625, "learning_rate": 4.838648419968728e-07, "loss": 0.0018, "rewards_raw/CoT ICLG": 0.034284867346286774, "rewards_raw/accuracy_reward": 0.5782454013824463, "rewards_raw/clip_reward_wrapper": 0.30154481530189514, "rewards_raw/format_reward": 5.0, "step": 333 }, { "Total_reward": 2.2777490471526107e-07, "advantage_std": 1.2686371803283691, "completion_length": 455.3750305175781, "epoch": 0.11533149171270718, "grad_norm": 4.2780309684544084, "kl": 0.0419921875, "learning_rate": 4.837688526786832e-07, "loss": 0.0017, "rewards_raw/CoT ICLG": 0.07597360759973526, "rewards_raw/accuracy_reward": 0.579462468624115, "rewards_raw/clip_reward_wrapper": 0.3057003617286682, "rewards_raw/format_reward": 5.0, "step": 334 }, { "Total_reward": -5.86467137964064e-07, "advantage_std": 1.5423966646194458, "completion_length": 437.3214416503906, "epoch": 0.1156767955801105, "grad_norm": 4.095457865077001, "kl": 0.044189453125, "learning_rate": 4.836725882613532e-07, "loss": 0.0018, "rewards_raw/CoT ICLG": 0.7779014110565186, "rewards_raw/accuracy_reward": 0.5379716157913208, "rewards_raw/clip_reward_wrapper": 0.3007616102695465, "rewards_raw/format_reward": 5.0, "step": 335 }, { "Total_reward": 3.83172746865057e-08, "advantage_std": 1.5037565231323242, "completion_length": 456.3214416503906, "epoch": 0.11602209944751381, "grad_norm": 10.460562978266545, "kl": 0.044189453125, "learning_rate": 4.835760488581669e-07, "loss": 0.0018, "rewards_raw/CoT ICLG": 0.16357117891311646, "rewards_raw/accuracy_reward": 0.5525510311126709, "rewards_raw/clip_reward_wrapper": 0.3066498935222626, "rewards_raw/format_reward": 5.0, "step": 336 }, { "Total_reward": -3.7678651665373764e-07, "advantage_std": 1.4362744092941284, "completion_length": 456.5714416503906, "epoch": 0.11636740331491713, "grad_norm": 4.307923681918063, "kl": 0.044189453125, "learning_rate": 4.834792345827319e-07, "loss": 0.0018, "rewards_raw/CoT ICLG": 0.08342233300209045, "rewards_raw/accuracy_reward": 0.5744428634643555, "rewards_raw/clip_reward_wrapper": 0.31791701912879944, "rewards_raw/format_reward": 5.0, "step": 337 }, { "Total_reward": 6.833246857240738e-07, "advantage_std": 1.617093563079834, "completion_length": 459.5357360839844, "epoch": 0.11671270718232044, "grad_norm": 5.43174445024682, "kl": 0.041748046875, "learning_rate": 4.833821455489792e-07, "loss": 0.0017, "rewards_raw/CoT ICLG": 0.27943775057792664, "rewards_raw/accuracy_reward": 0.5517279505729675, "rewards_raw/clip_reward_wrapper": 0.32243552803993225, "rewards_raw/format_reward": 5.0, "step": 338 }, { "Total_reward": 1.9158636632710113e-07, "advantage_std": 1.60775887966156, "completion_length": 448.4285888671875, "epoch": 0.11705801104972376, "grad_norm": 6.214022949559544, "kl": 0.0419921875, "learning_rate": 4.832847818711632e-07, "loss": 0.0017, "rewards_raw/CoT ICLG": 0.594567596912384, "rewards_raw/accuracy_reward": 0.5535534620285034, "rewards_raw/clip_reward_wrapper": 0.3122454285621643, "rewards_raw/format_reward": 5.0, "step": 339 }, { "Total_reward": -7.493155749216385e-07, "advantage_std": 1.4830412864685059, "completion_length": 463.39288330078125, "epoch": 0.11740331491712708, "grad_norm": 3.842215569064926, "kl": 0.04296875, "learning_rate": 4.831871436638613e-07, "loss": 0.0017, "rewards_raw/CoT ICLG": 0.20816101133823395, "rewards_raw/accuracy_reward": 0.5593048334121704, "rewards_raw/clip_reward_wrapper": 0.3058202266693115, "rewards_raw/format_reward": 5.0, "step": 340 }, { "Total_reward": -2.4267606590910873e-07, "advantage_std": 1.4846549034118652, "completion_length": 455.71429443359375, "epoch": 0.11774861878453038, "grad_norm": 3.29735700235273, "kl": 0.044677734375, "learning_rate": 4.830892310419742e-07, "loss": 0.0018, "rewards_raw/CoT ICLG": 0.291630357503891, "rewards_raw/accuracy_reward": 0.5665277242660522, "rewards_raw/clip_reward_wrapper": 0.31411170959472656, "rewards_raw/format_reward": 5.0, "step": 341 }, { "Total_reward": 2.3203237731195259e-07, "advantage_std": 1.4398752450942993, "completion_length": 468.3750305175781, "epoch": 0.1180939226519337, "grad_norm": 3.4040633787620207, "kl": 0.04541015625, "learning_rate": 4.829910441207255e-07, "loss": 0.0018, "rewards_raw/CoT ICLG": 0.32537734508514404, "rewards_raw/accuracy_reward": 0.5481268763542175, "rewards_raw/clip_reward_wrapper": 0.2980944812297821, "rewards_raw/format_reward": 5.0, "step": 342 }, { "Total_reward": -1.9158636632710113e-07, "advantage_std": 1.5125396251678467, "completion_length": 445.46429443359375, "epoch": 0.11843922651933701, "grad_norm": 3.7198039958816667, "kl": 0.047119140625, "learning_rate": 4.828925830156615e-07, "loss": 0.0019, "rewards_raw/CoT ICLG": 0.372651070356369, "rewards_raw/accuracy_reward": 0.5590583086013794, "rewards_raw/clip_reward_wrapper": 0.3091210126876831, "rewards_raw/format_reward": 5.0, "step": 343 }, { "Total_reward": 1.3943230214863434e-07, "advantage_std": 1.4384615421295166, "completion_length": 456.1250305175781, "epoch": 0.11878453038674033, "grad_norm": 3.819331586087766, "kl": 0.045166015625, "learning_rate": 4.827938478426512e-07, "loss": 0.0018, "rewards_raw/CoT ICLG": 0.26821720600128174, "rewards_raw/accuracy_reward": 0.5779155492782593, "rewards_raw/clip_reward_wrapper": 0.3102033734321594, "rewards_raw/format_reward": 5.0, "step": 344 }, { "Total_reward": -7.823110195204208e-07, "advantage_std": 1.5515189170837402, "completion_length": 457.83929443359375, "epoch": 0.11912983425414364, "grad_norm": 5.946320943103355, "kl": 0.0478515625, "learning_rate": 4.826948387178862e-07, "loss": 0.0019, "rewards_raw/CoT ICLG": 0.24795551598072052, "rewards_raw/accuracy_reward": 0.5649167895317078, "rewards_raw/clip_reward_wrapper": 0.317526251077652, "rewards_raw/format_reward": 5.0, "step": 345 }, { "Total_reward": 6.875822009533294e-07, "advantage_std": 1.396708607673645, "completion_length": 453.4285888671875, "epoch": 0.11947513812154696, "grad_norm": 4.3202744195389, "kl": 0.045654296875, "learning_rate": 4.825955557578803e-07, "loss": 0.0018, "rewards_raw/CoT ICLG": 0.42154058814048767, "rewards_raw/accuracy_reward": 0.5484341979026794, "rewards_raw/clip_reward_wrapper": 0.3084968328475952, "rewards_raw/format_reward": 5.0, "step": 346 }, { "Total_reward": 2.7673586089349556e-08, "advantage_std": 1.7510443925857544, "completion_length": 475.5714416503906, "epoch": 0.11982044198895028, "grad_norm": 4.652583964385986, "kl": 0.046875, "learning_rate": 4.824959990794698e-07, "loss": 0.0019, "rewards_raw/CoT ICLG": 0.3349984288215637, "rewards_raw/accuracy_reward": 0.5690367817878723, "rewards_raw/clip_reward_wrapper": 0.30275601148605347, "rewards_raw/format_reward": 5.0, "step": 347 }, { "Total_reward": 3.83172746865057e-08, "advantage_std": 1.5481137037277222, "completion_length": 451.3750305175781, "epoch": 0.12016574585635359, "grad_norm": 5.645291247071203, "kl": 0.046630859375, "learning_rate": 4.82396168799813e-07, "loss": 0.0019, "rewards_raw/CoT ICLG": 0.09681502729654312, "rewards_raw/accuracy_reward": 0.564751386642456, "rewards_raw/clip_reward_wrapper": 0.30931735038757324, "rewards_raw/format_reward": 5.0, "step": 348 }, { "Total_reward": 3.491129234589607e-07, "advantage_std": 1.5185033082962036, "completion_length": 460.982177734375, "epoch": 0.12051104972375691, "grad_norm": 6.9978975993060155, "kl": 0.046630859375, "learning_rate": 4.8229606503639e-07, "loss": 0.0019, "rewards_raw/CoT ICLG": 0.41912299394607544, "rewards_raw/accuracy_reward": 0.5741943717002869, "rewards_raw/clip_reward_wrapper": 0.3240135908126831, "rewards_raw/format_reward": 5.0, "step": 349 }, { "Total_reward": -1.1282308065574398e-07, "advantage_std": 1.5664138793945312, "completion_length": 455.14288330078125, "epoch": 0.12085635359116022, "grad_norm": 8.334423628362934, "kl": 0.044189453125, "learning_rate": 4.821956879070029e-07, "loss": 0.0018, "rewards_raw/CoT ICLG": 0.371111661195755, "rewards_raw/accuracy_reward": 0.5358504056930542, "rewards_raw/clip_reward_wrapper": 0.31573086977005005, "rewards_raw/format_reward": 5.0, "step": 350 }, { "Total_reward": -6.258487701416016e-07, "advantage_std": 1.7082550525665283, "completion_length": 446.1250305175781, "epoch": 0.12120165745856354, "grad_norm": 6.662730051051169, "kl": 0.0478515625, "learning_rate": 4.820950375297756e-07, "loss": 0.0019, "rewards_raw/CoT ICLG": 0.598520040512085, "rewards_raw/accuracy_reward": 0.585035502910614, "rewards_raw/clip_reward_wrapper": 0.3070496618747711, "rewards_raw/format_reward": 5.0, "step": 351 }, { "Total_reward": 3.895589486546669e-07, "advantage_std": 1.482184886932373, "completion_length": 457.8214416503906, "epoch": 0.12154696132596685, "grad_norm": 4.087029461687562, "kl": 0.049072265625, "learning_rate": 4.819941140231535e-07, "loss": 0.002, "rewards_raw/CoT ICLG": 0.2526724934577942, "rewards_raw/accuracy_reward": 0.5693168044090271, "rewards_raw/clip_reward_wrapper": 0.3152865469455719, "rewards_raw/format_reward": 5.0, "step": 352 }, { "Total_reward": -4.204256356388214e-07, "advantage_std": 1.3792798519134521, "completion_length": 450.51788330078125, "epoch": 0.12189226519337017, "grad_norm": 10.871219264471565, "kl": 0.046630859375, "learning_rate": 4.818929175059033e-07, "loss": 0.0019, "rewards_raw/CoT ICLG": 0.11039688438177109, "rewards_raw/accuracy_reward": 0.5599657297134399, "rewards_raw/clip_reward_wrapper": 0.32731908559799194, "rewards_raw/format_reward": 5.0, "step": 353 }, { "Total_reward": 9.685754776000977e-07, "advantage_std": 1.2935912609100342, "completion_length": 455.1250305175781, "epoch": 0.12223756906077347, "grad_norm": 2.7496804272033573, "kl": 0.0498046875, "learning_rate": 4.817914480971128e-07, "loss": 0.002, "rewards_raw/CoT ICLG": 0.24696815013885498, "rewards_raw/accuracy_reward": 0.5443284511566162, "rewards_raw/clip_reward_wrapper": 0.3155564069747925, "rewards_raw/format_reward": 5.0, "step": 354 }, { "Total_reward": 7.876328567135715e-08, "advantage_std": 1.2638931274414062, "completion_length": 443.5357360839844, "epoch": 0.1225828729281768, "grad_norm": 3.2771461597290807, "kl": 0.046875, "learning_rate": 4.816897059161914e-07, "loss": 0.0019, "rewards_raw/CoT ICLG": 0.859632670879364, "rewards_raw/accuracy_reward": 0.5308187007904053, "rewards_raw/clip_reward_wrapper": 0.31302309036254883, "rewards_raw/format_reward": 5.0, "step": 355 }, { "Total_reward": -1.2133803295455436e-07, "advantage_std": 1.4851760864257812, "completion_length": 441.0000305175781, "epoch": 0.12292817679558012, "grad_norm": 4.477910179638184, "kl": 0.0498046875, "learning_rate": 4.815876910828694e-07, "loss": 0.002, "rewards_raw/CoT ICLG": 0.149089053273201, "rewards_raw/accuracy_reward": 0.5767229795455933, "rewards_raw/clip_reward_wrapper": 0.31566840410232544, "rewards_raw/format_reward": 5.0, "step": 356 }, { "Total_reward": -2.618347139105026e-07, "advantage_std": 1.3853037357330322, "completion_length": 439.732177734375, "epoch": 0.12327348066298342, "grad_norm": 3.6116052109562795, "kl": 0.049072265625, "learning_rate": 4.814854037171976e-07, "loss": 0.002, "rewards_raw/CoT ICLG": 0.3045960068702698, "rewards_raw/accuracy_reward": 0.5866701602935791, "rewards_raw/clip_reward_wrapper": 0.31422603130340576, "rewards_raw/format_reward": 5.0, "step": 357 }, { "Total_reward": -2.3416111716301202e-08, "advantage_std": 1.326438546180725, "completion_length": 455.3750305175781, "epoch": 0.12361878453038674, "grad_norm": 3.712434140609507, "kl": 0.052978515625, "learning_rate": 4.813828439395479e-07, "loss": 0.0021, "rewards_raw/CoT ICLG": 0.5484907627105713, "rewards_raw/accuracy_reward": 0.5754675269126892, "rewards_raw/clip_reward_wrapper": 0.31872716546058655, "rewards_raw/format_reward": 5.0, "step": 358 }, { "Total_reward": 4.406486482366745e-07, "advantage_std": 1.5325772762298584, "completion_length": 451.5714416503906, "epoch": 0.12396408839779005, "grad_norm": 6.808507576121466, "kl": 0.047119140625, "learning_rate": 4.812800118706125e-07, "loss": 0.0019, "rewards_raw/CoT ICLG": 0.09975136071443558, "rewards_raw/accuracy_reward": 0.5629042387008667, "rewards_raw/clip_reward_wrapper": 0.32826125621795654, "rewards_raw/format_reward": 5.0, "step": 359 }, { "Total_reward": 2.682209014892578e-07, "advantage_std": 1.5399667024612427, "completion_length": 438.0535888671875, "epoch": 0.12430939226519337, "grad_norm": 3.7669432856876086, "kl": 0.049072265625, "learning_rate": 4.811769076314043e-07, "loss": 0.002, "rewards_raw/CoT ICLG": 0.13135580718517303, "rewards_raw/accuracy_reward": 0.5665808916091919, "rewards_raw/clip_reward_wrapper": 0.3161417841911316, "rewards_raw/format_reward": 5.0, "step": 360 }, { "Total_reward": 3.5230604567004775e-07, "advantage_std": 1.54036545753479, "completion_length": 465.39288330078125, "epoch": 0.12465469613259668, "grad_norm": 3.7273582140163404, "kl": 0.049560546875, "learning_rate": 4.810735313432563e-07, "loss": 0.002, "rewards_raw/CoT ICLG": 0.14452722668647766, "rewards_raw/accuracy_reward": 0.5544950366020203, "rewards_raw/clip_reward_wrapper": 0.3227481245994568, "rewards_raw/format_reward": 5.0, "step": 361 }, { "Total_reward": 3.193106010712654e-07, "advantage_std": 1.5025979280471802, "completion_length": 446.76788330078125, "epoch": 0.125, "grad_norm": 3.9460475260350774, "kl": 0.047119140625, "learning_rate": 4.809698831278217e-07, "loss": 0.0019, "rewards_raw/CoT ICLG": 0.20901533961296082, "rewards_raw/accuracy_reward": 0.5730556845664978, "rewards_raw/clip_reward_wrapper": 0.3175392746925354, "rewards_raw/format_reward": 5.0, "step": 362 }, { "Total_reward": -2.362898499086441e-07, "advantage_std": 1.412079930305481, "completion_length": 448.3214416503906, "epoch": 0.1253453038674033, "grad_norm": 19.316260430781224, "kl": 0.0498046875, "learning_rate": 4.808659631070737e-07, "loss": 0.002, "rewards_raw/CoT ICLG": 0.5937616229057312, "rewards_raw/accuracy_reward": 0.6042376756668091, "rewards_raw/clip_reward_wrapper": 0.31450772285461426, "rewards_raw/format_reward": 5.0, "step": 363 }, { "Total_reward": 7.85504084888089e-07, "advantage_std": 1.290867567062378, "completion_length": 457.232177734375, "epoch": 0.12569060773480664, "grad_norm": 7.031392320521405, "kl": 0.05029296875, "learning_rate": 4.807617714033053e-07, "loss": 0.002, "rewards_raw/CoT ICLG": 0.3960224688053131, "rewards_raw/accuracy_reward": 0.5597541928291321, "rewards_raw/clip_reward_wrapper": 0.29630643129348755, "rewards_raw/format_reward": 5.0, "step": 364 }, { "Total_reward": -5.311200084179291e-07, "advantage_std": 1.4406386613845825, "completion_length": 466.0535888671875, "epoch": 0.12603591160220995, "grad_norm": 4.289843447840519, "kl": 0.0498046875, "learning_rate": 4.806573081391292e-07, "loss": 0.002, "rewards_raw/CoT ICLG": 0.3474736511707306, "rewards_raw/accuracy_reward": 0.5854519009590149, "rewards_raw/clip_reward_wrapper": 0.3088424801826477, "rewards_raw/format_reward": 5.0, "step": 365 }, { "Total_reward": -4.853521318182175e-07, "advantage_std": 1.5077213048934937, "completion_length": 441.9464416503906, "epoch": 0.12638121546961326, "grad_norm": 17.53814885166433, "kl": 0.049560546875, "learning_rate": 4.805525734374779e-07, "loss": 0.002, "rewards_raw/CoT ICLG": 0.23129302263259888, "rewards_raw/accuracy_reward": 0.5536507368087769, "rewards_raw/clip_reward_wrapper": 0.31382879614830017, "rewards_raw/format_reward": 5.0, "step": 366 }, { "Total_reward": -2.341611207157257e-07, "advantage_std": 1.5394542217254639, "completion_length": 436.39288330078125, "epoch": 0.12672651933701656, "grad_norm": 5.199316310285512, "kl": 0.050537109375, "learning_rate": 4.804475674216032e-07, "loss": 0.002, "rewards_raw/CoT ICLG": 0.3003024756908417, "rewards_raw/accuracy_reward": 0.5698545575141907, "rewards_raw/clip_reward_wrapper": 0.31266042590141296, "rewards_raw/format_reward": 5.0, "step": 367 }, { "Total_reward": 6.865178079351608e-07, "advantage_std": 1.3565443754196167, "completion_length": 450.0000305175781, "epoch": 0.1270718232044199, "grad_norm": 4.670325344847007, "kl": 0.05126953125, "learning_rate": 4.803422902150761e-07, "loss": 0.002, "rewards_raw/CoT ICLG": 0.21758373081684113, "rewards_raw/accuracy_reward": 0.5636027455329895, "rewards_raw/clip_reward_wrapper": 0.31154462695121765, "rewards_raw/format_reward": 5.0, "step": 368 }, { "Total_reward": -5.130257250129944e-07, "advantage_std": 1.5643366575241089, "completion_length": 453.9107360839844, "epoch": 0.1274171270718232, "grad_norm": 5.404540099696659, "kl": 0.049560546875, "learning_rate": 4.802367419417868e-07, "loss": 0.002, "rewards_raw/CoT ICLG": 0.2610170841217041, "rewards_raw/accuracy_reward": 0.5538026690483093, "rewards_raw/clip_reward_wrapper": 0.30225440859794617, "rewards_raw/format_reward": 5.0, "step": 369 }, { "Total_reward": -4.683222414314514e-07, "advantage_std": 1.266538143157959, "completion_length": 460.1785888671875, "epoch": 0.1277624309392265, "grad_norm": 7.330027220075778, "kl": 0.05126953125, "learning_rate": 4.801309227259445e-07, "loss": 0.0021, "rewards_raw/CoT ICLG": -0.013755620457231998, "rewards_raw/accuracy_reward": 0.5629403591156006, "rewards_raw/clip_reward_wrapper": 0.3174271285533905, "rewards_raw/format_reward": 5.0, "step": 370 }, { "Total_reward": -3.4059798537100505e-08, "advantage_std": 1.3884882926940918, "completion_length": 440.8750305175781, "epoch": 0.12810773480662985, "grad_norm": 3.0253717750671942, "kl": 0.04833984375, "learning_rate": 4.800248326920772e-07, "loss": 0.0019, "rewards_raw/CoT ICLG": 0.24163490533828735, "rewards_raw/accuracy_reward": 0.5743927359580994, "rewards_raw/clip_reward_wrapper": 0.3071926534175873, "rewards_raw/format_reward": 5.0, "step": 371 }, { "Total_reward": 2.2990363390817947e-07, "advantage_std": 1.685203194618225, "completion_length": 451.2500305175781, "epoch": 0.12845303867403315, "grad_norm": 7.684153238555647, "kl": 0.05419921875, "learning_rate": 4.799184719650316e-07, "loss": 0.0022, "rewards_raw/CoT ICLG": 0.4718620777130127, "rewards_raw/accuracy_reward": 0.5940422415733337, "rewards_raw/clip_reward_wrapper": 0.31659066677093506, "rewards_raw/format_reward": 5.0, "step": 372 }, { "Total_reward": -1.1090721727669006e-06, "advantage_std": 1.5097712278366089, "completion_length": 465.89288330078125, "epoch": 0.12879834254143646, "grad_norm": 5.141379621272639, "kl": 0.04931640625, "learning_rate": 4.79811840669973e-07, "loss": 0.002, "rewards_raw/CoT ICLG": -0.055673688650131226, "rewards_raw/accuracy_reward": 0.563222348690033, "rewards_raw/clip_reward_wrapper": 0.318238228559494, "rewards_raw/format_reward": 5.0, "step": 373 }, { "Total_reward": 1.3027872682869202e-06, "advantage_std": 1.5275964736938477, "completion_length": 442.0535888671875, "epoch": 0.12914364640883977, "grad_norm": 10.510091224359114, "kl": 0.04833984375, "learning_rate": 4.797049389323852e-07, "loss": 0.0019, "rewards_raw/CoT ICLG": 0.8123337030410767, "rewards_raw/accuracy_reward": 0.5549421310424805, "rewards_raw/clip_reward_wrapper": 0.30583852529525757, "rewards_raw/format_reward": 5.0, "step": 374 }, { "Total_reward": -2.1819558071456413e-07, "advantage_std": 1.4649276733398438, "completion_length": 457.4285888671875, "epoch": 0.1294889502762431, "grad_norm": 4.016134051794887, "kl": 0.05078125, "learning_rate": 4.795977668780701e-07, "loss": 0.002, "rewards_raw/CoT ICLG": 0.2881270945072174, "rewards_raw/accuracy_reward": 0.570600688457489, "rewards_raw/clip_reward_wrapper": 0.3212699890136719, "rewards_raw/format_reward": 5.0, "step": 375 }, { "Total_reward": -5.843384087711456e-07, "advantage_std": 1.5797759294509888, "completion_length": 432.1071472167969, "epoch": 0.1298342541436464, "grad_norm": 4.958313646713683, "kl": 0.0498046875, "learning_rate": 4.794903246331477e-07, "loss": 0.002, "rewards_raw/CoT ICLG": 1.6108285188674927, "rewards_raw/accuracy_reward": 0.5520976185798645, "rewards_raw/clip_reward_wrapper": 0.3169943690299988, "rewards_raw/format_reward": 5.0, "step": 376 }, { "Total_reward": -8.089202196970291e-07, "advantage_std": 1.5643202066421509, "completion_length": 455.4464416503906, "epoch": 0.13017955801104972, "grad_norm": 4.374523269556359, "kl": 0.05322265625, "learning_rate": 4.793826123240559e-07, "loss": 0.0021, "rewards_raw/CoT ICLG": 0.41890719532966614, "rewards_raw/accuracy_reward": 0.5622228980064392, "rewards_raw/clip_reward_wrapper": 0.31840723752975464, "rewards_raw/format_reward": 5.0, "step": 377 }, { "Total_reward": 2.905726432800293e-07, "advantage_std": 1.515367865562439, "completion_length": 440.9285888671875, "epoch": 0.13052486187845305, "grad_norm": 4.383385190634479, "kl": 0.057373046875, "learning_rate": 4.792746300775507e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.3625803291797638, "rewards_raw/accuracy_reward": 0.558616042137146, "rewards_raw/clip_reward_wrapper": 0.31724652647972107, "rewards_raw/format_reward": 5.0, "step": 378 }, { "Total_reward": -1.4156103134155273e-07, "advantage_std": 1.3997361660003662, "completion_length": 438.39288330078125, "epoch": 0.13087016574585636, "grad_norm": 5.838141704402139, "kl": 0.0537109375, "learning_rate": 4.791663780207056e-07, "loss": 0.0021, "rewards_raw/CoT ICLG": 0.7209718227386475, "rewards_raw/accuracy_reward": 0.5684518814086914, "rewards_raw/clip_reward_wrapper": 0.3042653501033783, "rewards_raw/format_reward": 5.0, "step": 379 }, { "Total_reward": 2.682209014892578e-07, "advantage_std": 1.5770753622055054, "completion_length": 460.51788330078125, "epoch": 0.13121546961325967, "grad_norm": 4.630200208515466, "kl": 0.0517578125, "learning_rate": 4.790578562809116e-07, "loss": 0.0021, "rewards_raw/CoT ICLG": 0.3002437949180603, "rewards_raw/accuracy_reward": 0.5522132515907288, "rewards_raw/clip_reward_wrapper": 0.3107164800167084, "rewards_raw/format_reward": 5.0, "step": 380 }, { "Total_reward": 1.798783131334858e-07, "advantage_std": 1.3932750225067139, "completion_length": 438.8750305175781, "epoch": 0.13156077348066297, "grad_norm": 3.78575959786405, "kl": 0.051513671875, "learning_rate": 4.789490649858769e-07, "loss": 0.0021, "rewards_raw/CoT ICLG": 0.31676068902015686, "rewards_raw/accuracy_reward": 0.5826098322868347, "rewards_raw/clip_reward_wrapper": 0.31722593307495117, "rewards_raw/format_reward": 5.0, "step": 381 }, { "Total_reward": -3.5656350405588455e-07, "advantage_std": 1.3118300437927246, "completion_length": 438.71429443359375, "epoch": 0.1319060773480663, "grad_norm": 3.762050701728037, "kl": 0.0556640625, "learning_rate": 4.788400042636272e-07, "loss": 0.0022, "rewards_raw/CoT ICLG": 0.3789746165275574, "rewards_raw/accuracy_reward": 0.584889829158783, "rewards_raw/clip_reward_wrapper": 0.3305624723434448, "rewards_raw/format_reward": 5.0, "step": 382 }, { "Total_reward": 3.70400300653273e-07, "advantage_std": 1.5695068836212158, "completion_length": 456.5535888671875, "epoch": 0.13225138121546962, "grad_norm": 5.097080443231711, "kl": 0.052001953125, "learning_rate": 4.787306742425051e-07, "loss": 0.0021, "rewards_raw/CoT ICLG": 0.10683376342058182, "rewards_raw/accuracy_reward": 0.5604339241981506, "rewards_raw/clip_reward_wrapper": 0.32933884859085083, "rewards_raw/format_reward": 5.0, "step": 383 }, { "Total_reward": -3.107956558778824e-07, "advantage_std": 1.5816205739974976, "completion_length": 447.01788330078125, "epoch": 0.13259668508287292, "grad_norm": 4.784557904999775, "kl": 0.052978515625, "learning_rate": 4.786210750511701e-07, "loss": 0.0021, "rewards_raw/CoT ICLG": 0.25704044103622437, "rewards_raw/accuracy_reward": 0.55012047290802, "rewards_raw/clip_reward_wrapper": 0.31541985273361206, "rewards_raw/format_reward": 5.0, "step": 384 }, { "Total_reward": -8.876834840521042e-07, "advantage_std": 1.4598817825317383, "completion_length": 461.3035888671875, "epoch": 0.13294198895027623, "grad_norm": 4.625044417935108, "kl": 0.05322265625, "learning_rate": 4.785112068185986e-07, "loss": 0.0021, "rewards_raw/CoT ICLG": 0.5275108814239502, "rewards_raw/accuracy_reward": 0.5651334524154663, "rewards_raw/clip_reward_wrapper": 0.33271369338035583, "rewards_raw/format_reward": 5.0, "step": 385 }, { "Total_reward": 4.959958346262283e-07, "advantage_std": 1.4687784910202026, "completion_length": 442.0357360839844, "epoch": 0.13328729281767956, "grad_norm": 3.8129726070731547, "kl": 0.056884765625, "learning_rate": 4.784010696740835e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.4061514139175415, "rewards_raw/accuracy_reward": 0.5547969341278076, "rewards_raw/clip_reward_wrapper": 0.3216894268989563, "rewards_raw/format_reward": 5.0, "step": 386 }, { "Total_reward": -3.916876778475853e-07, "advantage_std": 1.2755340337753296, "completion_length": 438.4107360839844, "epoch": 0.13363259668508287, "grad_norm": 4.344067563843441, "kl": 0.055419921875, "learning_rate": 4.782906637472341e-07, "loss": 0.0022, "rewards_raw/CoT ICLG": 0.6638297438621521, "rewards_raw/accuracy_reward": 0.547423243522644, "rewards_raw/clip_reward_wrapper": 0.31933221220970154, "rewards_raw/format_reward": 5.0, "step": 387 }, { "Total_reward": 4.512923226229759e-07, "advantage_std": 1.5923675298690796, "completion_length": 448.107177734375, "epoch": 0.13397790055248618, "grad_norm": 5.8231038673807145, "kl": 0.053955078125, "learning_rate": 4.781799891679762e-07, "loss": 0.0022, "rewards_raw/CoT ICLG": 0.035950567573308945, "rewards_raw/accuracy_reward": 0.5462799072265625, "rewards_raw/clip_reward_wrapper": 0.3044072091579437, "rewards_raw/format_reward": 5.0, "step": 388 }, { "Total_reward": 5.364418029785156e-07, "advantage_std": 1.3658510446548462, "completion_length": 437.4285888671875, "epoch": 0.1343232044198895, "grad_norm": 6.084239364288135, "kl": 0.056396484375, "learning_rate": 4.780690460665516e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.4687812328338623, "rewards_raw/accuracy_reward": 0.5616902709007263, "rewards_raw/clip_reward_wrapper": 0.3165057897567749, "rewards_raw/format_reward": 5.0, "step": 389 }, { "Total_reward": 1.44754142183956e-06, "advantage_std": 1.6194337606430054, "completion_length": 446.1250305175781, "epoch": 0.13466850828729282, "grad_norm": 3.850213212041215, "kl": 0.056884765625, "learning_rate": 4.779578345735179e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.7227287888526917, "rewards_raw/accuracy_reward": 0.5508358478546143, "rewards_raw/clip_reward_wrapper": 0.3261364698410034, "rewards_raw/format_reward": 5.0, "step": 390 }, { "Total_reward": -3.576278970740532e-07, "advantage_std": 1.4868683815002441, "completion_length": 450.982177734375, "epoch": 0.13501381215469613, "grad_norm": 10.51547728213159, "kl": 0.0556640625, "learning_rate": 4.77846354819749e-07, "loss": 0.0022, "rewards_raw/CoT ICLG": -0.011315916664898396, "rewards_raw/accuracy_reward": 0.5627358555793762, "rewards_raw/clip_reward_wrapper": 0.32263490557670593, "rewards_raw/format_reward": 4.9464287757873535, "step": 391 }, { "Total_reward": 1.585909359391735e-07, "advantage_std": 1.5577092170715332, "completion_length": 441.9107360839844, "epoch": 0.13535911602209943, "grad_norm": 5.425147512531063, "kl": 0.059814453125, "learning_rate": 4.777346069364342e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.03963257744908333, "rewards_raw/accuracy_reward": 0.5630874037742615, "rewards_raw/clip_reward_wrapper": 0.32358723878860474, "rewards_raw/format_reward": 5.0, "step": 392 }, { "Total_reward": 5.619866669803741e-07, "advantage_std": 1.4708112478256226, "completion_length": 461.3214416503906, "epoch": 0.13570441988950277, "grad_norm": 5.846286871805291, "kl": 0.054443359375, "learning_rate": 4.776225910550784e-07, "loss": 0.0022, "rewards_raw/CoT ICLG": 0.25690457224845886, "rewards_raw/accuracy_reward": 0.5702058672904968, "rewards_raw/clip_reward_wrapper": 0.31930011510849, "rewards_raw/format_reward": 5.0, "step": 393 }, { "Total_reward": -3.4059798537100505e-08, "advantage_std": 1.481074333190918, "completion_length": 433.2500305175781, "epoch": 0.13604972375690608, "grad_norm": 6.119054712676368, "kl": 0.0556640625, "learning_rate": 4.775103073075018e-07, "loss": 0.0022, "rewards_raw/CoT ICLG": 0.39408910274505615, "rewards_raw/accuracy_reward": 0.5593748092651367, "rewards_raw/clip_reward_wrapper": 0.30965325236320496, "rewards_raw/format_reward": 5.0, "step": 394 }, { "Total_reward": 6.599086077585525e-08, "advantage_std": 1.5334464311599731, "completion_length": 452.9107360839844, "epoch": 0.13639502762430938, "grad_norm": 5.956192281815152, "kl": 0.057861328125, "learning_rate": 4.7739775582584e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.23453658819198608, "rewards_raw/accuracy_reward": 0.562127947807312, "rewards_raw/clip_reward_wrapper": 0.30737823247909546, "rewards_raw/format_reward": 5.0, "step": 395 }, { "Total_reward": 4.5342108023760375e-07, "advantage_std": 1.3719441890716553, "completion_length": 437.4285888671875, "epoch": 0.13674033149171272, "grad_norm": 5.352676585523958, "kl": 0.0595703125, "learning_rate": 4.772849367425433e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.6191343069076538, "rewards_raw/accuracy_reward": 0.5609598159790039, "rewards_raw/clip_reward_wrapper": 0.3206453323364258, "rewards_raw/format_reward": 5.0, "step": 396 }, { "Total_reward": -2.8099333349018707e-07, "advantage_std": 1.5549676418304443, "completion_length": 448.83929443359375, "epoch": 0.13708563535911603, "grad_norm": 4.2050608800127804, "kl": 0.0576171875, "learning_rate": 4.771718501903776e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.506341278553009, "rewards_raw/accuracy_reward": 0.562127411365509, "rewards_raw/clip_reward_wrapper": 0.31923747062683105, "rewards_raw/format_reward": 5.0, "step": 397 }, { "Total_reward": 1.7029899268550253e-08, "advantage_std": 1.4607689380645752, "completion_length": 438.5000305175781, "epoch": 0.13743093922651933, "grad_norm": 9.211282468288166, "kl": 0.058837890625, "learning_rate": 4.770584963024227e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.6171085238456726, "rewards_raw/accuracy_reward": 0.5393346548080444, "rewards_raw/clip_reward_wrapper": 0.3149954080581665, "rewards_raw/format_reward": 5.0, "step": 398 }, { "Total_reward": 2.07551892117408e-07, "advantage_std": 1.5847374200820923, "completion_length": 438.2857360839844, "epoch": 0.13777624309392264, "grad_norm": 7.1854975750684495, "kl": 0.060302734375, "learning_rate": 4.769448752120737e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.405801922082901, "rewards_raw/accuracy_reward": 0.5805927515029907, "rewards_raw/clip_reward_wrapper": 0.3078697919845581, "rewards_raw/format_reward": 5.0, "step": 399 }, { "Total_reward": 5.736947059631348e-07, "advantage_std": 1.3782562017440796, "completion_length": 468.1607360839844, "epoch": 0.13812154696132597, "grad_norm": 10.631400628469917, "kl": 0.056884765625, "learning_rate": 4.7683098705303993e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.15153436362743378, "rewards_raw/accuracy_reward": 0.5614911317825317, "rewards_raw/clip_reward_wrapper": 0.30971166491508484, "rewards_raw/format_reward": 5.0, "step": 400 }, { "Total_reward": 4.981245638191467e-07, "advantage_std": 1.474144697189331, "completion_length": 444.982177734375, "epoch": 0.13846685082872928, "grad_norm": 7.3351127001742835, "kl": 0.055419921875, "learning_rate": 4.767168319593447e-07, "loss": 0.0022, "rewards_raw/CoT ICLG": 0.134607195854187, "rewards_raw/accuracy_reward": 0.5658748149871826, "rewards_raw/clip_reward_wrapper": 0.31970566511154175, "rewards_raw/format_reward": 5.0, "step": 401 }, { "Total_reward": -1.553978279389412e-07, "advantage_std": 1.7872536182403564, "completion_length": 452.3035888671875, "epoch": 0.1388121546961326, "grad_norm": 4.264829497889408, "kl": 0.056884765625, "learning_rate": 4.7660241006532585e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.321233332157135, "rewards_raw/accuracy_reward": 0.5704050660133362, "rewards_raw/clip_reward_wrapper": 0.30780351161956787, "rewards_raw/format_reward": 5.0, "step": 402 }, { "Total_reward": -1.4475413934178505e-07, "advantage_std": 1.6497883796691895, "completion_length": 445.7500305175781, "epoch": 0.13915745856353592, "grad_norm": 5.0633215426165, "kl": 0.057861328125, "learning_rate": 4.7648772150563504e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.27337154746055603, "rewards_raw/accuracy_reward": 0.5789650082588196, "rewards_raw/clip_reward_wrapper": 0.31503430008888245, "rewards_raw/format_reward": 5.0, "step": 403 }, { "Total_reward": -3.8210836805774306e-07, "advantage_std": 1.5184171199798584, "completion_length": 437.1964416503906, "epoch": 0.13950276243093923, "grad_norm": 3.3742536121333364, "kl": 0.05908203125, "learning_rate": 4.7637276641523773e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.3370208144187927, "rewards_raw/accuracy_reward": 0.580776572227478, "rewards_raw/clip_reward_wrapper": 0.30345264077186584, "rewards_raw/format_reward": 5.0, "step": 404 }, { "Total_reward": -3.7997963886482466e-07, "advantage_std": 1.4141324758529663, "completion_length": 447.5357360839844, "epoch": 0.13984806629834254, "grad_norm": 4.432374071711113, "kl": 0.05712890625, "learning_rate": 4.7625754492941303e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.3701781928539276, "rewards_raw/accuracy_reward": 0.5508327484130859, "rewards_raw/clip_reward_wrapper": 0.3161197900772095, "rewards_raw/format_reward": 5.0, "step": 405 }, { "Total_reward": -1.6178404393940582e-07, "advantage_std": 1.554304599761963, "completion_length": 461.1607360839844, "epoch": 0.14019337016574585, "grad_norm": 3.954069640183794, "kl": 0.056884765625, "learning_rate": 4.7614205718375366e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.290854811668396, "rewards_raw/accuracy_reward": 0.5605118274688721, "rewards_raw/clip_reward_wrapper": 0.31828731298446655, "rewards_raw/format_reward": 5.0, "step": 406 }, { "Total_reward": 4.140394196383568e-07, "advantage_std": 1.4887316226959229, "completion_length": 431.8035888671875, "epoch": 0.14053867403314918, "grad_norm": 7.084202270093796, "kl": 0.061767578125, "learning_rate": 4.760263033141654e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 1.195564866065979, "rewards_raw/accuracy_reward": 0.5693578124046326, "rewards_raw/clip_reward_wrapper": 0.3174130916595459, "rewards_raw/format_reward": 4.9375, "step": 407 }, { "Total_reward": 1.4688288274555816e-07, "advantage_std": 1.5374854803085327, "completion_length": 449.982177734375, "epoch": 0.1408839779005525, "grad_norm": 3.7391379774869526, "kl": 0.05810546875, "learning_rate": 4.7591028345686755e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.38366618752479553, "rewards_raw/accuracy_reward": 0.5753451585769653, "rewards_raw/clip_reward_wrapper": 0.31572818756103516, "rewards_raw/format_reward": 5.0, "step": 408 }, { "Total_reward": -3.8104400346128386e-07, "advantage_std": 1.4322490692138672, "completion_length": 443.9464416503906, "epoch": 0.1412292817679558, "grad_norm": 4.161682572696815, "kl": 0.06201171875, "learning_rate": 4.757939977483922e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.3429080545902252, "rewards_raw/accuracy_reward": 0.5666734576225281, "rewards_raw/clip_reward_wrapper": 0.32130399346351624, "rewards_raw/format_reward": 5.0, "step": 409 }, { "Total_reward": -1.9903694692402496e-07, "advantage_std": 1.3562194108963013, "completion_length": 417.76788330078125, "epoch": 0.14157458563535913, "grad_norm": 22.097402113195855, "kl": 0.0576171875, "learning_rate": 4.7567744632558425e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.12817198038101196, "rewards_raw/accuracy_reward": 0.5461201667785645, "rewards_raw/clip_reward_wrapper": 0.3185567259788513, "rewards_raw/format_reward": 5.0, "step": 410 }, { "Total_reward": 5.960464477539062e-07, "advantage_std": 1.6557443141937256, "completion_length": 439.89288330078125, "epoch": 0.14191988950276244, "grad_norm": 5.034134658722893, "kl": 0.0615234375, "learning_rate": 4.7556062932560154e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.2667384147644043, "rewards_raw/accuracy_reward": 0.5614387392997742, "rewards_raw/clip_reward_wrapper": 0.32628312706947327, "rewards_raw/format_reward": 5.0, "step": 411 }, { "Total_reward": -8.30207568469632e-07, "advantage_std": 1.5145444869995117, "completion_length": 429.01788330078125, "epoch": 0.14226519337016574, "grad_norm": 7.281686615564344, "kl": 0.06201171875, "learning_rate": 4.754435468859142e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.6230738162994385, "rewards_raw/accuracy_reward": 0.5693857669830322, "rewards_raw/clip_reward_wrapper": 0.3105386197566986, "rewards_raw/format_reward": 5.0, "step": 412 }, { "Total_reward": -7.173845233410248e-07, "advantage_std": 1.9286891222000122, "completion_length": 450.76788330078125, "epoch": 0.14261049723756905, "grad_norm": 5.472237594712195, "kl": 0.060546875, "learning_rate": 4.753261991443048e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.2326909452676773, "rewards_raw/accuracy_reward": 0.5619385838508606, "rewards_raw/clip_reward_wrapper": 0.3021019697189331, "rewards_raw/format_reward": 4.928571701049805, "step": 413 }, { "Total_reward": 8.259501100837952e-07, "advantage_std": 1.4230315685272217, "completion_length": 437.2500305175781, "epoch": 0.14295580110497239, "grad_norm": 4.478184490234214, "kl": 0.05859375, "learning_rate": 4.752085862388683e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.40753456950187683, "rewards_raw/accuracy_reward": 0.573880672454834, "rewards_raw/clip_reward_wrapper": 0.3105466365814209, "rewards_raw/format_reward": 5.0, "step": 414 }, { "Total_reward": -1.3623919414840202e-07, "advantage_std": 1.4347363710403442, "completion_length": 433.8750305175781, "epoch": 0.1433011049723757, "grad_norm": 4.127791874378704, "kl": 0.056640625, "learning_rate": 4.750907083080114e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.4479934275150299, "rewards_raw/accuracy_reward": 0.562504768371582, "rewards_raw/clip_reward_wrapper": 0.31931987404823303, "rewards_raw/format_reward": 5.0, "step": 415 }, { "Total_reward": -2.2351743211856956e-07, "advantage_std": 1.368327021598816, "completion_length": 438.8035888671875, "epoch": 0.143646408839779, "grad_norm": 5.6170524024638695, "kl": 0.0634765625, "learning_rate": 4.7497256549045284e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.3139260411262512, "rewards_raw/accuracy_reward": 0.5497061610221863, "rewards_raw/clip_reward_wrapper": 0.3106935918331146, "rewards_raw/format_reward": 5.0, "step": 416 }, { "Total_reward": -2.0223005492425727e-07, "advantage_std": 1.3429014682769775, "completion_length": 430.5714416503906, "epoch": 0.1439917127071823, "grad_norm": 5.0889796045646865, "kl": 0.0634765625, "learning_rate": 4.7485415792522307e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.6185749173164368, "rewards_raw/accuracy_reward": 0.5775672197341919, "rewards_raw/clip_reward_wrapper": 0.3305520713329315, "rewards_raw/format_reward": 5.0, "step": 417 }, { "Total_reward": -4.0339574525205535e-07, "advantage_std": 1.3150970935821533, "completion_length": 435.26788330078125, "epoch": 0.14433701657458564, "grad_norm": 4.045981326428929, "kl": 0.06298828125, "learning_rate": 4.747354857516641e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.4679158329963684, "rewards_raw/accuracy_reward": 0.5689992308616638, "rewards_raw/clip_reward_wrapper": 0.33069607615470886, "rewards_raw/format_reward": 5.0, "step": 418 }, { "Total_reward": 1.0026353720604675e-06, "advantage_std": 1.2138147354125977, "completion_length": 445.482177734375, "epoch": 0.14468232044198895, "grad_norm": 7.133077528195124, "kl": 0.06005859375, "learning_rate": 4.746165491094294e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.4778520166873932, "rewards_raw/accuracy_reward": 0.5437167882919312, "rewards_raw/clip_reward_wrapper": 0.31490790843963623, "rewards_raw/format_reward": 5.0, "step": 419 }, { "Total_reward": 2.937657654911163e-07, "advantage_std": 1.5115724802017212, "completion_length": 443.6964416503906, "epoch": 0.14502762430939226, "grad_norm": 3.2277697761094197, "kl": 0.06201171875, "learning_rate": 4.744973481384834e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.23991112411022186, "rewards_raw/accuracy_reward": 0.5691211223602295, "rewards_raw/clip_reward_wrapper": 0.32024824619293213, "rewards_raw/format_reward": 5.0, "step": 420 }, { "Total_reward": -1.27724248955019e-08, "advantage_std": 1.5410692691802979, "completion_length": 448.76788330078125, "epoch": 0.1453729281767956, "grad_norm": 4.044122948565563, "kl": 0.061767578125, "learning_rate": 4.74377882979102e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.34530696272850037, "rewards_raw/accuracy_reward": 0.5698425769805908, "rewards_raw/clip_reward_wrapper": 0.3178122937679291, "rewards_raw/format_reward": 5.0, "step": 421 }, { "Total_reward": 1.1069434435739822e-07, "advantage_std": 1.4920568466186523, "completion_length": 439.46429443359375, "epoch": 0.1457182320441989, "grad_norm": 4.539293999686086, "kl": 0.06494140625, "learning_rate": 4.742581537718716e-07, "loss": 0.0026, "rewards_raw/CoT ICLG": 0.4757397770881653, "rewards_raw/accuracy_reward": 0.5829147100448608, "rewards_raw/clip_reward_wrapper": 0.3197004795074463, "rewards_raw/format_reward": 5.0, "step": 422 }, { "Total_reward": 8.089202196970291e-07, "advantage_std": 1.528633713722229, "completion_length": 436.4107360839844, "epoch": 0.1460635359116022, "grad_norm": 5.970550337662029, "kl": 0.06005859375, "learning_rate": 4.741381606576896e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.41829192638397217, "rewards_raw/accuracy_reward": 0.5409230589866638, "rewards_raw/clip_reward_wrapper": 0.3264676034450531, "rewards_raw/format_reward": 5.0, "step": 423 }, { "Total_reward": 1.4688288274555816e-07, "advantage_std": 1.5537703037261963, "completion_length": 451.4464416503906, "epoch": 0.1464088397790055, "grad_norm": 4.184164104113364, "kl": 0.05859375, "learning_rate": 4.7401790377776383e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.5113832950592041, "rewards_raw/accuracy_reward": 0.5639119148254395, "rewards_raw/clip_reward_wrapper": 0.33821332454681396, "rewards_raw/format_reward": 5.0, "step": 424 }, { "Total_reward": 1.6923462453632965e-07, "advantage_std": 1.4175664186477661, "completion_length": 453.0535888671875, "epoch": 0.14675414364640885, "grad_norm": 3.736104193131642, "kl": 0.0625, "learning_rate": 4.738973832736126e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.4402121305465698, "rewards_raw/accuracy_reward": 0.5525981783866882, "rewards_raw/clip_reward_wrapper": 0.3179781138896942, "rewards_raw/format_reward": 5.0, "step": 425 }, { "Total_reward": 1.2559550555124588e-07, "advantage_std": 1.6731315851211548, "completion_length": 444.357177734375, "epoch": 0.14709944751381215, "grad_norm": 9.448205357133721, "kl": 0.05712890625, "learning_rate": 4.737765992870644e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.08029665052890778, "rewards_raw/accuracy_reward": 0.5480138659477234, "rewards_raw/clip_reward_wrapper": 0.32439911365509033, "rewards_raw/format_reward": 5.0, "step": 426 }, { "Total_reward": -5.587935802964239e-08, "advantage_std": 1.3132669925689697, "completion_length": 452.0714416503906, "epoch": 0.14744475138121546, "grad_norm": 22.28581129473355, "kl": 0.055908203125, "learning_rate": 4.7365555196025783e-07, "loss": 0.0022, "rewards_raw/CoT ICLG": 0.5080060958862305, "rewards_raw/accuracy_reward": 0.5658339858055115, "rewards_raw/clip_reward_wrapper": 0.3375600278377533, "rewards_raw/format_reward": 5.0, "step": 427 }, { "Total_reward": -2.937657654911163e-07, "advantage_std": 1.4468064308166504, "completion_length": 452.76788330078125, "epoch": 0.1477900552486188, "grad_norm": 5.326486642833565, "kl": 0.057373046875, "learning_rate": 4.7353424143564123e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.34983426332473755, "rewards_raw/accuracy_reward": 0.5659460425376892, "rewards_raw/clip_reward_wrapper": 0.33144113421440125, "rewards_raw/format_reward": 5.0, "step": 428 }, { "Total_reward": 4.874808610111359e-07, "advantage_std": 1.378032922744751, "completion_length": 437.5357360839844, "epoch": 0.1481353591160221, "grad_norm": 3.901643310223378, "kl": 0.058349609375, "learning_rate": 4.73412667855973e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.43477320671081543, "rewards_raw/accuracy_reward": 0.560208797454834, "rewards_raw/clip_reward_wrapper": 0.3131537139415741, "rewards_raw/format_reward": 5.0, "step": 429 }, { "Total_reward": 4.225543648317398e-07, "advantage_std": 1.4548715353012085, "completion_length": 449.3750305175781, "epoch": 0.1484806629834254, "grad_norm": 8.606862257722844, "kl": 0.06103515625, "learning_rate": 4.732908313643208e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.8800152540206909, "rewards_raw/accuracy_reward": 0.5412046313285828, "rewards_raw/clip_reward_wrapper": 0.32508182525634766, "rewards_raw/format_reward": 5.0, "step": 430 }, { "Total_reward": -5.534717502087005e-07, "advantage_std": 1.6232714653015137, "completion_length": 447.0714416503906, "epoch": 0.14882596685082872, "grad_norm": 4.453432165169153, "kl": 0.060302734375, "learning_rate": 4.731687321040617e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.3363937437534332, "rewards_raw/accuracy_reward": 0.5664727091789246, "rewards_raw/clip_reward_wrapper": 0.32397863268852234, "rewards_raw/format_reward": 5.0, "step": 431 }, { "Total_reward": 3.469841942660423e-07, "advantage_std": 1.4672495126724243, "completion_length": 452.5000305175781, "epoch": 0.14917127071823205, "grad_norm": 4.569429349609941, "kl": 0.062255859375, "learning_rate": 4.730463702188823e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.13392099738121033, "rewards_raw/accuracy_reward": 0.5552876591682434, "rewards_raw/clip_reward_wrapper": 0.3418704867362976, "rewards_raw/format_reward": 5.0, "step": 432 }, { "Total_reward": 6.428787173717865e-07, "advantage_std": 1.4049975872039795, "completion_length": 443.5714416503906, "epoch": 0.14951657458563536, "grad_norm": 3.3665206624346284, "kl": 0.06396484375, "learning_rate": 4.7292374585277785e-07, "loss": 0.0026, "rewards_raw/CoT ICLG": 0.026544494554400444, "rewards_raw/accuracy_reward": 0.564974308013916, "rewards_raw/clip_reward_wrapper": 0.34172290563583374, "rewards_raw/format_reward": 5.0, "step": 433 }, { "Total_reward": -7.237706967089252e-08, "advantage_std": 1.6076710224151611, "completion_length": 445.3214416503906, "epoch": 0.14986187845303867, "grad_norm": 11.386192330238003, "kl": 0.0625, "learning_rate": 4.728008591500528e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.28500160574913025, "rewards_raw/accuracy_reward": 0.5581965446472168, "rewards_raw/clip_reward_wrapper": 0.32988810539245605, "rewards_raw/format_reward": 5.0, "step": 434 }, { "Total_reward": -4.0446010984851455e-08, "advantage_std": 1.7158688306808472, "completion_length": 433.6785888671875, "epoch": 0.150207182320442, "grad_norm": 4.83766597847915, "kl": 0.0576171875, "learning_rate": 4.726777102553201e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.10167525708675385, "rewards_raw/accuracy_reward": 0.5664137005805969, "rewards_raw/clip_reward_wrapper": 0.3112179934978485, "rewards_raw/format_reward": 5.0, "step": 435 }, { "Total_reward": 1.3836793755217514e-07, "advantage_std": 1.327372670173645, "completion_length": 454.4285888671875, "epoch": 0.1505524861878453, "grad_norm": 5.694187604187382, "kl": 0.057373046875, "learning_rate": 4.7255429931350146e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.35978829860687256, "rewards_raw/accuracy_reward": 0.5457972288131714, "rewards_raw/clip_reward_wrapper": 0.3226257860660553, "rewards_raw/format_reward": 5.0, "step": 436 }, { "Total_reward": 8.685248644724197e-07, "advantage_std": 1.5789051055908203, "completion_length": 456.1785888671875, "epoch": 0.15089779005524862, "grad_norm": 4.249227266221657, "kl": 0.05908203125, "learning_rate": 4.724306264698268e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.18078280985355377, "rewards_raw/accuracy_reward": 0.5707497596740723, "rewards_raw/clip_reward_wrapper": 0.3128490447998047, "rewards_raw/format_reward": 5.0, "step": 437 }, { "Total_reward": 1.3560057823269744e-06, "advantage_std": 1.691962480545044, "completion_length": 450.0357360839844, "epoch": 0.15124309392265192, "grad_norm": 15.081221914015233, "kl": 0.06201171875, "learning_rate": 4.7230669186983416e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.3203507661819458, "rewards_raw/accuracy_reward": 0.5625797510147095, "rewards_raw/clip_reward_wrapper": 0.32101982831954956, "rewards_raw/format_reward": 5.0, "step": 438 }, { "Total_reward": -5.364418029785156e-07, "advantage_std": 1.668884515762329, "completion_length": 455.1250305175781, "epoch": 0.15158839779005526, "grad_norm": 20.25381743300785, "kl": 0.060302734375, "learning_rate": 4.7218249565937003e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.17774637043476105, "rewards_raw/accuracy_reward": 0.5656126737594604, "rewards_raw/clip_reward_wrapper": 0.32764169573783875, "rewards_raw/format_reward": 5.0, "step": 439 }, { "Total_reward": 5.832740725963959e-07, "advantage_std": 1.5562750101089478, "completion_length": 452.33929443359375, "epoch": 0.15193370165745856, "grad_norm": 4.47398344084163, "kl": 0.05908203125, "learning_rate": 4.720580379845883e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.13449452817440033, "rewards_raw/accuracy_reward": 0.5552942752838135, "rewards_raw/clip_reward_wrapper": 0.32201242446899414, "rewards_raw/format_reward": 5.0, "step": 440 }, { "Total_reward": 2.043587983280304e-07, "advantage_std": 1.4206970930099487, "completion_length": 453.232177734375, "epoch": 0.15227900552486187, "grad_norm": 6.293569505503512, "kl": 0.05908203125, "learning_rate": 4.719333189919508e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.19011129438877106, "rewards_raw/accuracy_reward": 0.5633864402770996, "rewards_raw/clip_reward_wrapper": 0.3137384355068207, "rewards_raw/format_reward": 5.0, "step": 441 }, { "Total_reward": 5.10896995820076e-08, "advantage_std": 1.7320771217346191, "completion_length": 441.1785888671875, "epoch": 0.1526243093922652, "grad_norm": 3.502024594557158, "kl": 0.061279296875, "learning_rate": 4.718083388282267e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.28087353706359863, "rewards_raw/accuracy_reward": 0.5768600702285767, "rewards_raw/clip_reward_wrapper": 0.3336651027202606, "rewards_raw/format_reward": 5.0, "step": 442 }, { "Total_reward": 2.980232238769531e-07, "advantage_std": 1.4253689050674438, "completion_length": 456.1964416503906, "epoch": 0.1529696132596685, "grad_norm": 6.575089575304649, "kl": 0.060546875, "learning_rate": 4.7168309764049276e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.7049564719200134, "rewards_raw/accuracy_reward": 0.5614770650863647, "rewards_raw/clip_reward_wrapper": 0.3464341461658478, "rewards_raw/format_reward": 5.0, "step": 443 }, { "Total_reward": -6.386212021425308e-07, "advantage_std": 1.4255484342575073, "completion_length": 443.6964416503906, "epoch": 0.15331491712707182, "grad_norm": 3.904609779286531, "kl": 0.062255859375, "learning_rate": 4.7155759557613273e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.2904828190803528, "rewards_raw/accuracy_reward": 0.5533674359321594, "rewards_raw/clip_reward_wrapper": 0.31497448682785034, "rewards_raw/format_reward": 5.0, "step": 444 }, { "Total_reward": -1.3943230214863434e-07, "advantage_std": 1.5027199983596802, "completion_length": 457.8035888671875, "epoch": 0.15366022099447513, "grad_norm": 6.197703849446052, "kl": 0.058837890625, "learning_rate": 4.714318327828375e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.5623321533203125, "rewards_raw/accuracy_reward": 0.5581948161125183, "rewards_raw/clip_reward_wrapper": 0.31340062618255615, "rewards_raw/format_reward": 5.0, "step": 445 }, { "Total_reward": -7.237706967089252e-08, "advantage_std": 1.321351170539856, "completion_length": 458.2857360839844, "epoch": 0.15400552486187846, "grad_norm": 4.243253363773003, "kl": 0.05859375, "learning_rate": 4.713058094086046e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.13702984154224396, "rewards_raw/accuracy_reward": 0.5613176226615906, "rewards_raw/clip_reward_wrapper": 0.3388264775276184, "rewards_raw/format_reward": 5.0, "step": 446 }, { "Total_reward": 8.131776780828659e-07, "advantage_std": 1.5530592203140259, "completion_length": 437.3750305175781, "epoch": 0.15435082872928177, "grad_norm": 5.054718214819947, "kl": 0.0595703125, "learning_rate": 4.7117952560173833e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.39048731327056885, "rewards_raw/accuracy_reward": 0.5836292505264282, "rewards_raw/clip_reward_wrapper": 0.309700071811676, "rewards_raw/format_reward": 5.0, "step": 447 }, { "Total_reward": 6.971615107431717e-07, "advantage_std": 1.5514330863952637, "completion_length": 431.2500305175781, "epoch": 0.15469613259668508, "grad_norm": 7.143439650888979, "kl": 0.060791015625, "learning_rate": 4.710529815108495e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.40976274013519287, "rewards_raw/accuracy_reward": 0.5390679836273193, "rewards_raw/clip_reward_wrapper": 0.31200456619262695, "rewards_raw/format_reward": 5.0, "step": 448 }, { "Total_reward": 3.3208303307219467e-07, "advantage_std": 1.41318941116333, "completion_length": 441.83929443359375, "epoch": 0.1550414364640884, "grad_norm": 9.348845649548831, "kl": 0.0634765625, "learning_rate": 4.7092617728485506e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.6665231585502625, "rewards_raw/accuracy_reward": 0.5637893676757812, "rewards_raw/clip_reward_wrapper": 0.32462114095687866, "rewards_raw/format_reward": 5.0, "step": 449 }, { "Total_reward": -1.4368977474532585e-07, "advantage_std": 1.4849655628204346, "completion_length": 441.58929443359375, "epoch": 0.15538674033149172, "grad_norm": 4.7084164895475, "kl": 0.061279296875, "learning_rate": 4.7079911307297837e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.24140799045562744, "rewards_raw/accuracy_reward": 0.5508139729499817, "rewards_raw/clip_reward_wrapper": 0.3235432803630829, "rewards_raw/format_reward": 5.0, "step": 450 }, { "Total_reward": -3.6188535545989e-07, "advantage_std": 1.7022550106048584, "completion_length": 441.1607360839844, "epoch": 0.15573204419889503, "grad_norm": 24.35973436205371, "kl": 0.06201171875, "learning_rate": 4.7067178902474845e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.2852487564086914, "rewards_raw/accuracy_reward": 0.5722317695617676, "rewards_raw/clip_reward_wrapper": 0.344575434923172, "rewards_raw/format_reward": 5.0, "step": 451 }, { "Total_reward": -2.064875275209488e-07, "advantage_std": 1.5235915184020996, "completion_length": 450.6785888671875, "epoch": 0.15607734806629833, "grad_norm": 6.405671627829519, "kl": 0.06103515625, "learning_rate": 4.7054420529000024e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.30427151918411255, "rewards_raw/accuracy_reward": 0.5534210205078125, "rewards_raw/clip_reward_wrapper": 0.32526886463165283, "rewards_raw/format_reward": 5.0, "step": 452 }, { "Total_reward": -1.830714211337181e-07, "advantage_std": 1.4701168537139893, "completion_length": 448.3750305175781, "epoch": 0.15642265193370167, "grad_norm": 3.5301732153885403, "kl": 0.061279296875, "learning_rate": 4.7041636201887426e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.14621911942958832, "rewards_raw/accuracy_reward": 0.5597549676895142, "rewards_raw/clip_reward_wrapper": 0.31547337770462036, "rewards_raw/format_reward": 5.0, "step": 453 }, { "Total_reward": 8.131776780828659e-07, "advantage_std": 1.7870911359786987, "completion_length": 451.8035888671875, "epoch": 0.15676795580110497, "grad_norm": 4.771293858180879, "kl": 0.057373046875, "learning_rate": 4.7028825936181637e-07, "loss": 0.0023, "rewards_raw/CoT ICLG": 0.45401322841644287, "rewards_raw/accuracy_reward": 0.5576215982437134, "rewards_raw/clip_reward_wrapper": 0.31555289030075073, "rewards_raw/format_reward": 5.0, "step": 454 }, { "Total_reward": 2.63963443103421e-07, "advantage_std": 1.293202519416809, "completion_length": 447.5535888671875, "epoch": 0.15711325966850828, "grad_norm": 3.418618173487864, "kl": 0.061767578125, "learning_rate": 4.701598974695779e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.2952612042427063, "rewards_raw/accuracy_reward": 0.55046546459198, "rewards_raw/clip_reward_wrapper": 0.32491132616996765, "rewards_raw/format_reward": 5.0, "step": 455 }, { "Total_reward": 3.8530146184712066e-07, "advantage_std": 1.2871301174163818, "completion_length": 425.5357360839844, "epoch": 0.1574585635359116, "grad_norm": 3.33455043590802, "kl": 0.064453125, "learning_rate": 4.7003127649321506e-07, "loss": 0.0026, "rewards_raw/CoT ICLG": 0.3430202603340149, "rewards_raw/accuracy_reward": 0.5855758190155029, "rewards_raw/clip_reward_wrapper": 0.33398672938346863, "rewards_raw/format_reward": 5.0, "step": 456 }, { "Total_reward": -5.3218435880353354e-08, "advantage_std": 1.670328974723816, "completion_length": 454.982177734375, "epoch": 0.15780386740331492, "grad_norm": 5.516586962440457, "kl": 0.061767578125, "learning_rate": 4.6990239658408883e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.16122207045555115, "rewards_raw/accuracy_reward": 0.5747489929199219, "rewards_raw/clip_reward_wrapper": 0.3240935504436493, "rewards_raw/format_reward": 5.0, "step": 457 }, { "Total_reward": 3.672072068638954e-08, "advantage_std": 1.4694757461547852, "completion_length": 457.08929443359375, "epoch": 0.15814917127071823, "grad_norm": 3.676792331480519, "kl": 0.06103515625, "learning_rate": 4.697732578938653e-07, "loss": 0.0024, "rewards_raw/CoT ICLG": 0.16191695630550385, "rewards_raw/accuracy_reward": 0.5642678737640381, "rewards_raw/clip_reward_wrapper": 0.3165898025035858, "rewards_raw/format_reward": 5.0, "step": 458 }, { "Total_reward": -3.2143935868589324e-07, "advantage_std": 1.4542741775512695, "completion_length": 439.89288330078125, "epoch": 0.15849447513812154, "grad_norm": 3.5743451360007117, "kl": 0.06396484375, "learning_rate": 4.6964386057451464e-07, "loss": 0.0026, "rewards_raw/CoT ICLG": 0.8887079954147339, "rewards_raw/accuracy_reward": 0.5563722848892212, "rewards_raw/clip_reward_wrapper": 0.3289461135864258, "rewards_raw/format_reward": 5.0, "step": 459 }, { "Total_reward": 7.620880069225677e-07, "advantage_std": 1.4490143060684204, "completion_length": 446.58929443359375, "epoch": 0.15883977900552487, "grad_norm": 4.060631572467296, "kl": 0.06396484375, "learning_rate": 4.6951420477831176e-07, "loss": 0.0026, "rewards_raw/CoT ICLG": 0.4623803496360779, "rewards_raw/accuracy_reward": 0.5644280314445496, "rewards_raw/clip_reward_wrapper": 0.31434908509254456, "rewards_raw/format_reward": 5.0, "step": 460 }, { "Total_reward": 1.9797258232756576e-07, "advantage_std": 1.5761315822601318, "completion_length": 445.9107360839844, "epoch": 0.15918508287292818, "grad_norm": 3.7479841260366866, "kl": 0.06298828125, "learning_rate": 4.693842906578355e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.6468855142593384, "rewards_raw/accuracy_reward": 0.5760092735290527, "rewards_raw/clip_reward_wrapper": 0.33183303475379944, "rewards_raw/format_reward": 5.0, "step": 461 }, { "Total_reward": 4.480992288335983e-07, "advantage_std": 1.4998509883880615, "completion_length": 436.39288330078125, "epoch": 0.1595303867403315, "grad_norm": 3.4468072797030396, "kl": 0.0673828125, "learning_rate": 4.692541183659688e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.6806396842002869, "rewards_raw/accuracy_reward": 0.568527340888977, "rewards_raw/clip_reward_wrapper": 0.32930415868759155, "rewards_raw/format_reward": 5.0, "step": 462 }, { "Total_reward": -6.258487701416016e-07, "advantage_std": 1.4986534118652344, "completion_length": 437.96429443359375, "epoch": 0.1598756906077348, "grad_norm": 6.228546344489907, "kl": 0.0693359375, "learning_rate": 4.691236880558983e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.27348458766937256, "rewards_raw/accuracy_reward": 0.5743311643600464, "rewards_raw/clip_reward_wrapper": 0.31419214606285095, "rewards_raw/format_reward": 5.0, "step": 463 }, { "Total_reward": 5.939177185609879e-07, "advantage_std": 1.3868895769119263, "completion_length": 428.9285888671875, "epoch": 0.16022099447513813, "grad_norm": 5.439909717187579, "kl": 0.06298828125, "learning_rate": 4.689929998811145e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.17694313824176788, "rewards_raw/accuracy_reward": 0.5417176485061646, "rewards_raw/clip_reward_wrapper": 0.34130316972732544, "rewards_raw/format_reward": 5.0, "step": 464 }, { "Total_reward": 1.2559550555124588e-07, "advantage_std": 1.073121190071106, "completion_length": 437.6785888671875, "epoch": 0.16056629834254144, "grad_norm": 7.488129586907247, "kl": 0.0654296875, "learning_rate": 4.688620539954111e-07, "loss": 0.0026, "rewards_raw/CoT ICLG": 0.9014226794242859, "rewards_raw/accuracy_reward": 0.5754728317260742, "rewards_raw/clip_reward_wrapper": 0.3231405019760132, "rewards_raw/format_reward": 5.0, "step": 465 }, { "Total_reward": 6.599086077585525e-07, "advantage_std": 1.6232160329818726, "completion_length": 438.3214416503906, "epoch": 0.16091160220994474, "grad_norm": 8.486611535378495, "kl": 0.0615234375, "learning_rate": 4.6873085055288525e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.35984355211257935, "rewards_raw/accuracy_reward": 0.572829008102417, "rewards_raw/clip_reward_wrapper": 0.3206540644168854, "rewards_raw/format_reward": 5.0, "step": 466 }, { "Total_reward": 2.6875309799834213e-07, "advantage_std": 1.4155025482177734, "completion_length": 449.58929443359375, "epoch": 0.16125690607734808, "grad_norm": 5.082424779795104, "kl": 0.06494140625, "learning_rate": 4.685993897079371e-07, "loss": 0.0026, "rewards_raw/CoT ICLG": 0.15870733559131622, "rewards_raw/accuracy_reward": 0.5716299414634705, "rewards_raw/clip_reward_wrapper": 0.31643611192703247, "rewards_raw/format_reward": 5.0, "step": 467 }, { "Total_reward": 5.534717217869911e-08, "advantage_std": 1.5686393976211548, "completion_length": 434.1071472167969, "epoch": 0.16160220994475138, "grad_norm": 5.402074173523079, "kl": 0.06591796875, "learning_rate": 4.684676716152698e-07, "loss": 0.0026, "rewards_raw/CoT ICLG": 0.22509153187274933, "rewards_raw/accuracy_reward": 0.562059223651886, "rewards_raw/clip_reward_wrapper": 0.3267841339111328, "rewards_raw/format_reward": 5.0, "step": 468 }, { "Total_reward": -1.3623919414840202e-07, "advantage_std": 1.6248705387115479, "completion_length": 431.9464416503906, "epoch": 0.1619475138121547, "grad_norm": 7.147841792001373, "kl": 0.0634765625, "learning_rate": 4.683356964298891e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.29509803652763367, "rewards_raw/accuracy_reward": 0.5825488567352295, "rewards_raw/clip_reward_wrapper": 0.32458510994911194, "rewards_raw/format_reward": 5.0, "step": 469 }, { "Total_reward": -3.7891524584665603e-07, "advantage_std": 1.3422492742538452, "completion_length": 448.607177734375, "epoch": 0.162292817679558, "grad_norm": 3.722541892785056, "kl": 0.064453125, "learning_rate": 4.6820346430710354e-07, "loss": 0.0026, "rewards_raw/CoT ICLG": 0.2886432707309723, "rewards_raw/accuracy_reward": 0.5742741823196411, "rewards_raw/clip_reward_wrapper": 0.31685125827789307, "rewards_raw/format_reward": 5.0, "step": 470 }, { "Total_reward": 1.1282308065574398e-07, "advantage_std": 1.5111218690872192, "completion_length": 439.6607360839844, "epoch": 0.16263812154696133, "grad_norm": 4.190271078759806, "kl": 0.06494140625, "learning_rate": 4.6807097540252363e-07, "loss": 0.0026, "rewards_raw/CoT ICLG": 0.514638364315033, "rewards_raw/accuracy_reward": 0.5583035945892334, "rewards_raw/clip_reward_wrapper": 0.3233713209629059, "rewards_raw/format_reward": 5.0, "step": 471 }, { "Total_reward": -2.4267606590910873e-07, "advantage_std": 1.533048391342163, "completion_length": 454.1785888671875, "epoch": 0.16298342541436464, "grad_norm": 3.3510136868294076, "kl": 0.06591796875, "learning_rate": 4.6793822987206243e-07, "loss": 0.0026, "rewards_raw/CoT ICLG": 0.08275468647480011, "rewards_raw/accuracy_reward": 0.5750724673271179, "rewards_raw/clip_reward_wrapper": 0.33216676115989685, "rewards_raw/format_reward": 5.0, "step": 472 }, { "Total_reward": 2.650278076998802e-07, "advantage_std": 1.5545586347579956, "completion_length": 449.5000305175781, "epoch": 0.16332872928176795, "grad_norm": 3.692527838196315, "kl": 0.0634765625, "learning_rate": 4.6780522787193487e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.2170134335756302, "rewards_raw/accuracy_reward": 0.5676096081733704, "rewards_raw/clip_reward_wrapper": 0.33317407965660095, "rewards_raw/format_reward": 5.0, "step": 473 }, { "Total_reward": 5.577292085945373e-07, "advantage_std": 1.4056721925735474, "completion_length": 445.3035888671875, "epoch": 0.16367403314917128, "grad_norm": 3.112924214159205, "kl": 0.06298828125, "learning_rate": 4.6767196955865756e-07, "loss": 0.0025, "rewards_raw/CoT ICLG": 0.3807436525821686, "rewards_raw/accuracy_reward": 0.5664501190185547, "rewards_raw/clip_reward_wrapper": 0.33365634083747864, "rewards_raw/format_reward": 5.0, "step": 474 }, { "Total_reward": -1.6178404393940582e-07, "advantage_std": 1.337088942527771, "completion_length": 441.107177734375, "epoch": 0.1640193370165746, "grad_norm": 7.226041100966363, "kl": 0.06640625, "learning_rate": 4.675384550890489e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.454963356256485, "rewards_raw/accuracy_reward": 0.565360426902771, "rewards_raw/clip_reward_wrapper": 0.32388612627983093, "rewards_raw/format_reward": 5.0, "step": 475 }, { "Total_reward": 3.427267358802055e-07, "advantage_std": 1.3585413694381714, "completion_length": 421.6250305175781, "epoch": 0.1643646408839779, "grad_norm": 4.351325219165247, "kl": 0.0693359375, "learning_rate": 4.674046846202288e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.7005253434181213, "rewards_raw/accuracy_reward": 0.5712074041366577, "rewards_raw/clip_reward_wrapper": 0.3167927861213684, "rewards_raw/format_reward": 5.0, "step": 476 }, { "Total_reward": -3.129243850708008e-07, "advantage_std": 1.5438131093978882, "completion_length": 446.482177734375, "epoch": 0.1647099447513812, "grad_norm": 7.017034955854733, "kl": 0.064453125, "learning_rate": 4.672706583096182e-07, "loss": 0.0026, "rewards_raw/CoT ICLG": 0.40680405497550964, "rewards_raw/accuracy_reward": 0.5775997042655945, "rewards_raw/clip_reward_wrapper": 0.3128611743450165, "rewards_raw/format_reward": 5.0, "step": 477 }, { "Total_reward": 8.472375156998169e-07, "advantage_std": 1.4783183336257935, "completion_length": 430.7857360839844, "epoch": 0.16505524861878454, "grad_norm": 6.4059896973383035, "kl": 0.06787109375, "learning_rate": 4.671363763149393e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 1.1204746961593628, "rewards_raw/accuracy_reward": 0.5492931008338928, "rewards_raw/clip_reward_wrapper": 0.337563693523407, "rewards_raw/format_reward": 5.0, "step": 478 }, { "Total_reward": -3.363405198797409e-07, "advantage_std": 1.5034610033035278, "completion_length": 445.1785888671875, "epoch": 0.16540055248618785, "grad_norm": 3.3039933681518465, "kl": 0.06640625, "learning_rate": 4.67001838794215e-07, "loss": 0.0026, "rewards_raw/CoT ICLG": 0.24820347130298615, "rewards_raw/accuracy_reward": 0.5583263039588928, "rewards_raw/clip_reward_wrapper": 0.3302724361419678, "rewards_raw/format_reward": 5.0, "step": 479 }, { "Total_reward": 4.512923226229759e-07, "advantage_std": 1.5737029314041138, "completion_length": 436.1607360839844, "epoch": 0.16574585635359115, "grad_norm": 5.942376448993384, "kl": 0.0673828125, "learning_rate": 4.668670459057692e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.4927780032157898, "rewards_raw/accuracy_reward": 0.5705714821815491, "rewards_raw/clip_reward_wrapper": 0.33833616971969604, "rewards_raw/format_reward": 5.0, "step": 480 }, { "Total_reward": 1.4901161193847656e-07, "advantage_std": 1.6698979139328003, "completion_length": 447.89288330078125, "epoch": 0.1660911602209945, "grad_norm": 7.048230115213298, "kl": 0.0673828125, "learning_rate": 4.66731997808226e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.2240028828382492, "rewards_raw/accuracy_reward": 0.5592538118362427, "rewards_raw/clip_reward_wrapper": 0.3337118327617645, "rewards_raw/format_reward": 5.0, "step": 481 }, { "Total_reward": -5.790165573671402e-07, "advantage_std": 1.4762619733810425, "completion_length": 444.1607360839844, "epoch": 0.1664364640883978, "grad_norm": 6.044737092374796, "kl": 0.0673828125, "learning_rate": 4.665966946605102e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.408319890499115, "rewards_raw/accuracy_reward": 0.5647752285003662, "rewards_raw/clip_reward_wrapper": 0.3206164836883545, "rewards_raw/format_reward": 5.0, "step": 482 }, { "Total_reward": 7.91890329310263e-07, "advantage_std": 1.4887733459472656, "completion_length": 441.7857360839844, "epoch": 0.1667817679558011, "grad_norm": 6.215921021626891, "kl": 0.068359375, "learning_rate": 4.664611366218463e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.6850703358650208, "rewards_raw/accuracy_reward": 0.5524253845214844, "rewards_raw/clip_reward_wrapper": 0.32112202048301697, "rewards_raw/format_reward": 5.0, "step": 483 }, { "Total_reward": -2.8312206268310547e-07, "advantage_std": 1.5271602869033813, "completion_length": 450.4464416503906, "epoch": 0.1671270718232044, "grad_norm": 3.8538896614647813, "kl": 0.06689453125, "learning_rate": 4.663253238517589e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.2882113456726074, "rewards_raw/accuracy_reward": 0.5808643698692322, "rewards_raw/clip_reward_wrapper": 0.33116596937179565, "rewards_raw/format_reward": 5.0, "step": 484 }, { "Total_reward": -5.768878281742218e-07, "advantage_std": 1.5905500650405884, "completion_length": 442.3214416503906, "epoch": 0.16747237569060774, "grad_norm": 4.217529202984156, "kl": 0.06640625, "learning_rate": 4.6618925651007273e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.3674963712692261, "rewards_raw/accuracy_reward": 0.5373079180717468, "rewards_raw/clip_reward_wrapper": 0.3378807008266449, "rewards_raw/format_reward": 5.0, "step": 485 }, { "Total_reward": -7.40800601306546e-07, "advantage_std": 1.6064181327819824, "completion_length": 443.4464416503906, "epoch": 0.16781767955801105, "grad_norm": 5.402125564220951, "kl": 0.06640625, "learning_rate": 4.6605293475691163e-07, "loss": 0.0026, "rewards_raw/CoT ICLG": 0.35429584980010986, "rewards_raw/accuracy_reward": 0.5762593746185303, "rewards_raw/clip_reward_wrapper": 0.3183068633079529, "rewards_raw/format_reward": 5.0, "step": 486 }, { "Total_reward": -2.543841333135788e-07, "advantage_std": 1.126257300376892, "completion_length": 452.0000305175781, "epoch": 0.16816298342541436, "grad_norm": 3.1319311268568897, "kl": 0.06640625, "learning_rate": 4.659163587526991e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.5182269215583801, "rewards_raw/accuracy_reward": 0.5573713183403015, "rewards_raw/clip_reward_wrapper": 0.32393836975097656, "rewards_raw/format_reward": 5.0, "step": 487 }, { "Total_reward": 8.089202196970291e-08, "advantage_std": 1.5209404230117798, "completion_length": 446.857177734375, "epoch": 0.1685082872928177, "grad_norm": 5.123712822904718, "kl": 0.0673828125, "learning_rate": 4.657795286581576e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.2805684804916382, "rewards_raw/accuracy_reward": 0.5829735994338989, "rewards_raw/clip_reward_wrapper": 0.3255199193954468, "rewards_raw/format_reward": 5.0, "step": 488 }, { "Total_reward": 3.08666926684964e-07, "advantage_std": 1.6072566509246826, "completion_length": 439.6785888671875, "epoch": 0.168853591160221, "grad_norm": 15.935342806997214, "kl": 0.06982421875, "learning_rate": 4.6564244463430883e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.25123727321624756, "rewards_raw/accuracy_reward": 0.5757566690444946, "rewards_raw/clip_reward_wrapper": 0.32386499643325806, "rewards_raw/format_reward": 5.0, "step": 489 }, { "Total_reward": 9.047133744388702e-07, "advantage_std": 1.6888763904571533, "completion_length": 443.1785888671875, "epoch": 0.1691988950276243, "grad_norm": 5.350600278027105, "kl": 0.0703125, "learning_rate": 4.6550510684247334e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.39798372983932495, "rewards_raw/accuracy_reward": 0.550414502620697, "rewards_raw/clip_reward_wrapper": 0.32305869460105896, "rewards_raw/format_reward": 5.0, "step": 490 }, { "Total_reward": -8.727823086474018e-08, "advantage_std": 1.6048985719680786, "completion_length": 443.1250305175781, "epoch": 0.16954419889502761, "grad_norm": 6.91095735173027, "kl": 0.0693359375, "learning_rate": 4.6536751544427013e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 1.0009284019470215, "rewards_raw/accuracy_reward": 0.5509869456291199, "rewards_raw/clip_reward_wrapper": 0.3376578092575073, "rewards_raw/format_reward": 5.0, "step": 491 }, { "Total_reward": -2.1287374085687816e-09, "advantage_std": 1.6341239213943481, "completion_length": 433.1607360839844, "epoch": 0.16988950276243095, "grad_norm": 6.060249252794381, "kl": 0.06884765625, "learning_rate": 4.6522967060161676e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.13109354674816132, "rewards_raw/accuracy_reward": 0.5855809450149536, "rewards_raw/clip_reward_wrapper": 0.32131490111351013, "rewards_raw/format_reward": 5.0, "step": 492 }, { "Total_reward": 3.352761552832817e-07, "advantage_std": 1.4349297285079956, "completion_length": 442.2857360839844, "epoch": 0.17023480662983426, "grad_norm": 3.5907277799288115, "kl": 0.064453125, "learning_rate": 4.6509157247672897e-07, "loss": 0.0026, "rewards_raw/CoT ICLG": 0.7603251338005066, "rewards_raw/accuracy_reward": 0.5544420480728149, "rewards_raw/clip_reward_wrapper": 0.3286711573600769, "rewards_raw/format_reward": 5.0, "step": 493 }, { "Total_reward": -1.2304102483540191e-06, "advantage_std": 1.6714314222335815, "completion_length": 443.96429443359375, "epoch": 0.17058011049723756, "grad_norm": 5.317920264372711, "kl": 0.07080078125, "learning_rate": 4.6495322123212067e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.6996525526046753, "rewards_raw/accuracy_reward": 0.5708977580070496, "rewards_raw/clip_reward_wrapper": 0.31563299894332886, "rewards_raw/format_reward": 5.0, "step": 494 }, { "Total_reward": 7.876328567135715e-08, "advantage_std": 1.7407996654510498, "completion_length": 436.4107360839844, "epoch": 0.17092541436464087, "grad_norm": 4.001970830824344, "kl": 0.06787109375, "learning_rate": 4.648146170306035e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.37693318724632263, "rewards_raw/accuracy_reward": 0.5761905908584595, "rewards_raw/clip_reward_wrapper": 0.3261945843696594, "rewards_raw/format_reward": 5.0, "step": 495 }, { "Total_reward": 5.939177185609879e-07, "advantage_std": 1.431555986404419, "completion_length": 437.0000305175781, "epoch": 0.1712707182320442, "grad_norm": 3.4574589980174317, "kl": 0.0673828125, "learning_rate": 4.6467576003528694e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.4553976356983185, "rewards_raw/accuracy_reward": 0.5706729888916016, "rewards_raw/clip_reward_wrapper": 0.34186044335365295, "rewards_raw/format_reward": 5.0, "step": 496 }, { "Total_reward": 6.780028911634872e-07, "advantage_std": 1.3024988174438477, "completion_length": 448.107177734375, "epoch": 0.1716160220994475, "grad_norm": 7.7952112132939835, "kl": 0.0703125, "learning_rate": 4.645366504095778e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.7531046271324158, "rewards_raw/accuracy_reward": 0.5827212333679199, "rewards_raw/clip_reward_wrapper": 0.3286226689815521, "rewards_raw/format_reward": 5.0, "step": 497 }, { "Total_reward": -5.045107513979019e-07, "advantage_std": 1.2926299571990967, "completion_length": 435.46429443359375, "epoch": 0.17196132596685082, "grad_norm": 3.808575729986425, "kl": 0.06982421875, "learning_rate": 4.6439728831718034e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.5325521230697632, "rewards_raw/accuracy_reward": 0.5659823417663574, "rewards_raw/clip_reward_wrapper": 0.33093637228012085, "rewards_raw/format_reward": 5.0, "step": 498 }, { "Total_reward": -5.332487376108475e-07, "advantage_std": 1.1819274425506592, "completion_length": 434.89288330078125, "epoch": 0.17230662983425415, "grad_norm": 4.882467643075281, "kl": 0.0732421875, "learning_rate": 4.642576739220958e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.3638683259487152, "rewards_raw/accuracy_reward": 0.5585911870002747, "rewards_raw/clip_reward_wrapper": 0.330432653427124, "rewards_raw/format_reward": 5.0, "step": 499 }, { "Total_reward": 3.4059798537100505e-08, "advantage_std": 1.4748703241348267, "completion_length": 433.64288330078125, "epoch": 0.17265193370165746, "grad_norm": 5.606676820886971, "kl": 0.068359375, "learning_rate": 4.641178073886223e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.7882001996040344, "rewards_raw/accuracy_reward": 0.5861795544624329, "rewards_raw/clip_reward_wrapper": 0.3299439549446106, "rewards_raw/format_reward": 5.0, "step": 500 }, { "Total_reward": 5.747590847704487e-08, "advantage_std": 1.5766972303390503, "completion_length": 451.8214416503906, "epoch": 0.17299723756906077, "grad_norm": 5.372590450383778, "kl": 0.06689453125, "learning_rate": 4.639776888813549e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.3522337079048157, "rewards_raw/accuracy_reward": 0.5793513059616089, "rewards_raw/clip_reward_wrapper": 0.3348971903324127, "rewards_raw/format_reward": 5.0, "step": 501 }, { "Total_reward": -2.9163703629819793e-07, "advantage_std": 1.5468406677246094, "completion_length": 437.3214416503906, "epoch": 0.17334254143646408, "grad_norm": 8.424222371288305, "kl": 0.0673828125, "learning_rate": 4.638373185651849e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.628703236579895, "rewards_raw/accuracy_reward": 0.5592829585075378, "rewards_raw/clip_reward_wrapper": 0.3259853720664978, "rewards_raw/format_reward": 5.0, "step": 502 }, { "Total_reward": -9.132283480539627e-07, "advantage_std": 1.5314966440200806, "completion_length": 437.8035888671875, "epoch": 0.1736878453038674, "grad_norm": 3.820613834914182, "kl": 0.07177734375, "learning_rate": 4.636966966053002e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.40879449248313904, "rewards_raw/accuracy_reward": 0.5732408761978149, "rewards_raw/clip_reward_wrapper": 0.3155101537704468, "rewards_raw/format_reward": 5.0, "step": 503 }, { "Total_reward": 3.83172746865057e-08, "advantage_std": 1.4663772583007812, "completion_length": 443.8035888671875, "epoch": 0.17403314917127072, "grad_norm": 5.682590661301124, "kl": 0.07177734375, "learning_rate": 4.6355582316718455e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.28011295199394226, "rewards_raw/accuracy_reward": 0.5880357623100281, "rewards_raw/clip_reward_wrapper": 0.32293063402175903, "rewards_raw/format_reward": 5.0, "step": 504 }, { "Total_reward": -3.65078477670977e-07, "advantage_std": 1.5871450901031494, "completion_length": 434.08929443359375, "epoch": 0.17437845303867403, "grad_norm": 4.706538874357337, "kl": 0.06591796875, "learning_rate": 4.6341469841661785e-07, "loss": 0.0026, "rewards_raw/CoT ICLG": 0.28532034158706665, "rewards_raw/accuracy_reward": 0.5789362788200378, "rewards_raw/clip_reward_wrapper": 0.3261462450027466, "rewards_raw/format_reward": 5.0, "step": 505 }, { "Total_reward": -1.6497715193963813e-07, "advantage_std": 1.582392930984497, "completion_length": 456.0535888671875, "epoch": 0.17472375690607736, "grad_norm": 3.6425127143669798, "kl": 0.068359375, "learning_rate": 4.6327332251967565e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.34330031275749207, "rewards_raw/accuracy_reward": 0.5681192874908447, "rewards_raw/clip_reward_wrapper": 0.3317643404006958, "rewards_raw/format_reward": 5.0, "step": 506 }, { "Total_reward": 7.929546654850128e-07, "advantage_std": 1.3446388244628906, "completion_length": 450.357177734375, "epoch": 0.17506906077348067, "grad_norm": 4.740713630163017, "kl": 0.06884765625, "learning_rate": 4.6313169564272913e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.4062100350856781, "rewards_raw/accuracy_reward": 0.5638540983200073, "rewards_raw/clip_reward_wrapper": 0.33008018136024475, "rewards_raw/format_reward": 5.0, "step": 507 }, { "Total_reward": -4.5554980943052215e-07, "advantage_std": 1.3996021747589111, "completion_length": 445.4107360839844, "epoch": 0.17541436464088397, "grad_norm": 6.321921429405202, "kl": 0.0693359375, "learning_rate": 4.6298981795244473e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.19787350296974182, "rewards_raw/accuracy_reward": 0.5630084872245789, "rewards_raw/clip_reward_wrapper": 0.3310365378856659, "rewards_raw/format_reward": 5.0, "step": 508 }, { "Total_reward": 4.108463258489792e-07, "advantage_std": 1.8164440393447876, "completion_length": 441.96429443359375, "epoch": 0.17575966850828728, "grad_norm": 6.183086927152418, "kl": 0.0673828125, "learning_rate": 4.6284768961578414e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.5791165232658386, "rewards_raw/accuracy_reward": 0.5801845788955688, "rewards_raw/clip_reward_wrapper": 0.3327767252922058, "rewards_raw/format_reward": 5.0, "step": 509 }, { "Total_reward": -7.450580596923828e-08, "advantage_std": 1.3840277194976807, "completion_length": 427.9107360839844, "epoch": 0.17610497237569062, "grad_norm": 3.9214452752448667, "kl": 0.0703125, "learning_rate": 4.6270531080000385e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.15913993120193481, "rewards_raw/accuracy_reward": 0.5469112992286682, "rewards_raw/clip_reward_wrapper": 0.326695054769516, "rewards_raw/format_reward": 5.0, "step": 510 }, { "Total_reward": 2.2138868871479644e-07, "advantage_std": 1.581798791885376, "completion_length": 441.107177734375, "epoch": 0.17645027624309392, "grad_norm": 4.217750341683823, "kl": 0.0703125, "learning_rate": 4.6256268167265535e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.37858349084854126, "rewards_raw/accuracy_reward": 0.5717113614082336, "rewards_raw/clip_reward_wrapper": 0.31906986236572266, "rewards_raw/format_reward": 5.0, "step": 511 }, { "Total_reward": 1.4049666674509353e-07, "advantage_std": 1.4464881420135498, "completion_length": 447.9285888671875, "epoch": 0.17679558011049723, "grad_norm": 10.482939118228032, "kl": 0.06689453125, "learning_rate": 4.6241980240158447e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.21161991357803345, "rewards_raw/accuracy_reward": 0.5788668394088745, "rewards_raw/clip_reward_wrapper": 0.3392002284526825, "rewards_raw/format_reward": 5.0, "step": 512 }, { "Total_reward": -1.0196652056038147e-06, "advantage_std": 1.5496400594711304, "completion_length": 428.1964416503906, "epoch": 0.17714088397790057, "grad_norm": 12.865529860932284, "kl": 0.07275390625, "learning_rate": 4.6227667315493157e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.12558087706565857, "rewards_raw/accuracy_reward": 0.566423773765564, "rewards_raw/clip_reward_wrapper": 0.3373483121395111, "rewards_raw/format_reward": 5.0, "step": 513 }, { "Total_reward": 3.4059798537100505e-08, "advantage_std": 1.3932267427444458, "completion_length": 443.5000305175781, "epoch": 0.17748618784530387, "grad_norm": 3.681791476669103, "kl": 0.06884765625, "learning_rate": 4.6213329410113114e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.2886931002140045, "rewards_raw/accuracy_reward": 0.5632121562957764, "rewards_raw/clip_reward_wrapper": 0.3335984945297241, "rewards_raw/format_reward": 5.0, "step": 514 }, { "Total_reward": -8.089202196970291e-07, "advantage_std": 1.6969667673110962, "completion_length": 441.26788330078125, "epoch": 0.17783149171270718, "grad_norm": 10.29416412825298, "kl": 0.07275390625, "learning_rate": 4.619896654089115e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.5179938673973083, "rewards_raw/accuracy_reward": 0.5524289608001709, "rewards_raw/clip_reward_wrapper": 0.33634713292121887, "rewards_raw/format_reward": 5.0, "step": 515 }, { "Total_reward": 2.575772271029564e-07, "advantage_std": 1.5096927881240845, "completion_length": 451.21429443359375, "epoch": 0.1781767955801105, "grad_norm": 5.287722948887598, "kl": 0.0693359375, "learning_rate": 4.6184578724729496e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.236859992146492, "rewards_raw/accuracy_reward": 0.552129864692688, "rewards_raw/clip_reward_wrapper": 0.3283047378063202, "rewards_raw/format_reward": 5.0, "step": 516 }, { "Total_reward": 6.811959707420101e-08, "advantage_std": 1.209389567375183, "completion_length": 440.64288330078125, "epoch": 0.17852209944751382, "grad_norm": 2.7157254763953356, "kl": 0.0732421875, "learning_rate": 4.617016597855974e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.3938242793083191, "rewards_raw/accuracy_reward": 0.5678900480270386, "rewards_raw/clip_reward_wrapper": 0.31594178080558777, "rewards_raw/format_reward": 5.0, "step": 517 }, { "Total_reward": 3.874302194617485e-07, "advantage_std": 1.6502901315689087, "completion_length": 427.5000305175781, "epoch": 0.17886740331491713, "grad_norm": 11.426291456525785, "kl": 0.068359375, "learning_rate": 4.615572831934278e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.8070527911186218, "rewards_raw/accuracy_reward": 0.5678068995475769, "rewards_raw/clip_reward_wrapper": 0.32186031341552734, "rewards_raw/format_reward": 5.0, "step": 518 }, { "Total_reward": -1.6817025993987045e-07, "advantage_std": 1.3785450458526611, "completion_length": 432.3750305175781, "epoch": 0.17921270718232044, "grad_norm": 5.197506215812011, "kl": 0.07080078125, "learning_rate": 4.614126576406886e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.33971652388572693, "rewards_raw/accuracy_reward": 0.573936939239502, "rewards_raw/clip_reward_wrapper": 0.33309406042099, "rewards_raw/format_reward": 5.0, "step": 519 }, { "Total_reward": 4.959958346262283e-07, "advantage_std": 1.219915747642517, "completion_length": 443.9107360839844, "epoch": 0.17955801104972377, "grad_norm": 3.4774279127978254, "kl": 0.06982421875, "learning_rate": 4.612677832975751e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.16132208704948425, "rewards_raw/accuracy_reward": 0.5728501081466675, "rewards_raw/clip_reward_wrapper": 0.3326224684715271, "rewards_raw/format_reward": 5.0, "step": 520 }, { "Total_reward": -5.598579377874557e-07, "advantage_std": 1.5777055025100708, "completion_length": 452.64288330078125, "epoch": 0.17990331491712708, "grad_norm": 3.7218402520480787, "kl": 0.0712890625, "learning_rate": 4.611226603345755e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.27305954694747925, "rewards_raw/accuracy_reward": 0.5610136985778809, "rewards_raw/clip_reward_wrapper": 0.33865663409233093, "rewards_raw/format_reward": 5.0, "step": 521 }, { "Total_reward": 6.066901647727718e-08, "advantage_std": 1.450368881225586, "completion_length": 442.8214416503906, "epoch": 0.18024861878453038, "grad_norm": 3.3331251604903427, "kl": 0.0712890625, "learning_rate": 4.6097728892247034e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.23743599653244019, "rewards_raw/accuracy_reward": 0.5507409572601318, "rewards_raw/clip_reward_wrapper": 0.33375564217567444, "rewards_raw/format_reward": 5.0, "step": 522 }, { "Total_reward": 1.2357320429146057e-06, "advantage_std": 1.4281648397445679, "completion_length": 449.6964416503906, "epoch": 0.1805939226519337, "grad_norm": 2.9842389979965023, "kl": 0.07275390625, "learning_rate": 4.6083166923233266e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.1574126034975052, "rewards_raw/accuracy_reward": 0.5647486448287964, "rewards_raw/clip_reward_wrapper": 0.328411728143692, "rewards_raw/format_reward": 5.0, "step": 523 }, { "Total_reward": -3.938164070405037e-07, "advantage_std": 1.5621060132980347, "completion_length": 433.8035888671875, "epoch": 0.18093922651933703, "grad_norm": 3.3891436447403827, "kl": 0.0703125, "learning_rate": 4.606858014355278e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.2711154818534851, "rewards_raw/accuracy_reward": 0.5712386965751648, "rewards_raw/clip_reward_wrapper": 0.3194931745529175, "rewards_raw/format_reward": 5.0, "step": 524 }, { "Total_reward": -6.982258469179214e-07, "advantage_std": 1.2670762538909912, "completion_length": 447.5357360839844, "epoch": 0.18128453038674033, "grad_norm": 4.083824859062347, "kl": 0.0712890625, "learning_rate": 4.6053968570371295e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 1.063887119293213, "rewards_raw/accuracy_reward": 0.5743300318717957, "rewards_raw/clip_reward_wrapper": 0.3463861048221588, "rewards_raw/format_reward": 5.0, "step": 525 }, { "Total_reward": 8.02533975274855e-07, "advantage_std": 1.3924981355667114, "completion_length": 462.3214416503906, "epoch": 0.18162983425414364, "grad_norm": 6.485651415547561, "kl": 0.06787109375, "learning_rate": 4.6039332220883697e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.43583789467811584, "rewards_raw/accuracy_reward": 0.5622890591621399, "rewards_raw/clip_reward_wrapper": 0.3350850045681, "rewards_raw/format_reward": 5.0, "step": 526 }, { "Total_reward": 1.585909359391735e-07, "advantage_std": 1.5761709213256836, "completion_length": 451.21429443359375, "epoch": 0.18197513812154695, "grad_norm": 4.540664711439914, "kl": 0.06982421875, "learning_rate": 4.602467111231405e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.32135725021362305, "rewards_raw/accuracy_reward": 0.586490273475647, "rewards_raw/clip_reward_wrapper": 0.3272576928138733, "rewards_raw/format_reward": 5.0, "step": 527 }, { "Total_reward": 3.5337041026650695e-07, "advantage_std": 1.4920598268508911, "completion_length": 432.0000305175781, "epoch": 0.18232044198895028, "grad_norm": 4.010233056719489, "kl": 0.07373046875, "learning_rate": 4.600998526191553e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.3968745172023773, "rewards_raw/accuracy_reward": 0.5647526979446411, "rewards_raw/clip_reward_wrapper": 0.3133266568183899, "rewards_raw/format_reward": 5.0, "step": 528 }, { "Total_reward": 1.7029898913278885e-07, "advantage_std": 1.544987678527832, "completion_length": 454.0714416503906, "epoch": 0.1826657458563536, "grad_norm": 4.344286589581189, "kl": 0.0693359375, "learning_rate": 4.599527468697045e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.5195280909538269, "rewards_raw/accuracy_reward": 0.5521537065505981, "rewards_raw/clip_reward_wrapper": 0.3338105380535126, "rewards_raw/format_reward": 5.0, "step": 529 }, { "Total_reward": -1.2080585065632476e-06, "advantage_std": 1.3544769287109375, "completion_length": 441.1964416503906, "epoch": 0.1830110497237569, "grad_norm": 3.1217461939139115, "kl": 0.072265625, "learning_rate": 4.598053940479022e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.4586256742477417, "rewards_raw/accuracy_reward": 0.5694880485534668, "rewards_raw/clip_reward_wrapper": 0.33612459897994995, "rewards_raw/format_reward": 5.0, "step": 530 }, { "Total_reward": 3.70400300653273e-07, "advantage_std": 1.4912372827529907, "completion_length": 438.4285888671875, "epoch": 0.18335635359116023, "grad_norm": 3.5708175218482827, "kl": 0.07470703125, "learning_rate": 4.5965779432715305e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.5168972611427307, "rewards_raw/accuracy_reward": 0.5778011083602905, "rewards_raw/clip_reward_wrapper": 0.31218236684799194, "rewards_raw/format_reward": 5.0, "step": 531 }, { "Total_reward": -2.4054733671619033e-07, "advantage_std": 1.4233466386795044, "completion_length": 436.4107360839844, "epoch": 0.18370165745856354, "grad_norm": 4.946120789986409, "kl": 0.07373046875, "learning_rate": 4.595099478811525e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.5004131197929382, "rewards_raw/accuracy_reward": 0.5843676328659058, "rewards_raw/clip_reward_wrapper": 0.3251846432685852, "rewards_raw/format_reward": 5.0, "step": 532 }, { "Total_reward": 2.2351743211856956e-07, "advantage_std": 1.6360493898391724, "completion_length": 452.51788330078125, "epoch": 0.18404696132596685, "grad_norm": 4.394671943872684, "kl": 0.0673828125, "learning_rate": 4.593618548838861e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.41861215233802795, "rewards_raw/accuracy_reward": 0.581679105758667, "rewards_raw/clip_reward_wrapper": 0.314003586769104, "rewards_raw/format_reward": 5.0, "step": 533 }, { "Total_reward": 3.491129234589607e-07, "advantage_std": 1.4955848455429077, "completion_length": 452.107177734375, "epoch": 0.18439226519337015, "grad_norm": 3.5789021179950544, "kl": 0.07275390625, "learning_rate": 4.592135155096297e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 1.521062970161438, "rewards_raw/accuracy_reward": 0.5474309325218201, "rewards_raw/clip_reward_wrapper": 0.3281532824039459, "rewards_raw/format_reward": 5.0, "step": 534 }, { "Total_reward": 1.788139485370266e-07, "advantage_std": 1.5835143327713013, "completion_length": 447.732177734375, "epoch": 0.1847375690607735, "grad_norm": 4.188298725577162, "kl": 0.06982421875, "learning_rate": 4.59064929932949e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.28644952178001404, "rewards_raw/accuracy_reward": 0.5509669184684753, "rewards_raw/clip_reward_wrapper": 0.3131980895996094, "rewards_raw/format_reward": 5.0, "step": 535 }, { "Total_reward": 3.7891524584665603e-07, "advantage_std": 1.5414094924926758, "completion_length": 442.0535888671875, "epoch": 0.1850828729281768, "grad_norm": 7.566397514334021, "kl": 0.07177734375, "learning_rate": 4.589160983286996e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.34338122606277466, "rewards_raw/accuracy_reward": 0.5580800175666809, "rewards_raw/clip_reward_wrapper": 0.31717878580093384, "rewards_raw/format_reward": 5.0, "step": 536 }, { "Total_reward": -6.301062853708572e-07, "advantage_std": 1.3362113237380981, "completion_length": 446.64288330078125, "epoch": 0.1854281767955801, "grad_norm": 4.045131264039196, "kl": 0.07275390625, "learning_rate": 4.5876702087202636e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.36569875478744507, "rewards_raw/accuracy_reward": 0.5478742718696594, "rewards_raw/clip_reward_wrapper": 0.31693267822265625, "rewards_raw/format_reward": 5.0, "step": 537 }, { "Total_reward": 1.830714211337181e-07, "advantage_std": 1.3554086685180664, "completion_length": 438.1964416503906, "epoch": 0.18577348066298344, "grad_norm": 3.3071982691061597, "kl": 0.072265625, "learning_rate": 4.5861769773836367e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.5131389498710632, "rewards_raw/accuracy_reward": 0.569018542766571, "rewards_raw/clip_reward_wrapper": 0.3257881700992584, "rewards_raw/format_reward": 5.0, "step": 538 }, { "Total_reward": 2.096806355211811e-07, "advantage_std": 1.500698447227478, "completion_length": 433.0535888671875, "epoch": 0.18611878453038674, "grad_norm": 3.208616569209803, "kl": 0.07177734375, "learning_rate": 4.58468129103435e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.5166475176811218, "rewards_raw/accuracy_reward": 0.5634322166442871, "rewards_raw/clip_reward_wrapper": 0.34071311354637146, "rewards_raw/format_reward": 5.0, "step": 539 }, { "Total_reward": -5.130257250129944e-07, "advantage_std": 1.3975672721862793, "completion_length": 444.5714416503906, "epoch": 0.18646408839779005, "grad_norm": 3.3855304954007788, "kl": 0.06787109375, "learning_rate": 4.583183151432527e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 1.2484323978424072, "rewards_raw/accuracy_reward": 0.5588210821151733, "rewards_raw/clip_reward_wrapper": 0.3322688639163971, "rewards_raw/format_reward": 5.0, "step": 540 }, { "Total_reward": -4.704509706243698e-07, "advantage_std": 1.2699360847473145, "completion_length": 434.1250305175781, "epoch": 0.18680939226519336, "grad_norm": 6.8206155554739025, "kl": 0.07421875, "learning_rate": 4.5816825603411775e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.03827302157878876, "rewards_raw/accuracy_reward": 0.5643227100372314, "rewards_raw/clip_reward_wrapper": 0.3324783444404602, "rewards_raw/format_reward": 5.0, "step": 541 }, { "Total_reward": -7.652810722902359e-07, "advantage_std": 1.3478643894195557, "completion_length": 437.33929443359375, "epoch": 0.1871546961325967, "grad_norm": 4.224579247413797, "kl": 0.07080078125, "learning_rate": 4.580179519526197e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.40674659609794617, "rewards_raw/accuracy_reward": 0.5679541826248169, "rewards_raw/clip_reward_wrapper": 0.33227309584617615, "rewards_raw/format_reward": 5.0, "step": 542 }, { "Total_reward": -2.4267606590910873e-07, "advantage_std": 1.4117798805236816, "completion_length": 437.4464416503906, "epoch": 0.1875, "grad_norm": 3.290427862631747, "kl": 0.072265625, "learning_rate": 4.578674030756363e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.29739123582839966, "rewards_raw/accuracy_reward": 0.5728576183319092, "rewards_raw/clip_reward_wrapper": 0.3361184298992157, "rewards_raw/format_reward": 5.0, "step": 543 }, { "Total_reward": -9.834766387939453e-07, "advantage_std": 1.1943503618240356, "completion_length": 437.14288330078125, "epoch": 0.1878453038674033, "grad_norm": 4.513237477875812, "kl": 0.07177734375, "learning_rate": 4.5771660958033354e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.2662326991558075, "rewards_raw/accuracy_reward": 0.574838399887085, "rewards_raw/clip_reward_wrapper": 0.3300763964653015, "rewards_raw/format_reward": 5.0, "step": 544 }, { "Total_reward": -1.0196652056038147e-06, "advantage_std": 1.5080921649932861, "completion_length": 420.8750305175781, "epoch": 0.18819060773480664, "grad_norm": 3.589241459785867, "kl": 0.0732421875, "learning_rate": 4.575655716441651e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.4559037685394287, "rewards_raw/accuracy_reward": 0.582497239112854, "rewards_raw/clip_reward_wrapper": 0.32803940773010254, "rewards_raw/format_reward": 5.0, "step": 545 }, { "Total_reward": -1.6391278734317893e-07, "advantage_std": 1.4110974073410034, "completion_length": 427.89288330078125, "epoch": 0.18853591160220995, "grad_norm": 5.178480422864582, "kl": 0.076171875, "learning_rate": 4.5741428944487246e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 1.0124791860580444, "rewards_raw/accuracy_reward": 0.5629420280456543, "rewards_raw/clip_reward_wrapper": 0.32834959030151367, "rewards_raw/format_reward": 5.0, "step": 546 }, { "Total_reward": 2.07551892117408e-07, "advantage_std": 1.6067851781845093, "completion_length": 437.3571472167969, "epoch": 0.18888121546961326, "grad_norm": 5.388923708791108, "kl": 0.0771484375, "learning_rate": 4.5726276316048444e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.7566378712654114, "rewards_raw/accuracy_reward": 0.5401927828788757, "rewards_raw/clip_reward_wrapper": 0.3242201805114746, "rewards_raw/format_reward": 5.0, "step": 547 }, { "Total_reward": -7.642167361154861e-07, "advantage_std": 1.5218032598495483, "completion_length": 462.3214416503906, "epoch": 0.18922651933701656, "grad_norm": 3.7881219968103355, "kl": 0.06982421875, "learning_rate": 4.571109929693172e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.5371250510215759, "rewards_raw/accuracy_reward": 0.5457433462142944, "rewards_raw/clip_reward_wrapper": 0.3304416835308075, "rewards_raw/format_reward": 4.901785850524902, "step": 548 }, { "Total_reward": -8.344650836988876e-07, "advantage_std": 1.449181079864502, "completion_length": 428.2321472167969, "epoch": 0.1895718232044199, "grad_norm": 3.914866608979685, "kl": 0.07421875, "learning_rate": 4.569589790499737e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.4881933927536011, "rewards_raw/accuracy_reward": 0.5639755129814148, "rewards_raw/clip_reward_wrapper": 0.33271723985671997, "rewards_raw/format_reward": 5.0, "step": 549 }, { "Total_reward": 2.0223005492425727e-08, "advantage_std": 1.3773967027664185, "completion_length": 460.1250305175781, "epoch": 0.1899171270718232, "grad_norm": 5.294528326908603, "kl": 0.06884765625, "learning_rate": 4.5680672158134416e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.4921768307685852, "rewards_raw/accuracy_reward": 0.5657678842544556, "rewards_raw/clip_reward_wrapper": 0.3395371735095978, "rewards_raw/format_reward": 5.0, "step": 550 }, { "Total_reward": 3.2995430387927627e-07, "advantage_std": 1.2851771116256714, "completion_length": 439.5000305175781, "epoch": 0.1902624309392265, "grad_norm": 5.942552018390594, "kl": 0.072265625, "learning_rate": 4.5665422074260497e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.40595772862434387, "rewards_raw/accuracy_reward": 0.5659180283546448, "rewards_raw/clip_reward_wrapper": 0.328233540058136, "rewards_raw/format_reward": 5.0, "step": 551 }, { "Total_reward": 5.406993182077713e-07, "advantage_std": 1.4985593557357788, "completion_length": 437.4821472167969, "epoch": 0.19060773480662985, "grad_norm": 6.31206275265564, "kl": 0.072265625, "learning_rate": 4.5650147671321905e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.29699596762657166, "rewards_raw/accuracy_reward": 0.54787278175354, "rewards_raw/clip_reward_wrapper": 0.3178224563598633, "rewards_raw/format_reward": 5.0, "step": 552 }, { "Total_reward": 1.2559551123558776e-06, "advantage_std": 1.3219753503799438, "completion_length": 442.51788330078125, "epoch": 0.19095303867403315, "grad_norm": 3.6385971167052706, "kl": 0.07177734375, "learning_rate": 4.563484896729356e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.45953255891799927, "rewards_raw/accuracy_reward": 0.5757498145103455, "rewards_raw/clip_reward_wrapper": 0.32392802834510803, "rewards_raw/format_reward": 5.0, "step": 553 }, { "Total_reward": 3.5337041026650695e-07, "advantage_std": 1.6540477275848389, "completion_length": 444.71429443359375, "epoch": 0.19129834254143646, "grad_norm": 4.485184193533417, "kl": 0.07421875, "learning_rate": 4.5619525980178977e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.7291872501373291, "rewards_raw/accuracy_reward": 0.5819417238235474, "rewards_raw/clip_reward_wrapper": 0.3317033052444458, "rewards_raw/format_reward": 5.0, "step": 554 }, { "Total_reward": -2.969588592804939e-07, "advantage_std": 1.4389969110488892, "completion_length": 454.14288330078125, "epoch": 0.19164364640883977, "grad_norm": 3.8557156932931003, "kl": 0.0703125, "learning_rate": 4.5604178728010244e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.6485934257507324, "rewards_raw/accuracy_reward": 0.5475444793701172, "rewards_raw/clip_reward_wrapper": 0.34420183300971985, "rewards_raw/format_reward": 5.0, "step": 555 }, { "Total_reward": -7.035476983219269e-07, "advantage_std": 1.5649664402008057, "completion_length": 435.14288330078125, "epoch": 0.1919889502762431, "grad_norm": 6.4666118339175895, "kl": 0.0712890625, "learning_rate": 4.5588807228848e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.6686511635780334, "rewards_raw/accuracy_reward": 0.5510862469673157, "rewards_raw/clip_reward_wrapper": 0.3294479250907898, "rewards_raw/format_reward": 5.0, "step": 556 }, { "Total_reward": -1.2069940567016602e-06, "advantage_std": 1.5376726388931274, "completion_length": 451.232177734375, "epoch": 0.1923342541436464, "grad_norm": 6.7728544728670625, "kl": 0.07275390625, "learning_rate": 4.557341150078142e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.4878000020980835, "rewards_raw/accuracy_reward": 0.5759657621383667, "rewards_raw/clip_reward_wrapper": 0.33545419573783875, "rewards_raw/format_reward": 5.0, "step": 557 }, { "Total_reward": 2.8312206268310547e-07, "advantage_std": 1.354146957397461, "completion_length": 442.9285888671875, "epoch": 0.19267955801104972, "grad_norm": 2.807924814239708, "kl": 0.0732421875, "learning_rate": 4.5557991561928206e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.16788062453269958, "rewards_raw/accuracy_reward": 0.5758447051048279, "rewards_raw/clip_reward_wrapper": 0.3260328471660614, "rewards_raw/format_reward": 5.0, "step": 558 }, { "Total_reward": 7.344144137277908e-07, "advantage_std": 1.3814142942428589, "completion_length": 444.5535888671875, "epoch": 0.19302486187845305, "grad_norm": 4.512666828353547, "kl": 0.07080078125, "learning_rate": 4.554254743043453e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.38721710443496704, "rewards_raw/accuracy_reward": 0.5736751556396484, "rewards_raw/clip_reward_wrapper": 0.3239467740058899, "rewards_raw/format_reward": 5.0, "step": 559 }, { "Total_reward": 6.599086077585525e-07, "advantage_std": 1.5481314659118652, "completion_length": 458.2500305175781, "epoch": 0.19337016574585636, "grad_norm": 6.661410158599612, "kl": 0.0732421875, "learning_rate": 4.5527079124475037e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.4461416006088257, "rewards_raw/accuracy_reward": 0.5701175928115845, "rewards_raw/clip_reward_wrapper": 0.3066847026348114, "rewards_raw/format_reward": 5.0, "step": 560 }, { "Total_reward": -1.1431319535404327e-06, "advantage_std": 1.6413952112197876, "completion_length": 446.3750305175781, "epoch": 0.19371546961325967, "grad_norm": 5.224171242616052, "kl": 0.06884765625, "learning_rate": 4.5511586662252845e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.34370407462120056, "rewards_raw/accuracy_reward": 0.5364734530448914, "rewards_raw/clip_reward_wrapper": 0.3253558874130249, "rewards_raw/format_reward": 5.0, "step": 561 }, { "Total_reward": 1.4688288274555816e-07, "advantage_std": 1.483642816543579, "completion_length": 425.3750305175781, "epoch": 0.19406077348066297, "grad_norm": 5.637940294217447, "kl": 0.07275390625, "learning_rate": 4.549607006199946e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.7376362681388855, "rewards_raw/accuracy_reward": 0.5501946210861206, "rewards_raw/clip_reward_wrapper": 0.3345881402492523, "rewards_raw/format_reward": 5.0, "step": 562 }, { "Total_reward": 4.087175966560608e-07, "advantage_std": 1.4340542554855347, "completion_length": 425.0357360839844, "epoch": 0.1944060773480663, "grad_norm": 30.232882886503702, "kl": 0.07275390625, "learning_rate": 4.548052934197482e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.4016592800617218, "rewards_raw/accuracy_reward": 0.5840474367141724, "rewards_raw/clip_reward_wrapper": 0.3200751841068268, "rewards_raw/format_reward": 5.0, "step": 563 }, { "Total_reward": -5.045107513979019e-07, "advantage_std": 1.5045115947723389, "completion_length": 455.1607360839844, "epoch": 0.19475138121546962, "grad_norm": 2.9607193696333383, "kl": 0.068359375, "learning_rate": 4.5464964520467254e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.4067589342594147, "rewards_raw/accuracy_reward": 0.5614211559295654, "rewards_raw/clip_reward_wrapper": 0.32862964272499084, "rewards_raw/format_reward": 5.0, "step": 564 }, { "Total_reward": 1.0239226639896515e-06, "advantage_std": 1.3538601398468018, "completion_length": 451.1964416503906, "epoch": 0.19509668508287292, "grad_norm": 4.470449161266737, "kl": 0.07177734375, "learning_rate": 4.544937561579343e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.49027907848358154, "rewards_raw/accuracy_reward": 0.5821580290794373, "rewards_raw/clip_reward_wrapper": 0.3272359073162079, "rewards_raw/format_reward": 5.0, "step": 565 }, { "Total_reward": 6.183981895446777e-07, "advantage_std": 1.6577861309051514, "completion_length": 434.0714416503906, "epoch": 0.19544198895027623, "grad_norm": 5.7953657464396215, "kl": 0.06884765625, "learning_rate": 4.5433762646298365e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 1.0073174238204956, "rewards_raw/accuracy_reward": 0.5923590660095215, "rewards_raw/clip_reward_wrapper": 0.32679319381713867, "rewards_raw/format_reward": 5.0, "step": 566 }, { "Total_reward": 1.2304102483540191e-06, "advantage_std": 1.2527692317962646, "completion_length": 450.51788330078125, "epoch": 0.19578729281767956, "grad_norm": 3.6107386729169795, "kl": 0.07421875, "learning_rate": 4.5418125630355396e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.41450273990631104, "rewards_raw/accuracy_reward": 0.5803530216217041, "rewards_raw/clip_reward_wrapper": 0.32985931634902954, "rewards_raw/format_reward": 5.0, "step": 567 }, { "Total_reward": -5.960464477539063e-08, "advantage_std": 1.5621482133865356, "completion_length": 452.3214416503906, "epoch": 0.19613259668508287, "grad_norm": 4.002246179289324, "kl": 0.07421875, "learning_rate": 4.5402464586366185e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.4815128445625305, "rewards_raw/accuracy_reward": 0.5705705881118774, "rewards_raw/clip_reward_wrapper": 0.3300084173679352, "rewards_raw/format_reward": 5.0, "step": 568 }, { "Total_reward": -2.0223005492425727e-08, "advantage_std": 1.3189817667007446, "completion_length": 450.5714416503906, "epoch": 0.19647790055248618, "grad_norm": 3.8305464608548925, "kl": 0.0712890625, "learning_rate": 4.538677953276061e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.6368156671524048, "rewards_raw/accuracy_reward": 0.561733067035675, "rewards_raw/clip_reward_wrapper": 0.32294192910194397, "rewards_raw/format_reward": 5.0, "step": 569 }, { "Total_reward": 1.596553005356327e-07, "advantage_std": 1.3277523517608643, "completion_length": 446.107177734375, "epoch": 0.1968232044198895, "grad_norm": 4.180199619216709, "kl": 0.07275390625, "learning_rate": 4.537107048799687e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.2531523108482361, "rewards_raw/accuracy_reward": 0.5603834986686707, "rewards_raw/clip_reward_wrapper": 0.3299972116947174, "rewards_raw/format_reward": 5.0, "step": 570 }, { "Total_reward": 1.4475413934178505e-07, "advantage_std": 1.4705581665039062, "completion_length": 444.08929443359375, "epoch": 0.19716850828729282, "grad_norm": 3.9377890787417997, "kl": 0.07275390625, "learning_rate": 4.5355337470561343e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.33396998047828674, "rewards_raw/accuracy_reward": 0.5924159288406372, "rewards_raw/clip_reward_wrapper": 0.321883887052536, "rewards_raw/format_reward": 5.0, "step": 571 }, { "Total_reward": 2.9163703629819793e-07, "advantage_std": 1.3978301286697388, "completion_length": 454.357177734375, "epoch": 0.19751381215469613, "grad_norm": 4.2063483400102015, "kl": 0.072265625, "learning_rate": 4.533958049896865e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.39008083939552307, "rewards_raw/accuracy_reward": 0.5484225749969482, "rewards_raw/clip_reward_wrapper": 0.33511030673980713, "rewards_raw/format_reward": 5.0, "step": 572 }, { "Total_reward": -3.2782557468635787e-07, "advantage_std": 1.3355250358581543, "completion_length": 449.0357360839844, "epoch": 0.19785911602209943, "grad_norm": 8.185890591041778, "kl": 0.0693359375, "learning_rate": 4.53237995917616e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.4988299310207367, "rewards_raw/accuracy_reward": 0.5637601613998413, "rewards_raw/clip_reward_wrapper": 0.33023473620414734, "rewards_raw/format_reward": 5.0, "step": 573 }, { "Total_reward": -7.91890329310263e-07, "advantage_std": 1.6112802028656006, "completion_length": 457.96429443359375, "epoch": 0.19820441988950277, "grad_norm": 9.470079373224353, "kl": 0.07080078125, "learning_rate": 4.530799476751116e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.521317720413208, "rewards_raw/accuracy_reward": 0.5626752972602844, "rewards_raw/clip_reward_wrapper": 0.32033246755599976, "rewards_raw/format_reward": 5.0, "step": 574 }, { "Total_reward": 1.0771411780297058e-06, "advantage_std": 1.6035364866256714, "completion_length": 445.5535888671875, "epoch": 0.19854972375690608, "grad_norm": 3.2961319847843407, "kl": 0.07421875, "learning_rate": 4.529216604481643e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.2915028929710388, "rewards_raw/accuracy_reward": 0.5753402709960938, "rewards_raw/clip_reward_wrapper": 0.320139616727829, "rewards_raw/format_reward": 5.0, "step": 575 }, { "Total_reward": -5.396349251896027e-07, "advantage_std": 1.674692988395691, "completion_length": 430.0357360839844, "epoch": 0.19889502762430938, "grad_norm": 4.559104326548097, "kl": 0.07421875, "learning_rate": 4.527631344230465e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.45805978775024414, "rewards_raw/accuracy_reward": 0.5838735699653625, "rewards_raw/clip_reward_wrapper": 0.3245656192302704, "rewards_raw/format_reward": 5.0, "step": 576 }, { "Total_reward": 2.1287374085687816e-09, "advantage_std": 1.5110087394714355, "completion_length": 448.89288330078125, "epoch": 0.19924033149171272, "grad_norm": 5.808823949565064, "kl": 0.07080078125, "learning_rate": 4.526043697863117e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.32865554094314575, "rewards_raw/accuracy_reward": 0.5465580821037292, "rewards_raw/clip_reward_wrapper": 0.34932199120521545, "rewards_raw/format_reward": 5.0, "step": 577 }, { "Total_reward": 1.6391278734317893e-07, "advantage_std": 1.7621270418167114, "completion_length": 441.607177734375, "epoch": 0.19958563535911603, "grad_norm": 5.007672604877489, "kl": 0.07568359375, "learning_rate": 4.5244536672479405e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.5990458726882935, "rewards_raw/accuracy_reward": 0.5853875279426575, "rewards_raw/clip_reward_wrapper": 0.3250400722026825, "rewards_raw/format_reward": 5.0, "step": 578 }, { "Total_reward": -5.89660260175151e-07, "advantage_std": 1.5188497304916382, "completion_length": 454.2857360839844, "epoch": 0.19993093922651933, "grad_norm": 11.37335241187051, "kl": 0.0712890625, "learning_rate": 4.522861254256083e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.392141193151474, "rewards_raw/accuracy_reward": 0.5749825239181519, "rewards_raw/clip_reward_wrapper": 0.33215564489364624, "rewards_raw/format_reward": 5.0, "step": 579 }, { "Total_reward": 7.791178973093338e-07, "advantage_std": 1.379551649093628, "completion_length": 452.357177734375, "epoch": 0.20027624309392264, "grad_norm": 3.1029747387500715, "kl": 0.0751953125, "learning_rate": 4.5212664607614966e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.45978105068206787, "rewards_raw/accuracy_reward": 0.5816820859909058, "rewards_raw/clip_reward_wrapper": 0.3387944996356964, "rewards_raw/format_reward": 5.0, "step": 580 }, { "Total_reward": 1.532690987460228e-07, "advantage_std": 1.580193042755127, "completion_length": 432.83929443359375, "epoch": 0.20062154696132597, "grad_norm": 5.7713338539462935, "kl": 0.072265625, "learning_rate": 4.5196692886409327e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.4161943793296814, "rewards_raw/accuracy_reward": 0.5907906293869019, "rewards_raw/clip_reward_wrapper": 0.326529324054718, "rewards_raw/format_reward": 5.0, "step": 581 }, { "Total_reward": -7.450581165358017e-07, "advantage_std": 1.391299843788147, "completion_length": 446.3214416503906, "epoch": 0.20096685082872928, "grad_norm": 4.5797576834909455, "kl": 0.068359375, "learning_rate": 4.5180697397739444e-07, "loss": 0.0027, "rewards_raw/CoT ICLG": 0.5870707631111145, "rewards_raw/accuracy_reward": 0.5573599934577942, "rewards_raw/clip_reward_wrapper": 0.32540977001190186, "rewards_raw/format_reward": 5.0, "step": 582 }, { "Total_reward": -2.8099333349018707e-07, "advantage_std": 1.3728832006454468, "completion_length": 458.0535888671875, "epoch": 0.2013121546961326, "grad_norm": 3.644150577870711, "kl": 0.072265625, "learning_rate": 4.5164678160428794e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.8093313574790955, "rewards_raw/accuracy_reward": 0.5706555247306824, "rewards_raw/clip_reward_wrapper": 0.3306457996368408, "rewards_raw/format_reward": 5.0, "step": 583 }, { "Total_reward": -7.535730333074753e-07, "advantage_std": 1.4921663999557495, "completion_length": 450.71429443359375, "epoch": 0.20165745856353592, "grad_norm": 4.268721681891539, "kl": 0.07421875, "learning_rate": 4.5148635193328814e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.6584336161613464, "rewards_raw/accuracy_reward": 0.5680747628211975, "rewards_raw/clip_reward_wrapper": 0.32514309883117676, "rewards_raw/format_reward": 5.0, "step": 584 }, { "Total_reward": 2.5119101110249176e-07, "advantage_std": 1.778481364250183, "completion_length": 453.9464416503906, "epoch": 0.20200276243093923, "grad_norm": 6.298736136992049, "kl": 0.0693359375, "learning_rate": 4.5132568515318874e-07, "loss": 0.0028, "rewards_raw/CoT ICLG": 0.2516214847564697, "rewards_raw/accuracy_reward": 0.5739642977714539, "rewards_raw/clip_reward_wrapper": 0.31664523482322693, "rewards_raw/format_reward": 5.0, "step": 585 }, { "Total_reward": 4.427773774295929e-07, "advantage_std": 1.2656824588775635, "completion_length": 436.96429443359375, "epoch": 0.20234806629834254, "grad_norm": 4.25832100144064, "kl": 0.07470703125, "learning_rate": 4.511647814530623e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.5058553218841553, "rewards_raw/accuracy_reward": 0.5632805824279785, "rewards_raw/clip_reward_wrapper": 0.3268832564353943, "rewards_raw/format_reward": 5.0, "step": 586 }, { "Total_reward": -3.0440943987741775e-07, "advantage_std": 1.5666447877883911, "completion_length": 435.5357360839844, "epoch": 0.20269337016574585, "grad_norm": 5.799413219998373, "kl": 0.07421875, "learning_rate": 4.510036410222603e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.6061381697654724, "rewards_raw/accuracy_reward": 0.5576309561729431, "rewards_raw/clip_reward_wrapper": 0.33206018805503845, "rewards_raw/format_reward": 5.0, "step": 587 }, { "Total_reward": -3.7891524584665603e-07, "advantage_std": 1.4686052799224854, "completion_length": 437.6071472167969, "epoch": 0.20303867403314918, "grad_norm": 10.120117141895633, "kl": 0.076171875, "learning_rate": 4.508422640504127e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.646115779876709, "rewards_raw/accuracy_reward": 0.554591178894043, "rewards_raw/clip_reward_wrapper": 0.34062620997428894, "rewards_raw/format_reward": 5.0, "step": 588 }, { "Total_reward": -1.27724248955019e-07, "advantage_std": 1.5388007164001465, "completion_length": 431.3035888671875, "epoch": 0.2033839779005525, "grad_norm": 5.674648122838814, "kl": 0.07177734375, "learning_rate": 4.5068065072742796e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.7272828221321106, "rewards_raw/accuracy_reward": 0.5928778052330017, "rewards_raw/clip_reward_wrapper": 0.3310244083404541, "rewards_raw/format_reward": 5.0, "step": 589 }, { "Total_reward": -6.258487701416016e-07, "advantage_std": 1.3506261110305786, "completion_length": 422.39288330078125, "epoch": 0.2037292817679558, "grad_norm": 5.905071871038781, "kl": 0.07568359375, "learning_rate": 4.5051880124349257e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.17147554457187653, "rewards_raw/accuracy_reward": 0.5849850177764893, "rewards_raw/clip_reward_wrapper": 0.33610662817955017, "rewards_raw/format_reward": 5.0, "step": 590 }, { "Total_reward": 4.725796998172882e-07, "advantage_std": 1.3407179117202759, "completion_length": 440.8035888671875, "epoch": 0.20407458563535913, "grad_norm": 4.1220377075578165, "kl": 0.07373046875, "learning_rate": 4.5035671578907097e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.4305424988269806, "rewards_raw/accuracy_reward": 0.5667902231216431, "rewards_raw/clip_reward_wrapper": 0.3232891261577606, "rewards_raw/format_reward": 5.0, "step": 591 }, { "Total_reward": 2.1287374352141342e-07, "advantage_std": 1.5223674774169922, "completion_length": 435.2857360839844, "epoch": 0.20441988950276244, "grad_norm": 3.507940688021166, "kl": 0.076171875, "learning_rate": 4.501943945549053e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.7317143082618713, "rewards_raw/accuracy_reward": 0.5901099443435669, "rewards_raw/clip_reward_wrapper": 0.33160966634750366, "rewards_raw/format_reward": 5.0, "step": 592 }, { "Total_reward": -5.832740725963959e-07, "advantage_std": 1.45575749874115, "completion_length": 434.8571472167969, "epoch": 0.20476519337016574, "grad_norm": 5.311765060740555, "kl": 0.07568359375, "learning_rate": 4.500318377320152e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.6180932521820068, "rewards_raw/accuracy_reward": 0.5634167790412903, "rewards_raw/clip_reward_wrapper": 0.33383357524871826, "rewards_raw/format_reward": 5.0, "step": 593 }, { "Total_reward": -2.4054733671619033e-07, "advantage_std": 1.3330860137939453, "completion_length": 448.9107360839844, "epoch": 0.20511049723756905, "grad_norm": 3.2633681073239535, "kl": 0.076171875, "learning_rate": 4.4986904551169744e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.6148188710212708, "rewards_raw/accuracy_reward": 0.5661656856536865, "rewards_raw/clip_reward_wrapper": 0.32632482051849365, "rewards_raw/format_reward": 5.0, "step": 594 }, { "Total_reward": 5.598579377874557e-07, "advantage_std": 1.4137859344482422, "completion_length": 434.3035888671875, "epoch": 0.20545580110497239, "grad_norm": 4.888679785434669, "kl": 0.07421875, "learning_rate": 4.497060180855259e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.2793789803981781, "rewards_raw/accuracy_reward": 0.5926229357719421, "rewards_raw/clip_reward_wrapper": 0.3306388258934021, "rewards_raw/format_reward": 5.0, "step": 595 }, { "Total_reward": 5.981752337902435e-07, "advantage_std": 1.5748389959335327, "completion_length": 423.0714416503906, "epoch": 0.2058011049723757, "grad_norm": 3.7530713226030654, "kl": 0.07861328125, "learning_rate": 4.4954275564535125e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.5430573225021362, "rewards_raw/accuracy_reward": 0.5601636171340942, "rewards_raw/clip_reward_wrapper": 0.33267033100128174, "rewards_raw/format_reward": 5.0, "step": 596 }, { "Total_reward": 4.5342108023760375e-07, "advantage_std": 1.449849247932434, "completion_length": 441.0714416503906, "epoch": 0.206146408839779, "grad_norm": 3.874952776373435, "kl": 0.0751953125, "learning_rate": 4.4937925838330065e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.6293926239013672, "rewards_raw/accuracy_reward": 0.5914362072944641, "rewards_raw/clip_reward_wrapper": 0.3285520672798157, "rewards_raw/format_reward": 5.0, "step": 597 }, { "Total_reward": 1.1175871605928478e-07, "advantage_std": 1.443724274635315, "completion_length": 436.3035888671875, "epoch": 0.2064917127071823, "grad_norm": 3.3514322031716226, "kl": 0.0712890625, "learning_rate": 4.4921552649177766e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.3261721134185791, "rewards_raw/accuracy_reward": 0.5682811141014099, "rewards_raw/clip_reward_wrapper": 0.3273636996746063, "rewards_raw/format_reward": 5.0, "step": 598 }, { "Total_reward": -2.2883926931172027e-07, "advantage_std": 1.5039794445037842, "completion_length": 421.46429443359375, "epoch": 0.20683701657458564, "grad_norm": 16.05609134029662, "kl": 0.07666015625, "learning_rate": 4.4905156016346196e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.8422358632087708, "rewards_raw/accuracy_reward": 0.5538257360458374, "rewards_raw/clip_reward_wrapper": 0.3284357190132141, "rewards_raw/format_reward": 5.0, "step": 599 }, { "Total_reward": -3.5124168107358855e-07, "advantage_std": 1.4941272735595703, "completion_length": 442.6250305175781, "epoch": 0.20718232044198895, "grad_norm": 4.880429122670788, "kl": 0.078125, "learning_rate": 4.488873595913091e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.28842175006866455, "rewards_raw/accuracy_reward": 0.5636979937553406, "rewards_raw/clip_reward_wrapper": 0.3321177065372467, "rewards_raw/format_reward": 5.0, "step": 600 }, { "Total_reward": 6.38621244775095e-09, "advantage_std": 1.7334030866622925, "completion_length": 438.08929443359375, "epoch": 0.20752762430939226, "grad_norm": 9.090873602037727, "kl": 0.07958984375, "learning_rate": 4.4872292496855023e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.5243692398071289, "rewards_raw/accuracy_reward": 0.5586430430412292, "rewards_raw/clip_reward_wrapper": 0.31770408153533936, "rewards_raw/format_reward": 5.0, "step": 601 }, { "Total_reward": -1.575265713427143e-07, "advantage_std": 1.5972225666046143, "completion_length": 432.7321472167969, "epoch": 0.2078729281767956, "grad_norm": 5.526497064605558, "kl": 0.0751953125, "learning_rate": 4.4855825648869197e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.6765104532241821, "rewards_raw/accuracy_reward": 0.5846251249313354, "rewards_raw/clip_reward_wrapper": 0.32696864008903503, "rewards_raw/format_reward": 5.0, "step": 602 }, { "Total_reward": 2.8312206268310547e-07, "advantage_std": 1.3453279733657837, "completion_length": 442.107177734375, "epoch": 0.2082182320441989, "grad_norm": 3.6559634695036376, "kl": 0.07470703125, "learning_rate": 4.483933543455163e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.2403029501438141, "rewards_raw/accuracy_reward": 0.5492545366287231, "rewards_raw/clip_reward_wrapper": 0.32868465781211853, "rewards_raw/format_reward": 5.0, "step": 603 }, { "Total_reward": -2.852508202977333e-07, "advantage_std": 1.7174460887908936, "completion_length": 420.3571472167969, "epoch": 0.2085635359116022, "grad_norm": 5.192348174206617, "kl": 0.0771484375, "learning_rate": 4.4822821873307984e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.6209709644317627, "rewards_raw/accuracy_reward": 0.5664071440696716, "rewards_raw/clip_reward_wrapper": 0.3225328028202057, "rewards_raw/format_reward": 5.0, "step": 604 }, { "Total_reward": 8.621386768936645e-07, "advantage_std": 1.6530317068099976, "completion_length": 428.5000305175781, "epoch": 0.2089088397790055, "grad_norm": 4.06591639029775, "kl": 0.07568359375, "learning_rate": 4.4806284984571424e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 2.406606435775757, "rewards_raw/accuracy_reward": 0.5713672637939453, "rewards_raw/clip_reward_wrapper": 0.32244881987571716, "rewards_raw/format_reward": 5.0, "step": 605 }, { "Total_reward": 2.2671054011880187e-07, "advantage_std": 1.4203745126724243, "completion_length": 454.26788330078125, "epoch": 0.20925414364640885, "grad_norm": 5.7252086495645615, "kl": 0.072265625, "learning_rate": 4.478972478780256e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.7120653986930847, "rewards_raw/accuracy_reward": 0.548219621181488, "rewards_raw/clip_reward_wrapper": 0.32987362146377563, "rewards_raw/format_reward": 5.0, "step": 606 }, { "Total_reward": -3.4858075537158584e-08, "advantage_std": 1.4815577268600464, "completion_length": 424.9464416503906, "epoch": 0.20959944751381215, "grad_norm": 4.682717871121193, "kl": 0.07373046875, "learning_rate": 4.4773141302489424e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.617394208908081, "rewards_raw/accuracy_reward": 0.5900124907493591, "rewards_raw/clip_reward_wrapper": 0.32755225896835327, "rewards_raw/format_reward": 5.0, "step": 607 }, { "Total_reward": 7.929546654850128e-07, "advantage_std": 1.4898664951324463, "completion_length": 427.5357360839844, "epoch": 0.20994475138121546, "grad_norm": 3.3612738137636335, "kl": 0.0771484375, "learning_rate": 4.475653454814746e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.4742981791496277, "rewards_raw/accuracy_reward": 0.5520723462104797, "rewards_raw/clip_reward_wrapper": 0.33865848183631897, "rewards_raw/format_reward": 5.0, "step": 608 }, { "Total_reward": -2.575772271029564e-07, "advantage_std": 1.5968034267425537, "completion_length": 434.9285888671875, "epoch": 0.2102900552486188, "grad_norm": 5.036708443779723, "kl": 0.07177734375, "learning_rate": 4.473990454431949e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.5250948667526245, "rewards_raw/accuracy_reward": 0.5406753420829773, "rewards_raw/clip_reward_wrapper": 0.33607012033462524, "rewards_raw/format_reward": 5.0, "step": 609 }, { "Total_reward": -6.567154855474655e-07, "advantage_std": 1.5653061866760254, "completion_length": 436.51788330078125, "epoch": 0.2106353591160221, "grad_norm": 3.7110666551559346, "kl": 0.076171875, "learning_rate": 4.4723251310575707e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.28038400411605835, "rewards_raw/accuracy_reward": 0.5635663270950317, "rewards_raw/clip_reward_wrapper": 0.341429740190506, "rewards_raw/format_reward": 5.0, "step": 610 }, { "Total_reward": -2.565128625064972e-07, "advantage_std": 1.4977200031280518, "completion_length": 419.2500305175781, "epoch": 0.2109806629834254, "grad_norm": 5.296018373988461, "kl": 0.07861328125, "learning_rate": 4.4706574866513625e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.6681316494941711, "rewards_raw/accuracy_reward": 0.574141263961792, "rewards_raw/clip_reward_wrapper": 0.34618884325027466, "rewards_raw/format_reward": 5.0, "step": 611 }, { "Total_reward": 2.2138868871479644e-07, "advantage_std": 1.5239766836166382, "completion_length": 439.357177734375, "epoch": 0.21132596685082872, "grad_norm": 4.703863816395777, "kl": 0.07421875, "learning_rate": 4.468987523175808e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.2659923732280731, "rewards_raw/accuracy_reward": 0.5580873489379883, "rewards_raw/clip_reward_wrapper": 0.3249052166938782, "rewards_raw/format_reward": 5.0, "step": 612 }, { "Total_reward": -4.18296906445903e-07, "advantage_std": 1.4604591131210327, "completion_length": 428.1071472167969, "epoch": 0.21167127071823205, "grad_norm": 3.937580508328592, "kl": 0.076171875, "learning_rate": 4.46731524259612e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 1.3995569944381714, "rewards_raw/accuracy_reward": 0.5568622946739197, "rewards_raw/clip_reward_wrapper": 0.33735665678977966, "rewards_raw/format_reward": 5.0, "step": 613 }, { "Total_reward": 4.5767853862344055e-07, "advantage_std": 1.4965286254882812, "completion_length": 449.6250305175781, "epoch": 0.21201657458563536, "grad_norm": 5.012158052591613, "kl": 0.07470703125, "learning_rate": 4.4656406468802387e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.2719619572162628, "rewards_raw/accuracy_reward": 0.5930473208427429, "rewards_raw/clip_reward_wrapper": 0.33446913957595825, "rewards_raw/format_reward": 5.0, "step": 614 }, { "Total_reward": -7.109982789188507e-07, "advantage_std": 1.5100198984146118, "completion_length": 433.3571472167969, "epoch": 0.21236187845303867, "grad_norm": 23.228133008675744, "kl": 0.07470703125, "learning_rate": 4.463963737998828e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.5456578731536865, "rewards_raw/accuracy_reward": 0.5733681917190552, "rewards_raw/clip_reward_wrapper": 0.32383328676223755, "rewards_raw/format_reward": 5.0, "step": 615 }, { "Total_reward": -4.725796998172882e-07, "advantage_std": 1.2173097133636475, "completion_length": 426.26788330078125, "epoch": 0.212707182320442, "grad_norm": 3.64229934715252, "kl": 0.0732421875, "learning_rate": 4.4622845179252726e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.873700737953186, "rewards_raw/accuracy_reward": 0.5901567339897156, "rewards_raw/clip_reward_wrapper": 0.32417216897010803, "rewards_raw/format_reward": 5.0, "step": 616 }, { "Total_reward": -3.693359360568138e-07, "advantage_std": 1.58429753780365, "completion_length": 437.6071472167969, "epoch": 0.2130524861878453, "grad_norm": 3.925871301029353, "kl": 0.07421875, "learning_rate": 4.46060298863568e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.7620180249214172, "rewards_raw/accuracy_reward": 0.5415608286857605, "rewards_raw/clip_reward_wrapper": 0.33776673674583435, "rewards_raw/format_reward": 5.0, "step": 617 }, { "Total_reward": -3.437911004766647e-07, "advantage_std": 1.4334747791290283, "completion_length": 421.9464416503906, "epoch": 0.21339779005524862, "grad_norm": 3.6972607745134884, "kl": 0.07763671875, "learning_rate": 4.458919152108872e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.458770751953125, "rewards_raw/accuracy_reward": 0.5749247074127197, "rewards_raw/clip_reward_wrapper": 0.33290615677833557, "rewards_raw/format_reward": 5.0, "step": 618 }, { "Total_reward": -1.2006079259663238e-06, "advantage_std": 1.5340938568115234, "completion_length": 439.2500305175781, "epoch": 0.21374309392265192, "grad_norm": 6.91162339051431, "kl": 0.07470703125, "learning_rate": 4.4572330103263883e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.26388636231422424, "rewards_raw/accuracy_reward": 0.5640138983726501, "rewards_raw/clip_reward_wrapper": 0.3301646411418915, "rewards_raw/format_reward": 5.0, "step": 619 }, { "Total_reward": -2.0116569032779807e-07, "advantage_std": 1.4269688129425049, "completion_length": 437.8750305175781, "epoch": 0.21408839779005526, "grad_norm": 3.155748569734843, "kl": 0.0771484375, "learning_rate": 4.455544565272479e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.7447850108146667, "rewards_raw/accuracy_reward": 0.5792397260665894, "rewards_raw/clip_reward_wrapper": 0.3221827745437622, "rewards_raw/format_reward": 5.0, "step": 620 }, { "Total_reward": -2.054231629244896e-07, "advantage_std": 1.487518072128296, "completion_length": 429.9285888671875, "epoch": 0.21443370165745856, "grad_norm": 4.470643682420728, "kl": 0.078125, "learning_rate": 4.453853818934108e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.3314288556575775, "rewards_raw/accuracy_reward": 0.5768072605133057, "rewards_raw/clip_reward_wrapper": 0.33020415902137756, "rewards_raw/format_reward": 5.0, "step": 621 }, { "Total_reward": -1.0643687176070671e-07, "advantage_std": 1.6269367933273315, "completion_length": 455.1964416503906, "epoch": 0.21477900552486187, "grad_norm": 8.28716259088619, "kl": 0.0751953125, "learning_rate": 4.4521607733009427e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.29192477464675903, "rewards_raw/accuracy_reward": 0.5736732482910156, "rewards_raw/clip_reward_wrapper": 0.321793794631958, "rewards_raw/format_reward": 5.0, "step": 622 }, { "Total_reward": 1.021793991640152e-06, "advantage_std": 1.6295397281646729, "completion_length": 434.6785888671875, "epoch": 0.2151243093922652, "grad_norm": 3.4366205657054967, "kl": 0.076171875, "learning_rate": 4.4504654303653606e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.34127700328826904, "rewards_raw/accuracy_reward": 0.5819838643074036, "rewards_raw/clip_reward_wrapper": 0.33707746863365173, "rewards_raw/format_reward": 5.0, "step": 623 }, { "Total_reward": 5.534717217869911e-08, "advantage_std": 1.4871467351913452, "completion_length": 438.6071472167969, "epoch": 0.2154696132596685, "grad_norm": 10.425988556573312, "kl": 0.07470703125, "learning_rate": 4.4487677921224397e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.273954838514328, "rewards_raw/accuracy_reward": 0.5757924914360046, "rewards_raw/clip_reward_wrapper": 0.33398476243019104, "rewards_raw/format_reward": 5.0, "step": 624 }, { "Total_reward": 3.970095292515907e-07, "advantage_std": 1.503245234489441, "completion_length": 429.4464416503906, "epoch": 0.21581491712707182, "grad_norm": 4.953332087897882, "kl": 0.07763671875, "learning_rate": 4.4470678605699616e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.8326533436775208, "rewards_raw/accuracy_reward": 0.5762260556221008, "rewards_raw/clip_reward_wrapper": 0.336600661277771, "rewards_raw/format_reward": 5.0, "step": 625 }, { "Total_reward": 5.683729114025482e-07, "advantage_std": 1.6528730392456055, "completion_length": 440.6607360839844, "epoch": 0.21616022099447513, "grad_norm": 6.261531205916692, "kl": 0.0751953125, "learning_rate": 4.445365637708406e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.30316177010536194, "rewards_raw/accuracy_reward": 0.5616545677185059, "rewards_raw/clip_reward_wrapper": 0.3259187638759613, "rewards_raw/format_reward": 5.0, "step": 626 }, { "Total_reward": -5.960464477539062e-07, "advantage_std": 1.620489239692688, "completion_length": 438.7857360839844, "epoch": 0.21650552486187846, "grad_norm": 10.271055430111863, "kl": 0.07275390625, "learning_rate": 4.443661125540946e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.36034542322158813, "rewards_raw/accuracy_reward": 0.5708785057067871, "rewards_raw/clip_reward_wrapper": 0.3330661356449127, "rewards_raw/format_reward": 5.0, "step": 627 }, { "Total_reward": -7.972121807142685e-07, "advantage_std": 1.4585657119750977, "completion_length": 440.4285888671875, "epoch": 0.21685082872928177, "grad_norm": 9.1686695285429, "kl": 0.07275390625, "learning_rate": 4.4419543260734525e-07, "loss": 0.0029, "rewards_raw/CoT ICLG": 0.6441864371299744, "rewards_raw/accuracy_reward": 0.5809730291366577, "rewards_raw/clip_reward_wrapper": 0.32665449380874634, "rewards_raw/format_reward": 5.0, "step": 628 }, { "Total_reward": 1.4901161193847656e-07, "advantage_std": 1.4075983762741089, "completion_length": 456.1607360839844, "epoch": 0.21719613259668508, "grad_norm": 3.3872519979945843, "kl": 0.07421875, "learning_rate": 4.4402452413144874e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.38377144932746887, "rewards_raw/accuracy_reward": 0.5771465301513672, "rewards_raw/clip_reward_wrapper": 0.33686357736587524, "rewards_raw/format_reward": 5.0, "step": 629 }, { "Total_reward": -4.981245638191467e-07, "advantage_std": 1.4665706157684326, "completion_length": 444.83929443359375, "epoch": 0.2175414364640884, "grad_norm": 4.149082661891975, "kl": 0.07421875, "learning_rate": 4.4385338732752996e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 1.4072728157043457, "rewards_raw/accuracy_reward": 0.5536662340164185, "rewards_raw/clip_reward_wrapper": 0.33063602447509766, "rewards_raw/format_reward": 5.0, "step": 630 }, { "Total_reward": 3.17181871878347e-07, "advantage_std": 1.2300593852996826, "completion_length": 438.7321472167969, "epoch": 0.21788674033149172, "grad_norm": 3.408620960571551, "kl": 0.078125, "learning_rate": 4.4368202239698294e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 1.0489017963409424, "rewards_raw/accuracy_reward": 0.5420864820480347, "rewards_raw/clip_reward_wrapper": 0.3326546847820282, "rewards_raw/format_reward": 5.0, "step": 631 }, { "Total_reward": -5.226050348028366e-07, "advantage_std": 1.4751052856445312, "completion_length": 438.0714416503906, "epoch": 0.21823204419889503, "grad_norm": 4.055488374902955, "kl": 0.0751953125, "learning_rate": 4.435104295414697e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.7911843657493591, "rewards_raw/accuracy_reward": 0.547519326210022, "rewards_raw/clip_reward_wrapper": 0.3176402449607849, "rewards_raw/format_reward": 5.0, "step": 632 }, { "Total_reward": -2.043587983280304e-07, "advantage_std": 1.470329999923706, "completion_length": 438.08929443359375, "epoch": 0.21857734806629833, "grad_norm": 5.047374987990915, "kl": 0.0771484375, "learning_rate": 4.4333860896292064e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.33412060141563416, "rewards_raw/accuracy_reward": 0.5763630270957947, "rewards_raw/clip_reward_wrapper": 0.3336237668991089, "rewards_raw/format_reward": 5.0, "step": 633 }, { "Total_reward": -2.384185791015625e-07, "advantage_std": 1.5354790687561035, "completion_length": 430.9464416503906, "epoch": 0.21892265193370167, "grad_norm": 4.7100973114833735, "kl": 0.07861328125, "learning_rate": 4.431665608635342e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.3910280764102936, "rewards_raw/accuracy_reward": 0.5763459205627441, "rewards_raw/clip_reward_wrapper": 0.3274538218975067, "rewards_raw/format_reward": 5.0, "step": 634 }, { "Total_reward": -5.534717502087005e-07, "advantage_std": 1.429479956626892, "completion_length": 449.0535888671875, "epoch": 0.21926795580110497, "grad_norm": 4.47843645453551, "kl": 0.07568359375, "learning_rate": 4.4299428544577677e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.6349534392356873, "rewards_raw/accuracy_reward": 0.5670593976974487, "rewards_raw/clip_reward_wrapper": 0.3410910665988922, "rewards_raw/format_reward": 5.0, "step": 635 }, { "Total_reward": 6.418143243536178e-07, "advantage_std": 1.4461170434951782, "completion_length": 435.9464416503906, "epoch": 0.21961325966850828, "grad_norm": 20.841736930413553, "kl": 0.0751953125, "learning_rate": 4.4282178291238175e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.6270133256912231, "rewards_raw/accuracy_reward": 0.5793912410736084, "rewards_raw/clip_reward_wrapper": 0.33558282256126404, "rewards_raw/format_reward": 5.0, "step": 636 }, { "Total_reward": -4.832234026252991e-07, "advantage_std": 1.4924176931381226, "completion_length": 435.4285888671875, "epoch": 0.2199585635359116, "grad_norm": 4.994843006818974, "kl": 0.07470703125, "learning_rate": 4.426490534663503e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.6683196425437927, "rewards_raw/accuracy_reward": 0.571051299571991, "rewards_raw/clip_reward_wrapper": 0.33758580684661865, "rewards_raw/format_reward": 5.0, "step": 637 }, { "Total_reward": 5.194119125917496e-07, "advantage_std": 1.4232367277145386, "completion_length": 449.857177734375, "epoch": 0.22030386740331492, "grad_norm": 3.3779153734165805, "kl": 0.07666015625, "learning_rate": 4.424760973109502e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.39166322350502014, "rewards_raw/accuracy_reward": 0.5741276144981384, "rewards_raw/clip_reward_wrapper": 0.335317462682724, "rewards_raw/format_reward": 5.0, "step": 638 }, { "Total_reward": -1.788139485370266e-07, "advantage_std": 1.662345290184021, "completion_length": 430.9821472167969, "epoch": 0.22064917127071823, "grad_norm": 4.806580252504798, "kl": 0.07421875, "learning_rate": 4.423029146497164e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.3911301791667938, "rewards_raw/accuracy_reward": 0.5544816255569458, "rewards_raw/clip_reward_wrapper": 0.33270347118377686, "rewards_raw/format_reward": 5.0, "step": 639 }, { "Total_reward": -4.747084290102066e-07, "advantage_std": 1.5004146099090576, "completion_length": 440.8750305175781, "epoch": 0.22099447513812154, "grad_norm": 4.758337635540029, "kl": 0.07568359375, "learning_rate": 4.4212950568645e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.5704633593559265, "rewards_raw/accuracy_reward": 0.5424942970275879, "rewards_raw/clip_reward_wrapper": 0.34348174929618835, "rewards_raw/format_reward": 5.0, "step": 640 }, { "Total_reward": 4.5554980943052215e-07, "advantage_std": 1.4604436159133911, "completion_length": 428.2857360839844, "epoch": 0.22133977900552487, "grad_norm": 4.024897587077256, "kl": 0.07470703125, "learning_rate": 4.419558706252189e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.5083439350128174, "rewards_raw/accuracy_reward": 0.5716124773025513, "rewards_raw/clip_reward_wrapper": 0.3403078019618988, "rewards_raw/format_reward": 5.0, "step": 641 }, { "Total_reward": -3.7465778746081924e-07, "advantage_std": 1.5637894868850708, "completion_length": 432.6071472167969, "epoch": 0.22168508287292818, "grad_norm": 7.48373573924043, "kl": 0.0771484375, "learning_rate": 4.417820096703567e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 1.0371713638305664, "rewards_raw/accuracy_reward": 0.5928967595100403, "rewards_raw/clip_reward_wrapper": 0.3304900527000427, "rewards_raw/format_reward": 5.0, "step": 642 }, { "Total_reward": 6.662947953373077e-07, "advantage_std": 1.5090241432189941, "completion_length": 427.0535888671875, "epoch": 0.2220303867403315, "grad_norm": 5.746226772346815, "kl": 0.07958984375, "learning_rate": 4.416079230264629e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.5062500834465027, "rewards_raw/accuracy_reward": 0.5876129269599915, "rewards_raw/clip_reward_wrapper": 0.34069469571113586, "rewards_raw/format_reward": 5.0, "step": 643 }, { "Total_reward": 1.7668520513325348e-07, "advantage_std": 1.491289496421814, "completion_length": 449.8035888671875, "epoch": 0.2223756906077348, "grad_norm": 14.927492687743527, "kl": 0.07861328125, "learning_rate": 4.4143361089840266e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.5517281889915466, "rewards_raw/accuracy_reward": 0.5655099749565125, "rewards_raw/clip_reward_wrapper": 0.3146819472312927, "rewards_raw/format_reward": 5.0, "step": 644 }, { "Total_reward": -7.525086971327255e-07, "advantage_std": 1.4245994091033936, "completion_length": 445.7857360839844, "epoch": 0.22272099447513813, "grad_norm": 5.779240108074459, "kl": 0.07763671875, "learning_rate": 4.412590734913064e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.7999697923660278, "rewards_raw/accuracy_reward": 0.5732467174530029, "rewards_raw/clip_reward_wrapper": 0.33580631017684937, "rewards_raw/format_reward": 5.0, "step": 645 }, { "Total_reward": -2.1287373641598606e-08, "advantage_std": 1.4597254991531372, "completion_length": 429.5535888671875, "epoch": 0.22306629834254144, "grad_norm": 3.289205493304059, "kl": 0.076171875, "learning_rate": 4.4108431101056975e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.429294615983963, "rewards_raw/accuracy_reward": 0.5851942300796509, "rewards_raw/clip_reward_wrapper": 0.3416387736797333, "rewards_raw/format_reward": 5.0, "step": 646 }, { "Total_reward": 2.043587983280304e-07, "advantage_std": 1.527022123336792, "completion_length": 438.71429443359375, "epoch": 0.22341160220994474, "grad_norm": 3.8515837632463823, "kl": 0.07568359375, "learning_rate": 4.4090932366185326e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.9490936398506165, "rewards_raw/accuracy_reward": 0.5833876729011536, "rewards_raw/clip_reward_wrapper": 0.3384642004966736, "rewards_raw/format_reward": 5.0, "step": 647 }, { "Total_reward": 5.438923835754395e-07, "advantage_std": 1.47248375415802, "completion_length": 421.6785888671875, "epoch": 0.22375690607734808, "grad_norm": 3.8133457322066255, "kl": 0.07568359375, "learning_rate": 4.4073411165108176e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 1.087242603302002, "rewards_raw/accuracy_reward": 0.5781072378158569, "rewards_raw/clip_reward_wrapper": 0.3171955943107605, "rewards_raw/format_reward": 5.0, "step": 648 }, { "Total_reward": 8.33400690680719e-07, "advantage_std": 1.3415926694869995, "completion_length": 436.1785888671875, "epoch": 0.22410220994475138, "grad_norm": 3.5597947576754185, "kl": 0.07666015625, "learning_rate": 4.40558675184445e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.5345603823661804, "rewards_raw/accuracy_reward": 0.5672888159751892, "rewards_raw/clip_reward_wrapper": 0.3332783877849579, "rewards_raw/format_reward": 5.0, "step": 649 }, { "Total_reward": -6.045614213689987e-07, "advantage_std": 1.665565013885498, "completion_length": 445.482177734375, "epoch": 0.2244475138121547, "grad_norm": 5.288107655425798, "kl": 0.07763671875, "learning_rate": 4.403830144683964e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.2491813451051712, "rewards_raw/accuracy_reward": 0.5816328525543213, "rewards_raw/clip_reward_wrapper": 0.3252033591270447, "rewards_raw/format_reward": 5.0, "step": 650 }, { "Total_reward": -9.5367431640625e-07, "advantage_std": 1.3754503726959229, "completion_length": 435.96429443359375, "epoch": 0.224792817679558, "grad_norm": 4.478625537944209, "kl": 0.080078125, "learning_rate": 4.402071297096536e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.39359715580940247, "rewards_raw/accuracy_reward": 0.5709383487701416, "rewards_raw/clip_reward_wrapper": 0.3324747681617737, "rewards_raw/format_reward": 5.0, "step": 651 }, { "Total_reward": 4.5874290321989974e-07, "advantage_std": 1.2300381660461426, "completion_length": 426.9285888671875, "epoch": 0.22513812154696133, "grad_norm": 5.4237606820362085, "kl": 0.0771484375, "learning_rate": 4.400310211151975e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.3147200345993042, "rewards_raw/accuracy_reward": 0.5549740791320801, "rewards_raw/clip_reward_wrapper": 0.3319878578186035, "rewards_raw/format_reward": 5.0, "step": 652 }, { "Total_reward": -3.193106223875475e-08, "advantage_std": 1.4517804384231567, "completion_length": 449.96429443359375, "epoch": 0.22548342541436464, "grad_norm": 8.866826480879485, "kl": 0.07861328125, "learning_rate": 4.3985468889227284e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.4355836808681488, "rewards_raw/accuracy_reward": 0.5703524947166443, "rewards_raw/clip_reward_wrapper": 0.3262089192867279, "rewards_raw/format_reward": 5.0, "step": 653 }, { "Total_reward": 3.107956558778824e-07, "advantage_std": 1.3629248142242432, "completion_length": 429.1607360839844, "epoch": 0.22582872928176795, "grad_norm": 21.094550951983035, "kl": 0.078125, "learning_rate": 4.396781332483873e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.29141709208488464, "rewards_raw/accuracy_reward": 0.5568325519561768, "rewards_raw/clip_reward_wrapper": 0.32736825942993164, "rewards_raw/format_reward": 5.0, "step": 654 }, { "Total_reward": 9.749617220222717e-07, "advantage_std": 1.5267809629440308, "completion_length": 438.1250305175781, "epoch": 0.22617403314917128, "grad_norm": 4.715653246000095, "kl": 0.07861328125, "learning_rate": 4.395013543913116e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.36699607968330383, "rewards_raw/accuracy_reward": 0.5694707036018372, "rewards_raw/clip_reward_wrapper": 0.32215768098831177, "rewards_raw/format_reward": 5.0, "step": 655 }, { "Total_reward": 2.2564616131148796e-07, "advantage_std": 1.402866244316101, "completion_length": 443.0535888671875, "epoch": 0.2265193370165746, "grad_norm": 4.561735939496992, "kl": 0.078125, "learning_rate": 4.3932435252907907e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.35127776861190796, "rewards_raw/accuracy_reward": 0.5531808733940125, "rewards_raw/clip_reward_wrapper": 0.3371788561344147, "rewards_raw/format_reward": 5.0, "step": 656 }, { "Total_reward": -1.7029898913278885e-07, "advantage_std": 1.4025719165802002, "completion_length": 410.2857360839844, "epoch": 0.2268646408839779, "grad_norm": 3.197698470189237, "kl": 0.080078125, "learning_rate": 4.3914712786998534e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.9561932682991028, "rewards_raw/accuracy_reward": 0.5794209241867065, "rewards_raw/clip_reward_wrapper": 0.33086463809013367, "rewards_raw/format_reward": 5.0, "step": 657 }, { "Total_reward": -6.641660661443893e-07, "advantage_std": 1.4734739065170288, "completion_length": 434.4107360839844, "epoch": 0.2272099447513812, "grad_norm": 4.363108189739717, "kl": 0.0791015625, "learning_rate": 4.389696806225884e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.5919471979141235, "rewards_raw/accuracy_reward": 0.5790037512779236, "rewards_raw/clip_reward_wrapper": 0.3299410939216614, "rewards_raw/format_reward": 5.0, "step": 658 }, { "Total_reward": -1.2091228427379974e-06, "advantage_std": 1.4813414812088013, "completion_length": 439.89288330078125, "epoch": 0.22755524861878454, "grad_norm": 7.855149655119633, "kl": 0.078125, "learning_rate": 4.3879201099570815e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.8726793527603149, "rewards_raw/accuracy_reward": 0.5800173282623291, "rewards_raw/clip_reward_wrapper": 0.31468623876571655, "rewards_raw/format_reward": 5.0, "step": 659 }, { "Total_reward": -1.1282308065574398e-07, "advantage_std": 1.5976499319076538, "completion_length": 437.7857360839844, "epoch": 0.22790055248618785, "grad_norm": 3.9070522312584033, "kl": 0.08056640625, "learning_rate": 4.3861411919842614e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.3626839220523834, "rewards_raw/accuracy_reward": 0.5836876630783081, "rewards_raw/clip_reward_wrapper": 0.3411295711994171, "rewards_raw/format_reward": 5.0, "step": 660 }, { "Total_reward": 2.682209014892578e-07, "advantage_std": 1.3341583013534546, "completion_length": 424.7857360839844, "epoch": 0.22824585635359115, "grad_norm": 3.130150655196806, "kl": 0.076171875, "learning_rate": 4.384360054400855e-07, "loss": 0.003, "rewards_raw/CoT ICLG": 0.3160604238510132, "rewards_raw/accuracy_reward": 0.5761183500289917, "rewards_raw/clip_reward_wrapper": 0.33164262771606445, "rewards_raw/format_reward": 5.0, "step": 661 }, { "Total_reward": 1.4262541014886665e-07, "advantage_std": 1.3485279083251953, "completion_length": 445.51788330078125, "epoch": 0.2285911602209945, "grad_norm": 5.5785602778101655, "kl": 0.0791015625, "learning_rate": 4.3825766993029043e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.5743488669395447, "rewards_raw/accuracy_reward": 0.5792351961135864, "rewards_raw/clip_reward_wrapper": 0.3403504192829132, "rewards_raw/format_reward": 5.0, "step": 662 }, { "Total_reward": 2.2883926931172027e-07, "advantage_std": 1.3817214965820312, "completion_length": 436.9285888671875, "epoch": 0.2289364640883978, "grad_norm": 5.345124622390749, "kl": 0.07958984375, "learning_rate": 4.380791128789061e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.5186638236045837, "rewards_raw/accuracy_reward": 0.5739473700523376, "rewards_raw/clip_reward_wrapper": 0.3508679270744324, "rewards_raw/format_reward": 5.0, "step": 663 }, { "Total_reward": -4.6832223432602405e-08, "advantage_std": 1.6384357213974, "completion_length": 426.1607360839844, "epoch": 0.2292817679558011, "grad_norm": 4.343270450236609, "kl": 0.078125, "learning_rate": 4.379003344960584e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.4133918881416321, "rewards_raw/accuracy_reward": 0.583294153213501, "rewards_raw/clip_reward_wrapper": 0.31087982654571533, "rewards_raw/format_reward": 5.0, "step": 664 }, { "Total_reward": -1.0675618113964447e-06, "advantage_std": 1.5341297388076782, "completion_length": 420.2321472167969, "epoch": 0.2296270718232044, "grad_norm": 10.477938681979243, "kl": 0.08154296875, "learning_rate": 4.377213349921337e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.5872915387153625, "rewards_raw/accuracy_reward": 0.5457739233970642, "rewards_raw/clip_reward_wrapper": 0.32939788699150085, "rewards_raw/format_reward": 5.0, "step": 665 }, { "Total_reward": 1.1708055858150601e-08, "advantage_std": 1.7332912683486938, "completion_length": 429.08929443359375, "epoch": 0.22997237569060774, "grad_norm": 5.552877688598618, "kl": 0.080078125, "learning_rate": 4.3754211457777857e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.8418976068496704, "rewards_raw/accuracy_reward": 0.5802225470542908, "rewards_raw/clip_reward_wrapper": 0.34298980236053467, "rewards_raw/format_reward": 5.0, "step": 666 }, { "Total_reward": 7.66345493730114e-08, "advantage_std": 1.8548338413238525, "completion_length": 439.9107360839844, "epoch": 0.23031767955801105, "grad_norm": 4.6979355616082135, "kl": 0.078125, "learning_rate": 4.373626734638996e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.5288639068603516, "rewards_raw/accuracy_reward": 0.5647509098052979, "rewards_raw/clip_reward_wrapper": 0.33993837237358093, "rewards_raw/format_reward": 5.0, "step": 667 }, { "Total_reward": 5.045107513979019e-07, "advantage_std": 1.6497142314910889, "completion_length": 426.1785888671875, "epoch": 0.23066298342541436, "grad_norm": 5.232878797499411, "kl": 0.08251953125, "learning_rate": 4.371830118616631e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.30923739075660706, "rewards_raw/accuracy_reward": 0.5789275169372559, "rewards_raw/clip_reward_wrapper": 0.31915557384490967, "rewards_raw/format_reward": 5.0, "step": 668 }, { "Total_reward": -6.599086077585525e-07, "advantage_std": 1.5132428407669067, "completion_length": 418.71429443359375, "epoch": 0.2310082872928177, "grad_norm": 4.250219706790569, "kl": 0.080078125, "learning_rate": 4.370031299824946e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.7314479351043701, "rewards_raw/accuracy_reward": 0.575946033000946, "rewards_raw/clip_reward_wrapper": 0.33320552110671997, "rewards_raw/format_reward": 5.0, "step": 669 }, { "Total_reward": -5.662441253662109e-07, "advantage_std": 1.39900541305542, "completion_length": 421.3214416503906, "epoch": 0.231353591160221, "grad_norm": 4.520081020178533, "kl": 0.08349609375, "learning_rate": 4.3682302803807936e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.7834073901176453, "rewards_raw/accuracy_reward": 0.5755942463874817, "rewards_raw/clip_reward_wrapper": 0.33447226881980896, "rewards_raw/format_reward": 5.0, "step": 670 }, { "Total_reward": 1.4954380844756088e-07, "advantage_std": 1.3912327289581299, "completion_length": 424.8214416503906, "epoch": 0.2316988950276243, "grad_norm": 5.896202242883098, "kl": 0.08056640625, "learning_rate": 4.3664270624036105e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.4681762158870697, "rewards_raw/accuracy_reward": 0.5867847800254822, "rewards_raw/clip_reward_wrapper": 0.32918426394462585, "rewards_raw/format_reward": 5.0, "step": 671 }, { "Total_reward": -4.874808610111359e-07, "advantage_std": 1.668772578239441, "completion_length": 412.3214416503906, "epoch": 0.23204419889502761, "grad_norm": 4.52184708092664, "kl": 0.080078125, "learning_rate": 4.364621648015425e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.3644625246524811, "rewards_raw/accuracy_reward": 0.573078989982605, "rewards_raw/clip_reward_wrapper": 0.31929922103881836, "rewards_raw/format_reward": 5.0, "step": 672 }, { "Total_reward": 3.5337041026650695e-07, "advantage_std": 1.634837031364441, "completion_length": 429.9821472167969, "epoch": 0.23238950276243095, "grad_norm": 3.8530368138068884, "kl": 0.0791015625, "learning_rate": 4.362814039340847e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.5097426772117615, "rewards_raw/accuracy_reward": 0.5690760016441345, "rewards_raw/clip_reward_wrapper": 0.33913367986679077, "rewards_raw/format_reward": 5.0, "step": 673 }, { "Total_reward": -7.024833337254677e-08, "advantage_std": 1.7100800275802612, "completion_length": 429.5714416503906, "epoch": 0.23273480662983426, "grad_norm": 4.192057304329793, "kl": 0.0830078125, "learning_rate": 4.361004238507071e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.3333194851875305, "rewards_raw/accuracy_reward": 0.5856325030326843, "rewards_raw/clip_reward_wrapper": 0.32834506034851074, "rewards_raw/format_reward": 5.0, "step": 674 }, { "Total_reward": 2.4906228190957336e-07, "advantage_std": 1.4563474655151367, "completion_length": 436.4107360839844, "epoch": 0.23308011049723756, "grad_norm": 3.2769838195917327, "kl": 0.0830078125, "learning_rate": 4.359192247643871e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.8248478174209595, "rewards_raw/accuracy_reward": 0.5682269930839539, "rewards_raw/clip_reward_wrapper": 0.33698704838752747, "rewards_raw/format_reward": 5.0, "step": 675 }, { "Total_reward": 3.299543038792763e-08, "advantage_std": 1.3350675106048584, "completion_length": 418.1607360839844, "epoch": 0.23342541436464087, "grad_norm": 5.589008998355867, "kl": 0.078125, "learning_rate": 4.357378068883595e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.5053751468658447, "rewards_raw/accuracy_reward": 0.5835597515106201, "rewards_raw/clip_reward_wrapper": 0.32472535967826843, "rewards_raw/format_reward": 5.0, "step": 676 }, { "Total_reward": -3.938164070405037e-07, "advantage_std": 1.7014633417129517, "completion_length": 422.0714416503906, "epoch": 0.2337707182320442, "grad_norm": 4.41944363253679, "kl": 0.078125, "learning_rate": 4.3555617043611714e-07, "loss": 0.0031, "rewards_raw/CoT ICLG": 0.4329032897949219, "rewards_raw/accuracy_reward": 0.5660704374313354, "rewards_raw/clip_reward_wrapper": 0.32797783613204956, "rewards_raw/format_reward": 5.0, "step": 677 }, { "Total_reward": 4.2361875784990843e-07, "advantage_std": 1.5853427648544312, "completion_length": 412.2857360839844, "epoch": 0.2341160220994475, "grad_norm": 15.179380344168269, "kl": 0.0791015625, "learning_rate": 4.3537431562140947e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.4314135015010834, "rewards_raw/accuracy_reward": 0.5753260850906372, "rewards_raw/clip_reward_wrapper": 0.3336518108844757, "rewards_raw/format_reward": 5.0, "step": 678 }, { "Total_reward": -9.366444686520481e-08, "advantage_std": 1.4938278198242188, "completion_length": 430.6607360839844, "epoch": 0.23446132596685082, "grad_norm": 4.434498831279218, "kl": 0.08251953125, "learning_rate": 4.351922426582434e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 1.0297057628631592, "rewards_raw/accuracy_reward": 0.5458470582962036, "rewards_raw/clip_reward_wrapper": 0.3361104726791382, "rewards_raw/format_reward": 5.0, "step": 679 }, { "Total_reward": -4.480992288335983e-07, "advantage_std": 1.602687954902649, "completion_length": 412.4821472167969, "epoch": 0.23480662983425415, "grad_norm": 3.6487371544617924, "kl": 0.083984375, "learning_rate": 4.350099517608823e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.6552144885063171, "rewards_raw/accuracy_reward": 0.5579483509063721, "rewards_raw/clip_reward_wrapper": 0.3307570219039917, "rewards_raw/format_reward": 5.0, "step": 680 }, { "Total_reward": 5.768878281742218e-07, "advantage_std": 1.5312362909317017, "completion_length": 421.2321472167969, "epoch": 0.23515193370165746, "grad_norm": 3.559791542564137, "kl": 0.080078125, "learning_rate": 4.34827443143846e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.8263453245162964, "rewards_raw/accuracy_reward": 0.5754793882369995, "rewards_raw/clip_reward_wrapper": 0.3178001940250397, "rewards_raw/format_reward": 5.0, "step": 681 }, { "Total_reward": 3.2143935868589324e-07, "advantage_std": 1.6031248569488525, "completion_length": 416.7321472167969, "epoch": 0.23549723756906077, "grad_norm": 8.5488122954751, "kl": 0.08154296875, "learning_rate": 4.346447170219106e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.5108211040496826, "rewards_raw/accuracy_reward": 0.5501519441604614, "rewards_raw/clip_reward_wrapper": 0.3209180235862732, "rewards_raw/format_reward": 4.964285850524902, "step": 682 }, { "Total_reward": 1.2346677635832748e-07, "advantage_std": 1.3814654350280762, "completion_length": 436.83929443359375, "epoch": 0.23584254143646408, "grad_norm": 5.449892009380607, "kl": 0.07958984375, "learning_rate": 4.344617736101083e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 1.001635193824768, "rewards_raw/accuracy_reward": 0.5781042575836182, "rewards_raw/clip_reward_wrapper": 0.32870927453041077, "rewards_raw/format_reward": 5.0, "step": 683 }, { "Total_reward": -9.387732120558212e-07, "advantage_std": 1.4838550090789795, "completion_length": 408.3571472167969, "epoch": 0.2361878453038674, "grad_norm": 6.508018760000977, "kl": 0.08251953125, "learning_rate": 4.3427861312372684e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 1.1262578964233398, "rewards_raw/accuracy_reward": 0.5390703678131104, "rewards_raw/clip_reward_wrapper": 0.3290938138961792, "rewards_raw/format_reward": 5.0, "step": 684 }, { "Total_reward": 4.002026230409683e-07, "advantage_std": 1.442219853401184, "completion_length": 412.51788330078125, "epoch": 0.23653314917127072, "grad_norm": 5.6797709184513625, "kl": 0.080078125, "learning_rate": 4.3409523577830933e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.9032351970672607, "rewards_raw/accuracy_reward": 0.550250232219696, "rewards_raw/clip_reward_wrapper": 0.34575286507606506, "rewards_raw/format_reward": 5.0, "step": 685 }, { "Total_reward": 1.1191837074875366e-06, "advantage_std": 1.3769477605819702, "completion_length": 414.6071472167969, "epoch": 0.23687845303867403, "grad_norm": 3.262848136908389, "kl": 0.0830078125, "learning_rate": 4.339116417896542e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.27452516555786133, "rewards_raw/accuracy_reward": 0.5654136538505554, "rewards_raw/clip_reward_wrapper": 0.33241280913352966, "rewards_raw/format_reward": 5.0, "step": 686 }, { "Total_reward": -8.98327186860115e-07, "advantage_std": 1.4119000434875488, "completion_length": 412.21429443359375, "epoch": 0.23722375690607736, "grad_norm": 3.9106261872051853, "kl": 0.080078125, "learning_rate": 4.33727831373815e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.6692761778831482, "rewards_raw/accuracy_reward": 0.5668579936027527, "rewards_raw/clip_reward_wrapper": 0.3353329002857208, "rewards_raw/format_reward": 5.0, "step": 687 }, { "Total_reward": 3.661428422674362e-07, "advantage_std": 1.6071704626083374, "completion_length": 421.7857360839844, "epoch": 0.23756906077348067, "grad_norm": 4.254657900071004, "kl": 0.0830078125, "learning_rate": 4.335438047470995e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.47459661960601807, "rewards_raw/accuracy_reward": 0.572016716003418, "rewards_raw/clip_reward_wrapper": 0.3378041982650757, "rewards_raw/format_reward": 5.0, "step": 688 }, { "Total_reward": 1.3836793755217514e-07, "advantage_std": 1.420822024345398, "completion_length": 416.6964416503906, "epoch": 0.23791436464088397, "grad_norm": 4.791820648197948, "kl": 0.07958984375, "learning_rate": 4.333595621260705e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.8644685745239258, "rewards_raw/accuracy_reward": 0.544415295124054, "rewards_raw/clip_reward_wrapper": 0.3269104063510895, "rewards_raw/format_reward": 5.0, "step": 689 }, { "Total_reward": -5.832740725963959e-07, "advantage_std": 1.524141788482666, "completion_length": 410.3750305175781, "epoch": 0.23825966850828728, "grad_norm": 4.39386755070784, "kl": 0.08056640625, "learning_rate": 4.331751037275445e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.3892049491405487, "rewards_raw/accuracy_reward": 0.5636399388313293, "rewards_raw/clip_reward_wrapper": 0.3311688303947449, "rewards_raw/format_reward": 5.0, "step": 690 }, { "Total_reward": -3.8530146184712066e-07, "advantage_std": 1.5359220504760742, "completion_length": 406.2500305175781, "epoch": 0.23860497237569062, "grad_norm": 3.829293349612586, "kl": 0.08447265625, "learning_rate": 4.329904297685922e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.48413532972335815, "rewards_raw/accuracy_reward": 0.5550255179405212, "rewards_raw/clip_reward_wrapper": 0.3420310318470001, "rewards_raw/format_reward": 5.0, "step": 691 }, { "Total_reward": -1.7668520513325348e-07, "advantage_std": 1.6229164600372314, "completion_length": 401.9285888671875, "epoch": 0.23895027624309392, "grad_norm": 4.588749417882401, "kl": 0.08544921875, "learning_rate": 4.3280554046653785e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.7829667925834656, "rewards_raw/accuracy_reward": 0.5581058859825134, "rewards_raw/clip_reward_wrapper": 0.3458772599697113, "rewards_raw/format_reward": 5.0, "step": 692 }, { "Total_reward": 2.5331974029541016e-07, "advantage_std": 1.610653281211853, "completion_length": 412.6785888671875, "epoch": 0.23929558011049723, "grad_norm": 3.939097263959233, "kl": 0.08203125, "learning_rate": 4.3262043603895903e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.5873022079467773, "rewards_raw/accuracy_reward": 0.5824048519134521, "rewards_raw/clip_reward_wrapper": 0.3370353579521179, "rewards_raw/format_reward": 5.0, "step": 693 }, { "Total_reward": -2.4693355271665496e-07, "advantage_std": 1.3817799091339111, "completion_length": 400.5357360839844, "epoch": 0.23964088397790057, "grad_norm": 3.639372456348153, "kl": 0.08154296875, "learning_rate": 4.3243511670368675e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.5892846584320068, "rewards_raw/accuracy_reward": 0.5522379875183105, "rewards_raw/clip_reward_wrapper": 0.3403666317462921, "rewards_raw/format_reward": 5.0, "step": 694 }, { "Total_reward": 7.663454653084045e-07, "advantage_std": 1.4173392057418823, "completion_length": 388.3571472167969, "epoch": 0.23998618784530387, "grad_norm": 4.10313407214455, "kl": 0.0849609375, "learning_rate": 4.3224958267880473e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.4511701464653015, "rewards_raw/accuracy_reward": 0.5414924025535583, "rewards_raw/clip_reward_wrapper": 0.33563363552093506, "rewards_raw/format_reward": 5.0, "step": 695 }, { "Total_reward": 5.736947059631348e-07, "advantage_std": 1.3204028606414795, "completion_length": 408.0714416503906, "epoch": 0.24033149171270718, "grad_norm": 4.456761270394652, "kl": 0.08447265625, "learning_rate": 4.3206383418264937e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.2992619574069977, "rewards_raw/accuracy_reward": 0.5653776526451111, "rewards_raw/clip_reward_wrapper": 0.3318824768066406, "rewards_raw/format_reward": 5.0, "step": 696 }, { "Total_reward": -2.043587983280304e-07, "advantage_std": 1.6363959312438965, "completion_length": 400.6785888671875, "epoch": 0.2406767955801105, "grad_norm": 4.262165138111262, "kl": 0.08740234375, "learning_rate": 4.318778714338094e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 1.0206903219223022, "rewards_raw/accuracy_reward": 0.5347051024436951, "rewards_raw/clip_reward_wrapper": 0.3254697918891907, "rewards_raw/format_reward": 5.0, "step": 697 }, { "Total_reward": -7.003546329542587e-07, "advantage_std": 1.6012632846832275, "completion_length": 394.8214416503906, "epoch": 0.24102209944751382, "grad_norm": 3.7675895618920165, "kl": 0.08642578125, "learning_rate": 4.316916946511258e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.6740745902061462, "rewards_raw/accuracy_reward": 0.5408438444137573, "rewards_raw/clip_reward_wrapper": 0.343290239572525, "rewards_raw/format_reward": 4.928571701049805, "step": 698 }, { "Total_reward": -5.013176860302337e-07, "advantage_std": 1.677384614944458, "completion_length": 399.5535888671875, "epoch": 0.24136740331491713, "grad_norm": 6.740835055326527, "kl": 0.08642578125, "learning_rate": 4.315053040536914e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.6288641095161438, "rewards_raw/accuracy_reward": 0.5711355209350586, "rewards_raw/clip_reward_wrapper": 0.31790992617607117, "rewards_raw/format_reward": 5.0, "step": 699 }, { "Total_reward": 8.930053354561096e-07, "advantage_std": 1.465105652809143, "completion_length": 400.33929443359375, "epoch": 0.24171270718232044, "grad_norm": 8.959019035148224, "kl": 0.08544921875, "learning_rate": 4.3131869986085056e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.6164993643760681, "rewards_raw/accuracy_reward": 0.5736009478569031, "rewards_raw/clip_reward_wrapper": 0.3412093222141266, "rewards_raw/format_reward": 5.0, "step": 700 }, { "Total_reward": 6.780028911634872e-07, "advantage_std": 1.520957350730896, "completion_length": 405.6250305175781, "epoch": 0.24205801104972377, "grad_norm": 6.2953575378172255, "kl": 0.08544921875, "learning_rate": 4.311318822921991e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.4770594537258148, "rewards_raw/accuracy_reward": 0.5803462862968445, "rewards_raw/clip_reward_wrapper": 0.3220349848270416, "rewards_raw/format_reward": 5.0, "step": 701 }, { "Total_reward": 1.4262541014886665e-07, "advantage_std": 1.3885575532913208, "completion_length": 396.6250305175781, "epoch": 0.24240331491712708, "grad_norm": 3.4477258866561744, "kl": 0.0888671875, "learning_rate": 4.3094485156758385e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.388342022895813, "rewards_raw/accuracy_reward": 0.5721907615661621, "rewards_raw/clip_reward_wrapper": 0.3268672227859497, "rewards_raw/format_reward": 5.0, "step": 702 }, { "Total_reward": -2.7247838829680404e-07, "advantage_std": 1.4849430322647095, "completion_length": 404.7857360839844, "epoch": 0.24274861878453038, "grad_norm": 3.8901397776486837, "kl": 0.0849609375, "learning_rate": 4.307576079071026e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.38888153433799744, "rewards_raw/accuracy_reward": 0.5736090540885925, "rewards_raw/clip_reward_wrapper": 0.3310030400753021, "rewards_raw/format_reward": 5.0, "step": 703 }, { "Total_reward": -1.0430813546236095e-07, "advantage_std": 1.5697340965270996, "completion_length": 408.83929443359375, "epoch": 0.2430939226519337, "grad_norm": 6.350772036469267, "kl": 0.08935546875, "learning_rate": 4.305701515311037e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.4438205361366272, "rewards_raw/accuracy_reward": 0.5652906894683838, "rewards_raw/clip_reward_wrapper": 0.337604820728302, "rewards_raw/format_reward": 5.0, "step": 704 }, { "Total_reward": 3.5549913945942535e-07, "advantage_std": 1.6526801586151123, "completion_length": 397.58929443359375, "epoch": 0.24343922651933703, "grad_norm": 9.837220780985549, "kl": 0.0859375, "learning_rate": 4.303824826601855e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.6267166137695312, "rewards_raw/accuracy_reward": 0.5561797022819519, "rewards_raw/clip_reward_wrapper": 0.3405975103378296, "rewards_raw/format_reward": 5.0, "step": 705 }, { "Total_reward": -8.834260256662674e-08, "advantage_std": 1.442255973815918, "completion_length": 402.4107360839844, "epoch": 0.24378453038674033, "grad_norm": 5.420231788219902, "kl": 0.0859375, "learning_rate": 4.30194601515197e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.4855906069278717, "rewards_raw/accuracy_reward": 0.5696464776992798, "rewards_raw/clip_reward_wrapper": 0.33439144492149353, "rewards_raw/format_reward": 5.0, "step": 706 }, { "Total_reward": 2.341611207157257e-07, "advantage_std": 1.3592802286148071, "completion_length": 408.1607360839844, "epoch": 0.24412983425414364, "grad_norm": 6.606009870400867, "kl": 0.0849609375, "learning_rate": 4.300065083172366e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.44230562448501587, "rewards_raw/accuracy_reward": 0.5819227695465088, "rewards_raw/clip_reward_wrapper": 0.3380810022354126, "rewards_raw/format_reward": 5.0, "step": 707 }, { "Total_reward": -2.5225537569895096e-07, "advantage_std": 1.5186655521392822, "completion_length": 400.6785888671875, "epoch": 0.24447513812154695, "grad_norm": 3.423503989823395, "kl": 0.0859375, "learning_rate": 4.298182032876523e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.36258944869041443, "rewards_raw/accuracy_reward": 0.5555305480957031, "rewards_raw/clip_reward_wrapper": 0.33528751134872437, "rewards_raw/format_reward": 5.0, "step": 708 }, { "Total_reward": 1.7455646172948036e-07, "advantage_std": 1.4568166732788086, "completion_length": 411.6071472167969, "epoch": 0.24482044198895028, "grad_norm": 4.7379355112087955, "kl": 0.08251953125, "learning_rate": 4.2962968664804134e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.6145675778388977, "rewards_raw/accuracy_reward": 0.568229079246521, "rewards_raw/clip_reward_wrapper": 0.339507520198822, "rewards_raw/format_reward": 5.0, "step": 709 }, { "Total_reward": 6.599086077585525e-08, "advantage_std": 1.614667296409607, "completion_length": 409.0714416503906, "epoch": 0.2451657458563536, "grad_norm": 4.6733322530602095, "kl": 0.0859375, "learning_rate": 4.2944095862025025e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.46377649903297424, "rewards_raw/accuracy_reward": 0.5903094410896301, "rewards_raw/clip_reward_wrapper": 0.32502269744873047, "rewards_raw/format_reward": 5.0, "step": 710 }, { "Total_reward": -4.193612710423622e-07, "advantage_std": 1.2299604415893555, "completion_length": 409.5535888671875, "epoch": 0.2455110497237569, "grad_norm": 3.7820864328283603, "kl": 0.0849609375, "learning_rate": 4.2925201942637407e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.9539964199066162, "rewards_raw/accuracy_reward": 0.541843831539154, "rewards_raw/clip_reward_wrapper": 0.31908172369003296, "rewards_raw/format_reward": 5.0, "step": 711 }, { "Total_reward": 1.7242773253656196e-07, "advantage_std": 1.7005072832107544, "completion_length": 399.01788330078125, "epoch": 0.24585635359116023, "grad_norm": 13.2484833715593, "kl": 0.08837890625, "learning_rate": 4.290628692887563e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 1.5053422451019287, "rewards_raw/accuracy_reward": 0.5734744071960449, "rewards_raw/clip_reward_wrapper": 0.33595171570777893, "rewards_raw/format_reward": 5.0, "step": 712 }, { "Total_reward": 2.7034965910388564e-07, "advantage_std": 1.4999009370803833, "completion_length": 401.2857360839844, "epoch": 0.24620165745856354, "grad_norm": 6.769336024248552, "kl": 0.0859375, "learning_rate": 4.28873508429989e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.7513207793235779, "rewards_raw/accuracy_reward": 0.5631080865859985, "rewards_raw/clip_reward_wrapper": 0.33440718054771423, "rewards_raw/format_reward": 5.0, "step": 713 }, { "Total_reward": -1.1708056035786285e-07, "advantage_std": 1.5483921766281128, "completion_length": 409.6607360839844, "epoch": 0.24654696132596685, "grad_norm": 7.767063087367301, "kl": 0.08544921875, "learning_rate": 4.286839370729118e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.5407043099403381, "rewards_raw/accuracy_reward": 0.5760980844497681, "rewards_raw/clip_reward_wrapper": 0.3438534438610077, "rewards_raw/format_reward": 5.0, "step": 714 }, { "Total_reward": -3.916876778475853e-07, "advantage_std": 1.4770509004592896, "completion_length": 403.89288330078125, "epoch": 0.24689226519337015, "grad_norm": 7.235664437782266, "kl": 0.0859375, "learning_rate": 4.284941554406123e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.3995078206062317, "rewards_raw/accuracy_reward": 0.5668747425079346, "rewards_raw/clip_reward_wrapper": 0.3333122432231903, "rewards_raw/format_reward": 5.0, "step": 715 }, { "Total_reward": 3.129243850708008e-07, "advantage_std": 1.4141122102737427, "completion_length": 400.8750305175781, "epoch": 0.2472375690607735, "grad_norm": 5.8880433215628525, "kl": 0.08544921875, "learning_rate": 4.283041637564254e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.5777363181114197, "rewards_raw/accuracy_reward": 0.5761338472366333, "rewards_raw/clip_reward_wrapper": 0.33669087290763855, "rewards_raw/format_reward": 5.0, "step": 716 }, { "Total_reward": 4.257474728319721e-08, "advantage_std": 1.376519799232483, "completion_length": 421.7321472167969, "epoch": 0.2475828729281768, "grad_norm": 5.3256051317558795, "kl": 0.0849609375, "learning_rate": 4.281139622439335e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.8114364743232727, "rewards_raw/accuracy_reward": 0.5614389777183533, "rewards_raw/clip_reward_wrapper": 0.3441278636455536, "rewards_raw/format_reward": 5.0, "step": 717 }, { "Total_reward": -1.0899135531872162e-06, "advantage_std": 1.6298285722732544, "completion_length": 405.3035888671875, "epoch": 0.2479281767955801, "grad_norm": 5.365229509853473, "kl": 0.083984375, "learning_rate": 4.2792355112696553e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.948458731174469, "rewards_raw/accuracy_reward": 0.5476477742195129, "rewards_raw/clip_reward_wrapper": 0.3350324034690857, "rewards_raw/format_reward": 5.0, "step": 718 }, { "Total_reward": 5.747590847704487e-08, "advantage_std": 1.359853744506836, "completion_length": 399.6250305175781, "epoch": 0.24827348066298344, "grad_norm": 4.941304116805569, "kl": 0.08642578125, "learning_rate": 4.277329306295973e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.6112940907478333, "rewards_raw/accuracy_reward": 0.58827143907547, "rewards_raw/clip_reward_wrapper": 0.3367104232311249, "rewards_raw/format_reward": 5.0, "step": 719 }, { "Total_reward": 4.257474817137563e-09, "advantage_std": 1.3932044506072998, "completion_length": 407.3214416503906, "epoch": 0.24861878453038674, "grad_norm": 26.694930828892794, "kl": 0.0830078125, "learning_rate": 4.275421009761509e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.47871366143226624, "rewards_raw/accuracy_reward": 0.5676301717758179, "rewards_raw/clip_reward_wrapper": 0.3290289640426636, "rewards_raw/format_reward": 5.0, "step": 720 }, { "Total_reward": 1.7455646172948036e-07, "advantage_std": 1.535068154335022, "completion_length": 400.1607360839844, "epoch": 0.24896408839779005, "grad_norm": 10.406970786920859, "kl": 0.08251953125, "learning_rate": 4.273510623911947e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.7945005297660828, "rewards_raw/accuracy_reward": 0.5781158208847046, "rewards_raw/clip_reward_wrapper": 0.3307519853115082, "rewards_raw/format_reward": 5.0, "step": 721 }, { "Total_reward": 4.0233138065559615e-07, "advantage_std": 1.3180031776428223, "completion_length": 393.5357360839844, "epoch": 0.24930939226519336, "grad_norm": 4.561775753434488, "kl": 0.0869140625, "learning_rate": 4.2715981509954293e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.50372713804245, "rewards_raw/accuracy_reward": 0.5628446340560913, "rewards_raw/clip_reward_wrapper": 0.33524924516677856, "rewards_raw/format_reward": 5.0, "step": 722 }, { "Total_reward": 1.6029392782002105e-06, "advantage_std": 1.7576826810836792, "completion_length": 407.0000305175781, "epoch": 0.2496546961325967, "grad_norm": 5.131283774245999, "kl": 0.08447265625, "learning_rate": 4.269683593262553e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.5691848993301392, "rewards_raw/accuracy_reward": 0.5667464733123779, "rewards_raw/clip_reward_wrapper": 0.329283207654953, "rewards_raw/format_reward": 5.0, "step": 723 }, { "Total_reward": -6.652304307408485e-08, "advantage_std": 1.5332108736038208, "completion_length": 398.0000305175781, "epoch": 0.25, "grad_norm": 6.442796057279418, "kl": 0.08154296875, "learning_rate": 4.2677669529663686e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.9401519298553467, "rewards_raw/accuracy_reward": 0.6085683703422546, "rewards_raw/clip_reward_wrapper": 0.3252945840358734, "rewards_raw/format_reward": 5.0, "step": 724 }, { "Total_reward": -6.641660661443893e-07, "advantage_std": 1.5263172388076782, "completion_length": 405.76788330078125, "epoch": 0.25034530386740333, "grad_norm": 4.922246512972632, "kl": 0.08447265625, "learning_rate": 4.265848232362379e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.44060707092285156, "rewards_raw/accuracy_reward": 0.5616682171821594, "rewards_raw/clip_reward_wrapper": 0.33511245250701904, "rewards_raw/format_reward": 5.0, "step": 725 }, { "Total_reward": 6.599086077585525e-08, "advantage_std": 1.421678900718689, "completion_length": 418.1250305175781, "epoch": 0.2506906077348066, "grad_norm": 3.7551879214742856, "kl": 0.0830078125, "learning_rate": 4.263927433708534e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.7328078150749207, "rewards_raw/accuracy_reward": 0.5823341608047485, "rewards_raw/clip_reward_wrapper": 0.332045316696167, "rewards_raw/format_reward": 5.0, "step": 726 }, { "Total_reward": 1.852001503266365e-07, "advantage_std": 1.4201538562774658, "completion_length": 395.0535888671875, "epoch": 0.25103591160220995, "grad_norm": 3.719602047362917, "kl": 0.08203125, "learning_rate": 4.2620045592652286e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.3553244471549988, "rewards_raw/accuracy_reward": 0.5744941830635071, "rewards_raw/clip_reward_wrapper": 0.3292195498943329, "rewards_raw/format_reward": 5.0, "step": 727 }, { "Total_reward": 4.747084290102066e-07, "advantage_std": 1.5129132270812988, "completion_length": 409.2321472167969, "epoch": 0.2513812154696133, "grad_norm": 11.930296827540486, "kl": 0.0859375, "learning_rate": 4.260079611295302e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.9127243161201477, "rewards_raw/accuracy_reward": 0.5698628425598145, "rewards_raw/clip_reward_wrapper": 0.3417019248008728, "rewards_raw/format_reward": 5.0, "step": 728 }, { "Total_reward": -3.469841942660423e-07, "advantage_std": 1.2104333639144897, "completion_length": 395.8571472167969, "epoch": 0.25172651933701656, "grad_norm": 3.8563774527218215, "kl": 0.083984375, "learning_rate": 4.258152592064033e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.6271888017654419, "rewards_raw/accuracy_reward": 0.5412231087684631, "rewards_raw/clip_reward_wrapper": 0.3341067433357239, "rewards_raw/format_reward": 5.0, "step": 729 }, { "Total_reward": 1.7668520513325348e-07, "advantage_std": 1.5211106538772583, "completion_length": 410.1250305175781, "epoch": 0.2520718232044199, "grad_norm": 5.856742060698542, "kl": 0.07958984375, "learning_rate": 4.2562235038391347e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.5872772932052612, "rewards_raw/accuracy_reward": 0.5635443925857544, "rewards_raw/clip_reward_wrapper": 0.3257964253425598, "rewards_raw/format_reward": 5.0, "step": 730 }, { "Total_reward": 1.9371510973087425e-07, "advantage_std": 1.5086969137191772, "completion_length": 401.64288330078125, "epoch": 0.2524171270718232, "grad_norm": 4.403895460534149, "kl": 0.08447265625, "learning_rate": 4.254292348890761e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.651349663734436, "rewards_raw/accuracy_reward": 0.5707278251647949, "rewards_raw/clip_reward_wrapper": 0.34322869777679443, "rewards_raw/format_reward": 5.0, "step": 731 }, { "Total_reward": 9.100352258428757e-07, "advantage_std": 1.570406436920166, "completion_length": 403.6071472167969, "epoch": 0.2527624309392265, "grad_norm": 4.035343530466818, "kl": 0.08056640625, "learning_rate": 4.2523591294914924e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.8137880563735962, "rewards_raw/accuracy_reward": 0.5813875198364258, "rewards_raw/clip_reward_wrapper": 0.32724371552467346, "rewards_raw/format_reward": 5.0, "step": 732 }, { "Total_reward": -4.129750550418976e-07, "advantage_std": 1.5932297706604004, "completion_length": 395.8571472167969, "epoch": 0.25310773480662985, "grad_norm": 5.657737471008914, "kl": 0.0830078125, "learning_rate": 4.2504238479163426e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.9274182915687561, "rewards_raw/accuracy_reward": 0.5404269695281982, "rewards_raw/clip_reward_wrapper": 0.3379921019077301, "rewards_raw/format_reward": 5.0, "step": 733 }, { "Total_reward": 1.5114035534224968e-07, "advantage_std": 1.45504891872406, "completion_length": 399.8571472167969, "epoch": 0.2534530386740331, "grad_norm": 4.583417184960308, "kl": 0.08349609375, "learning_rate": 4.248486506442749e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.45263922214508057, "rewards_raw/accuracy_reward": 0.5541473627090454, "rewards_raw/clip_reward_wrapper": 0.34276077151298523, "rewards_raw/format_reward": 5.0, "step": 734 }, { "Total_reward": 7.195132525339432e-07, "advantage_std": 1.470884084701538, "completion_length": 404.6071472167969, "epoch": 0.25379834254143646, "grad_norm": 8.033478386288495, "kl": 0.0830078125, "learning_rate": 4.2465471073505765e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.6379591226577759, "rewards_raw/accuracy_reward": 0.5649864673614502, "rewards_raw/clip_reward_wrapper": 0.34015682339668274, "rewards_raw/format_reward": 5.0, "step": 735 }, { "Total_reward": -7.237706967089252e-08, "advantage_std": 1.3640937805175781, "completion_length": 394.7857360839844, "epoch": 0.2541436464088398, "grad_norm": 4.2153103194151536, "kl": 0.08447265625, "learning_rate": 4.2446056529221076e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.8453678488731384, "rewards_raw/accuracy_reward": 0.5801388621330261, "rewards_raw/clip_reward_wrapper": 0.3292672038078308, "rewards_raw/format_reward": 5.0, "step": 736 }, { "Total_reward": 8.302075826804867e-08, "advantage_std": 1.331101894378662, "completion_length": 403.5357360839844, "epoch": 0.2544889502762431, "grad_norm": 4.672333219893779, "kl": 0.083984375, "learning_rate": 4.2426621454420475e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.6092886924743652, "rewards_raw/accuracy_reward": 0.5754157900810242, "rewards_raw/clip_reward_wrapper": 0.34515291452407837, "rewards_raw/format_reward": 5.0, "step": 737 }, { "Total_reward": -8.98327186860115e-07, "advantage_std": 1.4359771013259888, "completion_length": 422.8571472167969, "epoch": 0.2548342541436464, "grad_norm": 13.198119832974875, "kl": 0.08251953125, "learning_rate": 4.2407165871975147e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.6054542064666748, "rewards_raw/accuracy_reward": 0.5578186511993408, "rewards_raw/clip_reward_wrapper": 0.33607685565948486, "rewards_raw/format_reward": 5.0, "step": 738 }, { "Total_reward": -3.576278970740532e-07, "advantage_std": 1.41061270236969, "completion_length": 405.1071472167969, "epoch": 0.25517955801104975, "grad_norm": 3.106927489147587, "kl": 0.080078125, "learning_rate": 4.238768980478041e-07, "loss": 0.0032, "rewards_raw/CoT ICLG": 0.5764760375022888, "rewards_raw/accuracy_reward": 0.5718164443969727, "rewards_raw/clip_reward_wrapper": 0.3259594142436981, "rewards_raw/format_reward": 5.0, "step": 739 }, { "Total_reward": -3.70400300653273e-07, "advantage_std": 1.5016757249832153, "completion_length": 396.8035888671875, "epoch": 0.255524861878453, "grad_norm": 5.438755025607763, "kl": 0.0830078125, "learning_rate": 4.2368193275755706e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.4146691858768463, "rewards_raw/accuracy_reward": 0.5634137988090515, "rewards_raw/clip_reward_wrapper": 0.340781033039093, "rewards_raw/format_reward": 5.0, "step": 740 }, { "Total_reward": -4.810946734323807e-07, "advantage_std": 1.3097426891326904, "completion_length": 399.51788330078125, "epoch": 0.25587016574585636, "grad_norm": 3.6579238361705366, "kl": 0.08544921875, "learning_rate": 4.234867630784453e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.750364363193512, "rewards_raw/accuracy_reward": 0.559660017490387, "rewards_raw/clip_reward_wrapper": 0.3331848680973053, "rewards_raw/format_reward": 5.0, "step": 741 }, { "Total_reward": -5.492142349794449e-07, "advantage_std": 1.5288335084915161, "completion_length": 413.6250305175781, "epoch": 0.2562154696132597, "grad_norm": 4.374970263539008, "kl": 0.08154296875, "learning_rate": 4.2329138924014473e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.5482150912284851, "rewards_raw/accuracy_reward": 0.5694777965545654, "rewards_raw/clip_reward_wrapper": 0.3421280086040497, "rewards_raw/format_reward": 5.0, "step": 742 }, { "Total_reward": -6.024326921760803e-07, "advantage_std": 1.4436935186386108, "completion_length": 413.51788330078125, "epoch": 0.256560773480663, "grad_norm": 3.7774339607293093, "kl": 0.083984375, "learning_rate": 4.2309581147257103e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.48677361011505127, "rewards_raw/accuracy_reward": 0.5840131044387817, "rewards_raw/clip_reward_wrapper": 0.3394657075405121, "rewards_raw/format_reward": 5.0, "step": 743 }, { "Total_reward": 1.4049666674509353e-07, "advantage_std": 1.2274305820465088, "completion_length": 408.1250305175781, "epoch": 0.2569060773480663, "grad_norm": 3.719569444843775, "kl": 0.0830078125, "learning_rate": 4.2290003000588017e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.5845762491226196, "rewards_raw/accuracy_reward": 0.5666618347167969, "rewards_raw/clip_reward_wrapper": 0.32589396834373474, "rewards_raw/format_reward": 5.0, "step": 744 }, { "Total_reward": 3.618853483544626e-08, "advantage_std": 1.2644810676574707, "completion_length": 413.1964416503906, "epoch": 0.2572513812154696, "grad_norm": 3.5482614515071313, "kl": 0.083984375, "learning_rate": 4.2270404507046774e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.8442367315292358, "rewards_raw/accuracy_reward": 0.5938887000083923, "rewards_raw/clip_reward_wrapper": 0.3447147309780121, "rewards_raw/format_reward": 5.0, "step": 745 }, { "Total_reward": -7.43993723517633e-07, "advantage_std": 1.3996460437774658, "completion_length": 393.96429443359375, "epoch": 0.2575966850828729, "grad_norm": 4.172109303304764, "kl": 0.08837890625, "learning_rate": 4.2250785689696866e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.671063244342804, "rewards_raw/accuracy_reward": 0.5647596716880798, "rewards_raw/clip_reward_wrapper": 0.34272387623786926, "rewards_raw/format_reward": 5.0, "step": 746 }, { "Total_reward": -8.94069742685133e-08, "advantage_std": 1.601873517036438, "completion_length": 397.2500305175781, "epoch": 0.25794198895027626, "grad_norm": 3.7142712033462746, "kl": 0.08251953125, "learning_rate": 4.223114657162573e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.4992524981498718, "rewards_raw/accuracy_reward": 0.5481897592544556, "rewards_raw/clip_reward_wrapper": 0.32476481795310974, "rewards_raw/format_reward": 5.0, "step": 747 }, { "Total_reward": 4.470348642371391e-07, "advantage_std": 1.537921667098999, "completion_length": 394.26788330078125, "epoch": 0.25828729281767954, "grad_norm": 5.076622996528353, "kl": 0.08251953125, "learning_rate": 4.221148717594467e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.6205014586448669, "rewards_raw/accuracy_reward": 0.5644075870513916, "rewards_raw/clip_reward_wrapper": 0.3414846658706665, "rewards_raw/format_reward": 5.0, "step": 748 }, { "Total_reward": -3.895589486546669e-07, "advantage_std": 1.503740668296814, "completion_length": 390.7321472167969, "epoch": 0.25863259668508287, "grad_norm": 7.493938076352404, "kl": 0.083984375, "learning_rate": 4.2191807525788856e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.9759231805801392, "rewards_raw/accuracy_reward": 0.5942220687866211, "rewards_raw/clip_reward_wrapper": 0.3251858651638031, "rewards_raw/format_reward": 5.0, "step": 749 }, { "Total_reward": 1.09736413378414e-06, "advantage_std": 1.5248522758483887, "completion_length": 392.5357360839844, "epoch": 0.2589779005524862, "grad_norm": 5.186865004819536, "kl": 0.0859375, "learning_rate": 4.217210764431729e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.6255504488945007, "rewards_raw/accuracy_reward": 0.5705285668373108, "rewards_raw/clip_reward_wrapper": 0.31769463419914246, "rewards_raw/format_reward": 5.0, "step": 750 }, { "Total_reward": -3.0547380447387695e-07, "advantage_std": 1.4712783098220825, "completion_length": 391.8035888671875, "epoch": 0.2593232044198895, "grad_norm": 3.3027344964725747, "kl": 0.091796875, "learning_rate": 4.215238755471281e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7119957804679871, "rewards_raw/accuracy_reward": 0.5670508146286011, "rewards_raw/clip_reward_wrapper": 0.34529367089271545, "rewards_raw/format_reward": 5.0, "step": 751 }, { "Total_reward": -2.895082786835701e-07, "advantage_std": 1.5632957220077515, "completion_length": 392.76788330078125, "epoch": 0.2596685082872928, "grad_norm": 5.97035295429586, "kl": 0.08447265625, "learning_rate": 4.2132647280182e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.539847731590271, "rewards_raw/accuracy_reward": 0.5473163723945618, "rewards_raw/clip_reward_wrapper": 0.32204926013946533, "rewards_raw/format_reward": 5.0, "step": 752 }, { "Total_reward": -3.08666926684964e-07, "advantage_std": 1.3756095170974731, "completion_length": 393.5535888671875, "epoch": 0.26001381215469616, "grad_norm": 7.2687470127789275, "kl": 0.08349609375, "learning_rate": 4.21128868439552e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.6444995403289795, "rewards_raw/accuracy_reward": 0.5679816007614136, "rewards_raw/clip_reward_wrapper": 0.33634746074676514, "rewards_raw/format_reward": 5.0, "step": 753 }, { "Total_reward": 3.384692490726593e-07, "advantage_std": 1.7278945446014404, "completion_length": 397.7857360839844, "epoch": 0.26035911602209943, "grad_norm": 8.737333188544785, "kl": 0.08251953125, "learning_rate": 4.209310626928651e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.7540964484214783, "rewards_raw/accuracy_reward": 0.5617657899856567, "rewards_raw/clip_reward_wrapper": 0.34093591570854187, "rewards_raw/format_reward": 5.0, "step": 754 }, { "Total_reward": -1.021793991640152e-07, "advantage_std": 1.5202101469039917, "completion_length": 395.5714416503906, "epoch": 0.26070441988950277, "grad_norm": 8.079537551114697, "kl": 0.08642578125, "learning_rate": 4.2073305579453686e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.781819760799408, "rewards_raw/accuracy_reward": 0.5872286558151245, "rewards_raw/clip_reward_wrapper": 0.3249175250530243, "rewards_raw/format_reward": 5.0, "step": 755 }, { "Total_reward": 4.5661417402698135e-07, "advantage_std": 1.644812822341919, "completion_length": 404.5000305175781, "epoch": 0.2610497237569061, "grad_norm": 5.226834993042888, "kl": 0.083984375, "learning_rate": 4.20534847977582e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.5881560444831848, "rewards_raw/accuracy_reward": 0.5808561444282532, "rewards_raw/clip_reward_wrapper": 0.32654252648353577, "rewards_raw/format_reward": 5.0, "step": 756 }, { "Total_reward": -9.025846452459518e-07, "advantage_std": 1.4803227186203003, "completion_length": 405.39288330078125, "epoch": 0.2613950276243094, "grad_norm": 7.9443364370535114, "kl": 0.08154296875, "learning_rate": 4.203364394752512e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.4671494960784912, "rewards_raw/accuracy_reward": 0.5500861406326294, "rewards_raw/clip_reward_wrapper": 0.3421996831893921, "rewards_raw/format_reward": 5.0, "step": 757 }, { "Total_reward": 8.302075826804867e-08, "advantage_std": 1.4946092367172241, "completion_length": 393.2500305175781, "epoch": 0.2617403314917127, "grad_norm": 19.18440597119343, "kl": 0.0849609375, "learning_rate": 4.201378305210317e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.6285279393196106, "rewards_raw/accuracy_reward": 0.569988489151001, "rewards_raw/clip_reward_wrapper": 0.33970746397972107, "rewards_raw/format_reward": 5.0, "step": 758 }, { "Total_reward": -6.875822009533294e-07, "advantage_std": 1.4081976413726807, "completion_length": 396.71429443359375, "epoch": 0.262085635359116, "grad_norm": 15.636809106325934, "kl": 0.08447265625, "learning_rate": 4.199390213486463e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.4915633797645569, "rewards_raw/accuracy_reward": 0.5720562934875488, "rewards_raw/clip_reward_wrapper": 0.3349844515323639, "rewards_raw/format_reward": 5.0, "step": 759 }, { "Total_reward": -4.896095902040543e-07, "advantage_std": 1.528503656387329, "completion_length": 407.0357360839844, "epoch": 0.26243093922651933, "grad_norm": 6.384117134036107, "kl": 0.0849609375, "learning_rate": 4.1974001219205383e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.46999940276145935, "rewards_raw/accuracy_reward": 0.5712990760803223, "rewards_raw/clip_reward_wrapper": 0.3385639488697052, "rewards_raw/format_reward": 5.0, "step": 760 }, { "Total_reward": 4.938671054333099e-07, "advantage_std": 1.3636596202850342, "completion_length": 400.51788330078125, "epoch": 0.26277624309392267, "grad_norm": 5.129330264918043, "kl": 0.0859375, "learning_rate": 4.195408032854481e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 1.0270529985427856, "rewards_raw/accuracy_reward": 0.5871038436889648, "rewards_raw/clip_reward_wrapper": 0.3413669466972351, "rewards_raw/format_reward": 5.0, "step": 761 }, { "Total_reward": 6.96097117725003e-07, "advantage_std": 1.5951218605041504, "completion_length": 405.9464416503906, "epoch": 0.26312154696132595, "grad_norm": 5.081175838458157, "kl": 0.0830078125, "learning_rate": 4.1934139486325815e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.8554710745811462, "rewards_raw/accuracy_reward": 0.5687920451164246, "rewards_raw/clip_reward_wrapper": 0.3337930738925934, "rewards_raw/format_reward": 5.0, "step": 762 }, { "Total_reward": -2.863151848941925e-07, "advantage_std": 1.6093223094940186, "completion_length": 411.6071472167969, "epoch": 0.2634668508287293, "grad_norm": 5.040390464661536, "kl": 0.0849609375, "learning_rate": 4.191417871601477e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.7564957737922668, "rewards_raw/accuracy_reward": 0.5844998359680176, "rewards_raw/clip_reward_wrapper": 0.31337589025497437, "rewards_raw/format_reward": 5.0, "step": 763 }, { "Total_reward": -1.8200704516857513e-06, "advantage_std": 1.6216788291931152, "completion_length": 399.7500305175781, "epoch": 0.2638121546961326, "grad_norm": 7.411456364458404, "kl": 0.0830078125, "learning_rate": 4.189419804110151e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 1.002011775970459, "rewards_raw/accuracy_reward": 0.564172625541687, "rewards_raw/clip_reward_wrapper": 0.34054532647132874, "rewards_raw/format_reward": 5.0, "step": 764 }, { "Total_reward": 3.4059798537100505e-08, "advantage_std": 1.6355384588241577, "completion_length": 396.3214416503906, "epoch": 0.2641574585635359, "grad_norm": 4.720441126461771, "kl": 0.0869140625, "learning_rate": 4.1874197485099295e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.690654993057251, "rewards_raw/accuracy_reward": 0.573634147644043, "rewards_raw/clip_reward_wrapper": 0.3358813524246216, "rewards_raw/format_reward": 5.0, "step": 765 }, { "Total_reward": -1.9797258232756576e-07, "advantage_std": 1.3006218671798706, "completion_length": 405.39288330078125, "epoch": 0.26450276243093923, "grad_norm": 3.3422234916586846, "kl": 0.08349609375, "learning_rate": 4.1854177071544765e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.4739272892475128, "rewards_raw/accuracy_reward": 0.5674034357070923, "rewards_raw/clip_reward_wrapper": 0.35158780217170715, "rewards_raw/format_reward": 5.0, "step": 766 }, { "Total_reward": -2.9802322387695312e-08, "advantage_std": 1.7712994813919067, "completion_length": 396.9285888671875, "epoch": 0.26484806629834257, "grad_norm": 5.21862206928116, "kl": 0.0849609375, "learning_rate": 4.1834136823997943e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.5017749667167664, "rewards_raw/accuracy_reward": 0.568305492401123, "rewards_raw/clip_reward_wrapper": 0.33479589223861694, "rewards_raw/format_reward": 5.0, "step": 767 }, { "Total_reward": 1.9158636632710113e-07, "advantage_std": 1.5901744365692139, "completion_length": 389.1250305175781, "epoch": 0.26519337016574585, "grad_norm": 5.669217732461096, "kl": 0.0859375, "learning_rate": 4.18140767660422e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.7001442313194275, "rewards_raw/accuracy_reward": 0.5780121088027954, "rewards_raw/clip_reward_wrapper": 0.3361395597457886, "rewards_raw/format_reward": 5.0, "step": 768 }, { "Total_reward": 9.110996188610443e-07, "advantage_std": 1.3161180019378662, "completion_length": 390.0357360839844, "epoch": 0.2655386740331492, "grad_norm": 2.9942865131426215, "kl": 0.08203125, "learning_rate": 4.179399692128419e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.669994056224823, "rewards_raw/accuracy_reward": 0.5810683965682983, "rewards_raw/clip_reward_wrapper": 0.3390684425830841, "rewards_raw/format_reward": 5.0, "step": 769 }, { "Total_reward": -2.064875275209488e-07, "advantage_std": 1.3776226043701172, "completion_length": 394.71429443359375, "epoch": 0.26588397790055246, "grad_norm": 3.559258353267909, "kl": 0.08544921875, "learning_rate": 4.177389731335388e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.5977256894111633, "rewards_raw/accuracy_reward": 0.5605403780937195, "rewards_raw/clip_reward_wrapper": 0.3427697718143463, "rewards_raw/format_reward": 5.0, "step": 770 }, { "Total_reward": 2.8099333349018707e-07, "advantage_std": 1.791774034500122, "completion_length": 406.1607360839844, "epoch": 0.2662292817679558, "grad_norm": 4.354138980850442, "kl": 0.0849609375, "learning_rate": 4.175377796590448e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.6127010583877563, "rewards_raw/accuracy_reward": 0.5863248705863953, "rewards_raw/clip_reward_wrapper": 0.3325352668762207, "rewards_raw/format_reward": 4.964285850524902, "step": 771 }, { "Total_reward": 1.2665987014770508e-07, "advantage_std": 1.6110563278198242, "completion_length": 383.3035888671875, "epoch": 0.26657458563535913, "grad_norm": 4.6454646040806145, "kl": 0.09033203125, "learning_rate": 4.173363890261244e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.5225434303283691, "rewards_raw/accuracy_reward": 0.5851406455039978, "rewards_raw/clip_reward_wrapper": 0.3192960023880005, "rewards_raw/format_reward": 5.0, "step": 772 }, { "Total_reward": -3.5656350405588455e-07, "advantage_std": 1.5145639181137085, "completion_length": 379.3214416503906, "epoch": 0.2669198895027624, "grad_norm": 5.8797054281327, "kl": 0.08447265625, "learning_rate": 4.1713480147177403e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.6184213161468506, "rewards_raw/accuracy_reward": 0.572783887386322, "rewards_raw/clip_reward_wrapper": 0.3317511975765228, "rewards_raw/format_reward": 5.0, "step": 773 }, { "Total_reward": 2.0010131152048416e-07, "advantage_std": 1.727892518043518, "completion_length": 399.33929443359375, "epoch": 0.26726519337016574, "grad_norm": 4.748669456614219, "kl": 0.08447265625, "learning_rate": 4.1693301723322184e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.5417773723602295, "rewards_raw/accuracy_reward": 0.5716093182563782, "rewards_raw/clip_reward_wrapper": 0.33471739292144775, "rewards_raw/format_reward": 5.0, "step": 774 }, { "Total_reward": -8.472375156998169e-07, "advantage_std": 1.6061561107635498, "completion_length": 415.46429443359375, "epoch": 0.2676104972375691, "grad_norm": 11.342486867018044, "kl": 0.083984375, "learning_rate": 4.167310365479275e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.509120762348175, "rewards_raw/accuracy_reward": 0.5650936961174011, "rewards_raw/clip_reward_wrapper": 0.3410378396511078, "rewards_raw/format_reward": 5.0, "step": 775 }, { "Total_reward": -5.24733763995755e-07, "advantage_std": 1.552770972251892, "completion_length": 403.46429443359375, "epoch": 0.26795580110497236, "grad_norm": 4.17619278377112, "kl": 0.0830078125, "learning_rate": 4.16528859653582e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.4448311924934387, "rewards_raw/accuracy_reward": 0.5753430724143982, "rewards_raw/clip_reward_wrapper": 0.3328169584274292, "rewards_raw/format_reward": 5.0, "step": 776 }, { "Total_reward": -3.437911004766647e-07, "advantage_std": 1.3463033437728882, "completion_length": 399.6964416503906, "epoch": 0.2683011049723757, "grad_norm": 3.6183951061761057, "kl": 0.08447265625, "learning_rate": 4.163264867881069e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.5912123918533325, "rewards_raw/accuracy_reward": 0.56659334897995, "rewards_raw/clip_reward_wrapper": 0.3235533833503723, "rewards_raw/format_reward": 5.0, "step": 777 }, { "Total_reward": -4.087175966560608e-07, "advantage_std": 1.3184483051300049, "completion_length": 400.96429443359375, "epoch": 0.268646408839779, "grad_norm": 5.023887164457188, "kl": 0.08544921875, "learning_rate": 4.1612391818965446e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.6268731355667114, "rewards_raw/accuracy_reward": 0.5580672025680542, "rewards_raw/clip_reward_wrapper": 0.321145623922348, "rewards_raw/format_reward": 5.0, "step": 778 }, { "Total_reward": 6.088188797548355e-07, "advantage_std": 1.5434589385986328, "completion_length": 388.6964416503906, "epoch": 0.2689917127071823, "grad_norm": 16.964679439984472, "kl": 0.083984375, "learning_rate": 4.159211540966076e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.47647982835769653, "rewards_raw/accuracy_reward": 0.5705327987670898, "rewards_raw/clip_reward_wrapper": 0.335281103849411, "rewards_raw/format_reward": 5.0, "step": 779 }, { "Total_reward": 1.021793991640152e-07, "advantage_std": 1.5744189023971558, "completion_length": 395.7500305175781, "epoch": 0.26933701657458564, "grad_norm": 3.715577065973244, "kl": 0.08837890625, "learning_rate": 4.15718194747579e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.5390633940696716, "rewards_raw/accuracy_reward": 0.5659973621368408, "rewards_raw/clip_reward_wrapper": 0.34682780504226685, "rewards_raw/format_reward": 5.0, "step": 780 }, { "Total_reward": -1.0430813546236095e-07, "advantage_std": 1.6400413513183594, "completion_length": 405.64288330078125, "epoch": 0.269682320441989, "grad_norm": 4.20633158993248, "kl": 0.083984375, "learning_rate": 4.15515040381411e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.7538851499557495, "rewards_raw/accuracy_reward": 0.5587775111198425, "rewards_raw/clip_reward_wrapper": 0.3466971516609192, "rewards_raw/format_reward": 5.0, "step": 781 }, { "Total_reward": -4.129750550418976e-07, "advantage_std": 1.4352695941925049, "completion_length": 410.4107360839844, "epoch": 0.27002762430939226, "grad_norm": 3.3093154170582633, "kl": 0.08447265625, "learning_rate": 4.153116912371758e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.39651674032211304, "rewards_raw/accuracy_reward": 0.5740615725517273, "rewards_raw/clip_reward_wrapper": 0.33971378207206726, "rewards_raw/format_reward": 5.0, "step": 782 }, { "Total_reward": -2.054231629244896e-07, "advantage_std": 1.306069016456604, "completion_length": 386.71429443359375, "epoch": 0.2703729281767956, "grad_norm": 3.0272824553501336, "kl": 0.08984375, "learning_rate": 4.1510814755417466e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.8236606121063232, "rewards_raw/accuracy_reward": 0.5911972522735596, "rewards_raw/clip_reward_wrapper": 0.3083385229110718, "rewards_raw/format_reward": 5.0, "step": 783 }, { "Total_reward": -5.534717217869911e-08, "advantage_std": 1.4922856092453003, "completion_length": 405.6607360839844, "epoch": 0.27071823204419887, "grad_norm": 4.894105466917194, "kl": 0.09033203125, "learning_rate": 4.149044095719376e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.5563490390777588, "rewards_raw/accuracy_reward": 0.562675416469574, "rewards_raw/clip_reward_wrapper": 0.31625911593437195, "rewards_raw/format_reward": 5.0, "step": 784 }, { "Total_reward": 2.043587983280304e-07, "advantage_std": 1.6706486940383911, "completion_length": 391.1964416503906, "epoch": 0.2710635359116022, "grad_norm": 4.528324359119487, "kl": 0.0859375, "learning_rate": 4.147004775302236e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 1.174998164176941, "rewards_raw/accuracy_reward": 0.5806798934936523, "rewards_raw/clip_reward_wrapper": 0.3260774314403534, "rewards_raw/format_reward": 5.0, "step": 785 }, { "Total_reward": 1.0196652056038147e-06, "advantage_std": 1.5062777996063232, "completion_length": 397.0357360839844, "epoch": 0.27140883977900554, "grad_norm": 4.310924888682362, "kl": 0.0859375, "learning_rate": 4.144963516690197e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.4989067018032074, "rewards_raw/accuracy_reward": 0.5722345113754272, "rewards_raw/clip_reward_wrapper": 0.33856362104415894, "rewards_raw/format_reward": 5.0, "step": 786 }, { "Total_reward": -3.980738938480499e-07, "advantage_std": 1.4821550846099854, "completion_length": 404.96429443359375, "epoch": 0.2717541436464088, "grad_norm": 7.711542847302677, "kl": 0.0830078125, "learning_rate": 4.1429203222854124e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 0.7976890802383423, "rewards_raw/accuracy_reward": 0.5818866491317749, "rewards_raw/clip_reward_wrapper": 0.32905545830726624, "rewards_raw/format_reward": 5.0, "step": 787 }, { "Total_reward": 1.5007599074579048e-07, "advantage_std": 1.4601867198944092, "completion_length": 394.0000305175781, "epoch": 0.27209944751381215, "grad_norm": 5.962802080589545, "kl": 0.0849609375, "learning_rate": 4.140875194492313e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.6580718159675598, "rewards_raw/accuracy_reward": 0.5761343240737915, "rewards_raw/clip_reward_wrapper": 0.33183014392852783, "rewards_raw/format_reward": 5.0, "step": 788 }, { "Total_reward": -2.1925995952187805e-07, "advantage_std": 1.5219441652297974, "completion_length": 402.7857360839844, "epoch": 0.2724447513812155, "grad_norm": 5.188097598903294, "kl": 0.08984375, "learning_rate": 4.1388281357176046e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.5325559973716736, "rewards_raw/accuracy_reward": 0.5474474430084229, "rewards_raw/clip_reward_wrapper": 0.33407989144325256, "rewards_raw/format_reward": 5.0, "step": 789 }, { "Total_reward": -6.471361757576233e-07, "advantage_std": 1.4320616722106934, "completion_length": 400.89288330078125, "epoch": 0.27279005524861877, "grad_norm": 5.172531831068349, "kl": 0.08740234375, "learning_rate": 4.136779148370266e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.6144100427627563, "rewards_raw/accuracy_reward": 0.5644487738609314, "rewards_raw/clip_reward_wrapper": 0.3263954520225525, "rewards_raw/format_reward": 5.0, "step": 790 }, { "Total_reward": -2.895082786835701e-07, "advantage_std": 1.5362051725387573, "completion_length": 411.6964416503906, "epoch": 0.2731353591160221, "grad_norm": 4.0144000984991095, "kl": 0.0859375, "learning_rate": 4.1347282348615443e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.49527209997177124, "rewards_raw/accuracy_reward": 0.5598151087760925, "rewards_raw/clip_reward_wrapper": 0.32847118377685547, "rewards_raw/format_reward": 5.0, "step": 791 }, { "Total_reward": -2.1287374085687816e-09, "advantage_std": 1.4676015377044678, "completion_length": 413.9285888671875, "epoch": 0.27348066298342544, "grad_norm": 6.991097775178609, "kl": 0.087890625, "learning_rate": 4.132675397604956e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.7629223465919495, "rewards_raw/accuracy_reward": 0.574193000793457, "rewards_raw/clip_reward_wrapper": 0.32541242241859436, "rewards_raw/format_reward": 5.0, "step": 792 }, { "Total_reward": 1.27724248955019e-07, "advantage_std": 1.410747766494751, "completion_length": 407.7857360839844, "epoch": 0.2738259668508287, "grad_norm": 4.044600480124063, "kl": 0.0859375, "learning_rate": 4.130620639016277e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.7228454947471619, "rewards_raw/accuracy_reward": 0.5898391604423523, "rewards_raw/clip_reward_wrapper": 0.3248770236968994, "rewards_raw/format_reward": 5.0, "step": 793 }, { "Total_reward": 1.4049666674509353e-07, "advantage_std": 1.4077519178390503, "completion_length": 391.89288330078125, "epoch": 0.27417127071823205, "grad_norm": 3.4130727637456384, "kl": 0.0859375, "learning_rate": 4.1285639615135474e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 1.1787388324737549, "rewards_raw/accuracy_reward": 0.5620880126953125, "rewards_raw/clip_reward_wrapper": 0.3378523290157318, "rewards_raw/format_reward": 5.0, "step": 794 }, { "Total_reward": 6.279775277562294e-08, "advantage_std": 1.378275990486145, "completion_length": 405.6071472167969, "epoch": 0.2745165745856354, "grad_norm": 5.093553450557287, "kl": 0.087890625, "learning_rate": 4.126505367517067e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.6204645037651062, "rewards_raw/accuracy_reward": 0.5615687966346741, "rewards_raw/clip_reward_wrapper": 0.32926270365715027, "rewards_raw/format_reward": 5.0, "step": 795 }, { "Total_reward": -3.3314739766865387e-07, "advantage_std": 1.342635989189148, "completion_length": 389.1785888671875, "epoch": 0.27486187845303867, "grad_norm": 3.5418307433988505, "kl": 0.0849609375, "learning_rate": 4.1244448594493865e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.5038427114486694, "rewards_raw/accuracy_reward": 0.5620173215866089, "rewards_raw/clip_reward_wrapper": 0.3254021406173706, "rewards_raw/format_reward": 5.0, "step": 796 }, { "Total_reward": 8.419156642958114e-07, "advantage_std": 1.6057186126708984, "completion_length": 398.14288330078125, "epoch": 0.275207182320442, "grad_norm": 6.406539862036025, "kl": 0.08544921875, "learning_rate": 4.122382439735312e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.5270516872406006, "rewards_raw/accuracy_reward": 0.5616673231124878, "rewards_raw/clip_reward_wrapper": 0.3247607350349426, "rewards_raw/format_reward": 5.0, "step": 797 }, { "Total_reward": 1.0643686820799303e-08, "advantage_std": 1.3022689819335938, "completion_length": 384.96429443359375, "epoch": 0.2755524861878453, "grad_norm": 11.511870404149876, "kl": 0.08740234375, "learning_rate": 4.120318110801899e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.6275606155395508, "rewards_raw/accuracy_reward": 0.5817038416862488, "rewards_raw/clip_reward_wrapper": 0.3364339768886566, "rewards_raw/format_reward": 5.0, "step": 798 }, { "Total_reward": -3.3208303307219467e-07, "advantage_std": 1.4957311153411865, "completion_length": 401.83929443359375, "epoch": 0.2758977900552486, "grad_norm": 5.291052488987266, "kl": 0.08447265625, "learning_rate": 4.118251875078449e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.5611647367477417, "rewards_raw/accuracy_reward": 0.5776368975639343, "rewards_raw/clip_reward_wrapper": 0.33487215638160706, "rewards_raw/format_reward": 5.0, "step": 799 }, { "Total_reward": 1.9903694692402496e-07, "advantage_std": 1.2373199462890625, "completion_length": 409.6250305175781, "epoch": 0.27624309392265195, "grad_norm": 3.3074631083577604, "kl": 0.08642578125, "learning_rate": 4.1161837349965087e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.5882573127746582, "rewards_raw/accuracy_reward": 0.533612847328186, "rewards_raw/clip_reward_wrapper": 0.3263571858406067, "rewards_raw/format_reward": 5.0, "step": 800 }, { "Total_reward": -6.013682991579117e-07, "advantage_std": 1.4917032718658447, "completion_length": 403.1071472167969, "epoch": 0.27658839779005523, "grad_norm": 3.1772179497111632, "kl": 0.08740234375, "learning_rate": 4.114113692989863e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.6697127223014832, "rewards_raw/accuracy_reward": 0.5775914192199707, "rewards_raw/clip_reward_wrapper": 0.3384520709514618, "rewards_raw/format_reward": 5.0, "step": 801 }, { "Total_reward": 7.195132525339432e-07, "advantage_std": 1.6892234086990356, "completion_length": 395.39288330078125, "epoch": 0.27693370165745856, "grad_norm": 8.095778671987324, "kl": 0.09228515625, "learning_rate": 4.1120417514945397e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.6827903985977173, "rewards_raw/accuracy_reward": 0.5625448226928711, "rewards_raw/clip_reward_wrapper": 0.33429625630378723, "rewards_raw/format_reward": 5.0, "step": 802 }, { "Total_reward": 1.5114035534224968e-07, "advantage_std": 1.6193652153015137, "completion_length": 392.6785888671875, "epoch": 0.2772790055248619, "grad_norm": 4.853069009196087, "kl": 0.08447265625, "learning_rate": 4.1099679129487966e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.6772617101669312, "rewards_raw/accuracy_reward": 0.5704063177108765, "rewards_raw/clip_reward_wrapper": 0.3256899118423462, "rewards_raw/format_reward": 5.0, "step": 803 }, { "Total_reward": -2.628990785069618e-07, "advantage_std": 1.2626222372055054, "completion_length": 404.2857360839844, "epoch": 0.2776243093922652, "grad_norm": 3.336136208714006, "kl": 0.08544921875, "learning_rate": 4.107892179793128e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.7143016457557678, "rewards_raw/accuracy_reward": 0.5657127499580383, "rewards_raw/clip_reward_wrapper": 0.33481648564338684, "rewards_raw/format_reward": 5.0, "step": 804 }, { "Total_reward": 3.2676118166818924e-07, "advantage_std": 1.3291047811508179, "completion_length": 398.1250305175781, "epoch": 0.2779696132596685, "grad_norm": 2.929483856467132, "kl": 0.08935546875, "learning_rate": 4.105814554470256e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.5655125379562378, "rewards_raw/accuracy_reward": 0.5822200179100037, "rewards_raw/clip_reward_wrapper": 0.35012248158454895, "rewards_raw/format_reward": 5.0, "step": 805 }, { "Total_reward": 2.7673587510435027e-07, "advantage_std": 1.4173825979232788, "completion_length": 399.0714416503906, "epoch": 0.27831491712707185, "grad_norm": 3.6117631702312836, "kl": 0.08935546875, "learning_rate": 4.103735039425129e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.5551689863204956, "rewards_raw/accuracy_reward": 0.5770566463470459, "rewards_raw/clip_reward_wrapper": 0.33431631326675415, "rewards_raw/format_reward": 5.0, "step": 806 }, { "Total_reward": -2.873795494906517e-07, "advantage_std": 1.517518162727356, "completion_length": 390.89288330078125, "epoch": 0.2786602209944751, "grad_norm": 4.71579153550892, "kl": 0.08740234375, "learning_rate": 4.10165363710492e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.6168478727340698, "rewards_raw/accuracy_reward": 0.5678114295005798, "rewards_raw/clip_reward_wrapper": 0.3360911011695862, "rewards_raw/format_reward": 5.0, "step": 807 }, { "Total_reward": -4.002026230409683e-07, "advantage_std": 1.5441806316375732, "completion_length": 389.9821472167969, "epoch": 0.27900552486187846, "grad_norm": 4.193652224742023, "kl": 0.09033203125, "learning_rate": 4.099570349959024e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.39469575881958, "rewards_raw/accuracy_reward": 0.5391813516616821, "rewards_raw/clip_reward_wrapper": 0.33633509278297424, "rewards_raw/format_reward": 5.0, "step": 808 }, { "Total_reward": -9.856054248302826e-07, "advantage_std": 1.84494149684906, "completion_length": 410.3214416503906, "epoch": 0.27935082872928174, "grad_norm": 10.38614344970887, "kl": 0.087890625, "learning_rate": 4.097485180439052e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.26544636487960815, "rewards_raw/accuracy_reward": 0.5509117841720581, "rewards_raw/clip_reward_wrapper": 0.33710047602653503, "rewards_raw/format_reward": 5.0, "step": 809 }, { "Total_reward": 8.514949456639442e-08, "advantage_std": 1.5128884315490723, "completion_length": 394.5000305175781, "epoch": 0.2796961325966851, "grad_norm": 6.624770689154092, "kl": 0.0888671875, "learning_rate": 4.0953981309988303e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 1.08675217628479, "rewards_raw/accuracy_reward": 0.5572872161865234, "rewards_raw/clip_reward_wrapper": 0.3333841860294342, "rewards_raw/format_reward": 5.0, "step": 810 }, { "Total_reward": -7.365431429207092e-07, "advantage_std": 1.6159228086471558, "completion_length": 403.5000305175781, "epoch": 0.2800414364640884, "grad_norm": 7.038474205832381, "kl": 0.087890625, "learning_rate": 4.093309204094401e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.735274076461792, "rewards_raw/accuracy_reward": 0.5604504346847534, "rewards_raw/clip_reward_wrapper": 0.3244097828865051, "rewards_raw/format_reward": 5.0, "step": 811 }, { "Total_reward": -1.27724248955019e-08, "advantage_std": 1.4792978763580322, "completion_length": 387.83929443359375, "epoch": 0.2803867403314917, "grad_norm": 4.806637929843782, "kl": 0.08837890625, "learning_rate": 4.091218402184009e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.7318107485771179, "rewards_raw/accuracy_reward": 0.5940638780593872, "rewards_raw/clip_reward_wrapper": 0.3347480595111847, "rewards_raw/format_reward": 5.0, "step": 812 }, { "Total_reward": 3.874302194617485e-07, "advantage_std": 1.5573903322219849, "completion_length": 393.4464416503906, "epoch": 0.280732044198895, "grad_norm": 4.991085774171862, "kl": 0.0869140625, "learning_rate": 4.0891257277281125e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.7180336117744446, "rewards_raw/accuracy_reward": 0.5559701323509216, "rewards_raw/clip_reward_wrapper": 0.32386085391044617, "rewards_raw/format_reward": 5.0, "step": 813 }, { "Total_reward": -3.2143935868589324e-07, "advantage_std": 1.3735151290893555, "completion_length": 406.71429443359375, "epoch": 0.28107734806629836, "grad_norm": 4.359108859453009, "kl": 0.08984375, "learning_rate": 4.08703118318937e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.5635392069816589, "rewards_raw/accuracy_reward": 0.5858809351921082, "rewards_raw/clip_reward_wrapper": 0.3333945572376251, "rewards_raw/format_reward": 5.0, "step": 814 }, { "Total_reward": 2.852508202977333e-07, "advantage_std": 1.3738281726837158, "completion_length": 393.21429443359375, "epoch": 0.28142265193370164, "grad_norm": 5.548123119066764, "kl": 0.08740234375, "learning_rate": 4.08493477103264e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 1.3166872262954712, "rewards_raw/accuracy_reward": 0.5687722563743591, "rewards_raw/clip_reward_wrapper": 0.32218027114868164, "rewards_raw/format_reward": 5.0, "step": 815 }, { "Total_reward": -5.24733763995755e-07, "advantage_std": 1.6504524946212769, "completion_length": 384.01788330078125, "epoch": 0.281767955801105, "grad_norm": 4.455607913036356, "kl": 0.09130859375, "learning_rate": 4.0828364937249804e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.182715892791748, "rewards_raw/accuracy_reward": 0.5764458179473877, "rewards_raw/clip_reward_wrapper": 0.31732773780822754, "rewards_raw/format_reward": 5.0, "step": 816 }, { "Total_reward": -2.384185791015625e-07, "advantage_std": 1.4186707735061646, "completion_length": 403.0357360839844, "epoch": 0.2821132596685083, "grad_norm": 4.916771454813604, "kl": 0.08642578125, "learning_rate": 4.0807363537356443e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.7478644251823425, "rewards_raw/accuracy_reward": 0.5776563882827759, "rewards_raw/clip_reward_wrapper": 0.338210791349411, "rewards_raw/format_reward": 5.0, "step": 817 }, { "Total_reward": 3.384692490726593e-07, "advantage_std": 1.4823238849639893, "completion_length": 379.1250305175781, "epoch": 0.2824585635359116, "grad_norm": 5.607398525367613, "kl": 0.08984375, "learning_rate": 4.0786343535360753e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.2545359134674072, "rewards_raw/accuracy_reward": 0.5848453640937805, "rewards_raw/clip_reward_wrapper": 0.33865711092948914, "rewards_raw/format_reward": 5.0, "step": 818 }, { "Total_reward": 2.7247838829680404e-07, "advantage_std": 1.6356664896011353, "completion_length": 377.39288330078125, "epoch": 0.2828038674033149, "grad_norm": 5.7918769848440625, "kl": 0.09423828125, "learning_rate": 4.0765304955999066e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6544952392578125, "rewards_raw/accuracy_reward": 0.5768913626670837, "rewards_raw/clip_reward_wrapper": 0.3231552243232727, "rewards_raw/format_reward": 5.0, "step": 819 }, { "Total_reward": -9.5367431640625e-07, "advantage_std": 1.2848390340805054, "completion_length": 400.9464416503906, "epoch": 0.28314917127071826, "grad_norm": 3.8346615980363015, "kl": 0.08642578125, "learning_rate": 4.0744247824029577e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.9182012677192688, "rewards_raw/accuracy_reward": 0.5651845335960388, "rewards_raw/clip_reward_wrapper": 0.3484625220298767, "rewards_raw/format_reward": 5.0, "step": 820 }, { "Total_reward": -3.3101866847573547e-07, "advantage_std": 1.3642816543579102, "completion_length": 391.08929443359375, "epoch": 0.28349447513812154, "grad_norm": 3.4554790993397337, "kl": 0.09033203125, "learning_rate": 4.0723172164232327e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.018019199371338, "rewards_raw/accuracy_reward": 0.5880513191223145, "rewards_raw/clip_reward_wrapper": 0.33328351378440857, "rewards_raw/format_reward": 5.0, "step": 821 }, { "Total_reward": -3.7997963886482466e-07, "advantage_std": 1.406911015510559, "completion_length": 380.3571472167969, "epoch": 0.2838397790055249, "grad_norm": 7.908320544119741, "kl": 0.08642578125, "learning_rate": 4.0702078001409137e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.7376709580421448, "rewards_raw/accuracy_reward": 0.5484089255332947, "rewards_raw/clip_reward_wrapper": 0.3268074095249176, "rewards_raw/format_reward": 5.0, "step": 822 }, { "Total_reward": -7.514443041145569e-07, "advantage_std": 1.35984206199646, "completion_length": 385.3750305175781, "epoch": 0.28418508287292815, "grad_norm": 5.754250181733839, "kl": 0.08740234375, "learning_rate": 4.068096536038362e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.8111130595207214, "rewards_raw/accuracy_reward": 0.5591242909431458, "rewards_raw/clip_reward_wrapper": 0.3499821424484253, "rewards_raw/format_reward": 5.0, "step": 823 }, { "Total_reward": -3.2888993928281707e-07, "advantage_std": 1.1901445388793945, "completion_length": 393.1785888671875, "epoch": 0.2845303867403315, "grad_norm": 78.47319958893138, "kl": 0.08544921875, "learning_rate": 4.0659834266001127e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.6421501040458679, "rewards_raw/accuracy_reward": 0.5613644123077393, "rewards_raw/clip_reward_wrapper": 0.3299665153026581, "rewards_raw/format_reward": 5.0, "step": 824 }, { "Total_reward": 4.619360254309868e-07, "advantage_std": 1.3348617553710938, "completion_length": 393.33929443359375, "epoch": 0.2848756906077348, "grad_norm": 5.007627607631633, "kl": 0.08935546875, "learning_rate": 4.063868474312871e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.8819535374641418, "rewards_raw/accuracy_reward": 0.5549165606498718, "rewards_raw/clip_reward_wrapper": 0.33148542046546936, "rewards_raw/format_reward": 5.0, "step": 825 }, { "Total_reward": -3.118600204743416e-07, "advantage_std": 1.487017273902893, "completion_length": 392.9107360839844, "epoch": 0.2852209944751381, "grad_norm": 6.732863390992552, "kl": 0.08935546875, "learning_rate": 4.061751681665515e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.7066196799278259, "rewards_raw/accuracy_reward": 0.5681910514831543, "rewards_raw/clip_reward_wrapper": 0.3321356773376465, "rewards_raw/format_reward": 5.0, "step": 826 }, { "Total_reward": 8.514949456639442e-08, "advantage_std": 1.6278167963027954, "completion_length": 398.76788330078125, "epoch": 0.28556629834254144, "grad_norm": 4.081917411372425, "kl": 0.08837890625, "learning_rate": 4.059633051149084e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.7081433534622192, "rewards_raw/accuracy_reward": 0.5779547691345215, "rewards_raw/clip_reward_wrapper": 0.3349691331386566, "rewards_raw/format_reward": 5.0, "step": 827 }, { "Total_reward": -1.3623919414840202e-07, "advantage_std": 1.4181833267211914, "completion_length": 391.4285888671875, "epoch": 0.28591160220994477, "grad_norm": 3.444681880293168, "kl": 0.08984375, "learning_rate": 4.0575125852567834e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.6548174023628235, "rewards_raw/accuracy_reward": 0.5607983469963074, "rewards_raw/clip_reward_wrapper": 0.33647647500038147, "rewards_raw/format_reward": 5.0, "step": 828 }, { "Total_reward": -6.354281367748627e-07, "advantage_std": 1.4857163429260254, "completion_length": 399.6250305175781, "epoch": 0.28625690607734805, "grad_norm": 4.470643046481244, "kl": 0.0849609375, "learning_rate": 4.0553902864839757e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.7800909876823425, "rewards_raw/accuracy_reward": 0.5702164173126221, "rewards_raw/clip_reward_wrapper": 0.3360760807991028, "rewards_raw/format_reward": 5.0, "step": 829 }, { "Total_reward": 5.2154067731180476e-08, "advantage_std": 1.5068531036376953, "completion_length": 394.7857360839844, "epoch": 0.2866022099447514, "grad_norm": 4.402082462628039, "kl": 0.09033203125, "learning_rate": 4.053266157328182e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.6408942341804504, "rewards_raw/accuracy_reward": 0.567094087600708, "rewards_raw/clip_reward_wrapper": 0.3266555666923523, "rewards_raw/format_reward": 5.0, "step": 830 }, { "Total_reward": -1.9584383892379265e-07, "advantage_std": 1.8426668643951416, "completion_length": 401.51788330078125, "epoch": 0.2869475138121547, "grad_norm": 4.478646925040551, "kl": 0.0869140625, "learning_rate": 4.0511402002890773e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 1.1038285493850708, "rewards_raw/accuracy_reward": 0.5428762435913086, "rewards_raw/clip_reward_wrapper": 0.32016175985336304, "rewards_raw/format_reward": 4.866071701049805, "step": 831 }, { "Total_reward": 6.748097689524002e-07, "advantage_std": 1.6102465391159058, "completion_length": 388.1250305175781, "epoch": 0.287292817679558, "grad_norm": 8.258426628389158, "kl": 0.0869140625, "learning_rate": 4.049012417868488e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 2.067316770553589, "rewards_raw/accuracy_reward": 0.5476083159446716, "rewards_raw/clip_reward_wrapper": 0.3293572962284088, "rewards_raw/format_reward": 5.0, "step": 832 }, { "Total_reward": -2.7354275289326324e-07, "advantage_std": 1.571625828742981, "completion_length": 395.8571472167969, "epoch": 0.28763812154696133, "grad_norm": 4.87712492185979, "kl": 0.08740234375, "learning_rate": 4.046882812570387e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.6221417784690857, "rewards_raw/accuracy_reward": 0.5717906951904297, "rewards_raw/clip_reward_wrapper": 0.3427124321460724, "rewards_raw/format_reward": 5.0, "step": 833 }, { "Total_reward": 6.173338817916374e-08, "advantage_std": 1.409816026687622, "completion_length": 395.3035888671875, "epoch": 0.2879834254143646, "grad_norm": 3.736511516015224, "kl": 0.087890625, "learning_rate": 4.0447513869008936e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 1.0255577564239502, "rewards_raw/accuracy_reward": 0.551601767539978, "rewards_raw/clip_reward_wrapper": 0.3354845345020294, "rewards_raw/format_reward": 5.0, "step": 834 }, { "Total_reward": -1.6604151653609733e-07, "advantage_std": 1.4285236597061157, "completion_length": 395.26788330078125, "epoch": 0.28832872928176795, "grad_norm": 17.081546041955168, "kl": 0.08642578125, "learning_rate": 4.0426181433682695e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.7314209342002869, "rewards_raw/accuracy_reward": 0.5932456851005554, "rewards_raw/clip_reward_wrapper": 0.3444260358810425, "rewards_raw/format_reward": 5.0, "step": 835 }, { "Total_reward": -9.068421604752075e-07, "advantage_std": 1.4568300247192383, "completion_length": 396.64288330078125, "epoch": 0.2886740331491713, "grad_norm": 6.801098712872046, "kl": 0.0908203125, "learning_rate": 4.040483084482915e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.7018941044807434, "rewards_raw/accuracy_reward": 0.5618669986724854, "rewards_raw/clip_reward_wrapper": 0.32485610246658325, "rewards_raw/format_reward": 5.0, "step": 836 }, { "Total_reward": -9.664468052505981e-07, "advantage_std": 1.5819531679153442, "completion_length": 396.3214416503906, "epoch": 0.28901933701657456, "grad_norm": 9.470037131194555, "kl": 0.08447265625, "learning_rate": 4.0383462127573677e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 0.684551477432251, "rewards_raw/accuracy_reward": 0.5642340183258057, "rewards_raw/clip_reward_wrapper": 0.3237581253051758, "rewards_raw/format_reward": 5.0, "step": 837 }, { "Total_reward": 8.866191478773544e-07, "advantage_std": 1.4048001766204834, "completion_length": 391.1607360839844, "epoch": 0.2893646408839779, "grad_norm": 4.774025292615337, "kl": 0.08642578125, "learning_rate": 4.036207530706297e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.4890732169151306, "rewards_raw/accuracy_reward": 0.5671142935752869, "rewards_raw/clip_reward_wrapper": 0.3329223394393921, "rewards_raw/format_reward": 5.0, "step": 838 }, { "Total_reward": 4.406486482366745e-07, "advantage_std": 1.4980924129486084, "completion_length": 388.5000305175781, "epoch": 0.28970994475138123, "grad_norm": 4.991409769874811, "kl": 0.08837890625, "learning_rate": 4.034067040846503e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.8467835783958435, "rewards_raw/accuracy_reward": 0.5852500796318054, "rewards_raw/clip_reward_wrapper": 0.33388954401016235, "rewards_raw/format_reward": 5.0, "step": 839 }, { "Total_reward": 9.409019412487396e-07, "advantage_std": 1.366857886314392, "completion_length": 379.7321472167969, "epoch": 0.2900552486187845, "grad_norm": 5.746258321095552, "kl": 0.087890625, "learning_rate": 4.031924745696915e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.7406747937202454, "rewards_raw/accuracy_reward": 0.5808150768280029, "rewards_raw/clip_reward_wrapper": 0.32708561420440674, "rewards_raw/format_reward": 5.0, "step": 840 }, { "Total_reward": 2.8099333349018707e-07, "advantage_std": 1.470401644706726, "completion_length": 394.46429443359375, "epoch": 0.29040055248618785, "grad_norm": 9.619617344110525, "kl": 0.08837890625, "learning_rate": 4.0297806477785856e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.7615296244621277, "rewards_raw/accuracy_reward": 0.5946942567825317, "rewards_raw/clip_reward_wrapper": 0.3482295870780945, "rewards_raw/format_reward": 5.0, "step": 841 }, { "Total_reward": 4.747084290102066e-07, "advantage_std": 1.6923531293869019, "completion_length": 405.26788330078125, "epoch": 0.2907458563535912, "grad_norm": 5.752336614118759, "kl": 0.087890625, "learning_rate": 4.0276347496146873e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.527092456817627, "rewards_raw/accuracy_reward": 0.5659409761428833, "rewards_raw/clip_reward_wrapper": 0.33838409185409546, "rewards_raw/format_reward": 5.0, "step": 842 }, { "Total_reward": 3.437911004766647e-07, "advantage_std": 1.5509828329086304, "completion_length": 400.76788330078125, "epoch": 0.29109116022099446, "grad_norm": 4.541900019697067, "kl": 0.0927734375, "learning_rate": 4.025487053730514e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7178663611412048, "rewards_raw/accuracy_reward": 0.5847234725952148, "rewards_raw/clip_reward_wrapper": 0.32796669006347656, "rewards_raw/format_reward": 5.0, "step": 843 }, { "Total_reward": -7.046120913400955e-07, "advantage_std": 1.6419607400894165, "completion_length": 385.1964416503906, "epoch": 0.2914364640883978, "grad_norm": 7.287275460573095, "kl": 0.0888671875, "learning_rate": 4.023337562653473e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.6935532689094543, "rewards_raw/accuracy_reward": 0.5601106882095337, "rewards_raw/clip_reward_wrapper": 0.3277769684791565, "rewards_raw/format_reward": 5.0, "step": 844 }, { "Total_reward": -2.2564616131148796e-07, "advantage_std": 1.684162974357605, "completion_length": 390.8214416503906, "epoch": 0.29178176795580113, "grad_norm": 4.567549254385319, "kl": 0.087890625, "learning_rate": 4.021186278913086e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.7332141399383545, "rewards_raw/accuracy_reward": 0.5641300678253174, "rewards_raw/clip_reward_wrapper": 0.3360913097858429, "rewards_raw/format_reward": 5.0, "step": 845 }, { "Total_reward": 9.260007516331825e-08, "advantage_std": 1.5714658498764038, "completion_length": 391.71429443359375, "epoch": 0.2921270718232044, "grad_norm": 8.621791467299918, "kl": 0.09130859375, "learning_rate": 4.0190332050409826e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.6667961478233337, "rewards_raw/accuracy_reward": 0.5713000297546387, "rewards_raw/clip_reward_wrapper": 0.3369562327861786, "rewards_raw/format_reward": 5.0, "step": 846 }, { "Total_reward": -1.0005065860241302e-06, "advantage_std": 1.6888160705566406, "completion_length": 395.5000305175781, "epoch": 0.29247237569060774, "grad_norm": 4.829831128102735, "kl": 0.0869140625, "learning_rate": 4.0168783435708995e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.7940002083778381, "rewards_raw/accuracy_reward": 0.5783714652061462, "rewards_raw/clip_reward_wrapper": 0.33431729674339294, "rewards_raw/format_reward": 5.0, "step": 847 }, { "Total_reward": 4.917383193969727e-07, "advantage_std": 1.3774057626724243, "completion_length": 389.51788330078125, "epoch": 0.292817679558011, "grad_norm": 23.641987468433264, "kl": 0.09033203125, "learning_rate": 4.0147216970386774e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.7934406995773315, "rewards_raw/accuracy_reward": 0.5730579495429993, "rewards_raw/clip_reward_wrapper": 0.32953330874443054, "rewards_raw/format_reward": 5.0, "step": 848 }, { "Total_reward": 4.704509706243698e-07, "advantage_std": 1.6147942543029785, "completion_length": 387.8750305175781, "epoch": 0.29316298342541436, "grad_norm": 6.256803469007254, "kl": 0.09130859375, "learning_rate": 4.012563267982258e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.6921388506889343, "rewards_raw/accuracy_reward": 0.5570283532142639, "rewards_raw/clip_reward_wrapper": 0.33219990134239197, "rewards_raw/format_reward": 5.0, "step": 849 }, { "Total_reward": 3.437911004766647e-07, "advantage_std": 1.5335265398025513, "completion_length": 407.3214416503906, "epoch": 0.2935082872928177, "grad_norm": 6.403389430074387, "kl": 0.09228515625, "learning_rate": 4.0104030589416794e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.5745541453361511, "rewards_raw/accuracy_reward": 0.5784709453582764, "rewards_raw/clip_reward_wrapper": 0.33833450078964233, "rewards_raw/format_reward": 5.0, "step": 850 }, { "Total_reward": 6.556511493727157e-07, "advantage_std": 1.3878064155578613, "completion_length": 403.83929443359375, "epoch": 0.293853591160221, "grad_norm": 4.480273505740804, "kl": 0.08837890625, "learning_rate": 4.008241072459075e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.5364899635314941, "rewards_raw/accuracy_reward": 0.5596698522567749, "rewards_raw/clip_reward_wrapper": 0.32730937004089355, "rewards_raw/format_reward": 5.0, "step": 851 }, { "Total_reward": 6.939683885320846e-07, "advantage_std": 1.6430810689926147, "completion_length": 401.71429443359375, "epoch": 0.2941988950276243, "grad_norm": 5.689358531416143, "kl": 0.08984375, "learning_rate": 4.006077311078672e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.7605068683624268, "rewards_raw/accuracy_reward": 0.5675213932991028, "rewards_raw/clip_reward_wrapper": 0.31668105721473694, "rewards_raw/format_reward": 5.0, "step": 852 }, { "Total_reward": -2.7460711748972244e-07, "advantage_std": 1.5831233263015747, "completion_length": 395.9107360839844, "epoch": 0.29454419889502764, "grad_norm": 4.790926769414662, "kl": 0.09033203125, "learning_rate": 4.0039117773467824e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.4162624776363373, "rewards_raw/accuracy_reward": 0.5551201105117798, "rewards_raw/clip_reward_wrapper": 0.3273519277572632, "rewards_raw/format_reward": 5.0, "step": 853 }, { "Total_reward": -3.938164070405037e-07, "advantage_std": 1.581053614616394, "completion_length": 408.5714416503906, "epoch": 0.2948895027624309, "grad_norm": 5.437241665899135, "kl": 0.08740234375, "learning_rate": 4.0017444738118066e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.753330409526825, "rewards_raw/accuracy_reward": 0.5609087944030762, "rewards_raw/clip_reward_wrapper": 0.3268641233444214, "rewards_raw/format_reward": 5.0, "step": 854 }, { "Total_reward": 6.79067227338237e-07, "advantage_std": 1.5271071195602417, "completion_length": 394.26788330078125, "epoch": 0.29523480662983426, "grad_norm": 4.735162764980101, "kl": 0.087890625, "learning_rate": 3.999575403024227e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.959898054599762, "rewards_raw/accuracy_reward": 0.5715116858482361, "rewards_raw/clip_reward_wrapper": 0.35383373498916626, "rewards_raw/format_reward": 5.0, "step": 855 }, { "Total_reward": -6.450074465647049e-07, "advantage_std": 1.7223047018051147, "completion_length": 390.0535888671875, "epoch": 0.2955801104972376, "grad_norm": 3.734685696072309, "kl": 0.08447265625, "learning_rate": 3.9974045675366055e-07, "loss": 0.0034, "rewards_raw/CoT ICLG": 1.350847601890564, "rewards_raw/accuracy_reward": 0.5554801821708679, "rewards_raw/clip_reward_wrapper": 0.3280709981918335, "rewards_raw/format_reward": 5.0, "step": 856 }, { "Total_reward": 1.27724248955019e-07, "advantage_std": 1.4230543375015259, "completion_length": 391.3750305175781, "epoch": 0.29592541436464087, "grad_norm": 7.749060489793194, "kl": 0.08740234375, "learning_rate": 3.995231969903581e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.37625524401664734, "rewards_raw/accuracy_reward": 0.5684787034988403, "rewards_raw/clip_reward_wrapper": 0.3437236547470093, "rewards_raw/format_reward": 5.0, "step": 857 }, { "Total_reward": 8.663961352795013e-07, "advantage_std": 1.40294349193573, "completion_length": 386.2321472167969, "epoch": 0.2962707182320442, "grad_norm": 3.772263646298885, "kl": 0.091796875, "learning_rate": 3.9930576126818654e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.5803626775741577, "rewards_raw/accuracy_reward": 0.5780604481697083, "rewards_raw/clip_reward_wrapper": 0.3226465880870819, "rewards_raw/format_reward": 5.0, "step": 858 }, { "Total_reward": -8.749110520511749e-07, "advantage_std": 1.4412806034088135, "completion_length": 384.0535888671875, "epoch": 0.29661602209944754, "grad_norm": 4.083313076799628, "kl": 0.091796875, "learning_rate": 3.9908814984302433e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.8358511328697205, "rewards_raw/accuracy_reward": 0.5608384609222412, "rewards_raw/clip_reward_wrapper": 0.33404412865638733, "rewards_raw/format_reward": 5.0, "step": 859 }, { "Total_reward": -8.302075826804867e-08, "advantage_std": 1.5930297374725342, "completion_length": 387.3750305175781, "epoch": 0.2969613259668508, "grad_norm": 6.724220167710849, "kl": 0.087890625, "learning_rate": 3.988703629709563e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.6315144300460815, "rewards_raw/accuracy_reward": 0.5514834523200989, "rewards_raw/clip_reward_wrapper": 0.34466975927352905, "rewards_raw/format_reward": 5.0, "step": 860 }, { "Total_reward": 5.470855057865265e-07, "advantage_std": 1.511024832725525, "completion_length": 401.0357360839844, "epoch": 0.29730662983425415, "grad_norm": 3.5933382194780212, "kl": 0.0927734375, "learning_rate": 3.986524009082743e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.5848884582519531, "rewards_raw/accuracy_reward": 0.5674219131469727, "rewards_raw/clip_reward_wrapper": 0.34089526534080505, "rewards_raw/format_reward": 5.0, "step": 861 }, { "Total_reward": 9.068421604752075e-07, "advantage_std": 1.5304571390151978, "completion_length": 382.3035888671875, "epoch": 0.29765193370165743, "grad_norm": 4.291788483817667, "kl": 0.08740234375, "learning_rate": 3.9843426391147596e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 1.0704779624938965, "rewards_raw/accuracy_reward": 0.5664045810699463, "rewards_raw/clip_reward_wrapper": 0.3202856779098511, "rewards_raw/format_reward": 5.0, "step": 862 }, { "Total_reward": 2.3416111716301202e-08, "advantage_std": 1.317585825920105, "completion_length": 398.4821472167969, "epoch": 0.29799723756906077, "grad_norm": 29.596627132113966, "kl": 0.08642578125, "learning_rate": 3.9821595223726463e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.9079170823097229, "rewards_raw/accuracy_reward": 0.5631138682365417, "rewards_raw/clip_reward_wrapper": 0.33101093769073486, "rewards_raw/format_reward": 5.0, "step": 863 }, { "Total_reward": -4.1510379134024333e-08, "advantage_std": 1.2762939929962158, "completion_length": 395.3035888671875, "epoch": 0.2983425414364641, "grad_norm": 3.3402166791293166, "kl": 0.08984375, "learning_rate": 3.9799746614254967e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.6306407451629639, "rewards_raw/accuracy_reward": 0.5881571173667908, "rewards_raw/clip_reward_wrapper": 0.3505401909351349, "rewards_raw/format_reward": 5.0, "step": 864 }, { "Total_reward": -2.2990363390817947e-07, "advantage_std": 1.505048394203186, "completion_length": 409.58929443359375, "epoch": 0.2986878453038674, "grad_norm": 4.559426021611661, "kl": 0.087890625, "learning_rate": 3.9777880588444545e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.895627498626709, "rewards_raw/accuracy_reward": 0.5725167393684387, "rewards_raw/clip_reward_wrapper": 0.327450156211853, "rewards_raw/format_reward": 5.0, "step": 865 }, { "Total_reward": 2.2777490471526107e-07, "advantage_std": 1.4026033878326416, "completion_length": 394.2321472167969, "epoch": 0.2990331491712707, "grad_norm": 3.497921210242291, "kl": 0.08935546875, "learning_rate": 3.975599717202712e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.6045980453491211, "rewards_raw/accuracy_reward": 0.5689886808395386, "rewards_raw/clip_reward_wrapper": 0.32833331823349, "rewards_raw/format_reward": 4.973214626312256, "step": 866 }, { "Total_reward": 9.153571056685905e-08, "advantage_std": 1.359845519065857, "completion_length": 388.4107360839844, "epoch": 0.29937845303867405, "grad_norm": 8.519040391354283, "kl": 0.08935546875, "learning_rate": 3.973409639075511e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.9502390027046204, "rewards_raw/accuracy_reward": 0.5715659856796265, "rewards_raw/clip_reward_wrapper": 0.3410913646221161, "rewards_raw/format_reward": 5.0, "step": 867 }, { "Total_reward": 8.98327186860115e-07, "advantage_std": 1.4914348125457764, "completion_length": 393.2500305175781, "epoch": 0.29972375690607733, "grad_norm": 4.140083329274869, "kl": 0.08349609375, "learning_rate": 3.971217827040134e-07, "loss": 0.0033, "rewards_raw/CoT ICLG": 1.3154200315475464, "rewards_raw/accuracy_reward": 0.5651034712791443, "rewards_raw/clip_reward_wrapper": 0.33442577719688416, "rewards_raw/format_reward": 5.0, "step": 868 }, { "Total_reward": 4.789659158177528e-08, "advantage_std": 1.539738416671753, "completion_length": 394.83929443359375, "epoch": 0.30006906077348067, "grad_norm": 4.064389531361659, "kl": 0.08837890625, "learning_rate": 3.9690242836759047e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.4545159637928009, "rewards_raw/accuracy_reward": 0.5790842771530151, "rewards_raw/clip_reward_wrapper": 0.3220702111721039, "rewards_raw/format_reward": 5.0, "step": 869 }, { "Total_reward": -4.470348642371391e-07, "advantage_std": 1.4073392152786255, "completion_length": 394.3750305175781, "epoch": 0.300414364640884, "grad_norm": 5.234452373182205, "kl": 0.08935546875, "learning_rate": 3.966829011564185e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.7116085886955261, "rewards_raw/accuracy_reward": 0.570593535900116, "rewards_raw/clip_reward_wrapper": 0.34502145648002625, "rewards_raw/format_reward": 5.0, "step": 870 }, { "Total_reward": 4.0978196125252e-07, "advantage_std": 1.3881993293762207, "completion_length": 383.0535888671875, "epoch": 0.3007596685082873, "grad_norm": 3.7153851752573313, "kl": 0.09130859375, "learning_rate": 3.9646320132883705e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.48125159740448, "rewards_raw/accuracy_reward": 0.5712288618087769, "rewards_raw/clip_reward_wrapper": 0.33505523204803467, "rewards_raw/format_reward": 5.0, "step": 871 }, { "Total_reward": -1.0643687176070671e-07, "advantage_std": 1.4965378046035767, "completion_length": 378.8750305175781, "epoch": 0.3011049723756906, "grad_norm": 4.341183724808975, "kl": 0.09130859375, "learning_rate": 3.9624332914338887e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7225342392921448, "rewards_raw/accuracy_reward": 0.5837925672531128, "rewards_raw/clip_reward_wrapper": 0.32073649764060974, "rewards_raw/format_reward": 5.0, "step": 872 }, { "Total_reward": 2.7673586089349556e-08, "advantage_std": 1.3136131763458252, "completion_length": 395.5714416503906, "epoch": 0.3014502762430939, "grad_norm": 4.225104803582305, "kl": 0.0927734375, "learning_rate": 3.9602328485881943e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7178534865379333, "rewards_raw/accuracy_reward": 0.5863354802131653, "rewards_raw/clip_reward_wrapper": 0.3339875340461731, "rewards_raw/format_reward": 5.0, "step": 873 }, { "Total_reward": 7.322856845348724e-07, "advantage_std": 1.3220200538635254, "completion_length": 389.96429443359375, "epoch": 0.30179558011049723, "grad_norm": 3.559539026361347, "kl": 0.0908203125, "learning_rate": 3.9580306873407686e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.040814757347107, "rewards_raw/accuracy_reward": 0.5778698325157166, "rewards_raw/clip_reward_wrapper": 0.32883310317993164, "rewards_raw/format_reward": 5.0, "step": 874 }, { "Total_reward": 2.841864557012741e-07, "advantage_std": 1.360641598701477, "completion_length": 389.4107360839844, "epoch": 0.30214088397790057, "grad_norm": 3.116655941255359, "kl": 0.09326171875, "learning_rate": 3.955826810283114e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9693773984909058, "rewards_raw/accuracy_reward": 0.5688303112983704, "rewards_raw/clip_reward_wrapper": 0.33847370743751526, "rewards_raw/format_reward": 5.0, "step": 875 }, { "Total_reward": -1.7083117143101845e-07, "advantage_std": 1.6933305263519287, "completion_length": 391.26788330078125, "epoch": 0.30248618784530384, "grad_norm": 5.589918972334533, "kl": 0.08984375, "learning_rate": 3.953621220008754e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.6395165920257568, "rewards_raw/accuracy_reward": 0.561379611492157, "rewards_raw/clip_reward_wrapper": 0.3269845247268677, "rewards_raw/format_reward": 5.0, "step": 876 }, { "Total_reward": -6.109476089477539e-07, "advantage_std": 1.5171693563461304, "completion_length": 389.71429443359375, "epoch": 0.3028314917127072, "grad_norm": 3.681033737711319, "kl": 0.087890625, "learning_rate": 3.9514139191132253e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.9209499359130859, "rewards_raw/accuracy_reward": 0.5788331031799316, "rewards_raw/clip_reward_wrapper": 0.3288935720920563, "rewards_raw/format_reward": 5.0, "step": 877 }, { "Total_reward": -2.55448497910038e-08, "advantage_std": 1.2414230108261108, "completion_length": 385.6071472167969, "epoch": 0.3031767955801105, "grad_norm": 2.8559701083109394, "kl": 0.09130859375, "learning_rate": 3.949204910194079e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.8166112303733826, "rewards_raw/accuracy_reward": 0.5678706169128418, "rewards_raw/clip_reward_wrapper": 0.3321870267391205, "rewards_raw/format_reward": 5.0, "step": 878 }, { "Total_reward": -2.8099333349018707e-07, "advantage_std": 1.2932831048965454, "completion_length": 377.39288330078125, "epoch": 0.3035220994475138, "grad_norm": 3.7460721760007267, "kl": 0.09423828125, "learning_rate": 3.946994195850878e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7723363637924194, "rewards_raw/accuracy_reward": 0.5956028699874878, "rewards_raw/clip_reward_wrapper": 0.3588692843914032, "rewards_raw/format_reward": 5.0, "step": 879 }, { "Total_reward": 5.981752337902435e-07, "advantage_std": 1.4706833362579346, "completion_length": 395.7500305175781, "epoch": 0.30386740331491713, "grad_norm": 7.859769607637714, "kl": 0.09130859375, "learning_rate": 3.9447817786851886e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.6965933442115784, "rewards_raw/accuracy_reward": 0.6002820134162903, "rewards_raw/clip_reward_wrapper": 0.3360614478588104, "rewards_raw/format_reward": 5.0, "step": 880 }, { "Total_reward": -1.0962997976093902e-07, "advantage_std": 1.4581317901611328, "completion_length": 389.14288330078125, "epoch": 0.30421270718232046, "grad_norm": 7.80597917392553, "kl": 0.0966796875, "learning_rate": 3.942567661300584e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8964883685112, "rewards_raw/accuracy_reward": 0.6012165546417236, "rewards_raw/clip_reward_wrapper": 0.34309107065200806, "rewards_raw/format_reward": 5.0, "step": 881 }, { "Total_reward": 7.610236139043991e-07, "advantage_std": 1.5553420782089233, "completion_length": 397.51788330078125, "epoch": 0.30455801104972374, "grad_norm": 4.192676047737989, "kl": 0.091796875, "learning_rate": 3.9403518463026377e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.827355682849884, "rewards_raw/accuracy_reward": 0.576542854309082, "rewards_raw/clip_reward_wrapper": 0.3339764177799225, "rewards_raw/format_reward": 5.0, "step": 882 }, { "Total_reward": -3.193106010712654e-07, "advantage_std": 1.4824717044830322, "completion_length": 389.3035888671875, "epoch": 0.3049033149171271, "grad_norm": 8.273695771905137, "kl": 0.08837890625, "learning_rate": 3.938134336298919e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.6621831655502319, "rewards_raw/accuracy_reward": 0.5641152262687683, "rewards_raw/clip_reward_wrapper": 0.33736652135849, "rewards_raw/format_reward": 5.0, "step": 883 }, { "Total_reward": -8.073236585914856e-07, "advantage_std": 1.5161385536193848, "completion_length": 396.5714416503906, "epoch": 0.3052486187845304, "grad_norm": 4.104252075437537, "kl": 0.09033203125, "learning_rate": 3.935915133898994e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.6229212880134583, "rewards_raw/accuracy_reward": 0.5787351131439209, "rewards_raw/clip_reward_wrapper": 0.3274334669113159, "rewards_raw/format_reward": 5.0, "step": 884 }, { "Total_reward": -2.7673587510435027e-07, "advantage_std": 1.629791259765625, "completion_length": 400.0714416503906, "epoch": 0.3055939226519337, "grad_norm": 6.6140847037034, "kl": 0.091796875, "learning_rate": 3.933694241714419e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.612912654876709, "rewards_raw/accuracy_reward": 0.5657004117965698, "rewards_raw/clip_reward_wrapper": 0.3314764201641083, "rewards_raw/format_reward": 5.0, "step": 885 }, { "Total_reward": 6.173338817916374e-08, "advantage_std": 1.413130521774292, "completion_length": 387.89288330078125, "epoch": 0.305939226519337, "grad_norm": 4.35595685293282, "kl": 0.09228515625, "learning_rate": 3.9314716623587394e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.5954573750495911, "rewards_raw/accuracy_reward": 0.5803910493850708, "rewards_raw/clip_reward_wrapper": 0.33256232738494873, "rewards_raw/format_reward": 5.0, "step": 886 }, { "Total_reward": -1.220830881720758e-06, "advantage_std": 1.3491158485412598, "completion_length": 391.1607360839844, "epoch": 0.3062845303867403, "grad_norm": 3.6560405976299717, "kl": 0.09033203125, "learning_rate": 3.9292473984474886e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.6819786429405212, "rewards_raw/accuracy_reward": 0.5763400793075562, "rewards_raw/clip_reward_wrapper": 0.32444316148757935, "rewards_raw/format_reward": 5.0, "step": 887 }, { "Total_reward": -9.472881146166401e-08, "advantage_std": 1.414486289024353, "completion_length": 402.9107360839844, "epoch": 0.30662983425414364, "grad_norm": 4.749557524570573, "kl": 0.091796875, "learning_rate": 3.9270214525981763e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.6232962608337402, "rewards_raw/accuracy_reward": 0.5811835527420044, "rewards_raw/clip_reward_wrapper": 0.3514634966850281, "rewards_raw/format_reward": 5.0, "step": 888 }, { "Total_reward": -1.341104507446289e-07, "advantage_std": 1.6519261598587036, "completion_length": 403.51788330078125, "epoch": 0.306975138121547, "grad_norm": 4.39538200647189, "kl": 0.08740234375, "learning_rate": 3.9247938274302975e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.6913842558860779, "rewards_raw/accuracy_reward": 0.5885704755783081, "rewards_raw/clip_reward_wrapper": 0.34497615694999695, "rewards_raw/format_reward": 5.0, "step": 889 }, { "Total_reward": 3.8317273265420226e-07, "advantage_std": 1.5067957639694214, "completion_length": 398.5000305175781, "epoch": 0.30732044198895025, "grad_norm": 5.4461398702212644, "kl": 0.0908203125, "learning_rate": 3.922564525565321e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.6916304230690002, "rewards_raw/accuracy_reward": 0.566505491733551, "rewards_raw/clip_reward_wrapper": 0.33540135622024536, "rewards_raw/format_reward": 5.0, "step": 890 }, { "Total_reward": 2.895082786835701e-07, "advantage_std": 1.4203041791915894, "completion_length": 396.26788330078125, "epoch": 0.3076657458563536, "grad_norm": 4.0195296406376775, "kl": 0.09228515625, "learning_rate": 3.9203335496266876e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.6605275869369507, "rewards_raw/accuracy_reward": 0.5601542592048645, "rewards_raw/clip_reward_wrapper": 0.3388229310512543, "rewards_raw/format_reward": 5.0, "step": 891 }, { "Total_reward": -1.5284334722309723e-06, "advantage_std": 1.5105290412902832, "completion_length": 404.6964416503906, "epoch": 0.3080110497237569, "grad_norm": 6.5989224628636505, "kl": 0.091796875, "learning_rate": 3.91810090223981e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.5763274431228638, "rewards_raw/accuracy_reward": 0.5611540079116821, "rewards_raw/clip_reward_wrapper": 0.3369079530239105, "rewards_raw/format_reward": 5.0, "step": 892 }, { "Total_reward": -9.664468052505981e-07, "advantage_std": 1.7652226686477661, "completion_length": 400.3750305175781, "epoch": 0.3083563535911602, "grad_norm": 5.247534216810753, "kl": 0.0908203125, "learning_rate": 3.915866586032067e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.7077419757843018, "rewards_raw/accuracy_reward": 0.5644307732582092, "rewards_raw/clip_reward_wrapper": 0.3256644308567047, "rewards_raw/format_reward": 5.0, "step": 893 }, { "Total_reward": -2.7460711748972244e-07, "advantage_std": 1.5225454568862915, "completion_length": 398.33929443359375, "epoch": 0.30870165745856354, "grad_norm": 3.405384239187055, "kl": 0.09033203125, "learning_rate": 3.9136306036328027e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.5249378085136414, "rewards_raw/accuracy_reward": 0.554482102394104, "rewards_raw/clip_reward_wrapper": 0.32568952441215515, "rewards_raw/format_reward": 5.0, "step": 894 }, { "Total_reward": 2.2564616131148796e-07, "advantage_std": 1.3561756610870361, "completion_length": 387.1607360839844, "epoch": 0.3090469613259669, "grad_norm": 7.677766863473671, "kl": 0.09375, "learning_rate": 3.9113929576733184e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6595971584320068, "rewards_raw/accuracy_reward": 0.5805150866508484, "rewards_raw/clip_reward_wrapper": 0.3500036299228668, "rewards_raw/format_reward": 5.0, "step": 895 }, { "Total_reward": 2.948301300875755e-07, "advantage_std": 1.6486800909042358, "completion_length": 391.8214416503906, "epoch": 0.30939226519337015, "grad_norm": 5.8873508720755, "kl": 0.0927734375, "learning_rate": 3.9091536507868775e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9044158458709717, "rewards_raw/accuracy_reward": 0.5832611322402954, "rewards_raw/clip_reward_wrapper": 0.32158246636390686, "rewards_raw/format_reward": 5.0, "step": 896 }, { "Total_reward": -2.1287374352141342e-07, "advantage_std": 1.4643568992614746, "completion_length": 384.7500305175781, "epoch": 0.3097375690607735, "grad_norm": 4.971006651330213, "kl": 0.09033203125, "learning_rate": 3.9069126856086954e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.0163567066192627, "rewards_raw/accuracy_reward": 0.576076328754425, "rewards_raw/clip_reward_wrapper": 0.3411902189254761, "rewards_raw/format_reward": 5.0, "step": 897 }, { "Total_reward": -6.85453471760411e-07, "advantage_std": 1.5141571760177612, "completion_length": 382.58929443359375, "epoch": 0.3100828729281768, "grad_norm": 6.986804287674011, "kl": 0.09033203125, "learning_rate": 3.9046700647759383e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.03966224193573, "rewards_raw/accuracy_reward": 0.5666590929031372, "rewards_raw/clip_reward_wrapper": 0.3416875898838043, "rewards_raw/format_reward": 5.0, "step": 898 }, { "Total_reward": 4.6619348381682357e-07, "advantage_std": 1.4221853017807007, "completion_length": 385.6250305175781, "epoch": 0.3104281767955801, "grad_norm": 10.585017081608015, "kl": 0.08984375, "learning_rate": 3.9024257909277236e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.3666589260101318, "rewards_raw/accuracy_reward": 0.5553247928619385, "rewards_raw/clip_reward_wrapper": 0.33045923709869385, "rewards_raw/format_reward": 5.0, "step": 899 }, { "Total_reward": 8.81297296473349e-07, "advantage_std": 1.504276990890503, "completion_length": 394.21429443359375, "epoch": 0.31077348066298344, "grad_norm": 5.444817863201802, "kl": 0.09228515625, "learning_rate": 3.9001798667051117e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7443712949752808, "rewards_raw/accuracy_reward": 0.5863211750984192, "rewards_raw/clip_reward_wrapper": 0.35006627440452576, "rewards_raw/format_reward": 5.0, "step": 900 }, { "Total_reward": 7.876328567135715e-08, "advantage_std": 1.6597336530685425, "completion_length": 393.5535888671875, "epoch": 0.3111187845303867, "grad_norm": 5.130874990502571, "kl": 0.09521484375, "learning_rate": 3.8979322947511064e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6545028686523438, "rewards_raw/accuracy_reward": 0.5689395070075989, "rewards_raw/clip_reward_wrapper": 0.33848288655281067, "rewards_raw/format_reward": 5.0, "step": 901 }, { "Total_reward": 2.895082786835701e-07, "advantage_std": 1.4477282762527466, "completion_length": 401.1071472167969, "epoch": 0.31146408839779005, "grad_norm": 3.8743067043282693, "kl": 0.091796875, "learning_rate": 3.8956830777106475e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.0517328977584839, "rewards_raw/accuracy_reward": 0.5603158473968506, "rewards_raw/clip_reward_wrapper": 0.31632867455482483, "rewards_raw/format_reward": 5.0, "step": 902 }, { "Total_reward": -1.830714211337181e-07, "advantage_std": 1.5034619569778442, "completion_length": 400.64288330078125, "epoch": 0.3118093922651934, "grad_norm": 5.006153062385334, "kl": 0.0908203125, "learning_rate": 3.8934322182306155e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.7963905930519104, "rewards_raw/accuracy_reward": 0.5830144882202148, "rewards_raw/clip_reward_wrapper": 0.321885347366333, "rewards_raw/format_reward": 5.0, "step": 903 }, { "Total_reward": -2.086162709247219e-07, "advantage_std": 1.6293407678604126, "completion_length": 392.1250305175781, "epoch": 0.31215469613259667, "grad_norm": 3.8050515358779413, "kl": 0.09228515625, "learning_rate": 3.8911797189598217e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7925776243209839, "rewards_raw/accuracy_reward": 0.5793117880821228, "rewards_raw/clip_reward_wrapper": 0.3271775245666504, "rewards_raw/format_reward": 5.0, "step": 904 }, { "Total_reward": -3.491129234589607e-07, "advantage_std": 1.6419161558151245, "completion_length": 401.6785888671875, "epoch": 0.3125, "grad_norm": 4.854677348601797, "kl": 0.091796875, "learning_rate": 3.8889255825490053e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.4198931455612183, "rewards_raw/accuracy_reward": 0.5734173059463501, "rewards_raw/clip_reward_wrapper": 0.32358285784721375, "rewards_raw/format_reward": 5.0, "step": 905 }, { "Total_reward": 6.258487701416016e-07, "advantage_std": 1.645194172859192, "completion_length": 382.5000305175781, "epoch": 0.31284530386740333, "grad_norm": 5.273081497975018, "kl": 0.08935546875, "learning_rate": 3.8866698116508345e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.1432433128356934, "rewards_raw/accuracy_reward": 0.567961573600769, "rewards_raw/clip_reward_wrapper": 0.3400435149669647, "rewards_raw/format_reward": 5.0, "step": 906 }, { "Total_reward": 1.2346677635832748e-07, "advantage_std": 1.4453884363174438, "completion_length": 405.4107360839844, "epoch": 0.3131906077348066, "grad_norm": 39.76455368434327, "kl": 0.09228515625, "learning_rate": 3.8844124089198996e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7013117671012878, "rewards_raw/accuracy_reward": 0.5804420709609985, "rewards_raw/clip_reward_wrapper": 0.32347965240478516, "rewards_raw/format_reward": 5.0, "step": 907 }, { "Total_reward": -1.0856560805905247e-07, "advantage_std": 1.3755913972854614, "completion_length": 383.01788330078125, "epoch": 0.31353591160220995, "grad_norm": 7.490702267580401, "kl": 0.09326171875, "learning_rate": 3.8821533770127117e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9011903405189514, "rewards_raw/accuracy_reward": 0.5575656890869141, "rewards_raw/clip_reward_wrapper": 0.33793479204177856, "rewards_raw/format_reward": 5.0, "step": 908 }, { "Total_reward": 5.747590989813034e-07, "advantage_std": 1.6735525131225586, "completion_length": 381.6607360839844, "epoch": 0.3138812154696133, "grad_norm": 4.073041466426822, "kl": 0.09375, "learning_rate": 3.8798927185876994e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.9475728273391724, "rewards_raw/accuracy_reward": 0.5948116779327393, "rewards_raw/clip_reward_wrapper": 0.33490344882011414, "rewards_raw/format_reward": 4.964285850524902, "step": 909 }, { "Total_reward": 5.396349251896027e-07, "advantage_std": 1.563941240310669, "completion_length": 393.9821472167969, "epoch": 0.31422651933701656, "grad_norm": 7.602099948567651, "kl": 0.0966796875, "learning_rate": 3.877630436305205e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.5896079540252686, "rewards_raw/accuracy_reward": 0.5765087008476257, "rewards_raw/clip_reward_wrapper": 0.3574942350387573, "rewards_raw/format_reward": 5.0, "step": 910 }, { "Total_reward": 2.7460711748972244e-07, "advantage_std": 1.2932178974151611, "completion_length": 402.6607360839844, "epoch": 0.3145718232044199, "grad_norm": 3.857443980633665, "kl": 0.08935546875, "learning_rate": 3.875366532827482e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.6970550417900085, "rewards_raw/accuracy_reward": 0.5831248760223389, "rewards_raw/clip_reward_wrapper": 0.3306061029434204, "rewards_raw/format_reward": 5.0, "step": 911 }, { "Total_reward": 4.385199190437561e-07, "advantage_std": 1.3640695810317993, "completion_length": 374.46429443359375, "epoch": 0.3149171270718232, "grad_norm": 4.847410757539439, "kl": 0.0927734375, "learning_rate": 3.8731010108186916e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.5670883059501648, "rewards_raw/accuracy_reward": 0.5866973996162415, "rewards_raw/clip_reward_wrapper": 0.33262717723846436, "rewards_raw/format_reward": 5.0, "step": 912 }, { "Total_reward": 3.884945840582077e-07, "advantage_std": 1.3497989177703857, "completion_length": 393.8035888671875, "epoch": 0.3152624309392265, "grad_norm": 4.638684691880689, "kl": 0.09033203125, "learning_rate": 3.870833872944899e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.7350591421127319, "rewards_raw/accuracy_reward": 0.5758442878723145, "rewards_raw/clip_reward_wrapper": 0.33740919828414917, "rewards_raw/format_reward": 5.0, "step": 913 }, { "Total_reward": 1.2665987014770508e-07, "advantage_std": 1.4271079301834106, "completion_length": 395.3035888671875, "epoch": 0.31560773480662985, "grad_norm": 5.5553709993295515, "kl": 0.09033203125, "learning_rate": 3.868565121874072e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.6210782527923584, "rewards_raw/accuracy_reward": 0.5522550940513611, "rewards_raw/clip_reward_wrapper": 0.32494601607322693, "rewards_raw/format_reward": 5.0, "step": 914 }, { "Total_reward": -6.290418923526886e-07, "advantage_std": 1.3323578834533691, "completion_length": 391.0714416503906, "epoch": 0.3159530386740331, "grad_norm": 5.172411550778421, "kl": 0.09033203125, "learning_rate": 3.866294760276077e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.4396496117115021, "rewards_raw/accuracy_reward": 0.5564510226249695, "rewards_raw/clip_reward_wrapper": 0.3451986610889435, "rewards_raw/format_reward": 5.0, "step": 915 }, { "Total_reward": -5.502786279976135e-07, "advantage_std": 1.6065808534622192, "completion_length": 393.7857360839844, "epoch": 0.31629834254143646, "grad_norm": 4.446015915223838, "kl": 0.0908203125, "learning_rate": 3.864022790822674e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.7373534440994263, "rewards_raw/accuracy_reward": 0.5859401226043701, "rewards_raw/clip_reward_wrapper": 0.33223456144332886, "rewards_raw/format_reward": 5.0, "step": 916 }, { "Total_reward": -2.7034965910388564e-07, "advantage_std": 1.4800997972488403, "completion_length": 394.4821472167969, "epoch": 0.3166436464088398, "grad_norm": 3.9226937106131183, "kl": 0.0947265625, "learning_rate": 3.861749216187517e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7263472080230713, "rewards_raw/accuracy_reward": 0.5765458345413208, "rewards_raw/clip_reward_wrapper": 0.3437650799751282, "rewards_raw/format_reward": 5.0, "step": 917 }, { "Total_reward": 3.8530146184712066e-07, "advantage_std": 1.3802372217178345, "completion_length": 397.0357360839844, "epoch": 0.3169889502762431, "grad_norm": 3.926601326518575, "kl": 0.09130859375, "learning_rate": 3.8594740390461476e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.5297015905380249, "rewards_raw/accuracy_reward": 0.5625077486038208, "rewards_raw/clip_reward_wrapper": 0.33258888125419617, "rewards_raw/format_reward": 5.0, "step": 918 }, { "Total_reward": -3.395336136691185e-07, "advantage_std": 1.6987413167953491, "completion_length": 394.51788330078125, "epoch": 0.3173342541436464, "grad_norm": 4.513630827485901, "kl": 0.09228515625, "learning_rate": 3.857197262075995e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.6159130930900574, "rewards_raw/accuracy_reward": 0.5715625882148743, "rewards_raw/clip_reward_wrapper": 0.3277899920940399, "rewards_raw/format_reward": 5.0, "step": 919 }, { "Total_reward": 1.319817215517105e-07, "advantage_std": 1.4450243711471558, "completion_length": 389.89288330078125, "epoch": 0.31767955801104975, "grad_norm": 4.711556461586131, "kl": 0.0908203125, "learning_rate": 3.854918887956368e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.8583118319511414, "rewards_raw/accuracy_reward": 0.5657837986946106, "rewards_raw/clip_reward_wrapper": 0.34089744091033936, "rewards_raw/format_reward": 5.0, "step": 920 }, { "Total_reward": -6.38621244775095e-08, "advantage_std": 1.497093915939331, "completion_length": 390.1785888671875, "epoch": 0.318024861878453, "grad_norm": 3.4705188709402424, "kl": 0.09228515625, "learning_rate": 3.852638919368459e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7896062135696411, "rewards_raw/accuracy_reward": 0.5726189017295837, "rewards_raw/clip_reward_wrapper": 0.3336403965950012, "rewards_raw/format_reward": 5.0, "step": 921 }, { "Total_reward": -6.38621244775095e-09, "advantage_std": 1.6019847393035889, "completion_length": 393.9107360839844, "epoch": 0.31837016574585636, "grad_norm": 5.306444807208973, "kl": 0.09228515625, "learning_rate": 3.850357358995333e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.8708664178848267, "rewards_raw/accuracy_reward": 0.5711289644241333, "rewards_raw/clip_reward_wrapper": 0.32414546608924866, "rewards_raw/format_reward": 5.0, "step": 922 }, { "Total_reward": 8.621386768936645e-07, "advantage_std": 1.420058012008667, "completion_length": 378.76788330078125, "epoch": 0.3187154696132597, "grad_norm": 4.7754750622818705, "kl": 0.0927734375, "learning_rate": 3.8480742095219296e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7385242581367493, "rewards_raw/accuracy_reward": 0.5706889629364014, "rewards_raw/clip_reward_wrapper": 0.32851821184158325, "rewards_raw/format_reward": 5.0, "step": 923 }, { "Total_reward": -1.043081283569336e-06, "advantage_std": 1.5286110639572144, "completion_length": 406.2321472167969, "epoch": 0.319060773480663, "grad_norm": 6.087946825150305, "kl": 0.0927734375, "learning_rate": 3.8457894736350605e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.6892454028129578, "rewards_raw/accuracy_reward": 0.5846754312515259, "rewards_raw/clip_reward_wrapper": 0.34215372800827026, "rewards_raw/format_reward": 5.0, "step": 924 }, { "Total_reward": 5.311200084179291e-07, "advantage_std": 1.4966340065002441, "completion_length": 398.6607360839844, "epoch": 0.3194060773480663, "grad_norm": 9.677750133777682, "kl": 0.09130859375, "learning_rate": 3.8435031540234007e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.037735939025879, "rewards_raw/accuracy_reward": 0.5400132536888123, "rewards_raw/clip_reward_wrapper": 0.3346208930015564, "rewards_raw/format_reward": 5.0, "step": 925 }, { "Total_reward": -7.40800601306546e-07, "advantage_std": 1.3691799640655518, "completion_length": 397.3035888671875, "epoch": 0.3197513812154696, "grad_norm": 3.7853495863452338, "kl": 0.09326171875, "learning_rate": 3.841215253377491e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.8657176494598389, "rewards_raw/accuracy_reward": 0.5731519460678101, "rewards_raw/clip_reward_wrapper": 0.3344711661338806, "rewards_raw/format_reward": 5.0, "step": 926 }, { "Total_reward": -3.5549913945942535e-07, "advantage_std": 1.598483920097351, "completion_length": 410.5714416503906, "epoch": 0.3200966850828729, "grad_norm": 3.9840615649987625, "kl": 0.091796875, "learning_rate": 3.838925774389733e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7330837845802307, "rewards_raw/accuracy_reward": 0.5893265008926392, "rewards_raw/clip_reward_wrapper": 0.3407876193523407, "rewards_raw/format_reward": 5.0, "step": 927 }, { "Total_reward": -6.109476089477539e-07, "advantage_std": 1.4763730764389038, "completion_length": 395.6607360839844, "epoch": 0.32044198895027626, "grad_norm": 6.261465187136077, "kl": 0.0947265625, "learning_rate": 3.8366347197543846e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.771671712398529, "rewards_raw/accuracy_reward": 0.5890790224075317, "rewards_raw/clip_reward_wrapper": 0.3396197557449341, "rewards_raw/format_reward": 5.0, "step": 928 }, { "Total_reward": 3.405979782655777e-07, "advantage_std": 1.6281262636184692, "completion_length": 398.0535888671875, "epoch": 0.32078729281767954, "grad_norm": 10.102636262403063, "kl": 0.095703125, "learning_rate": 3.8343420921675595e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6575311422348022, "rewards_raw/accuracy_reward": 0.5603405237197876, "rewards_raw/clip_reward_wrapper": 0.33583682775497437, "rewards_raw/format_reward": 5.0, "step": 929 }, { "Total_reward": -2.4480479510202713e-07, "advantage_std": 1.3258968591690063, "completion_length": 395.8750305175781, "epoch": 0.32113259668508287, "grad_norm": 3.7656675735872107, "kl": 0.09423828125, "learning_rate": 3.832047894327219e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8297040462493896, "rewards_raw/accuracy_reward": 0.5791922807693481, "rewards_raw/clip_reward_wrapper": 0.3204248547554016, "rewards_raw/format_reward": 5.0, "step": 930 }, { "Total_reward": 4.087175966560608e-07, "advantage_std": 1.5460662841796875, "completion_length": 402.14288330078125, "epoch": 0.3214779005524862, "grad_norm": 4.907707354071762, "kl": 0.095703125, "learning_rate": 3.8297521289331774e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6593868136405945, "rewards_raw/accuracy_reward": 0.5789836049079895, "rewards_raw/clip_reward_wrapper": 0.3362160623073578, "rewards_raw/format_reward": 5.0, "step": 931 }, { "Total_reward": 1.798783131334858e-07, "advantage_std": 1.5940431356430054, "completion_length": 402.4107360839844, "epoch": 0.3218232044198895, "grad_norm": 5.53420157100366, "kl": 0.091796875, "learning_rate": 3.8274547986870896e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.5964808464050293, "rewards_raw/accuracy_reward": 0.5605298280715942, "rewards_raw/clip_reward_wrapper": 0.3334652781486511, "rewards_raw/format_reward": 5.0, "step": 932 }, { "Total_reward": -1.596553005356327e-07, "advantage_std": 1.4792836904525757, "completion_length": 396.0357360839844, "epoch": 0.3221685082872928, "grad_norm": 6.115954032942287, "kl": 0.08984375, "learning_rate": 3.8251559062924543e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.7678701281547546, "rewards_raw/accuracy_reward": 0.5768020749092102, "rewards_raw/clip_reward_wrapper": 0.3460942804813385, "rewards_raw/format_reward": 5.0, "step": 933 }, { "Total_reward": -8.30207568469632e-07, "advantage_std": 1.4747027158737183, "completion_length": 408.0535888671875, "epoch": 0.32251381215469616, "grad_norm": 5.100059124025581, "kl": 0.09130859375, "learning_rate": 3.8228554544546056e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.8894966840744019, "rewards_raw/accuracy_reward": 0.577842116355896, "rewards_raw/clip_reward_wrapper": 0.33906251192092896, "rewards_raw/format_reward": 5.0, "step": 934 }, { "Total_reward": -5.747590847704487e-08, "advantage_std": 1.3422532081604004, "completion_length": 393.2857360839844, "epoch": 0.32285911602209943, "grad_norm": 5.7433388009862885, "kl": 0.09130859375, "learning_rate": 3.8205534458807165e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9291842579841614, "rewards_raw/accuracy_reward": 0.5655277967453003, "rewards_raw/clip_reward_wrapper": 0.3311123549938202, "rewards_raw/format_reward": 5.0, "step": 935 }, { "Total_reward": -4.5554980943052215e-07, "advantage_std": 1.4391324520111084, "completion_length": 386.89288330078125, "epoch": 0.32320441988950277, "grad_norm": 4.626822499118225, "kl": 0.095703125, "learning_rate": 3.8182498832797905e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 2.1098878383636475, "rewards_raw/accuracy_reward": 0.5345036387443542, "rewards_raw/clip_reward_wrapper": 0.3398382365703583, "rewards_raw/format_reward": 5.0, "step": 936 }, { "Total_reward": 6.045614213689987e-07, "advantage_std": 1.5743145942687988, "completion_length": 407.7321472167969, "epoch": 0.3235497237569061, "grad_norm": 4.520681734705234, "kl": 0.0888671875, "learning_rate": 3.815944769362658e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.5747120976448059, "rewards_raw/accuracy_reward": 0.5600499510765076, "rewards_raw/clip_reward_wrapper": 0.32337844371795654, "rewards_raw/format_reward": 5.0, "step": 937 }, { "Total_reward": 3.363405198797409e-07, "advantage_std": 1.4786549806594849, "completion_length": 397.76788330078125, "epoch": 0.3238950276243094, "grad_norm": 4.750754970058856, "kl": 0.09228515625, "learning_rate": 3.8136381068419777e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7307180166244507, "rewards_raw/accuracy_reward": 0.5739142894744873, "rewards_raw/clip_reward_wrapper": 0.3290732800960541, "rewards_raw/format_reward": 5.0, "step": 938 }, { "Total_reward": 2.980232238769531e-07, "advantage_std": 1.3789180517196655, "completion_length": 397.01788330078125, "epoch": 0.3242403314917127, "grad_norm": 5.130123648582671, "kl": 0.09912109375, "learning_rate": 3.8113298984322296e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7187075018882751, "rewards_raw/accuracy_reward": 0.5775328874588013, "rewards_raw/clip_reward_wrapper": 0.3494146764278412, "rewards_raw/format_reward": 5.0, "step": 939 }, { "Total_reward": -2.2564616131148796e-07, "advantage_std": 1.2320425510406494, "completion_length": 415.0000305175781, "epoch": 0.324585635359116, "grad_norm": 3.889722177001953, "kl": 0.0927734375, "learning_rate": 3.8090201468497133e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7967102527618408, "rewards_raw/accuracy_reward": 0.5638729929924011, "rewards_raw/clip_reward_wrapper": 0.33047208189964294, "rewards_raw/format_reward": 5.0, "step": 940 }, { "Total_reward": -3.70400300653273e-07, "advantage_std": 1.3854656219482422, "completion_length": 383.3571472167969, "epoch": 0.32493093922651933, "grad_norm": 3.216091036318671, "kl": 0.0927734375, "learning_rate": 3.806708854812543e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7055135369300842, "rewards_raw/accuracy_reward": 0.5729870200157166, "rewards_raw/clip_reward_wrapper": 0.3398381471633911, "rewards_raw/format_reward": 5.0, "step": 941 }, { "Total_reward": -2.7247838829680404e-07, "advantage_std": 1.5423349142074585, "completion_length": 397.7321472167969, "epoch": 0.32527624309392267, "grad_norm": 5.922381116369994, "kl": 0.09228515625, "learning_rate": 3.804396025040647e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.8074697256088257, "rewards_raw/accuracy_reward": 0.5693724155426025, "rewards_raw/clip_reward_wrapper": 0.3416329324245453, "rewards_raw/format_reward": 5.0, "step": 942 }, { "Total_reward": -3.405979782655777e-07, "advantage_std": 1.6867401599884033, "completion_length": 392.9821472167969, "epoch": 0.32562154696132595, "grad_norm": 8.436759483052041, "kl": 0.0947265625, "learning_rate": 3.802081660255764e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.5108862519264221, "rewards_raw/accuracy_reward": 0.5714991688728333, "rewards_raw/clip_reward_wrapper": 0.33072519302368164, "rewards_raw/format_reward": 5.0, "step": 943 }, { "Total_reward": 4.3000494542866363e-07, "advantage_std": 1.3728179931640625, "completion_length": 406.2500305175781, "epoch": 0.3259668508287293, "grad_norm": 3.6901643943956173, "kl": 0.09130859375, "learning_rate": 3.7997657631814363e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.837020218372345, "rewards_raw/accuracy_reward": 0.5944533348083496, "rewards_raw/clip_reward_wrapper": 0.33729010820388794, "rewards_raw/format_reward": 5.0, "step": 944 }, { "Total_reward": 5.662441253662109e-07, "advantage_std": 1.4497283697128296, "completion_length": 399.58929443359375, "epoch": 0.3263121546961326, "grad_norm": 4.036488701337424, "kl": 0.091796875, "learning_rate": 3.797448336543011e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7596908807754517, "rewards_raw/accuracy_reward": 0.5873960852622986, "rewards_raw/clip_reward_wrapper": 0.3394981920719147, "rewards_raw/format_reward": 5.0, "step": 945 }, { "Total_reward": -9.366444686520481e-08, "advantage_std": 1.4958767890930176, "completion_length": 394.7321472167969, "epoch": 0.3266574585635359, "grad_norm": 6.280662362253622, "kl": 0.09423828125, "learning_rate": 3.795129383067638e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7169119119644165, "rewards_raw/accuracy_reward": 0.5372699499130249, "rewards_raw/clip_reward_wrapper": 0.34195104241371155, "rewards_raw/format_reward": 5.0, "step": 946 }, { "Total_reward": -4.5767853862344055e-07, "advantage_std": 1.5978318452835083, "completion_length": 395.2857360839844, "epoch": 0.32700276243093923, "grad_norm": 5.321808301894086, "kl": 0.09228515625, "learning_rate": 3.7928089054842587e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9262703657150269, "rewards_raw/accuracy_reward": 0.571365475654602, "rewards_raw/clip_reward_wrapper": 0.33605965971946716, "rewards_raw/format_reward": 5.0, "step": 947 }, { "Total_reward": 1.0643687176070671e-07, "advantage_std": 1.8349870443344116, "completion_length": 410.0357360839844, "epoch": 0.32734806629834257, "grad_norm": 29.31305639695012, "kl": 0.09423828125, "learning_rate": 3.7904869065236123e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6405332684516907, "rewards_raw/accuracy_reward": 0.560639500617981, "rewards_raw/clip_reward_wrapper": 0.33325013518333435, "rewards_raw/format_reward": 4.901785850524902, "step": 948 }, { "Total_reward": 2.7673586089349556e-08, "advantage_std": 1.3262075185775757, "completion_length": 405.51788330078125, "epoch": 0.32769337016574585, "grad_norm": 9.378901666166334, "kl": 0.09423828125, "learning_rate": 3.788163388918225e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7963139414787292, "rewards_raw/accuracy_reward": 0.5682004690170288, "rewards_raw/clip_reward_wrapper": 0.3275601863861084, "rewards_raw/format_reward": 5.0, "step": 949 }, { "Total_reward": 2.4480479510202713e-07, "advantage_std": 1.3077483177185059, "completion_length": 408.01788330078125, "epoch": 0.3280386740331492, "grad_norm": 4.3535891051591, "kl": 0.09619140625, "learning_rate": 3.785838355402413e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7246894240379333, "rewards_raw/accuracy_reward": 0.544762372970581, "rewards_raw/clip_reward_wrapper": 0.33875057101249695, "rewards_raw/format_reward": 5.0, "step": 950 }, { "Total_reward": -2.5331974029541016e-07, "advantage_std": 1.550296664237976, "completion_length": 391.9464416503906, "epoch": 0.32838397790055246, "grad_norm": 14.457053263440224, "kl": 0.09375, "learning_rate": 3.7835118087122764e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6418623328208923, "rewards_raw/accuracy_reward": 0.5930956602096558, "rewards_raw/clip_reward_wrapper": 0.3379431366920471, "rewards_raw/format_reward": 5.0, "step": 951 }, { "Total_reward": -6.162694603517593e-07, "advantage_std": 1.5412746667861938, "completion_length": 400.8214416503906, "epoch": 0.3287292817679558, "grad_norm": 7.459801930223705, "kl": 0.0947265625, "learning_rate": 3.7811837515856926e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8115230202674866, "rewards_raw/accuracy_reward": 0.5876190662384033, "rewards_raw/clip_reward_wrapper": 0.3467026948928833, "rewards_raw/format_reward": 5.0, "step": 952 }, { "Total_reward": 4.821590096071304e-07, "advantage_std": 1.3809024095535278, "completion_length": 401.01788330078125, "epoch": 0.32907458563535913, "grad_norm": 7.1918359752833645, "kl": 0.09326171875, "learning_rate": 3.7788541867623194e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7412067651748657, "rewards_raw/accuracy_reward": 0.5616207122802734, "rewards_raw/clip_reward_wrapper": 0.3308071494102478, "rewards_raw/format_reward": 5.0, "step": 953 }, { "Total_reward": -7.152557941481064e-07, "advantage_std": 1.6091997623443604, "completion_length": 386.9464416503906, "epoch": 0.3294198895027624, "grad_norm": 6.275425477405251, "kl": 0.1005859375, "learning_rate": 3.77652311698359e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8152272701263428, "rewards_raw/accuracy_reward": 0.5834676623344421, "rewards_raw/clip_reward_wrapper": 0.3314621150493622, "rewards_raw/format_reward": 5.0, "step": 954 }, { "Total_reward": -7.237706967089252e-08, "advantage_std": 1.5887000560760498, "completion_length": 392.0714416503906, "epoch": 0.32976519337016574, "grad_norm": 5.288752424303468, "kl": 0.09814453125, "learning_rate": 3.7741905449927035e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.0777952671051025, "rewards_raw/accuracy_reward": 0.5809422731399536, "rewards_raw/clip_reward_wrapper": 0.34179291129112244, "rewards_raw/format_reward": 5.0, "step": 955 }, { "Total_reward": -1.4901161193847656e-08, "advantage_std": 1.4303951263427734, "completion_length": 382.51788330078125, "epoch": 0.3301104972375691, "grad_norm": 4.291894964077345, "kl": 0.09521484375, "learning_rate": 3.7718564735346335e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.1571087837219238, "rewards_raw/accuracy_reward": 0.6147782802581787, "rewards_raw/clip_reward_wrapper": 0.33302146196365356, "rewards_raw/format_reward": 5.0, "step": 956 }, { "Total_reward": -1.004764044409967e-06, "advantage_std": 1.516563892364502, "completion_length": 392.7500305175781, "epoch": 0.33045580110497236, "grad_norm": 3.7642563615178135, "kl": 0.0947265625, "learning_rate": 3.7695209053561134e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.5932006239891052, "rewards_raw/accuracy_reward": 0.5936996340751648, "rewards_raw/clip_reward_wrapper": 0.33844104409217834, "rewards_raw/format_reward": 5.0, "step": 957 }, { "Total_reward": -6.556511493727157e-07, "advantage_std": 1.7176148891448975, "completion_length": 390.1071472167969, "epoch": 0.3308011049723757, "grad_norm": 4.489836266071229, "kl": 0.10009765625, "learning_rate": 3.767183843205639e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.5857340693473816, "rewards_raw/accuracy_reward": 0.5719856023788452, "rewards_raw/clip_reward_wrapper": 0.33956342935562134, "rewards_raw/format_reward": 5.0, "step": 958 }, { "Total_reward": -8.02533975274855e-07, "advantage_std": 1.5368387699127197, "completion_length": 392.83929443359375, "epoch": 0.331146408839779, "grad_norm": 5.131483913629763, "kl": 0.09375, "learning_rate": 3.7648452898334655e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7914391160011292, "rewards_raw/accuracy_reward": 0.5696622729301453, "rewards_raw/clip_reward_wrapper": 0.3307058811187744, "rewards_raw/format_reward": 5.0, "step": 959 }, { "Total_reward": -2.7460711748972244e-07, "advantage_std": 1.5498698949813843, "completion_length": 403.3571472167969, "epoch": 0.3314917127071823, "grad_norm": 4.014907195493015, "kl": 0.095703125, "learning_rate": 3.7625052479916007e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.5972995758056641, "rewards_raw/accuracy_reward": 0.5829796195030212, "rewards_raw/clip_reward_wrapper": 0.3360820412635803, "rewards_raw/format_reward": 5.0, "step": 960 }, { "Total_reward": -6.279775561779388e-07, "advantage_std": 1.5163488388061523, "completion_length": 406.1071472167969, "epoch": 0.33183701657458564, "grad_norm": 5.0976531051142935, "kl": 0.09326171875, "learning_rate": 3.7601637204338076e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.8346431255340576, "rewards_raw/accuracy_reward": 0.5951460599899292, "rewards_raw/clip_reward_wrapper": 0.34178054332733154, "rewards_raw/format_reward": 5.0, "step": 961 }, { "Total_reward": -2.064875275209488e-07, "advantage_std": 1.6503281593322754, "completion_length": 391.9464416503906, "epoch": 0.332182320441989, "grad_norm": 6.727370182142883, "kl": 0.09521484375, "learning_rate": 3.7578207099155936e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6900526285171509, "rewards_raw/accuracy_reward": 0.5710383653640747, "rewards_raw/clip_reward_wrapper": 0.3204174041748047, "rewards_raw/format_reward": 5.0, "step": 962 }, { "Total_reward": 3.916876778475853e-07, "advantage_std": 1.5208306312561035, "completion_length": 399.1250305175781, "epoch": 0.33252762430939226, "grad_norm": 4.858381170955087, "kl": 0.09521484375, "learning_rate": 3.7554762191942133e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.5718019008636475, "rewards_raw/accuracy_reward": 0.5748910903930664, "rewards_raw/clip_reward_wrapper": 0.3279130160808563, "rewards_raw/format_reward": 5.0, "step": 963 }, { "Total_reward": 1.830714211337181e-07, "advantage_std": 1.5206362009048462, "completion_length": 414.2321472167969, "epoch": 0.3328729281767956, "grad_norm": 4.485629988115284, "kl": 0.0947265625, "learning_rate": 3.7531302510286623e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6070653796195984, "rewards_raw/accuracy_reward": 0.5716564059257507, "rewards_raw/clip_reward_wrapper": 0.3395259976387024, "rewards_raw/format_reward": 5.0, "step": 964 }, { "Total_reward": 6.811959707420101e-08, "advantage_std": 1.6212210655212402, "completion_length": 407.83929443359375, "epoch": 0.33321823204419887, "grad_norm": 3.914149027035024, "kl": 0.09228515625, "learning_rate": 3.7507828081796775e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7178913950920105, "rewards_raw/accuracy_reward": 0.5740514993667603, "rewards_raw/clip_reward_wrapper": 0.33175456523895264, "rewards_raw/format_reward": 5.0, "step": 965 }, { "Total_reward": 2.4480479510202713e-07, "advantage_std": 1.3462437391281128, "completion_length": 409.2500305175781, "epoch": 0.3335635359116022, "grad_norm": 5.017824683825319, "kl": 0.10107421875, "learning_rate": 3.7484338934097274e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7462148070335388, "rewards_raw/accuracy_reward": 0.575872540473938, "rewards_raw/clip_reward_wrapper": 0.32692646980285645, "rewards_raw/format_reward": 5.0, "step": 966 }, { "Total_reward": -2.873795494906517e-07, "advantage_std": 1.3217359781265259, "completion_length": 406.4107360839844, "epoch": 0.33390883977900554, "grad_norm": 4.942520641967413, "kl": 0.0986328125, "learning_rate": 3.7460835094830155e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.5207958817481995, "rewards_raw/accuracy_reward": 0.5667352080345154, "rewards_raw/clip_reward_wrapper": 0.3406430184841156, "rewards_raw/format_reward": 5.0, "step": 967 }, { "Total_reward": -3.193106010712654e-07, "advantage_std": 1.4237641096115112, "completion_length": 405.8214416503906, "epoch": 0.3342541436464088, "grad_norm": 5.747484645758187, "kl": 0.09130859375, "learning_rate": 3.743731659165472e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.818856418132782, "rewards_raw/accuracy_reward": 0.5870510339736938, "rewards_raw/clip_reward_wrapper": 0.32724103331565857, "rewards_raw/format_reward": 5.0, "step": 968 }, { "Total_reward": 9.323869676336471e-07, "advantage_std": 1.5546281337738037, "completion_length": 403.01788330078125, "epoch": 0.33459944751381215, "grad_norm": 13.528834476361066, "kl": 0.09765625, "learning_rate": 3.741378345224755e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6835045218467712, "rewards_raw/accuracy_reward": 0.5694025754928589, "rewards_raw/clip_reward_wrapper": 0.3402523398399353, "rewards_raw/format_reward": 5.0, "step": 969 }, { "Total_reward": -3.193106010712654e-07, "advantage_std": 1.5207809209823608, "completion_length": 393.3214416503906, "epoch": 0.3349447513812155, "grad_norm": 5.0513915965598795, "kl": 0.10400390625, "learning_rate": 3.739023570430243e-07, "loss": 0.0042, "rewards_raw/CoT ICLG": 0.7263891696929932, "rewards_raw/accuracy_reward": 0.5644139647483826, "rewards_raw/clip_reward_wrapper": 0.3288811445236206, "rewards_raw/format_reward": 5.0, "step": 970 }, { "Total_reward": -9.281295092478103e-07, "advantage_std": 1.5030931234359741, "completion_length": 406.1071472167969, "epoch": 0.33529005524861877, "grad_norm": 4.089857628393601, "kl": 0.09814453125, "learning_rate": 3.7366673375530356e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6433321237564087, "rewards_raw/accuracy_reward": 0.5627593398094177, "rewards_raw/clip_reward_wrapper": 0.3307414948940277, "rewards_raw/format_reward": 5.0, "step": 971 }, { "Total_reward": 2.3416111716301202e-08, "advantage_std": 1.3162647485733032, "completion_length": 416.21429443359375, "epoch": 0.3356353591160221, "grad_norm": 3.3380227270825586, "kl": 0.09130859375, "learning_rate": 3.734309649365946e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7458072900772095, "rewards_raw/accuracy_reward": 0.5767785906791687, "rewards_raw/clip_reward_wrapper": 0.355459988117218, "rewards_raw/format_reward": 5.0, "step": 972 }, { "Total_reward": 9.941203416019562e-07, "advantage_std": 1.4195715188980103, "completion_length": 404.9821472167969, "epoch": 0.33598066298342544, "grad_norm": 4.621747794704135, "kl": 0.09130859375, "learning_rate": 3.7319505086435014e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.0604385137557983, "rewards_raw/accuracy_reward": 0.5493787527084351, "rewards_raw/clip_reward_wrapper": 0.33420073986053467, "rewards_raw/format_reward": 5.0, "step": 973 }, { "Total_reward": -7.078052135511825e-07, "advantage_std": 1.449569821357727, "completion_length": 393.3571472167969, "epoch": 0.3363259668508287, "grad_norm": 3.803039433805653, "kl": 0.0986328125, "learning_rate": 3.729589918161939e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9867491722106934, "rewards_raw/accuracy_reward": 0.5872175097465515, "rewards_raw/clip_reward_wrapper": 0.34432175755500793, "rewards_raw/format_reward": 5.0, "step": 974 }, { "Total_reward": -8.855547548591858e-07, "advantage_std": 1.4315625429153442, "completion_length": 406.9464416503906, "epoch": 0.33667127071823205, "grad_norm": 3.8154969898657303, "kl": 0.09521484375, "learning_rate": 3.727227880699201e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.5978944301605225, "rewards_raw/accuracy_reward": 0.5796730518341064, "rewards_raw/clip_reward_wrapper": 0.3205067813396454, "rewards_raw/format_reward": 5.0, "step": 975 }, { "Total_reward": -4.5554980943052215e-07, "advantage_std": 1.5447505712509155, "completion_length": 402.64288330078125, "epoch": 0.3370165745856354, "grad_norm": 9.576927392010873, "kl": 0.09326171875, "learning_rate": 3.7248643990349317e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.8147847652435303, "rewards_raw/accuracy_reward": 0.5784137845039368, "rewards_raw/clip_reward_wrapper": 0.3293212056159973, "rewards_raw/format_reward": 5.0, "step": 976 }, { "Total_reward": 1.5965531119377374e-09, "advantage_std": 1.4795682430267334, "completion_length": 398.6785888671875, "epoch": 0.33736187845303867, "grad_norm": 4.051347782938202, "kl": 0.0966796875, "learning_rate": 3.7224994759504767e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.5512099266052246, "rewards_raw/accuracy_reward": 0.5675889253616333, "rewards_raw/clip_reward_wrapper": 0.3438708484172821, "rewards_raw/format_reward": 5.0, "step": 977 }, { "Total_reward": 6.183981895446777e-07, "advantage_std": 1.3874174356460571, "completion_length": 391.2857360839844, "epoch": 0.337707182320442, "grad_norm": 11.376530917123295, "kl": 0.09521484375, "learning_rate": 3.720133114228877e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.49573737382888794, "rewards_raw/accuracy_reward": 0.5635129809379578, "rewards_raw/clip_reward_wrapper": 0.3375331461429596, "rewards_raw/format_reward": 5.0, "step": 978 }, { "Total_reward": 3.437911004766647e-07, "advantage_std": 1.4751391410827637, "completion_length": 405.33929443359375, "epoch": 0.3380524861878453, "grad_norm": 4.337326261851376, "kl": 0.0947265625, "learning_rate": 3.7177653166548666e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.0110090970993042, "rewards_raw/accuracy_reward": 0.5785683989524841, "rewards_raw/clip_reward_wrapper": 0.34649422764778137, "rewards_raw/format_reward": 5.0, "step": 979 }, { "Total_reward": 6.620373369514709e-07, "advantage_std": 1.4501922130584717, "completion_length": 381.08929443359375, "epoch": 0.3383977900552486, "grad_norm": 4.069925442549367, "kl": 0.09033203125, "learning_rate": 3.7153960860148684e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.5714892148971558, "rewards_raw/accuracy_reward": 0.5762577056884766, "rewards_raw/clip_reward_wrapper": 0.3263598084449768, "rewards_raw/format_reward": 5.0, "step": 980 }, { "Total_reward": 4.5980726781635894e-07, "advantage_std": 1.8258010149002075, "completion_length": 411.14288330078125, "epoch": 0.33874309392265195, "grad_norm": 6.952451432505853, "kl": 0.0966796875, "learning_rate": 3.713025425096994e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7297222018241882, "rewards_raw/accuracy_reward": 0.5516942739486694, "rewards_raw/clip_reward_wrapper": 0.33638566732406616, "rewards_raw/format_reward": 4.866071701049805, "step": 981 }, { "Total_reward": 7.024833337254677e-08, "advantage_std": 1.2736046314239502, "completion_length": 410.1785888671875, "epoch": 0.33908839779005523, "grad_norm": 3.469478004395586, "kl": 0.09619140625, "learning_rate": 3.710653336691036e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6776988506317139, "rewards_raw/accuracy_reward": 0.5698365569114685, "rewards_raw/clip_reward_wrapper": 0.32727012038230896, "rewards_raw/format_reward": 5.0, "step": 982 }, { "Total_reward": 9.451593996345764e-07, "advantage_std": 1.5555921792984009, "completion_length": 396.2500305175781, "epoch": 0.33943370165745856, "grad_norm": 7.854406295262351, "kl": 0.0927734375, "learning_rate": 3.708279823588466e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.6596636772155762, "rewards_raw/accuracy_reward": 0.565008282661438, "rewards_raw/clip_reward_wrapper": 0.32630854845046997, "rewards_raw/format_reward": 5.0, "step": 983 }, { "Total_reward": 7.780535042911652e-07, "advantage_std": 1.4892668724060059, "completion_length": 386.33929443359375, "epoch": 0.3397790055248619, "grad_norm": 4.346353060041535, "kl": 0.09912109375, "learning_rate": 3.705904888582436e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7215211391448975, "rewards_raw/accuracy_reward": 0.5804404616355896, "rewards_raw/clip_reward_wrapper": 0.34908583760261536, "rewards_raw/format_reward": 5.0, "step": 984 }, { "Total_reward": 4.087175966560608e-07, "advantage_std": 1.5697154998779297, "completion_length": 400.1250305175781, "epoch": 0.3401243093922652, "grad_norm": 13.105545776121822, "kl": 0.10009765625, "learning_rate": 3.703528534467769e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7920756936073303, "rewards_raw/accuracy_reward": 0.5657026171684265, "rewards_raw/clip_reward_wrapper": 0.34046831727027893, "rewards_raw/format_reward": 5.0, "step": 985 }, { "Total_reward": -3.395336136691185e-07, "advantage_std": 1.4141991138458252, "completion_length": 401.8214416503906, "epoch": 0.3404696132596685, "grad_norm": 3.998344760115057, "kl": 0.09375, "learning_rate": 3.7011507640409566e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.1770137548446655, "rewards_raw/accuracy_reward": 0.5799297094345093, "rewards_raw/clip_reward_wrapper": 0.34042027592658997, "rewards_raw/format_reward": 5.0, "step": 986 }, { "Total_reward": -4.6832223432602405e-08, "advantage_std": 1.3520365953445435, "completion_length": 394.5535888671875, "epoch": 0.34081491712707185, "grad_norm": 5.334752605507187, "kl": 0.099609375, "learning_rate": 3.6987715801001594e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7813053727149963, "rewards_raw/accuracy_reward": 0.5852755308151245, "rewards_raw/clip_reward_wrapper": 0.3291471600532532, "rewards_raw/format_reward": 5.0, "step": 987 }, { "Total_reward": 8.110489488899475e-07, "advantage_std": 1.5849170684814453, "completion_length": 387.64288330078125, "epoch": 0.3411602209944751, "grad_norm": 4.243693664786306, "kl": 0.09765625, "learning_rate": 3.6963909854452014e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.5541372299194336, "rewards_raw/accuracy_reward": 0.5588208436965942, "rewards_raw/clip_reward_wrapper": 0.34812185168266296, "rewards_raw/format_reward": 4.982142925262451, "step": 988 }, { "Total_reward": 3.938164070405037e-07, "advantage_std": 1.3144887685775757, "completion_length": 395.1964416503906, "epoch": 0.34150552486187846, "grad_norm": 4.45310689525705, "kl": 0.09716796875, "learning_rate": 3.694008982877564e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9649530649185181, "rewards_raw/accuracy_reward": 0.5721185207366943, "rewards_raw/clip_reward_wrapper": 0.3359457552433014, "rewards_raw/format_reward": 5.0, "step": 989 }, { "Total_reward": -1.788139485370266e-07, "advantage_std": 1.3341522216796875, "completion_length": 397.4821472167969, "epoch": 0.34185082872928174, "grad_norm": 3.784759553448307, "kl": 0.0908203125, "learning_rate": 3.691625575200389e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.1951024532318115, "rewards_raw/accuracy_reward": 0.5572327375411987, "rewards_raw/clip_reward_wrapper": 0.3270869255065918, "rewards_raw/format_reward": 5.0, "step": 990 }, { "Total_reward": 5.194119125917496e-07, "advantage_std": 1.4881539344787598, "completion_length": 400.4107360839844, "epoch": 0.3421961325966851, "grad_norm": 3.738386383421782, "kl": 0.0966796875, "learning_rate": 3.6892407652184685e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7269760370254517, "rewards_raw/accuracy_reward": 0.567929208278656, "rewards_raw/clip_reward_wrapper": 0.3329562544822693, "rewards_raw/format_reward": 5.0, "step": 991 }, { "Total_reward": 7.450581165358017e-07, "advantage_std": 1.4787954092025757, "completion_length": 394.3750305175781, "epoch": 0.3425414364640884, "grad_norm": 3.4442091904928844, "kl": 0.0966796875, "learning_rate": 3.6868545557382483e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.727889358997345, "rewards_raw/accuracy_reward": 0.5949465036392212, "rewards_raw/clip_reward_wrapper": 0.34140369296073914, "rewards_raw/format_reward": 5.0, "step": 992 }, { "Total_reward": 8.727823086474018e-08, "advantage_std": 1.381301760673523, "completion_length": 409.01788330078125, "epoch": 0.3428867403314917, "grad_norm": 8.852363384316096, "kl": 0.08984375, "learning_rate": 3.6844669495678186e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.8027522563934326, "rewards_raw/accuracy_reward": 0.5876168012619019, "rewards_raw/clip_reward_wrapper": 0.33497458696365356, "rewards_raw/format_reward": 5.0, "step": 993 }, { "Total_reward": -7.769891681164154e-07, "advantage_std": 1.4194964170455933, "completion_length": 395.1607360839844, "epoch": 0.343232044198895, "grad_norm": 4.2194816817936145, "kl": 0.0927734375, "learning_rate": 3.682077949516913e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9930485486984253, "rewards_raw/accuracy_reward": 0.568321168422699, "rewards_raw/clip_reward_wrapper": 0.34890612959861755, "rewards_raw/format_reward": 5.0, "step": 994 }, { "Total_reward": 8.514949634275126e-09, "advantage_std": 1.3292583227157593, "completion_length": 393.4285888671875, "epoch": 0.34357734806629836, "grad_norm": 3.0806889636295045, "kl": 0.09716796875, "learning_rate": 3.679687558396907e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9155890345573425, "rewards_raw/accuracy_reward": 0.5794689059257507, "rewards_raw/clip_reward_wrapper": 0.33492982387542725, "rewards_raw/format_reward": 5.0, "step": 995 }, { "Total_reward": 8.685248644724197e-07, "advantage_std": 1.6269557476043701, "completion_length": 401.0714416503906, "epoch": 0.34392265193370164, "grad_norm": 5.321137886233928, "kl": 0.08935546875, "learning_rate": 3.677295779020813e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.1826063394546509, "rewards_raw/accuracy_reward": 0.5800780057907104, "rewards_raw/clip_reward_wrapper": 0.32995712757110596, "rewards_raw/format_reward": 5.0, "step": 996 }, { "Total_reward": -1.021793991640152e-07, "advantage_std": 1.535452127456665, "completion_length": 395.3750305175781, "epoch": 0.344267955801105, "grad_norm": 4.655138645117936, "kl": 0.10205078125, "learning_rate": 3.674902614203275e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.875603437423706, "rewards_raw/accuracy_reward": 0.5937713980674744, "rewards_raw/clip_reward_wrapper": 0.3339821398258209, "rewards_raw/format_reward": 5.0, "step": 997 }, { "Total_reward": 1.3836793044674778e-08, "advantage_std": 1.3991501331329346, "completion_length": 398.6785888671875, "epoch": 0.3446132596685083, "grad_norm": 3.913834236601185, "kl": 0.091796875, "learning_rate": 3.67250806676057e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.6075766086578369, "rewards_raw/accuracy_reward": 0.5617969036102295, "rewards_raw/clip_reward_wrapper": 0.3349347710609436, "rewards_raw/format_reward": 5.0, "step": 998 }, { "Total_reward": -6.811959707420101e-08, "advantage_std": 1.5782806873321533, "completion_length": 411.71429443359375, "epoch": 0.3449585635359116, "grad_norm": 6.53995696488787, "kl": 0.0908203125, "learning_rate": 3.670112139510601e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.8768895268440247, "rewards_raw/accuracy_reward": 0.5883924961090088, "rewards_raw/clip_reward_wrapper": 0.32899394631385803, "rewards_raw/format_reward": 5.0, "step": 999 }, { "Total_reward": -8.387225420847244e-07, "advantage_std": 1.5500560998916626, "completion_length": 402.1607360839844, "epoch": 0.3453038674033149, "grad_norm": 4.6244238976049115, "kl": 0.09130859375, "learning_rate": 3.667714835272894e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.2947183847427368, "rewards_raw/accuracy_reward": 0.5653280019760132, "rewards_raw/clip_reward_wrapper": 0.3369752764701843, "rewards_raw/format_reward": 5.0, "step": 1000 }, { "Total_reward": -1.27724248955019e-07, "advantage_std": 1.5501240491867065, "completion_length": 393.6250305175781, "epoch": 0.34564917127071826, "grad_norm": 13.117911387768247, "kl": 0.09716796875, "learning_rate": 3.665316156868597e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6539449691772461, "rewards_raw/accuracy_reward": 0.5692280530929565, "rewards_raw/clip_reward_wrapper": 0.32919925451278687, "rewards_raw/format_reward": 5.0, "step": 1001 }, { "Total_reward": -5.10896995820076e-08, "advantage_std": 1.547314167022705, "completion_length": 384.83929443359375, "epoch": 0.34599447513812154, "grad_norm": 6.028471447149824, "kl": 0.09326171875, "learning_rate": 3.662916107120474e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.0005497932434082, "rewards_raw/accuracy_reward": 0.5880386829376221, "rewards_raw/clip_reward_wrapper": 0.3458556830883026, "rewards_raw/format_reward": 5.0, "step": 1002 }, { "Total_reward": 5.194119125917496e-07, "advantage_std": 1.813056230545044, "completion_length": 399.2857360839844, "epoch": 0.3463397790055249, "grad_norm": 8.738464440081023, "kl": 0.09130859375, "learning_rate": 3.660514688852902e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.598580539226532, "rewards_raw/accuracy_reward": 0.5751782655715942, "rewards_raw/clip_reward_wrapper": 0.34063461422920227, "rewards_raw/format_reward": 5.0, "step": 1003 }, { "Total_reward": -5.492142349794449e-07, "advantage_std": 1.6111655235290527, "completion_length": 397.39288330078125, "epoch": 0.34668508287292815, "grad_norm": 4.606904274791014, "kl": 0.09423828125, "learning_rate": 3.658111904891872e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6909615397453308, "rewards_raw/accuracy_reward": 0.5818840265274048, "rewards_raw/clip_reward_wrapper": 0.3490320146083832, "rewards_raw/format_reward": 5.0, "step": 1004 }, { "Total_reward": 3.2995430387927627e-07, "advantage_std": 1.2770525217056274, "completion_length": 415.1071472167969, "epoch": 0.3470303867403315, "grad_norm": 4.464602180117425, "kl": 0.09521484375, "learning_rate": 3.655707758064976e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.0867345333099365, "rewards_raw/accuracy_reward": 0.5558803081512451, "rewards_raw/clip_reward_wrapper": 0.33052858710289, "rewards_raw/format_reward": 5.0, "step": 1005 }, { "Total_reward": 2.2564616131148796e-07, "advantage_std": 1.488213062286377, "completion_length": 393.3214416503906, "epoch": 0.3473756906077348, "grad_norm": 4.6864171308270866, "kl": 0.09765625, "learning_rate": 3.653302251201417e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6116517186164856, "rewards_raw/accuracy_reward": 0.5782755613327026, "rewards_raw/clip_reward_wrapper": 0.33865657448768616, "rewards_raw/format_reward": 5.0, "step": 1006 }, { "Total_reward": 3.384692490726593e-07, "advantage_std": 1.5939801931381226, "completion_length": 409.9821472167969, "epoch": 0.3477209944751381, "grad_norm": 7.824846063571082, "kl": 0.09716796875, "learning_rate": 3.650895387131995e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6977150440216064, "rewards_raw/accuracy_reward": 0.5921452045440674, "rewards_raw/clip_reward_wrapper": 0.3399347960948944, "rewards_raw/format_reward": 5.0, "step": 1007 }, { "Total_reward": 8.514949634275126e-09, "advantage_std": 1.5889482498168945, "completion_length": 410.5535888671875, "epoch": 0.34806629834254144, "grad_norm": 6.3187417809527355, "kl": 0.0888671875, "learning_rate": 3.6484871686891037e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.7498425245285034, "rewards_raw/accuracy_reward": 0.5887622237205505, "rewards_raw/clip_reward_wrapper": 0.338446706533432, "rewards_raw/format_reward": 5.0, "step": 1008 }, { "Total_reward": 1.9903694692402496e-07, "advantage_std": 1.5927345752716064, "completion_length": 384.4285888671875, "epoch": 0.34841160220994477, "grad_norm": 3.8061722265518885, "kl": 0.09521484375, "learning_rate": 3.6460775987067364e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.4588267803192139, "rewards_raw/accuracy_reward": 0.5595804452896118, "rewards_raw/clip_reward_wrapper": 0.3409680426120758, "rewards_raw/format_reward": 5.0, "step": 1009 }, { "Total_reward": 7.76989139694706e-08, "advantage_std": 1.6118820905685425, "completion_length": 401.6607360839844, "epoch": 0.34875690607734805, "grad_norm": 4.528327983233865, "kl": 0.0947265625, "learning_rate": 3.6436666800204755e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8680607080459595, "rewards_raw/accuracy_reward": 0.5759990811347961, "rewards_raw/clip_reward_wrapper": 0.34095504879951477, "rewards_raw/format_reward": 5.0, "step": 1010 }, { "Total_reward": -1.820070423264042e-07, "advantage_std": 1.4421617984771729, "completion_length": 400.4285888671875, "epoch": 0.3491022099447514, "grad_norm": 5.0314451259629065, "kl": 0.09716796875, "learning_rate": 3.6412544154674886e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8640195727348328, "rewards_raw/accuracy_reward": 0.5621417164802551, "rewards_raw/clip_reward_wrapper": 0.3261560797691345, "rewards_raw/format_reward": 5.0, "step": 1011 }, { "Total_reward": 1.0090215027958038e-06, "advantage_std": 1.414439082145691, "completion_length": 407.0714416503906, "epoch": 0.3494475138121547, "grad_norm": 7.870514807962987, "kl": 0.09765625, "learning_rate": 3.638840807886527e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.1698241233825684, "rewards_raw/accuracy_reward": 0.5910929441452026, "rewards_raw/clip_reward_wrapper": 0.33982932567596436, "rewards_raw/format_reward": 5.0, "step": 1012 }, { "Total_reward": 3.906233132511261e-07, "advantage_std": 1.370673656463623, "completion_length": 391.7857360839844, "epoch": 0.349792817679558, "grad_norm": 3.3514701880905853, "kl": 0.0966796875, "learning_rate": 3.636425860117926e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8544042110443115, "rewards_raw/accuracy_reward": 0.573607325553894, "rewards_raw/clip_reward_wrapper": 0.3418276011943817, "rewards_raw/format_reward": 5.0, "step": 1013 }, { "Total_reward": 3.448554650731239e-07, "advantage_std": 1.636393666267395, "completion_length": 384.9464416503906, "epoch": 0.35013812154696133, "grad_norm": 6.937759619507588, "kl": 0.0986328125, "learning_rate": 3.6340095750035937e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7859578728675842, "rewards_raw/accuracy_reward": 0.5694650411605835, "rewards_raw/clip_reward_wrapper": 0.3350381553173065, "rewards_raw/format_reward": 5.0, "step": 1014 }, { "Total_reward": 2.895082786835701e-07, "advantage_std": 1.494495153427124, "completion_length": 398.0714416503906, "epoch": 0.3504834254143646, "grad_norm": 7.520389163047977, "kl": 0.09326171875, "learning_rate": 3.6315919553870154e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7354028224945068, "rewards_raw/accuracy_reward": 0.5569019913673401, "rewards_raw/clip_reward_wrapper": 0.33190667629241943, "rewards_raw/format_reward": 5.0, "step": 1015 }, { "Total_reward": -6.684235245302261e-07, "advantage_std": 1.67854642868042, "completion_length": 382.9107360839844, "epoch": 0.35082872928176795, "grad_norm": 5.0076106435164425, "kl": 0.09716796875, "learning_rate": 3.6291730041132446e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.851737916469574, "rewards_raw/accuracy_reward": 0.5891426205635071, "rewards_raw/clip_reward_wrapper": 0.3350731134414673, "rewards_raw/format_reward": 5.0, "step": 1016 }, { "Total_reward": -8.004052460819366e-07, "advantage_std": 1.333018183708191, "completion_length": 388.0000305175781, "epoch": 0.3511740331491713, "grad_norm": 24.098446603088608, "kl": 0.1015625, "learning_rate": 3.626752724028902e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.7233889698982239, "rewards_raw/accuracy_reward": 0.5812132954597473, "rewards_raw/clip_reward_wrapper": 0.3206089437007904, "rewards_raw/format_reward": 5.0, "step": 1017 }, { "Total_reward": 3.2676118166818924e-07, "advantage_std": 1.4641900062561035, "completion_length": 390.71429443359375, "epoch": 0.35151933701657456, "grad_norm": 11.339996006068008, "kl": 0.10009765625, "learning_rate": 3.6243311179821744e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8874677419662476, "rewards_raw/accuracy_reward": 0.5668880343437195, "rewards_raw/clip_reward_wrapper": 0.3273389935493469, "rewards_raw/format_reward": 5.0, "step": 1018 }, { "Total_reward": 2.8099333349018707e-07, "advantage_std": 1.409226417541504, "completion_length": 398.51788330078125, "epoch": 0.3518646408839779, "grad_norm": 4.6693446398903165, "kl": 0.0947265625, "learning_rate": 3.621908188822805e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.5783743262290955, "rewards_raw/accuracy_reward": 0.5549122095108032, "rewards_raw/clip_reward_wrapper": 0.3326837122440338, "rewards_raw/format_reward": 5.0, "step": 1019 }, { "Total_reward": -2.3416111716301202e-08, "advantage_std": 1.6004717350006104, "completion_length": 386.9821472167969, "epoch": 0.35220994475138123, "grad_norm": 7.915783271800077, "kl": 0.0966796875, "learning_rate": 3.6194839394020994e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.2150810956954956, "rewards_raw/accuracy_reward": 0.5700778365135193, "rewards_raw/clip_reward_wrapper": 0.34006938338279724, "rewards_raw/format_reward": 5.0, "step": 1020 }, { "Total_reward": 6.620373369514709e-07, "advantage_std": 1.5520466566085815, "completion_length": 400.71429443359375, "epoch": 0.3525552486187845, "grad_norm": 5.723858407062558, "kl": 0.09912109375, "learning_rate": 3.6170583725729114e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.3715542256832123, "rewards_raw/accuracy_reward": 0.5716044306755066, "rewards_raw/clip_reward_wrapper": 0.3304592967033386, "rewards_raw/format_reward": 5.0, "step": 1021 }, { "Total_reward": -7.982765168890182e-07, "advantage_std": 1.6557697057724, "completion_length": 392.3214416503906, "epoch": 0.35290055248618785, "grad_norm": 5.6505750195868565, "kl": 0.099609375, "learning_rate": 3.614631491189649e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.3586307466030121, "rewards_raw/accuracy_reward": 0.5630468130111694, "rewards_raw/clip_reward_wrapper": 0.34103530645370483, "rewards_raw/format_reward": 5.0, "step": 1022 }, { "Total_reward": -3.7465778746081924e-07, "advantage_std": 1.4868569374084473, "completion_length": 391.2321472167969, "epoch": 0.3532458563535912, "grad_norm": 4.11289240887852, "kl": 0.0927734375, "learning_rate": 3.612203298108264e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.5895349383354187, "rewards_raw/accuracy_reward": 0.5694319605827332, "rewards_raw/clip_reward_wrapper": 0.3482653796672821, "rewards_raw/format_reward": 5.0, "step": 1023 }, { "Total_reward": 8.344650836988876e-07, "advantage_std": 1.6208423376083374, "completion_length": 402.83929443359375, "epoch": 0.35359116022099446, "grad_norm": 4.570906572423611, "kl": 0.09814453125, "learning_rate": 3.6097737961862555e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 2.1844513416290283, "rewards_raw/accuracy_reward": 0.5641164779663086, "rewards_raw/clip_reward_wrapper": 0.3315040171146393, "rewards_raw/format_reward": 5.0, "step": 1024 }, { "Total_reward": 4.470348713425665e-08, "advantage_std": 1.5378693342208862, "completion_length": 391.33929443359375, "epoch": 0.3539364640883978, "grad_norm": 4.784609108202281, "kl": 0.09521484375, "learning_rate": 3.60734298828266e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6244123578071594, "rewards_raw/accuracy_reward": 0.5948618650436401, "rewards_raw/clip_reward_wrapper": 0.3418019413948059, "rewards_raw/format_reward": 5.0, "step": 1025 }, { "Total_reward": -1.2729849458992248e-06, "advantage_std": 1.7582218647003174, "completion_length": 397.0535888671875, "epoch": 0.35428176795580113, "grad_norm": 6.813127749828438, "kl": 0.0966796875, "learning_rate": 3.6049108772580516e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7822182178497314, "rewards_raw/accuracy_reward": 0.5886361598968506, "rewards_raw/clip_reward_wrapper": 0.3404032588005066, "rewards_raw/format_reward": 5.0, "step": 1026 }, { "Total_reward": 4.3319806763975066e-07, "advantage_std": 1.1732088327407837, "completion_length": 395.9285888671875, "epoch": 0.3546270718232044, "grad_norm": 4.222393797107041, "kl": 0.09423828125, "learning_rate": 3.6024774659745394e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.670820415019989, "rewards_raw/accuracy_reward": 0.5696675181388855, "rewards_raw/clip_reward_wrapper": 0.33243146538734436, "rewards_raw/format_reward": 5.0, "step": 1027 }, { "Total_reward": -5.492142349794449e-07, "advantage_std": 1.5180747509002686, "completion_length": 388.6785888671875, "epoch": 0.35497237569060774, "grad_norm": 11.884898738759619, "kl": 0.09765625, "learning_rate": 3.600042757295759e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9377602338790894, "rewards_raw/accuracy_reward": 0.579879641532898, "rewards_raw/clip_reward_wrapper": 0.34877780079841614, "rewards_raw/format_reward": 5.0, "step": 1028 }, { "Total_reward": -6.428787173717865e-07, "advantage_std": 1.344233512878418, "completion_length": 398.58929443359375, "epoch": 0.355317679558011, "grad_norm": 8.57472593672371, "kl": 0.0966796875, "learning_rate": 3.5976067540868763e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.0140266418457031, "rewards_raw/accuracy_reward": 0.5919602513313293, "rewards_raw/clip_reward_wrapper": 0.337520956993103, "rewards_raw/format_reward": 5.0, "step": 1029 }, { "Total_reward": -3.097312912814232e-07, "advantage_std": 1.3439607620239258, "completion_length": 399.4285888671875, "epoch": 0.35566298342541436, "grad_norm": 4.2171897271738805, "kl": 0.09326171875, "learning_rate": 3.5951694592145804e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.394866704940796, "rewards_raw/accuracy_reward": 0.5598806738853455, "rewards_raw/clip_reward_wrapper": 0.3365625739097595, "rewards_raw/format_reward": 5.0, "step": 1030 }, { "Total_reward": -8.238213808908768e-07, "advantage_std": 1.5000139474868774, "completion_length": 395.7857360839844, "epoch": 0.3560082872928177, "grad_norm": 20.32382544414772, "kl": 0.09423828125, "learning_rate": 3.5927308755470773e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6906936168670654, "rewards_raw/accuracy_reward": 0.5686338543891907, "rewards_raw/clip_reward_wrapper": 0.32561808824539185, "rewards_raw/format_reward": 5.0, "step": 1031 }, { "Total_reward": -1.3017229321121704e-06, "advantage_std": 1.6734652519226074, "completion_length": 384.9464416503906, "epoch": 0.356353591160221, "grad_norm": 4.820788625924606, "kl": 0.09326171875, "learning_rate": 3.5902910059540934e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7486112713813782, "rewards_raw/accuracy_reward": 0.5841070413589478, "rewards_raw/clip_reward_wrapper": 0.339571088552475, "rewards_raw/format_reward": 5.0, "step": 1032 }, { "Total_reward": 4.896095973094816e-08, "advantage_std": 1.3870809078216553, "completion_length": 385.96429443359375, "epoch": 0.3566988950276243, "grad_norm": 6.148839557984861, "kl": 0.0908203125, "learning_rate": 3.587849853306867e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.0690439939498901, "rewards_raw/accuracy_reward": 0.5697583556175232, "rewards_raw/clip_reward_wrapper": 0.33713647723197937, "rewards_raw/format_reward": 5.0, "step": 1033 }, { "Total_reward": -8.557524324714905e-07, "advantage_std": 1.3415913581848145, "completion_length": 389.33929443359375, "epoch": 0.35704419889502764, "grad_norm": 11.833630725138189, "kl": 0.0927734375, "learning_rate": 3.585407420478145e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.4139641523361206, "rewards_raw/accuracy_reward": 0.5662757158279419, "rewards_raw/clip_reward_wrapper": 0.328711599111557, "rewards_raw/format_reward": 5.0, "step": 1034 }, { "Total_reward": 3.916876778475853e-07, "advantage_std": 1.670214056968689, "completion_length": 399.3571472167969, "epoch": 0.3573895027624309, "grad_norm": 4.089592791522072, "kl": 0.09716796875, "learning_rate": 3.5829637103421816e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9865481853485107, "rewards_raw/accuracy_reward": 0.6049750447273254, "rewards_raw/clip_reward_wrapper": 0.34427526593208313, "rewards_raw/format_reward": 5.0, "step": 1035 }, { "Total_reward": 3.70400300653273e-07, "advantage_std": 1.6143543720245361, "completion_length": 394.14288330078125, "epoch": 0.35773480662983426, "grad_norm": 3.9056407602310648, "kl": 0.0947265625, "learning_rate": 3.5805187257747363e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8041512370109558, "rewards_raw/accuracy_reward": 0.5942519307136536, "rewards_raw/clip_reward_wrapper": 0.33023175597190857, "rewards_raw/format_reward": 5.0, "step": 1036 }, { "Total_reward": -7.833753556951706e-07, "advantage_std": 1.6232324838638306, "completion_length": 400.1071472167969, "epoch": 0.3580801104972376, "grad_norm": 3.6720304608565133, "kl": 0.10107421875, "learning_rate": 3.578072469653064e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.6608580350875854, "rewards_raw/accuracy_reward": 0.5743430256843567, "rewards_raw/clip_reward_wrapper": 0.32983532547950745, "rewards_raw/format_reward": 5.0, "step": 1037 }, { "Total_reward": 3.4059798537100505e-08, "advantage_std": 1.4822642803192139, "completion_length": 399.71429443359375, "epoch": 0.35842541436464087, "grad_norm": 4.053044373388318, "kl": 0.0947265625, "learning_rate": 3.5756249448559205e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6280899047851562, "rewards_raw/accuracy_reward": 0.5874684453010559, "rewards_raw/clip_reward_wrapper": 0.33145615458488464, "rewards_raw/format_reward": 5.0, "step": 1038 }, { "Total_reward": 1.3836793755217514e-07, "advantage_std": 1.2903727293014526, "completion_length": 399.51788330078125, "epoch": 0.3587707182320442, "grad_norm": 10.758215565296807, "kl": 0.0908203125, "learning_rate": 3.573176154263551e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.5742567181587219, "rewards_raw/accuracy_reward": 0.5598797798156738, "rewards_raw/clip_reward_wrapper": 0.33615460991859436, "rewards_raw/format_reward": 5.0, "step": 1039 }, { "Total_reward": 4.406486482366745e-07, "advantage_std": 1.729542851448059, "completion_length": 389.4285888671875, "epoch": 0.35911602209944754, "grad_norm": 5.674040452239322, "kl": 0.09912109375, "learning_rate": 3.5707261007576927e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7806130647659302, "rewards_raw/accuracy_reward": 0.5633066892623901, "rewards_raw/clip_reward_wrapper": 0.34652698040008545, "rewards_raw/format_reward": 5.0, "step": 1040 }, { "Total_reward": 1.8030406181424041e-06, "advantage_std": 1.5061851739883423, "completion_length": 397.3571472167969, "epoch": 0.3594613259668508, "grad_norm": 5.3868589479972995, "kl": 0.09716796875, "learning_rate": 3.5682747872215685e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.5263841152191162, "rewards_raw/accuracy_reward": 0.5880011320114136, "rewards_raw/clip_reward_wrapper": 0.34123143553733826, "rewards_raw/format_reward": 5.0, "step": 1041 }, { "Total_reward": 9.73897385847522e-07, "advantage_std": 1.56705904006958, "completion_length": 394.08929443359375, "epoch": 0.35980662983425415, "grad_norm": 6.505885706246142, "kl": 0.0947265625, "learning_rate": 3.5658222165398825e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6457535028457642, "rewards_raw/accuracy_reward": 0.5766680836677551, "rewards_raw/clip_reward_wrapper": 0.33232009410858154, "rewards_raw/format_reward": 5.0, "step": 1042 }, { "Total_reward": -1.9052200173064193e-07, "advantage_std": 1.5190364122390747, "completion_length": 393.3571472167969, "epoch": 0.36015193370165743, "grad_norm": 4.166764887789288, "kl": 0.09423828125, "learning_rate": 3.56336839159882e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6018354892730713, "rewards_raw/accuracy_reward": 0.5625125765800476, "rewards_raw/clip_reward_wrapper": 0.3401927053928375, "rewards_raw/format_reward": 5.0, "step": 1043 }, { "Total_reward": -1.1708056035786285e-07, "advantage_std": 1.2156511545181274, "completion_length": 399.2321472167969, "epoch": 0.36049723756906077, "grad_norm": 3.1879938766627425, "kl": 0.09716796875, "learning_rate": 3.5609133152860416e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.648985743522644, "rewards_raw/accuracy_reward": 0.5753857493400574, "rewards_raw/clip_reward_wrapper": 0.3278203010559082, "rewards_raw/format_reward": 5.0, "step": 1044 }, { "Total_reward": 1.0505318641662598e-06, "advantage_std": 1.4339747428894043, "completion_length": 391.33929443359375, "epoch": 0.3608425414364641, "grad_norm": 6.424478125908934, "kl": 0.09716796875, "learning_rate": 3.5584569904906805e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8072417378425598, "rewards_raw/accuracy_reward": 0.5818958282470703, "rewards_raw/clip_reward_wrapper": 0.33294880390167236, "rewards_raw/format_reward": 5.0, "step": 1045 }, { "Total_reward": -2.064875275209488e-07, "advantage_std": 1.443015694618225, "completion_length": 398.46429443359375, "epoch": 0.3611878453038674, "grad_norm": 4.695831145158034, "kl": 0.09375, "learning_rate": 3.55599942010334e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 2.444218873977661, "rewards_raw/accuracy_reward": 0.5518952012062073, "rewards_raw/clip_reward_wrapper": 0.33157050609588623, "rewards_raw/format_reward": 5.0, "step": 1046 }, { "Total_reward": 6.120120019659225e-07, "advantage_std": 1.4724352359771729, "completion_length": 389.46429443359375, "epoch": 0.3615331491712707, "grad_norm": 4.263749150885583, "kl": 0.0966796875, "learning_rate": 3.553540607016087e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8725473284721375, "rewards_raw/accuracy_reward": 0.5559917688369751, "rewards_raw/clip_reward_wrapper": 0.3275429308414459, "rewards_raw/format_reward": 5.0, "step": 1047 }, { "Total_reward": -1.4496702078758972e-06, "advantage_std": 1.4106448888778687, "completion_length": 390.1071472167969, "epoch": 0.36187845303867405, "grad_norm": 3.8929375967059916, "kl": 0.09521484375, "learning_rate": 3.551080554122453e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.5658079385757446, "rewards_raw/accuracy_reward": 0.5802099108695984, "rewards_raw/clip_reward_wrapper": 0.343095600605011, "rewards_raw/format_reward": 5.0, "step": 1048 }, { "Total_reward": 1.4262541014886665e-07, "advantage_std": 1.394457459449768, "completion_length": 388.21429443359375, "epoch": 0.36222375690607733, "grad_norm": 9.379676308819084, "kl": 0.095703125, "learning_rate": 3.5486192643174295e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.79714435338974, "rewards_raw/accuracy_reward": 0.5493902564048767, "rewards_raw/clip_reward_wrapper": 0.33989495038986206, "rewards_raw/format_reward": 5.0, "step": 1049 }, { "Total_reward": -5.460211696117767e-07, "advantage_std": 1.4351046085357666, "completion_length": 385.26788330078125, "epoch": 0.36256906077348067, "grad_norm": 8.666693161426428, "kl": 0.09423828125, "learning_rate": 3.546156740497459e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.242540955543518, "rewards_raw/accuracy_reward": 0.544265866279602, "rewards_raw/clip_reward_wrapper": 0.3257097601890564, "rewards_raw/format_reward": 5.0, "step": 1050 }, { "Total_reward": -3.150531426854286e-07, "advantage_std": 1.4735506772994995, "completion_length": 396.7857360839844, "epoch": 0.362914364640884, "grad_norm": 4.799829366306161, "kl": 0.09423828125, "learning_rate": 3.5436929855604395e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8284218311309814, "rewards_raw/accuracy_reward": 0.5727050304412842, "rewards_raw/clip_reward_wrapper": 0.32418009638786316, "rewards_raw/format_reward": 5.0, "step": 1051 }, { "Total_reward": -4.385199190437561e-07, "advantage_std": 1.362043023109436, "completion_length": 384.58929443359375, "epoch": 0.3632596685082873, "grad_norm": 5.885641997624871, "kl": 0.0927734375, "learning_rate": 3.5412280024057196e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9499467611312866, "rewards_raw/accuracy_reward": 0.5714467763900757, "rewards_raw/clip_reward_wrapper": 0.335281640291214, "rewards_raw/format_reward": 5.0, "step": 1052 }, { "Total_reward": 1.596553005356327e-07, "advantage_std": 1.294065237045288, "completion_length": 388.3750305175781, "epoch": 0.3636049723756906, "grad_norm": 3.9963148072597092, "kl": 0.09130859375, "learning_rate": 3.538761793934089e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.7565628290176392, "rewards_raw/accuracy_reward": 0.5657289624214172, "rewards_raw/clip_reward_wrapper": 0.3407335579395294, "rewards_raw/format_reward": 5.0, "step": 1053 }, { "Total_reward": 1.335782684463993e-07, "advantage_std": 1.5611811876296997, "completion_length": 393.2321472167969, "epoch": 0.3639502762430939, "grad_norm": 4.427402002149047, "kl": 0.09228515625, "learning_rate": 3.536294363047782e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 2.0171761512756348, "rewards_raw/accuracy_reward": 0.5527911186218262, "rewards_raw/clip_reward_wrapper": 0.3355309069156647, "rewards_raw/format_reward": 5.0, "step": 1054 }, { "Total_reward": 1.9797258232756576e-07, "advantage_std": 1.4332038164138794, "completion_length": 369.0357360839844, "epoch": 0.36429558011049723, "grad_norm": 4.546143260445499, "kl": 0.09619140625, "learning_rate": 3.5338257126504705e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6522722244262695, "rewards_raw/accuracy_reward": 0.5745084285736084, "rewards_raw/clip_reward_wrapper": 0.3410572409629822, "rewards_raw/format_reward": 5.0, "step": 1055 }, { "Total_reward": -6.907752663209976e-07, "advantage_std": 1.4154475927352905, "completion_length": 396.0357360839844, "epoch": 0.36464088397790057, "grad_norm": 6.321887847622712, "kl": 0.09228515625, "learning_rate": 3.531355845647262e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.2193005084991455, "rewards_raw/accuracy_reward": 0.5598713159561157, "rewards_raw/clip_reward_wrapper": 0.33166056871414185, "rewards_raw/format_reward": 5.0, "step": 1056 }, { "Total_reward": 9.643180192142609e-07, "advantage_std": 1.6868176460266113, "completion_length": 377.01788330078125, "epoch": 0.36498618784530384, "grad_norm": 11.879822865629935, "kl": 0.09619140625, "learning_rate": 3.5288847649446963e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.0832008123397827, "rewards_raw/accuracy_reward": 0.5735728144645691, "rewards_raw/clip_reward_wrapper": 0.3324853479862213, "rewards_raw/format_reward": 5.0, "step": 1057 }, { "Total_reward": 1.0622400168358581e-06, "advantage_std": 1.6035003662109375, "completion_length": 394.8750305175781, "epoch": 0.3653314917127072, "grad_norm": 6.071637522968954, "kl": 0.09716796875, "learning_rate": 3.5264124734507385e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6548418402671814, "rewards_raw/accuracy_reward": 0.5806660652160645, "rewards_raw/clip_reward_wrapper": 0.32104235887527466, "rewards_raw/format_reward": 5.0, "step": 1058 }, { "Total_reward": 1.2676631513386383e-06, "advantage_std": 1.2915273904800415, "completion_length": 379.9285888671875, "epoch": 0.3656767955801105, "grad_norm": 8.665404703052788, "kl": 0.09814453125, "learning_rate": 3.5239389740747827e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9519030451774597, "rewards_raw/accuracy_reward": 0.5707296133041382, "rewards_raw/clip_reward_wrapper": 0.32700076699256897, "rewards_raw/format_reward": 5.0, "step": 1059 }, { "Total_reward": -4.864165248363861e-07, "advantage_std": 1.5304070711135864, "completion_length": 371.8035888671875, "epoch": 0.3660220994475138, "grad_norm": 5.1660129447912135, "kl": 0.09814453125, "learning_rate": 3.521464269727642e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8268463015556335, "rewards_raw/accuracy_reward": 0.5595096349716187, "rewards_raw/clip_reward_wrapper": 0.34546715021133423, "rewards_raw/format_reward": 5.0, "step": 1060 }, { "Total_reward": -1.341104507446289e-07, "advantage_std": 1.5502270460128784, "completion_length": 390.3750305175781, "epoch": 0.36636740331491713, "grad_norm": 4.232855169304149, "kl": 0.09814453125, "learning_rate": 3.518988363321548e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8685215711593628, "rewards_raw/accuracy_reward": 0.551196813583374, "rewards_raw/clip_reward_wrapper": 0.3292270302772522, "rewards_raw/format_reward": 5.0, "step": 1061 }, { "Total_reward": 1.4581851814909896e-07, "advantage_std": 1.4204522371292114, "completion_length": 385.4285888671875, "epoch": 0.36671270718232046, "grad_norm": 13.083389392659607, "kl": 0.09423828125, "learning_rate": 3.516511257770146e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.1002013683319092, "rewards_raw/accuracy_reward": 0.5512761473655701, "rewards_raw/clip_reward_wrapper": 0.34099921584129333, "rewards_raw/format_reward": 5.0, "step": 1062 }, { "Total_reward": -2.6609217940176677e-08, "advantage_std": 1.4039448499679565, "completion_length": 394.6964416503906, "epoch": 0.36705801104972374, "grad_norm": 5.204559801483981, "kl": 0.09033203125, "learning_rate": 3.514032955988495e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.1730225086212158, "rewards_raw/accuracy_reward": 0.5618193745613098, "rewards_raw/clip_reward_wrapper": 0.3296624720096588, "rewards_raw/format_reward": 5.0, "step": 1063 }, { "Total_reward": 7.237706967089252e-08, "advantage_std": 1.74424147605896, "completion_length": 370.08929443359375, "epoch": 0.3674033149171271, "grad_norm": 8.635928632249804, "kl": 0.09716796875, "learning_rate": 3.5115534608930584e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.5476224422454834, "rewards_raw/accuracy_reward": 0.5522356033325195, "rewards_raw/clip_reward_wrapper": 0.3495386838912964, "rewards_raw/format_reward": 5.0, "step": 1064 }, { "Total_reward": 1.7242773253656196e-07, "advantage_std": 1.472771167755127, "completion_length": 376.4464416503906, "epoch": 0.3677486187845304, "grad_norm": 4.383596249772385, "kl": 0.095703125, "learning_rate": 3.509072775401707e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.0539765357971191, "rewards_raw/accuracy_reward": 0.5710672736167908, "rewards_raw/clip_reward_wrapper": 0.343030720949173, "rewards_raw/format_reward": 5.0, "step": 1065 }, { "Total_reward": 1.2133803295455436e-07, "advantage_std": 1.8020418882369995, "completion_length": 385.8214416503906, "epoch": 0.3680939226519337, "grad_norm": 6.365750271201249, "kl": 0.095703125, "learning_rate": 3.5065909024337096e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.9500254392623901, "rewards_raw/accuracy_reward": 0.5592392086982727, "rewards_raw/clip_reward_wrapper": 0.3176596164703369, "rewards_raw/format_reward": 5.0, "step": 1066 }, { "Total_reward": -5.140900611877441e-07, "advantage_std": 1.3622623682022095, "completion_length": 389.39288330078125, "epoch": 0.368439226519337, "grad_norm": 4.4521447307633935, "kl": 0.09619140625, "learning_rate": 3.504107844909735e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8803513646125793, "rewards_raw/accuracy_reward": 0.5234918594360352, "rewards_raw/clip_reward_wrapper": 0.34412866830825806, "rewards_raw/format_reward": 5.0, "step": 1067 }, { "Total_reward": -3.938164070405037e-07, "advantage_std": 1.4454481601715088, "completion_length": 381.76788330078125, "epoch": 0.3687845303867403, "grad_norm": 4.210083835519418, "kl": 0.09912109375, "learning_rate": 3.501623605751845e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7527850866317749, "rewards_raw/accuracy_reward": 0.5536106824874878, "rewards_raw/clip_reward_wrapper": 0.3358979821205139, "rewards_raw/format_reward": 5.0, "step": 1068 }, { "Total_reward": -4.108463258489792e-07, "advantage_std": 1.5095781087875366, "completion_length": 383.6785888671875, "epoch": 0.36912983425414364, "grad_norm": 3.8349877327509763, "kl": 0.09423828125, "learning_rate": 3.49913818788349e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.0481531620025635, "rewards_raw/accuracy_reward": 0.5803362131118774, "rewards_raw/clip_reward_wrapper": 0.33155736327171326, "rewards_raw/format_reward": 5.0, "step": 1069 }, { "Total_reward": 2.682209014892578e-07, "advantage_std": 1.3767145872116089, "completion_length": 373.08929443359375, "epoch": 0.369475138121547, "grad_norm": 8.867872430418364, "kl": 0.09912109375, "learning_rate": 3.496651594229511e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.6999751925468445, "rewards_raw/accuracy_reward": 0.5637660026550293, "rewards_raw/clip_reward_wrapper": 0.3420957326889038, "rewards_raw/format_reward": 5.0, "step": 1070 }, { "Total_reward": 5.406993182077713e-07, "advantage_std": 1.5165210962295532, "completion_length": 388.64288330078125, "epoch": 0.36982044198895025, "grad_norm": 4.152673514390028, "kl": 0.0966796875, "learning_rate": 3.49416382771613e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.0268363952636719, "rewards_raw/accuracy_reward": 0.5829629898071289, "rewards_raw/clip_reward_wrapper": 0.3453163504600525, "rewards_raw/format_reward": 5.0, "step": 1071 }, { "Total_reward": -3.448554650731239e-07, "advantage_std": 1.4790600538253784, "completion_length": 389.5714416503906, "epoch": 0.3701657458563536, "grad_norm": 12.984054937464544, "kl": 0.09228515625, "learning_rate": 3.4916748912709506e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9140881299972534, "rewards_raw/accuracy_reward": 0.5780982971191406, "rewards_raw/clip_reward_wrapper": 0.34716302156448364, "rewards_raw/format_reward": 5.0, "step": 1072 }, { "Total_reward": 1.0601112307995209e-06, "advantage_std": 1.4815516471862793, "completion_length": 382.9464416503906, "epoch": 0.3705110497237569, "grad_norm": 6.750924132478102, "kl": 0.099609375, "learning_rate": 3.489184787822952e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.6964079141616821, "rewards_raw/accuracy_reward": 0.5708523988723755, "rewards_raw/clip_reward_wrapper": 0.345550000667572, "rewards_raw/format_reward": 5.0, "step": 1073 }, { "Total_reward": 3.672072068638954e-07, "advantage_std": 1.4878820180892944, "completion_length": 394.4285888671875, "epoch": 0.3708563535911602, "grad_norm": 3.373206368949079, "kl": 0.0947265625, "learning_rate": 3.486693520302488e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6386649012565613, "rewards_raw/accuracy_reward": 0.5474326610565186, "rewards_raw/clip_reward_wrapper": 0.34298714995384216, "rewards_raw/format_reward": 5.0, "step": 1074 }, { "Total_reward": -4.512923226229759e-07, "advantage_std": 1.3904603719711304, "completion_length": 378.3750305175781, "epoch": 0.37120165745856354, "grad_norm": 7.433237012210028, "kl": 0.0947265625, "learning_rate": 3.484201091641282e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.507500171661377, "rewards_raw/accuracy_reward": 0.5761483907699585, "rewards_raw/clip_reward_wrapper": 0.34379422664642334, "rewards_raw/format_reward": 5.0, "step": 1075 }, { "Total_reward": 7.876328140810074e-07, "advantage_std": 1.3300601243972778, "completion_length": 392.96429443359375, "epoch": 0.3715469613259669, "grad_norm": 4.576909710544679, "kl": 0.09619140625, "learning_rate": 3.4817075047724234e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.9204672574996948, "rewards_raw/accuracy_reward": 0.5774645209312439, "rewards_raw/clip_reward_wrapper": 0.3356080651283264, "rewards_raw/format_reward": 5.0, "step": 1076 }, { "Total_reward": 2.054231629244896e-07, "advantage_std": 1.3964215517044067, "completion_length": 392.4464416503906, "epoch": 0.37189226519337015, "grad_norm": 45.36388056382634, "kl": 0.09228515625, "learning_rate": 3.479212762630365e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.055887222290039, "rewards_raw/accuracy_reward": 0.5967733263969421, "rewards_raw/clip_reward_wrapper": 0.3339694142341614, "rewards_raw/format_reward": 5.0, "step": 1077 }, { "Total_reward": 4.80030280414212e-07, "advantage_std": 1.5652191638946533, "completion_length": 375.08929443359375, "epoch": 0.3722375690607735, "grad_norm": 4.6037824986376545, "kl": 0.09521484375, "learning_rate": 3.476716868150917e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.9650023579597473, "rewards_raw/accuracy_reward": 0.5980260372161865, "rewards_raw/clip_reward_wrapper": 0.32590335607528687, "rewards_raw/format_reward": 5.0, "step": 1078 }, { "Total_reward": 6.769384981453186e-07, "advantage_std": 1.5595674514770508, "completion_length": 386.5357360839844, "epoch": 0.3725828729281768, "grad_norm": 3.8576619344152383, "kl": 0.09521484375, "learning_rate": 3.4742198242712506e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8125095963478088, "rewards_raw/accuracy_reward": 0.5692076683044434, "rewards_raw/clip_reward_wrapper": 0.31395450234413147, "rewards_raw/format_reward": 5.0, "step": 1079 }, { "Total_reward": -3.08666926684964e-07, "advantage_std": 1.403588056564331, "completion_length": 381.14288330078125, "epoch": 0.3729281767955801, "grad_norm": 8.136340405648504, "kl": 0.0927734375, "learning_rate": 3.4717216339298847e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.739967405796051, "rewards_raw/accuracy_reward": 0.5523242950439453, "rewards_raw/clip_reward_wrapper": 0.33866068720817566, "rewards_raw/format_reward": 5.0, "step": 1080 }, { "Total_reward": -9.962491276382934e-07, "advantage_std": 1.5940909385681152, "completion_length": 380.64288330078125, "epoch": 0.37327348066298344, "grad_norm": 4.353144998156966, "kl": 0.09228515625, "learning_rate": 3.469222300066689e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.235095739364624, "rewards_raw/accuracy_reward": 0.5712993144989014, "rewards_raw/clip_reward_wrapper": 0.3481753468513489, "rewards_raw/format_reward": 5.0, "step": 1081 }, { "Total_reward": 1.798783131334858e-07, "advantage_std": 1.3652013540267944, "completion_length": 386.0714416503906, "epoch": 0.3736187845303867, "grad_norm": 5.8279054808774005, "kl": 0.0908203125, "learning_rate": 3.4667218256228805e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.8600205183029175, "rewards_raw/accuracy_reward": 0.5774210691452026, "rewards_raw/clip_reward_wrapper": 0.3393635153770447, "rewards_raw/format_reward": 5.0, "step": 1082 }, { "Total_reward": -2.543841333135788e-07, "advantage_std": 1.4044159650802612, "completion_length": 384.1071472167969, "epoch": 0.37396408839779005, "grad_norm": 11.645306170037331, "kl": 0.09814453125, "learning_rate": 3.464220213541015e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7079973816871643, "rewards_raw/accuracy_reward": 0.5721370577812195, "rewards_raw/clip_reward_wrapper": 0.33375227451324463, "rewards_raw/format_reward": 5.0, "step": 1083 }, { "Total_reward": 6.258487701416016e-07, "advantage_std": 1.2153465747833252, "completion_length": 380.2321472167969, "epoch": 0.3743093922651934, "grad_norm": 5.280945066632217, "kl": 0.0927734375, "learning_rate": 3.4617174667649913e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9860513210296631, "rewards_raw/accuracy_reward": 0.5792597532272339, "rewards_raw/clip_reward_wrapper": 0.34193137288093567, "rewards_raw/format_reward": 5.0, "step": 1084 }, { "Total_reward": -8.514949740856537e-07, "advantage_std": 1.3880853652954102, "completion_length": 385.6250305175781, "epoch": 0.37465469613259667, "grad_norm": 4.315352715741295, "kl": 0.09765625, "learning_rate": 3.45921358824004e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7192370295524597, "rewards_raw/accuracy_reward": 0.5602087378501892, "rewards_raw/clip_reward_wrapper": 0.3419076204299927, "rewards_raw/format_reward": 5.0, "step": 1085 }, { "Total_reward": -1.596553005356327e-07, "advantage_std": 1.3606330156326294, "completion_length": 394.8750305175781, "epoch": 0.375, "grad_norm": 4.065904321467054, "kl": 0.0966796875, "learning_rate": 3.4567085809127245e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.1984211206436157, "rewards_raw/accuracy_reward": 0.5933369398117065, "rewards_raw/clip_reward_wrapper": 0.32611554861068726, "rewards_raw/format_reward": 5.0, "step": 1086 }, { "Total_reward": -5.492142349794449e-07, "advantage_std": 1.27505362033844, "completion_length": 377.08929443359375, "epoch": 0.37534530386740333, "grad_norm": 4.535166669960451, "kl": 0.09765625, "learning_rate": 3.4542024477309376e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.919786274433136, "rewards_raw/accuracy_reward": 0.5625916719436646, "rewards_raw/clip_reward_wrapper": 0.3493735194206238, "rewards_raw/format_reward": 5.0, "step": 1087 }, { "Total_reward": -2.7673587510435027e-07, "advantage_std": 1.4743982553482056, "completion_length": 390.1607360839844, "epoch": 0.3756906077348066, "grad_norm": 5.892876232326122, "kl": 0.095703125, "learning_rate": 3.451695191643897e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8744597434997559, "rewards_raw/accuracy_reward": 0.5586318969726562, "rewards_raw/clip_reward_wrapper": 0.32756075263023376, "rewards_raw/format_reward": 5.0, "step": 1088 }, { "Total_reward": -3.7119858120604476e-07, "advantage_std": 1.6094850301742554, "completion_length": 392.83929443359375, "epoch": 0.37603591160220995, "grad_norm": 4.444976708818609, "kl": 0.0908203125, "learning_rate": 3.4491868156021396e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.8860727548599243, "rewards_raw/accuracy_reward": 0.5843865871429443, "rewards_raw/clip_reward_wrapper": 0.33094698190689087, "rewards_raw/format_reward": 5.0, "step": 1089 }, { "Total_reward": 3.8157617154865875e-07, "advantage_std": 1.4905142784118652, "completion_length": 391.96429443359375, "epoch": 0.3763812154696133, "grad_norm": 12.542196327529382, "kl": 0.0986328125, "learning_rate": 3.4466773225575234e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.660491943359375, "rewards_raw/accuracy_reward": 0.5688398480415344, "rewards_raw/clip_reward_wrapper": 0.3436950445175171, "rewards_raw/format_reward": 5.0, "step": 1090 }, { "Total_reward": 4.512923226229759e-07, "advantage_std": 1.354580044746399, "completion_length": 385.5000305175781, "epoch": 0.37672651933701656, "grad_norm": 6.046481025333215, "kl": 0.09228515625, "learning_rate": 3.444166715463219e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9333595037460327, "rewards_raw/accuracy_reward": 0.5646392703056335, "rewards_raw/clip_reward_wrapper": 0.3286082446575165, "rewards_raw/format_reward": 5.0, "step": 1091 }, { "Total_reward": 4.853521318182175e-07, "advantage_std": 1.6586713790893555, "completion_length": 376.96429443359375, "epoch": 0.3770718232044199, "grad_norm": 3.8645585089673906, "kl": 0.09716796875, "learning_rate": 3.4416549972737067e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.4599694013595581, "rewards_raw/accuracy_reward": 0.5953254699707031, "rewards_raw/clip_reward_wrapper": 0.34352797269821167, "rewards_raw/format_reward": 5.0, "step": 1092 }, { "Total_reward": 2.980232238769531e-07, "advantage_std": 1.4383329153060913, "completion_length": 386.5357360839844, "epoch": 0.3774171270718232, "grad_norm": 3.5185290015652604, "kl": 0.09521484375, "learning_rate": 3.4391421709447785e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.0276635885238647, "rewards_raw/accuracy_reward": 0.6030562520027161, "rewards_raw/clip_reward_wrapper": 0.3275986611843109, "rewards_raw/format_reward": 5.0, "step": 1093 }, { "Total_reward": -5.556004794016189e-07, "advantage_std": 1.17344331741333, "completion_length": 388.5535888671875, "epoch": 0.3777624309392265, "grad_norm": 2.7934470286872193, "kl": 0.099609375, "learning_rate": 3.4366282394335263e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8520371317863464, "rewards_raw/accuracy_reward": 0.5965592265129089, "rewards_raw/clip_reward_wrapper": 0.32874736189842224, "rewards_raw/format_reward": 5.0, "step": 1094 }, { "Total_reward": -3.895589486546669e-07, "advantage_std": 1.5038834810256958, "completion_length": 389.2321472167969, "epoch": 0.37810773480662985, "grad_norm": 4.642438854353795, "kl": 0.09814453125, "learning_rate": 3.4341132056983443e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.5067803859710693, "rewards_raw/accuracy_reward": 0.5664127469062805, "rewards_raw/clip_reward_wrapper": 0.34411564469337463, "rewards_raw/format_reward": 5.0, "step": 1095 }, { "Total_reward": -2.2564616131148796e-07, "advantage_std": 1.3758708238601685, "completion_length": 394.89288330078125, "epoch": 0.3784530386740331, "grad_norm": 5.086512005401586, "kl": 0.10205078125, "learning_rate": 3.431597072698924e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.6564232707023621, "rewards_raw/accuracy_reward": 0.5529890060424805, "rewards_raw/clip_reward_wrapper": 0.3490208387374878, "rewards_raw/format_reward": 5.0, "step": 1096 }, { "Total_reward": 1.2197665455460083e-06, "advantage_std": 1.4835476875305176, "completion_length": 393.9107360839844, "epoch": 0.37879834254143646, "grad_norm": 4.197305450538823, "kl": 0.09912109375, "learning_rate": 3.4290798433962504e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 1.0808656215667725, "rewards_raw/accuracy_reward": 0.5972446203231812, "rewards_raw/clip_reward_wrapper": 0.3433789610862732, "rewards_raw/format_reward": 5.0, "step": 1097 }, { "Total_reward": -7.2377071091978e-07, "advantage_std": 1.5109636783599854, "completion_length": 374.89288330078125, "epoch": 0.3791436464088398, "grad_norm": 5.8875740284936935, "kl": 0.095703125, "learning_rate": 3.426561520752598e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8570935130119324, "rewards_raw/accuracy_reward": 0.6040900945663452, "rewards_raw/clip_reward_wrapper": 0.34788885712623596, "rewards_raw/format_reward": 5.0, "step": 1098 }, { "Total_reward": -6.482005687757919e-07, "advantage_std": 1.387825608253479, "completion_length": 386.26788330078125, "epoch": 0.3794889502762431, "grad_norm": 3.377851186058313, "kl": 0.0986328125, "learning_rate": 3.424042107731529e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.43638721108436584, "rewards_raw/accuracy_reward": 0.57939213514328, "rewards_raw/clip_reward_wrapper": 0.35543620586395264, "rewards_raw/format_reward": 5.0, "step": 1099 }, { "Total_reward": -1.0643686820799303e-08, "advantage_std": 1.5303834676742554, "completion_length": 395.4464416503906, "epoch": 0.3798342541436464, "grad_norm": 6.717590257816745, "kl": 0.09619140625, "learning_rate": 3.421521607297888e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7976701855659485, "rewards_raw/accuracy_reward": 0.5871354341506958, "rewards_raw/clip_reward_wrapper": 0.31653743982315063, "rewards_raw/format_reward": 5.0, "step": 1100 }, { "Total_reward": -7.663454653084045e-07, "advantage_std": 1.4114041328430176, "completion_length": 382.9464416503906, "epoch": 0.38017955801104975, "grad_norm": 3.9230145273188595, "kl": 0.0966796875, "learning_rate": 3.419000022417799e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8782612085342407, "rewards_raw/accuracy_reward": 0.5602288246154785, "rewards_raw/clip_reward_wrapper": 0.3439246714115143, "rewards_raw/format_reward": 5.0, "step": 1101 }, { "Total_reward": -4.257474817137563e-09, "advantage_std": 1.6297235488891602, "completion_length": 380.3571472167969, "epoch": 0.380524861878453, "grad_norm": 7.006415036601618, "kl": 0.1005859375, "learning_rate": 3.416477356058663e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.6674856543540955, "rewards_raw/accuracy_reward": 0.5750638842582703, "rewards_raw/clip_reward_wrapper": 0.3601267635822296, "rewards_raw/format_reward": 5.0, "step": 1102 }, { "Total_reward": 4.6832223432602405e-08, "advantage_std": 1.4688800573349, "completion_length": 387.6785888671875, "epoch": 0.38087016574585636, "grad_norm": 3.776050650866925, "kl": 0.1015625, "learning_rate": 3.413953611189153e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.9459290504455566, "rewards_raw/accuracy_reward": 0.6058123707771301, "rewards_raw/clip_reward_wrapper": 0.3421975076198578, "rewards_raw/format_reward": 5.0, "step": 1103 }, { "Total_reward": 5.470855057865265e-07, "advantage_std": 1.4971550703048706, "completion_length": 384.6071472167969, "epoch": 0.3812154696132597, "grad_norm": 8.200909523466537, "kl": 0.09912109375, "learning_rate": 3.411428790779211e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.68421471118927, "rewards_raw/accuracy_reward": 0.5846607089042664, "rewards_raw/clip_reward_wrapper": 0.3370647132396698, "rewards_raw/format_reward": 5.0, "step": 1104 }, { "Total_reward": 2.841864557012741e-07, "advantage_std": 1.4772323369979858, "completion_length": 382.89288330078125, "epoch": 0.381560773480663, "grad_norm": 4.062952705281447, "kl": 0.0966796875, "learning_rate": 3.4089028978000466e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6091896295547485, "rewards_raw/accuracy_reward": 0.5911723971366882, "rewards_raw/clip_reward_wrapper": 0.3409522771835327, "rewards_raw/format_reward": 5.0, "step": 1105 }, { "Total_reward": 1.4390265050678863e-06, "advantage_std": 1.4161778688430786, "completion_length": 398.26788330078125, "epoch": 0.3819060773480663, "grad_norm": 3.877742645490471, "kl": 0.09912109375, "learning_rate": 3.4063759352241293e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.39481088519096375, "rewards_raw/accuracy_reward": 0.5705912113189697, "rewards_raw/clip_reward_wrapper": 0.32875367999076843, "rewards_raw/format_reward": 5.0, "step": 1106 }, { "Total_reward": -3.150531426854286e-07, "advantage_std": 1.4363948106765747, "completion_length": 382.2857360839844, "epoch": 0.3822513812154696, "grad_norm": 5.732275833213769, "kl": 0.09375, "learning_rate": 3.403847906025188e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8134090304374695, "rewards_raw/accuracy_reward": 0.5848259925842285, "rewards_raw/clip_reward_wrapper": 0.3373074233531952, "rewards_raw/format_reward": 5.0, "step": 1107 }, { "Total_reward": -7.056764275148453e-07, "advantage_std": 1.3339428901672363, "completion_length": 386.1785888671875, "epoch": 0.3825966850828729, "grad_norm": 4.083385456445094, "kl": 0.10009765625, "learning_rate": 3.4013188131782064e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7265143990516663, "rewards_raw/accuracy_reward": 0.5790874361991882, "rewards_raw/clip_reward_wrapper": 0.3229750096797943, "rewards_raw/format_reward": 5.0, "step": 1108 }, { "Total_reward": 1.4049666674509353e-07, "advantage_std": 1.5748436450958252, "completion_length": 389.0357360839844, "epoch": 0.38294198895027626, "grad_norm": 4.222413339786742, "kl": 0.09326171875, "learning_rate": 3.3987886596594225e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.8158895969390869, "rewards_raw/accuracy_reward": 0.5721317529678345, "rewards_raw/clip_reward_wrapper": 0.3447437584400177, "rewards_raw/format_reward": 5.0, "step": 1109 }, { "Total_reward": 2.7247838829680404e-07, "advantage_std": 1.2632372379302979, "completion_length": 389.76788330078125, "epoch": 0.38328729281767954, "grad_norm": 3.4077496612699525, "kl": 0.09912109375, "learning_rate": 3.396257448446319e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7298018932342529, "rewards_raw/accuracy_reward": 0.5848328471183777, "rewards_raw/clip_reward_wrapper": 0.33902379870414734, "rewards_raw/format_reward": 5.0, "step": 1110 }, { "Total_reward": 1.2325389207035187e-06, "advantage_std": 1.3434581756591797, "completion_length": 389.3571472167969, "epoch": 0.38363259668508287, "grad_norm": 3.489615494213868, "kl": 0.09765625, "learning_rate": 3.3937251825176236e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8196491599082947, "rewards_raw/accuracy_reward": 0.5718607306480408, "rewards_raw/clip_reward_wrapper": 0.3564806282520294, "rewards_raw/format_reward": 5.0, "step": 1111 }, { "Total_reward": 8.621386626828098e-08, "advantage_std": 1.3545269966125488, "completion_length": 394.51788330078125, "epoch": 0.3839779005524862, "grad_norm": 3.4376254151582817, "kl": 0.09375, "learning_rate": 3.3911918648533087e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.1136627197265625, "rewards_raw/accuracy_reward": 0.5864284038543701, "rewards_raw/clip_reward_wrapper": 0.34832391142845154, "rewards_raw/format_reward": 5.0, "step": 1112 }, { "Total_reward": 6.386212021425308e-07, "advantage_std": 1.364665150642395, "completion_length": 384.9285888671875, "epoch": 0.3843232044198895, "grad_norm": 9.477828284728893, "kl": 0.09228515625, "learning_rate": 3.3886574984345815e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.2831902503967285, "rewards_raw/accuracy_reward": 0.5938315391540527, "rewards_raw/clip_reward_wrapper": 0.35199278593063354, "rewards_raw/format_reward": 5.0, "step": 1113 }, { "Total_reward": 3.938164070405037e-07, "advantage_std": 1.3920767307281494, "completion_length": 389.5535888671875, "epoch": 0.3846685082872928, "grad_norm": 5.109932846056742, "kl": 0.095703125, "learning_rate": 3.3861220862438833e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.5290070176124573, "rewards_raw/accuracy_reward": 0.5762976408004761, "rewards_raw/clip_reward_wrapper": 0.3355488181114197, "rewards_raw/format_reward": 5.0, "step": 1114 }, { "Total_reward": 6.976936788305466e-07, "advantage_std": 1.358636498451233, "completion_length": 388.8035888671875, "epoch": 0.38501381215469616, "grad_norm": 17.552306151310297, "kl": 0.09521484375, "learning_rate": 3.383585631264886e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.751416802406311, "rewards_raw/accuracy_reward": 0.5825870037078857, "rewards_raw/clip_reward_wrapper": 0.32995960116386414, "rewards_raw/format_reward": 5.0, "step": 1115 }, { "Total_reward": -4.2787621623574523e-07, "advantage_std": 1.584647297859192, "completion_length": 372.5000305175781, "epoch": 0.38535911602209943, "grad_norm": 4.469993248817608, "kl": 0.10009765625, "learning_rate": 3.3810481364824906e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 1.2969375848770142, "rewards_raw/accuracy_reward": 0.549954891204834, "rewards_raw/clip_reward_wrapper": 0.35362258553504944, "rewards_raw/format_reward": 5.0, "step": 1116 }, { "Total_reward": -6.939683885320846e-07, "advantage_std": 1.2533410787582397, "completion_length": 393.3571472167969, "epoch": 0.38570441988950277, "grad_norm": 9.734524452075238, "kl": 0.0966796875, "learning_rate": 3.3785096048828205e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9909890294075012, "rewards_raw/accuracy_reward": 0.5810480117797852, "rewards_raw/clip_reward_wrapper": 0.3418155610561371, "rewards_raw/format_reward": 5.0, "step": 1117 }, { "Total_reward": 4.991888999938965e-07, "advantage_std": 1.3842793703079224, "completion_length": 384.8035888671875, "epoch": 0.3860497237569061, "grad_norm": 4.798775705515169, "kl": 0.10205078125, "learning_rate": 3.375970039453218e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.6998640298843384, "rewards_raw/accuracy_reward": 0.5683870315551758, "rewards_raw/clip_reward_wrapper": 0.3288538157939911, "rewards_raw/format_reward": 5.0, "step": 1118 }, { "Total_reward": 9.153571056685905e-08, "advantage_std": 1.489267349243164, "completion_length": 384.58929443359375, "epoch": 0.3863950276243094, "grad_norm": 7.585852978909944, "kl": 0.09814453125, "learning_rate": 3.373429443182244e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8860324025154114, "rewards_raw/accuracy_reward": 0.5730555653572083, "rewards_raw/clip_reward_wrapper": 0.33293652534484863, "rewards_raw/format_reward": 5.0, "step": 1119 }, { "Total_reward": 2.1180937892495422e-07, "advantage_std": 1.4867572784423828, "completion_length": 379.5000305175781, "epoch": 0.3867403314917127, "grad_norm": 6.149696740277884, "kl": 0.0966796875, "learning_rate": 3.370887819059672e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9209190607070923, "rewards_raw/accuracy_reward": 0.6084794998168945, "rewards_raw/clip_reward_wrapper": 0.34630995988845825, "rewards_raw/format_reward": 5.0, "step": 1120 }, { "Total_reward": 4.725796998172882e-07, "advantage_std": 1.4392997026443481, "completion_length": 391.7857360839844, "epoch": 0.387085635359116, "grad_norm": 3.9748362287028587, "kl": 0.0947265625, "learning_rate": 3.368345170076484e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7521523237228394, "rewards_raw/accuracy_reward": 0.560940146446228, "rewards_raw/clip_reward_wrapper": 0.3381759226322174, "rewards_raw/format_reward": 5.0, "step": 1121 }, { "Total_reward": 1.7774956972971268e-07, "advantage_std": 1.455371618270874, "completion_length": 387.89288330078125, "epoch": 0.38743093922651933, "grad_norm": 3.6073961444621117, "kl": 0.099609375, "learning_rate": 3.365801499224869e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.727275013923645, "rewards_raw/accuracy_reward": 0.5764952898025513, "rewards_raw/clip_reward_wrapper": 0.33584120869636536, "rewards_raw/format_reward": 5.0, "step": 1122 }, { "Total_reward": -4.0658883904143295e-07, "advantage_std": 1.5126944780349731, "completion_length": 387.7500305175781, "epoch": 0.38777624309392267, "grad_norm": 3.5979045641113014, "kl": 0.0947265625, "learning_rate": 3.3632568094982187e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.9445397257804871, "rewards_raw/accuracy_reward": 0.5658018589019775, "rewards_raw/clip_reward_wrapper": 0.3457041382789612, "rewards_raw/format_reward": 5.0, "step": 1123 }, { "Total_reward": 2.384185791015625e-07, "advantage_std": 1.3798140287399292, "completion_length": 384.9464416503906, "epoch": 0.38812154696132595, "grad_norm": 5.344485847920319, "kl": 0.0966796875, "learning_rate": 3.360711103891124e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9712219834327698, "rewards_raw/accuracy_reward": 0.5533893704414368, "rewards_raw/clip_reward_wrapper": 0.32392892241477966, "rewards_raw/format_reward": 5.0, "step": 1124 }, { "Total_reward": 1.4560563386112335e-06, "advantage_std": 1.3906316757202148, "completion_length": 387.3750305175781, "epoch": 0.3884668508287293, "grad_norm": 3.2334042373908236, "kl": 0.09814453125, "learning_rate": 3.3581643853993696e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9581289291381836, "rewards_raw/accuracy_reward": 0.5837181806564331, "rewards_raw/clip_reward_wrapper": 0.32266318798065186, "rewards_raw/format_reward": 5.0, "step": 1125 }, { "Total_reward": -7.982765168890182e-07, "advantage_std": 1.4847989082336426, "completion_length": 381.2321472167969, "epoch": 0.3888121546961326, "grad_norm": 4.248989575510979, "kl": 0.0966796875, "learning_rate": 3.3556166570199335e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9239388108253479, "rewards_raw/accuracy_reward": 0.5671616792678833, "rewards_raw/clip_reward_wrapper": 0.32780200242996216, "rewards_raw/format_reward": 5.0, "step": 1126 }, { "Total_reward": 1.0430813546236095e-07, "advantage_std": 1.5854185819625854, "completion_length": 388.1964416503906, "epoch": 0.3891574585635359, "grad_norm": 8.336532569230407, "kl": 0.09619140625, "learning_rate": 3.353067921750983e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.0871376991271973, "rewards_raw/accuracy_reward": 0.5687026977539062, "rewards_raw/clip_reward_wrapper": 0.32829272747039795, "rewards_raw/format_reward": 5.0, "step": 1127 }, { "Total_reward": 5.151544542059128e-07, "advantage_std": 1.4562008380889893, "completion_length": 392.9464416503906, "epoch": 0.38950276243093923, "grad_norm": 11.651434600426763, "kl": 0.10205078125, "learning_rate": 3.3505181825918684e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.9106831550598145, "rewards_raw/accuracy_reward": 0.5495836734771729, "rewards_raw/clip_reward_wrapper": 0.3390108346939087, "rewards_raw/format_reward": 5.0, "step": 1128 }, { "Total_reward": -9.334513606518158e-07, "advantage_std": 1.4311786890029907, "completion_length": 394.96429443359375, "epoch": 0.38984806629834257, "grad_norm": 9.144607671606856, "kl": 0.09326171875, "learning_rate": 3.3479674425431217e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.8041301965713501, "rewards_raw/accuracy_reward": 0.5655585527420044, "rewards_raw/clip_reward_wrapper": 0.3192407786846161, "rewards_raw/format_reward": 5.0, "step": 1129 }, { "Total_reward": -7.109982789188507e-07, "advantage_std": 1.4431053400039673, "completion_length": 388.1785888671875, "epoch": 0.39019337016574585, "grad_norm": 7.251792975425606, "kl": 0.0986328125, "learning_rate": 3.345415704606455e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7775954008102417, "rewards_raw/accuracy_reward": 0.5848674178123474, "rewards_raw/clip_reward_wrapper": 0.3359393775463104, "rewards_raw/format_reward": 5.0, "step": 1130 }, { "Total_reward": -4.119106904454384e-07, "advantage_std": 1.5623559951782227, "completion_length": 391.0535888671875, "epoch": 0.3905386740331492, "grad_norm": 4.130035360281125, "kl": 0.09521484375, "learning_rate": 3.3428629717847506e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7261548638343811, "rewards_raw/accuracy_reward": 0.5628477334976196, "rewards_raw/clip_reward_wrapper": 0.3312404751777649, "rewards_raw/format_reward": 5.0, "step": 1131 }, { "Total_reward": 4.0233138065559615e-07, "advantage_std": 1.4906717538833618, "completion_length": 387.2857360839844, "epoch": 0.39088397790055246, "grad_norm": 18.005028191271844, "kl": 0.10107421875, "learning_rate": 3.3403092470820673e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7814752459526062, "rewards_raw/accuracy_reward": 0.6120812892913818, "rewards_raw/clip_reward_wrapper": 0.336764931678772, "rewards_raw/format_reward": 5.0, "step": 1132 }, { "Total_reward": -9.472881146166401e-08, "advantage_std": 1.271148920059204, "completion_length": 375.26788330078125, "epoch": 0.3912292817679558, "grad_norm": 6.282012251078625, "kl": 0.10107421875, "learning_rate": 3.337754533503624e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7702519297599792, "rewards_raw/accuracy_reward": 0.5776736736297607, "rewards_raw/clip_reward_wrapper": 0.33951276540756226, "rewards_raw/format_reward": 5.0, "step": 1133 }, { "Total_reward": 1.9584383892379265e-07, "advantage_std": 1.6214802265167236, "completion_length": 378.4285888671875, "epoch": 0.39157458563535913, "grad_norm": 5.646957202747601, "kl": 0.09814453125, "learning_rate": 3.3351988340558096e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7442247271537781, "rewards_raw/accuracy_reward": 0.5742603540420532, "rewards_raw/clip_reward_wrapper": 0.3325123190879822, "rewards_raw/format_reward": 5.0, "step": 1134 }, { "Total_reward": 2.7141402370034484e-07, "advantage_std": 1.3415013551712036, "completion_length": 374.9285888671875, "epoch": 0.3919198895027624, "grad_norm": 6.422320771823021, "kl": 0.09326171875, "learning_rate": 3.332642151746171e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.3834195137023926, "rewards_raw/accuracy_reward": 0.5796393752098083, "rewards_raw/clip_reward_wrapper": 0.3295932412147522, "rewards_raw/format_reward": 5.0, "step": 1135 }, { "Total_reward": -8.94069742685133e-08, "advantage_std": 1.6116784811019897, "completion_length": 370.2857360839844, "epoch": 0.39226519337016574, "grad_norm": 4.362321937686235, "kl": 0.1005859375, "learning_rate": 3.3300844895834094e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.9319472312927246, "rewards_raw/accuracy_reward": 0.5526047348976135, "rewards_raw/clip_reward_wrapper": 0.3369617760181427, "rewards_raw/format_reward": 5.0, "step": 1136 }, { "Total_reward": 1.021793991640152e-07, "advantage_std": 1.5913110971450806, "completion_length": 380.76788330078125, "epoch": 0.3926104972375691, "grad_norm": 4.274844242098809, "kl": 0.0986328125, "learning_rate": 3.3275258505773834e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8646712899208069, "rewards_raw/accuracy_reward": 0.5665826201438904, "rewards_raw/clip_reward_wrapper": 0.3437708020210266, "rewards_raw/format_reward": 5.0, "step": 1137 }, { "Total_reward": -3.491129234589607e-07, "advantage_std": 1.3492443561553955, "completion_length": 387.3035888671875, "epoch": 0.39295580110497236, "grad_norm": 4.250346241112365, "kl": 0.09912109375, "learning_rate": 3.3249662377390973e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 1.0363552570343018, "rewards_raw/accuracy_reward": 0.58806312084198, "rewards_raw/clip_reward_wrapper": 0.33334869146347046, "rewards_raw/format_reward": 5.0, "step": 1138 }, { "Total_reward": -1.3985804798721801e-06, "advantage_std": 1.5969104766845703, "completion_length": 386.0357360839844, "epoch": 0.3933011049723757, "grad_norm": 5.399438418708017, "kl": 0.09912109375, "learning_rate": 3.3224056540807035e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 1.0013645887374878, "rewards_raw/accuracy_reward": 0.5830726027488708, "rewards_raw/clip_reward_wrapper": 0.34187108278274536, "rewards_raw/format_reward": 5.0, "step": 1139 }, { "Total_reward": -3.491129234589607e-07, "advantage_std": 1.5712692737579346, "completion_length": 392.3035888671875, "epoch": 0.393646408839779, "grad_norm": 5.728307011715238, "kl": 0.09375, "learning_rate": 3.319844102615497e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.6180831789970398, "rewards_raw/accuracy_reward": 0.5873894095420837, "rewards_raw/clip_reward_wrapper": 0.3303833603858948, "rewards_raw/format_reward": 5.0, "step": 1140 }, { "Total_reward": 2.384185791015625e-07, "advantage_std": 1.351905345916748, "completion_length": 383.01788330078125, "epoch": 0.3939917127071823, "grad_norm": 3.5539603988010007, "kl": 0.09716796875, "learning_rate": 3.3172815863579093e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.775297999382019, "rewards_raw/accuracy_reward": 0.5704554319381714, "rewards_raw/clip_reward_wrapper": 0.3484300971031189, "rewards_raw/format_reward": 5.0, "step": 1141 }, { "Total_reward": -6.045614213689987e-07, "advantage_std": 1.3615890741348267, "completion_length": 385.76788330078125, "epoch": 0.39433701657458564, "grad_norm": 8.641653304373564, "kl": 0.10205078125, "learning_rate": 3.314718108323511e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.7539518475532532, "rewards_raw/accuracy_reward": 0.5518004298210144, "rewards_raw/clip_reward_wrapper": 0.3536350131034851, "rewards_raw/format_reward": 5.0, "step": 1142 }, { "Total_reward": 8.456409545942734e-07, "advantage_std": 1.4739567041397095, "completion_length": 368.1785888671875, "epoch": 0.394682320441989, "grad_norm": 3.7391466073484345, "kl": 0.0966796875, "learning_rate": 3.312153671529001e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.7843148708343506, "rewards_raw/accuracy_reward": 0.5528869032859802, "rewards_raw/clip_reward_wrapper": 0.3194051682949066, "rewards_raw/format_reward": 5.0, "step": 1143 }, { "Total_reward": -8.514949456639442e-08, "advantage_std": 1.5759789943695068, "completion_length": 382.26788330078125, "epoch": 0.39502762430939226, "grad_norm": 4.099158455816029, "kl": 0.09619140625, "learning_rate": 3.3095882789922095e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.9740016460418701, "rewards_raw/accuracy_reward": 0.6030513048171997, "rewards_raw/clip_reward_wrapper": 0.3420569598674774, "rewards_raw/format_reward": 5.0, "step": 1144 }, { "Total_reward": -1.6391278734317893e-07, "advantage_std": 1.433236837387085, "completion_length": 393.9821472167969, "epoch": 0.3953729281767956, "grad_norm": 5.1088730035863295, "kl": 0.0986328125, "learning_rate": 3.3070219337320884e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 1.0711398124694824, "rewards_raw/accuracy_reward": 0.5488006472587585, "rewards_raw/clip_reward_wrapper": 0.33882981538772583, "rewards_raw/format_reward": 5.0, "step": 1145 }, { "Total_reward": 8.408512712776428e-07, "advantage_std": 1.3806331157684326, "completion_length": 368.8571472167969, "epoch": 0.39571823204419887, "grad_norm": 4.17914872124629, "kl": 0.099609375, "learning_rate": 3.3044546387687143e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 1.143879771232605, "rewards_raw/accuracy_reward": 0.5808779001235962, "rewards_raw/clip_reward_wrapper": 0.3294501006603241, "rewards_raw/format_reward": 5.0, "step": 1146 }, { "Total_reward": -8.727823086474018e-08, "advantage_std": 1.4728213548660278, "completion_length": 384.6250305175781, "epoch": 0.3960635359116022, "grad_norm": 6.262486970051699, "kl": 0.09814453125, "learning_rate": 3.3018863971232766e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9663296341896057, "rewards_raw/accuracy_reward": 0.5666336417198181, "rewards_raw/clip_reward_wrapper": 0.33867666125297546, "rewards_raw/format_reward": 5.0, "step": 1147 }, { "Total_reward": 1.3730355874486122e-07, "advantage_std": 1.5088740587234497, "completion_length": 385.8035888671875, "epoch": 0.39640883977900554, "grad_norm": 6.2474460747811005, "kl": 0.10302734375, "learning_rate": 3.2993172118180846e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 1.5439903736114502, "rewards_raw/accuracy_reward": 0.5688212513923645, "rewards_raw/clip_reward_wrapper": 0.3437283933162689, "rewards_raw/format_reward": 5.0, "step": 1148 }, { "Total_reward": -5.534717502087005e-07, "advantage_std": 1.542738676071167, "completion_length": 373.8750305175781, "epoch": 0.3967541436464088, "grad_norm": 3.787181474692571, "kl": 0.0986328125, "learning_rate": 3.296747085876553e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.9568759799003601, "rewards_raw/accuracy_reward": 0.5800119638442993, "rewards_raw/clip_reward_wrapper": 0.33517614006996155, "rewards_raw/format_reward": 5.0, "step": 1149 }, { "Total_reward": 5.194119125917496e-07, "advantage_std": 1.3913904428482056, "completion_length": 384.0357360839844, "epoch": 0.39709944751381215, "grad_norm": 6.390655265995095, "kl": 0.095703125, "learning_rate": 3.294176022323206e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7816751003265381, "rewards_raw/accuracy_reward": 0.579309344291687, "rewards_raw/clip_reward_wrapper": 0.3299456834793091, "rewards_raw/format_reward": 5.0, "step": 1150 }, { "Total_reward": 4.129750550418976e-07, "advantage_std": 1.618220329284668, "completion_length": 390.0535888671875, "epoch": 0.3974447513812155, "grad_norm": 10.755344924204636, "kl": 0.0986328125, "learning_rate": 3.291604024183671e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6565871238708496, "rewards_raw/accuracy_reward": 0.5867640972137451, "rewards_raw/clip_reward_wrapper": 0.3407188653945923, "rewards_raw/format_reward": 5.0, "step": 1151 }, { "Total_reward": -2.9961978498249664e-07, "advantage_std": 1.4910746812820435, "completion_length": 381.1785888671875, "epoch": 0.39779005524861877, "grad_norm": 4.5176005041551175, "kl": 0.1015625, "learning_rate": 3.2890310944846746e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.9876865148544312, "rewards_raw/accuracy_reward": 0.5636480450630188, "rewards_raw/clip_reward_wrapper": 0.33212584257125854, "rewards_raw/format_reward": 5.0, "step": 1152 }, { "Total_reward": 2.7247838829680404e-07, "advantage_std": 1.5739628076553345, "completion_length": 378.71429443359375, "epoch": 0.3981353591160221, "grad_norm": 6.299808578251312, "kl": 0.0986328125, "learning_rate": 3.28645723625404e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6658258438110352, "rewards_raw/accuracy_reward": 0.5668642520904541, "rewards_raw/clip_reward_wrapper": 0.34377142786979675, "rewards_raw/format_reward": 5.0, "step": 1153 }, { "Total_reward": 8.536237032785721e-07, "advantage_std": 1.444591999053955, "completion_length": 373.6785888671875, "epoch": 0.39848066298342544, "grad_norm": 6.5884915522239735, "kl": 0.09814453125, "learning_rate": 3.283882452520683e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.3145718574523926, "rewards_raw/accuracy_reward": 0.5711095333099365, "rewards_raw/clip_reward_wrapper": 0.3370833992958069, "rewards_raw/format_reward": 5.0, "step": 1154 }, { "Total_reward": 3.395336136691185e-07, "advantage_std": 1.3799582719802856, "completion_length": 376.4285888671875, "epoch": 0.3988259668508287, "grad_norm": 3.959036259035352, "kl": 0.1015625, "learning_rate": 3.281306746314609e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.7334844470024109, "rewards_raw/accuracy_reward": 0.5835849642753601, "rewards_raw/clip_reward_wrapper": 0.3440260589122772, "rewards_raw/format_reward": 5.0, "step": 1155 }, { "Total_reward": 1.329928750237741e-06, "advantage_std": 1.467986822128296, "completion_length": 373.8750305175781, "epoch": 0.39917127071823205, "grad_norm": 6.858066437952529, "kl": 0.0966796875, "learning_rate": 3.278730120666908e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6983289122581482, "rewards_raw/accuracy_reward": 0.5826444625854492, "rewards_raw/clip_reward_wrapper": 0.34058722853660583, "rewards_raw/format_reward": 5.0, "step": 1156 }, { "Total_reward": 2.1713121611810493e-07, "advantage_std": 1.2851030826568604, "completion_length": 385.2321472167969, "epoch": 0.3995165745856354, "grad_norm": 6.4475718356117, "kl": 0.091796875, "learning_rate": 3.2761525786097534e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7293269038200378, "rewards_raw/accuracy_reward": 0.5754315853118896, "rewards_raw/clip_reward_wrapper": 0.3375600576400757, "rewards_raw/format_reward": 5.0, "step": 1157 }, { "Total_reward": 5.577292085945373e-07, "advantage_std": 1.4034790992736816, "completion_length": 379.64288330078125, "epoch": 0.39986187845303867, "grad_norm": 6.140415161169021, "kl": 0.09619140625, "learning_rate": 3.273574123176396e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6919199824333191, "rewards_raw/accuracy_reward": 0.5860464572906494, "rewards_raw/clip_reward_wrapper": 0.3281573951244354, "rewards_raw/format_reward": 5.0, "step": 1158 }, { "Total_reward": 3.7465778746081924e-07, "advantage_std": 1.6292330026626587, "completion_length": 373.76788330078125, "epoch": 0.400207182320442, "grad_norm": 5.571375676651009, "kl": 0.09619140625, "learning_rate": 3.27099475740116e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.0499800443649292, "rewards_raw/accuracy_reward": 0.5815932154655457, "rewards_raw/clip_reward_wrapper": 0.327210009098053, "rewards_raw/format_reward": 5.0, "step": 1159 }, { "Total_reward": -2.5331974029541016e-07, "advantage_std": 1.348635196685791, "completion_length": 373.01788330078125, "epoch": 0.4005524861878453, "grad_norm": 3.919054196822736, "kl": 0.09912109375, "learning_rate": 3.268414484319445e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.6701871752738953, "rewards_raw/accuracy_reward": 0.5705653429031372, "rewards_raw/clip_reward_wrapper": 0.34724169969558716, "rewards_raw/format_reward": 5.0, "step": 1160 }, { "Total_reward": -1.309173569552513e-07, "advantage_std": 1.453963041305542, "completion_length": 366.01788330078125, "epoch": 0.4008977900552486, "grad_norm": 5.775010689597276, "kl": 0.1083984375, "learning_rate": 3.2658333069677143e-07, "loss": 0.0043, "rewards_raw/CoT ICLG": 0.6103290319442749, "rewards_raw/accuracy_reward": 0.5788740515708923, "rewards_raw/clip_reward_wrapper": 0.3377813994884491, "rewards_raw/format_reward": 5.0, "step": 1161 }, { "Total_reward": 1.1708056035786285e-07, "advantage_std": 1.5399669408798218, "completion_length": 383.89288330078125, "epoch": 0.40124309392265195, "grad_norm": 4.069978099862844, "kl": 0.10205078125, "learning_rate": 3.263251228383497e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.7438085079193115, "rewards_raw/accuracy_reward": 0.5748376250267029, "rewards_raw/clip_reward_wrapper": 0.3415762186050415, "rewards_raw/format_reward": 5.0, "step": 1162 }, { "Total_reward": -4.6406475462390517e-07, "advantage_std": 1.3518860340118408, "completion_length": 385.6250305175781, "epoch": 0.40158839779005523, "grad_norm": 19.56026984634445, "kl": 0.09814453125, "learning_rate": 3.260668251605384e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6935922503471375, "rewards_raw/accuracy_reward": 0.5847386121749878, "rewards_raw/clip_reward_wrapper": 0.3393261134624481, "rewards_raw/format_reward": 5.0, "step": 1163 }, { "Total_reward": -3.182462364748062e-07, "advantage_std": 1.4349989891052246, "completion_length": 366.7321472167969, "epoch": 0.40193370165745856, "grad_norm": 7.31326434438726, "kl": 0.1025390625, "learning_rate": 3.2580843796730196e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.7019471526145935, "rewards_raw/accuracy_reward": 0.5717896223068237, "rewards_raw/clip_reward_wrapper": 0.3343375623226166, "rewards_raw/format_reward": 5.0, "step": 1164 }, { "Total_reward": 6.843890787422424e-07, "advantage_std": 1.526429533958435, "completion_length": 366.8571472167969, "epoch": 0.4022790055248619, "grad_norm": 4.025709861130286, "kl": 0.0986328125, "learning_rate": 3.2554996156271053e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.6745758652687073, "rewards_raw/accuracy_reward": 0.5944380164146423, "rewards_raw/clip_reward_wrapper": 0.3382430970668793, "rewards_raw/format_reward": 5.0, "step": 1165 }, { "Total_reward": -9.409019412487396e-07, "advantage_std": 1.4924917221069336, "completion_length": 369.4285888671875, "epoch": 0.4026243093922652, "grad_norm": 8.243664411072166, "kl": 0.1025390625, "learning_rate": 3.252913962509392e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.8416849374771118, "rewards_raw/accuracy_reward": 0.574336051940918, "rewards_raw/clip_reward_wrapper": 0.33151012659072876, "rewards_raw/format_reward": 5.0, "step": 1166 }, { "Total_reward": -2.3203237731195259e-07, "advantage_std": 1.232986569404602, "completion_length": 381.5000305175781, "epoch": 0.4029696132596685, "grad_norm": 3.7760282652647965, "kl": 0.09619140625, "learning_rate": 3.250327423362673e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.3459664583206177, "rewards_raw/accuracy_reward": 0.5926905870437622, "rewards_raw/clip_reward_wrapper": 0.33113893866539, "rewards_raw/format_reward": 5.0, "step": 1167 }, { "Total_reward": 3.5549913945942535e-07, "advantage_std": 1.2951371669769287, "completion_length": 372.0357360839844, "epoch": 0.40331491712707185, "grad_norm": 3.710289710601642, "kl": 0.1005859375, "learning_rate": 3.2477400012307883e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.6663191318511963, "rewards_raw/accuracy_reward": 0.5808044672012329, "rewards_raw/clip_reward_wrapper": 0.3382386565208435, "rewards_raw/format_reward": 5.0, "step": 1168 }, { "Total_reward": 3.895589486546669e-07, "advantage_std": 1.506561517715454, "completion_length": 368.5357360839844, "epoch": 0.4036602209944751, "grad_norm": 5.316752615526279, "kl": 0.1025390625, "learning_rate": 3.245151699158617e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.8087547421455383, "rewards_raw/accuracy_reward": 0.565905749797821, "rewards_raw/clip_reward_wrapper": 0.33564457297325134, "rewards_raw/format_reward": 5.0, "step": 1169 }, { "Total_reward": 5.226050348028366e-07, "advantage_std": 1.3562345504760742, "completion_length": 366.0000305175781, "epoch": 0.40400552486187846, "grad_norm": 3.7980712230598535, "kl": 0.10498046875, "learning_rate": 3.2425625201920714e-07, "loss": 0.0042, "rewards_raw/CoT ICLG": 1.1419506072998047, "rewards_raw/accuracy_reward": 0.5888822674751282, "rewards_raw/clip_reward_wrapper": 0.34188446402549744, "rewards_raw/format_reward": 5.0, "step": 1170 }, { "Total_reward": -3.395336136691185e-07, "advantage_std": 1.3827041387557983, "completion_length": 387.51788330078125, "epoch": 0.40435082872928174, "grad_norm": 3.7619714399088595, "kl": 0.09716796875, "learning_rate": 3.239972467378096e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.5904912948608398, "rewards_raw/accuracy_reward": 0.5805169939994812, "rewards_raw/clip_reward_wrapper": 0.3357308506965637, "rewards_raw/format_reward": 5.0, "step": 1171 }, { "Total_reward": 4.5980726781635894e-07, "advantage_std": 1.6959630250930786, "completion_length": 368.21429443359375, "epoch": 0.4046961325966851, "grad_norm": 5.111691870641891, "kl": 0.099609375, "learning_rate": 3.237381543764666e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.9115623235702515, "rewards_raw/accuracy_reward": 0.5824021100997925, "rewards_raw/clip_reward_wrapper": 0.34248802065849304, "rewards_raw/format_reward": 5.0, "step": 1172 }, { "Total_reward": 1.0154077472179779e-06, "advantage_std": 1.5484986305236816, "completion_length": 363.4107360839844, "epoch": 0.4050414364640884, "grad_norm": 5.094386387539188, "kl": 0.1025390625, "learning_rate": 3.234789752400779e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.6389118432998657, "rewards_raw/accuracy_reward": 0.5895706415176392, "rewards_raw/clip_reward_wrapper": 0.3440033495426178, "rewards_raw/format_reward": 5.0, "step": 1173 }, { "Total_reward": 1.27724248955019e-08, "advantage_std": 1.6775449514389038, "completion_length": 370.71429443359375, "epoch": 0.4053867403314917, "grad_norm": 5.3519455714732915, "kl": 0.09521484375, "learning_rate": 3.2321970963364545e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.199455738067627, "rewards_raw/accuracy_reward": 0.6047197580337524, "rewards_raw/clip_reward_wrapper": 0.32867011427879333, "rewards_raw/format_reward": 5.0, "step": 1174 }, { "Total_reward": -2.3203237731195259e-07, "advantage_std": 1.5181655883789062, "completion_length": 376.2321472167969, "epoch": 0.405732044198895, "grad_norm": 7.201786475687077, "kl": 0.095703125, "learning_rate": 3.229603578622729e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.9692156910896301, "rewards_raw/accuracy_reward": 0.5570322871208191, "rewards_raw/clip_reward_wrapper": 0.32196518778800964, "rewards_raw/format_reward": 5.0, "step": 1175 }, { "Total_reward": 1.3091735127090942e-06, "advantage_std": 1.4746253490447998, "completion_length": 365.6607360839844, "epoch": 0.40607734806629836, "grad_norm": 5.575596594584968, "kl": 0.10595703125, "learning_rate": 3.2270092023116556e-07, "loss": 0.0042, "rewards_raw/CoT ICLG": 0.5649694204330444, "rewards_raw/accuracy_reward": 0.5758671164512634, "rewards_raw/clip_reward_wrapper": 0.34799548983573914, "rewards_raw/format_reward": 5.0, "step": 1176 }, { "Total_reward": -3.7891524584665603e-07, "advantage_std": 1.7886219024658203, "completion_length": 374.1071472167969, "epoch": 0.40642265193370164, "grad_norm": 10.140229640586899, "kl": 0.09521484375, "learning_rate": 3.2244139704562957e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.9651070237159729, "rewards_raw/accuracy_reward": 0.5774250030517578, "rewards_raw/clip_reward_wrapper": 0.32320496439933777, "rewards_raw/format_reward": 5.0, "step": 1177 }, { "Total_reward": 6.173338817916374e-08, "advantage_std": 1.2462674379348755, "completion_length": 372.9107360839844, "epoch": 0.406767955801105, "grad_norm": 3.6595406611816856, "kl": 0.09619140625, "learning_rate": 3.221817886110715e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.815058708190918, "rewards_raw/accuracy_reward": 0.5536861419677734, "rewards_raw/clip_reward_wrapper": 0.32728371024131775, "rewards_raw/format_reward": 5.0, "step": 1178 }, { "Total_reward": 5.917889893680695e-07, "advantage_std": 1.4090174436569214, "completion_length": 367.08929443359375, "epoch": 0.4071132596685083, "grad_norm": 4.908676328611156, "kl": 0.1005859375, "learning_rate": 3.2192209523299864e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8764782547950745, "rewards_raw/accuracy_reward": 0.551200807094574, "rewards_raw/clip_reward_wrapper": 0.33276456594467163, "rewards_raw/format_reward": 5.0, "step": 1179 }, { "Total_reward": -3.437911004766647e-07, "advantage_std": 1.4796574115753174, "completion_length": 372.9464416503906, "epoch": 0.4074585635359116, "grad_norm": 5.372900687270982, "kl": 0.09619140625, "learning_rate": 3.2166231721701823e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.5021672248840332, "rewards_raw/accuracy_reward": 0.5565058588981628, "rewards_raw/clip_reward_wrapper": 0.3505651354789734, "rewards_raw/format_reward": 5.0, "step": 1180 }, { "Total_reward": 2.3203237731195259e-07, "advantage_std": 1.5242704153060913, "completion_length": 371.7500305175781, "epoch": 0.4078038674033149, "grad_norm": 20.53048521948171, "kl": 0.10009765625, "learning_rate": 3.2140245486883686e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7143098711967468, "rewards_raw/accuracy_reward": 0.5882434248924255, "rewards_raw/clip_reward_wrapper": 0.32861340045928955, "rewards_raw/format_reward": 4.964285850524902, "step": 1181 }, { "Total_reward": -9.792191946189632e-08, "advantage_std": 1.7424335479736328, "completion_length": 366.3035888671875, "epoch": 0.40814917127071826, "grad_norm": 5.205041530553817, "kl": 0.0986328125, "learning_rate": 3.211425084942606e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6969095468521118, "rewards_raw/accuracy_reward": 0.5732901692390442, "rewards_raw/clip_reward_wrapper": 0.35409805178642273, "rewards_raw/format_reward": 5.0, "step": 1182 }, { "Total_reward": -4.619360254309868e-07, "advantage_std": 1.5819717645645142, "completion_length": 366.4821472167969, "epoch": 0.40849447513812154, "grad_norm": 4.825940877417496, "kl": 0.09521484375, "learning_rate": 3.208824783991941e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.2877507209777832, "rewards_raw/accuracy_reward": 0.6062132120132446, "rewards_raw/clip_reward_wrapper": 0.33506453037261963, "rewards_raw/format_reward": 5.0, "step": 1183 }, { "Total_reward": 7.2377071091978e-07, "advantage_std": 1.5334538221359253, "completion_length": 373.0357360839844, "epoch": 0.4088397790055249, "grad_norm": 6.796620517839566, "kl": 0.09619140625, "learning_rate": 3.206223648896409e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.9420997500419617, "rewards_raw/accuracy_reward": 0.5911440253257751, "rewards_raw/clip_reward_wrapper": 0.32613053917884827, "rewards_raw/format_reward": 5.0, "step": 1184 }, { "Total_reward": 4.5342108023760375e-07, "advantage_std": 1.47234046459198, "completion_length": 376.6785888671875, "epoch": 0.40918508287292815, "grad_norm": 6.128361005423658, "kl": 0.095703125, "learning_rate": 3.203621682717025e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6819360256195068, "rewards_raw/accuracy_reward": 0.5724014043807983, "rewards_raw/clip_reward_wrapper": 0.3486975133419037, "rewards_raw/format_reward": 5.0, "step": 1185 }, { "Total_reward": 3.469841942660423e-07, "advantage_std": 1.523187279701233, "completion_length": 365.83929443359375, "epoch": 0.4095303867403315, "grad_norm": 4.27600212711586, "kl": 0.09521484375, "learning_rate": 3.201018888515781e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.0927177667617798, "rewards_raw/accuracy_reward": 0.5469434261322021, "rewards_raw/clip_reward_wrapper": 0.3279029428958893, "rewards_raw/format_reward": 5.0, "step": 1186 }, { "Total_reward": -6.471361757576233e-07, "advantage_std": 1.1634441614151, "completion_length": 378.5714416503906, "epoch": 0.4098756906077348, "grad_norm": 3.101477300611101, "kl": 0.099609375, "learning_rate": 3.1984152693556463e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7446898221969604, "rewards_raw/accuracy_reward": 0.5658752918243408, "rewards_raw/clip_reward_wrapper": 0.3388015925884247, "rewards_raw/format_reward": 5.0, "step": 1187 }, { "Total_reward": 3.4059798537100505e-08, "advantage_std": 1.4325809478759766, "completion_length": 367.7321472167969, "epoch": 0.4102209944751381, "grad_norm": 3.719903410276591, "kl": 0.095703125, "learning_rate": 3.19581082830056e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.0348315238952637, "rewards_raw/accuracy_reward": 0.5782496929168701, "rewards_raw/clip_reward_wrapper": 0.34040477871894836, "rewards_raw/format_reward": 5.0, "step": 1188 }, { "Total_reward": -7.876328567135715e-08, "advantage_std": 1.5543735027313232, "completion_length": 363.71429443359375, "epoch": 0.41056629834254144, "grad_norm": 4.608873533922045, "kl": 0.0986328125, "learning_rate": 3.193205568415425e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.4549049735069275, "rewards_raw/accuracy_reward": 0.5750587582588196, "rewards_raw/clip_reward_wrapper": 0.34973853826522827, "rewards_raw/format_reward": 5.0, "step": 1189 }, { "Total_reward": 2.55448497910038e-07, "advantage_std": 1.4017196893692017, "completion_length": 363.58929443359375, "epoch": 0.41091160220994477, "grad_norm": 3.8159881487495304, "kl": 0.09765625, "learning_rate": 3.190599492766113e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8704514503479004, "rewards_raw/accuracy_reward": 0.5656335353851318, "rewards_raw/clip_reward_wrapper": 0.3307619094848633, "rewards_raw/format_reward": 5.0, "step": 1190 }, { "Total_reward": 2.650278076998802e-07, "advantage_std": 1.4738719463348389, "completion_length": 356.8750305175781, "epoch": 0.41125690607734805, "grad_norm": 49.658618695041056, "kl": 0.09765625, "learning_rate": 3.1879926044194525e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.2098958492279053, "rewards_raw/accuracy_reward": 0.5513806939125061, "rewards_raw/clip_reward_wrapper": 0.3413276970386505, "rewards_raw/format_reward": 5.0, "step": 1191 }, { "Total_reward": -4.76837158203125e-07, "advantage_std": 1.4589816331863403, "completion_length": 358.5714416503906, "epoch": 0.4116022099447514, "grad_norm": 6.941254441537067, "kl": 0.09423828125, "learning_rate": 3.185384906443229e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.929984450340271, "rewards_raw/accuracy_reward": 0.5685974359512329, "rewards_raw/clip_reward_wrapper": 0.32355377078056335, "rewards_raw/format_reward": 5.0, "step": 1192 }, { "Total_reward": 5.960464477539062e-07, "advantage_std": 1.4090757369995117, "completion_length": 367.39288330078125, "epoch": 0.4119475138121547, "grad_norm": 3.6777713943902075, "kl": 0.103515625, "learning_rate": 3.182776401906181e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.9470440745353699, "rewards_raw/accuracy_reward": 0.5796799063682556, "rewards_raw/clip_reward_wrapper": 0.33493122458457947, "rewards_raw/format_reward": 5.0, "step": 1193 }, { "Total_reward": 4.5980726781635894e-07, "advantage_std": 1.5190593004226685, "completion_length": 377.6607360839844, "epoch": 0.412292817679558, "grad_norm": 5.640228008483026, "kl": 0.09228515625, "learning_rate": 3.1801670938779975e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.973463773727417, "rewards_raw/accuracy_reward": 0.5736001133918762, "rewards_raw/clip_reward_wrapper": 0.34139707684516907, "rewards_raw/format_reward": 5.0, "step": 1194 }, { "Total_reward": -3.916876778475853e-07, "advantage_std": 1.4994174242019653, "completion_length": 376.0535888671875, "epoch": 0.41263812154696133, "grad_norm": 4.695130868369815, "kl": 0.09423828125, "learning_rate": 3.17755698542931e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.3152318000793457, "rewards_raw/accuracy_reward": 0.6006848216056824, "rewards_raw/clip_reward_wrapper": 0.3390408158302307, "rewards_raw/format_reward": 5.0, "step": 1195 }, { "Total_reward": -5.172831833988312e-07, "advantage_std": 1.4323400259017944, "completion_length": 364.6250305175781, "epoch": 0.4129834254143646, "grad_norm": 4.172317075653123, "kl": 0.09765625, "learning_rate": 3.174946079631695e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.4914878606796265, "rewards_raw/accuracy_reward": 0.5739154815673828, "rewards_raw/clip_reward_wrapper": 0.3398689031600952, "rewards_raw/format_reward": 5.0, "step": 1196 }, { "Total_reward": 4.257474728319721e-08, "advantage_std": 1.4709336757659912, "completion_length": 373.4107360839844, "epoch": 0.41332872928176795, "grad_norm": 4.155745497336514, "kl": 0.09814453125, "learning_rate": 3.1723343795576637e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7035089135169983, "rewards_raw/accuracy_reward": 0.5645111203193665, "rewards_raw/clip_reward_wrapper": 0.33550992608070374, "rewards_raw/format_reward": 5.0, "step": 1197 }, { "Total_reward": 5.683729114025482e-07, "advantage_std": 1.4371869564056396, "completion_length": 363.1785888671875, "epoch": 0.4136740331491713, "grad_norm": 4.402473589453043, "kl": 0.099609375, "learning_rate": 3.169721888280665e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.6769904494285583, "rewards_raw/accuracy_reward": 0.5796505212783813, "rewards_raw/clip_reward_wrapper": 0.3439793288707733, "rewards_raw/format_reward": 5.0, "step": 1198 }, { "Total_reward": -1.3421689573078766e-06, "advantage_std": 1.4799600839614868, "completion_length": 359.14288330078125, "epoch": 0.41401933701657456, "grad_norm": 5.509592008756088, "kl": 0.0947265625, "learning_rate": 3.167108608875081e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6912339329719543, "rewards_raw/accuracy_reward": 0.5699777007102966, "rewards_raw/clip_reward_wrapper": 0.34210050106048584, "rewards_raw/format_reward": 5.0, "step": 1199 }, { "Total_reward": -4.395842836402153e-07, "advantage_std": 1.527847170829773, "completion_length": 374.0535888671875, "epoch": 0.4143646408839779, "grad_norm": 5.143150251511344, "kl": 0.095703125, "learning_rate": 3.164494544416215e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.6389877796173096, "rewards_raw/accuracy_reward": 0.5541645884513855, "rewards_raw/clip_reward_wrapper": 0.3368626534938812, "rewards_raw/format_reward": 5.0, "step": 1200 }, { "Total_reward": -4.938671054333099e-07, "advantage_std": 1.5225961208343506, "completion_length": 376.3750305175781, "epoch": 0.41470994475138123, "grad_norm": 4.965155886557071, "kl": 0.09765625, "learning_rate": 3.161879697980299e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7127535939216614, "rewards_raw/accuracy_reward": 0.5613603591918945, "rewards_raw/clip_reward_wrapper": 0.3458861708641052, "rewards_raw/format_reward": 5.0, "step": 1201 }, { "Total_reward": 2.0223005492425727e-07, "advantage_std": 1.3325185775756836, "completion_length": 366.89288330078125, "epoch": 0.4150552486187845, "grad_norm": 3.4724824803605783, "kl": 0.09423828125, "learning_rate": 3.159264072644483e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7462828755378723, "rewards_raw/accuracy_reward": 0.5923474431037903, "rewards_raw/clip_reward_wrapper": 0.3276159465312958, "rewards_raw/format_reward": 5.0, "step": 1202 }, { "Total_reward": -1.1708056035786285e-07, "advantage_std": 1.5347895622253418, "completion_length": 355.9821472167969, "epoch": 0.41540055248618785, "grad_norm": 5.200754991570365, "kl": 0.09619140625, "learning_rate": 3.156647671486835e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9541478753089905, "rewards_raw/accuracy_reward": 0.551925539970398, "rewards_raw/clip_reward_wrapper": 0.3455246686935425, "rewards_raw/format_reward": 5.0, "step": 1203 }, { "Total_reward": -1.2346677635832748e-07, "advantage_std": 1.529334545135498, "completion_length": 368.96429443359375, "epoch": 0.4157458563535912, "grad_norm": 16.721668933214367, "kl": 0.09765625, "learning_rate": 3.154030497586336e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7833651900291443, "rewards_raw/accuracy_reward": 0.5742172002792358, "rewards_raw/clip_reward_wrapper": 0.33850863575935364, "rewards_raw/format_reward": 5.0, "step": 1204 }, { "Total_reward": -1.7455646172948036e-07, "advantage_std": 1.4211721420288086, "completion_length": 371.21429443359375, "epoch": 0.41609116022099446, "grad_norm": 5.546852196394795, "kl": 0.0986328125, "learning_rate": 3.151412554022874e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8169043064117432, "rewards_raw/accuracy_reward": 0.5523706674575806, "rewards_raw/clip_reward_wrapper": 0.3346758782863617, "rewards_raw/format_reward": 5.0, "step": 1205 }, { "Total_reward": 1.9158636632710113e-07, "advantage_std": 1.4464548826217651, "completion_length": 374.8035888671875, "epoch": 0.4164364640883978, "grad_norm": 5.645752456164749, "kl": 0.09716796875, "learning_rate": 3.1487938438772463e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.1326690912246704, "rewards_raw/accuracy_reward": 0.5601951479911804, "rewards_raw/clip_reward_wrapper": 0.341044545173645, "rewards_raw/format_reward": 5.0, "step": 1206 }, { "Total_reward": 2.8099333349018707e-07, "advantage_std": 1.2872679233551025, "completion_length": 376.01788330078125, "epoch": 0.41678176795580113, "grad_norm": 3.4761622225094055, "kl": 0.09619140625, "learning_rate": 3.1461743702311496e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7762739658355713, "rewards_raw/accuracy_reward": 0.5729479193687439, "rewards_raw/clip_reward_wrapper": 0.3360241949558258, "rewards_raw/format_reward": 5.0, "step": 1207 }, { "Total_reward": -2.7460711748972244e-07, "advantage_std": 1.580241084098816, "completion_length": 371.6607360839844, "epoch": 0.4171270718232044, "grad_norm": 7.033383895377289, "kl": 0.099609375, "learning_rate": 3.1435541361671797e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8689915537834167, "rewards_raw/accuracy_reward": 0.5913233757019043, "rewards_raw/clip_reward_wrapper": 0.34846702218055725, "rewards_raw/format_reward": 5.0, "step": 1208 }, { "Total_reward": 3.129243850708008e-07, "advantage_std": 1.3027445077896118, "completion_length": 377.6607360839844, "epoch": 0.41747237569060774, "grad_norm": 10.501841319581585, "kl": 0.10205078125, "learning_rate": 3.1409331447688276e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.8301447033882141, "rewards_raw/accuracy_reward": 0.562793493270874, "rewards_raw/clip_reward_wrapper": 0.3450833261013031, "rewards_raw/format_reward": 5.0, "step": 1209 }, { "Total_reward": 4.986567319065216e-07, "advantage_std": 1.230034351348877, "completion_length": 361.2321472167969, "epoch": 0.417817679558011, "grad_norm": 7.512976198742234, "kl": 0.09033203125, "learning_rate": 3.1383113991204765e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.9333568215370178, "rewards_raw/accuracy_reward": 0.5709054470062256, "rewards_raw/clip_reward_wrapper": 0.3295460343360901, "rewards_raw/format_reward": 5.0, "step": 1210 }, { "Total_reward": 1.0175365332543151e-06, "advantage_std": 1.3857182264328003, "completion_length": 369.2321472167969, "epoch": 0.41816298342541436, "grad_norm": 4.300573200367529, "kl": 0.095703125, "learning_rate": 3.1356889023073947e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.2836517095565796, "rewards_raw/accuracy_reward": 0.5757349729537964, "rewards_raw/clip_reward_wrapper": 0.3421112298965454, "rewards_raw/format_reward": 5.0, "step": 1211 }, { "Total_reward": 2.07551892117408e-07, "advantage_std": 1.2901533842086792, "completion_length": 376.6071472167969, "epoch": 0.4185082872928177, "grad_norm": 6.997959287698715, "kl": 0.099609375, "learning_rate": 3.1330656574157366e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 1.0786988735198975, "rewards_raw/accuracy_reward": 0.573249101638794, "rewards_raw/clip_reward_wrapper": 0.34267815947532654, "rewards_raw/format_reward": 5.0, "step": 1212 }, { "Total_reward": -6.34363743756694e-07, "advantage_std": 1.477495551109314, "completion_length": 382.1785888671875, "epoch": 0.418853591160221, "grad_norm": 30.397028538562687, "kl": 0.09423828125, "learning_rate": 3.1304416675325367e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7226136326789856, "rewards_raw/accuracy_reward": 0.5661537051200867, "rewards_raw/clip_reward_wrapper": 0.3423718214035034, "rewards_raw/format_reward": 5.0, "step": 1213 }, { "Total_reward": 7.195132525339432e-07, "advantage_std": 1.5374081134796143, "completion_length": 374.6785888671875, "epoch": 0.4191988950276243, "grad_norm": 7.578023450134784, "kl": 0.09765625, "learning_rate": 3.1278169357457053e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7096868753433228, "rewards_raw/accuracy_reward": 0.563380241394043, "rewards_raw/clip_reward_wrapper": 0.3409733176231384, "rewards_raw/format_reward": 5.0, "step": 1214 }, { "Total_reward": 6.322350145637756e-07, "advantage_std": 1.6792601346969604, "completion_length": 364.0714416503906, "epoch": 0.41954419889502764, "grad_norm": 4.554394171198956, "kl": 0.099609375, "learning_rate": 3.1251914651440265e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.6468276381492615, "rewards_raw/accuracy_reward": 0.5640212297439575, "rewards_raw/clip_reward_wrapper": 0.3413218855857849, "rewards_raw/format_reward": 5.0, "step": 1215 }, { "Total_reward": 2.7673586089349556e-08, "advantage_std": 1.4868459701538086, "completion_length": 379.4285888671875, "epoch": 0.4198895027624309, "grad_norm": 7.842306562579576, "kl": 0.10009765625, "learning_rate": 3.122565258817153e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 1.2518447637557983, "rewards_raw/accuracy_reward": 0.5782238841056824, "rewards_raw/clip_reward_wrapper": 0.3599938452243805, "rewards_raw/format_reward": 5.0, "step": 1216 }, { "Total_reward": 2.1925995952187805e-07, "advantage_std": 1.42231285572052, "completion_length": 378.2857360839844, "epoch": 0.42023480662983426, "grad_norm": 12.431815625973629, "kl": 0.09375, "learning_rate": 3.119938319855605e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.4184850454330444, "rewards_raw/accuracy_reward": 0.5601073503494263, "rewards_raw/clip_reward_wrapper": 0.3275507390499115, "rewards_raw/format_reward": 5.0, "step": 1217 }, { "Total_reward": 1.1686768175422912e-06, "advantage_std": 1.3289506435394287, "completion_length": 375.1607360839844, "epoch": 0.4205801104972376, "grad_norm": 4.444727558517418, "kl": 0.09716796875, "learning_rate": 3.117310651350762e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.0047378540039062, "rewards_raw/accuracy_reward": 0.5757074356079102, "rewards_raw/clip_reward_wrapper": 0.3367236256599426, "rewards_raw/format_reward": 5.0, "step": 1218 }, { "Total_reward": 5.406993182077713e-07, "advantage_std": 1.4786931276321411, "completion_length": 371.0535888671875, "epoch": 0.42092541436464087, "grad_norm": 11.26984814435804, "kl": 0.10205078125, "learning_rate": 3.114682256394866e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.6084539294242859, "rewards_raw/accuracy_reward": 0.5619305372238159, "rewards_raw/clip_reward_wrapper": 0.33671504259109497, "rewards_raw/format_reward": 5.0, "step": 1219 }, { "Total_reward": 8.834260256662674e-08, "advantage_std": 1.2499959468841553, "completion_length": 378.6250305175781, "epoch": 0.4212707182320442, "grad_norm": 2.898661753307356, "kl": 0.1015625, "learning_rate": 3.11205313808101e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.8757355809211731, "rewards_raw/accuracy_reward": 0.5715436935424805, "rewards_raw/clip_reward_wrapper": 0.33590224385261536, "rewards_raw/format_reward": 5.0, "step": 1220 }, { "Total_reward": -2.086162709247219e-07, "advantage_std": 1.5421273708343506, "completion_length": 374.2500305175781, "epoch": 0.42161602209944754, "grad_norm": 4.312776895182523, "kl": 0.095703125, "learning_rate": 3.1094232995031404e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.9244780540466309, "rewards_raw/accuracy_reward": 0.583526611328125, "rewards_raw/clip_reward_wrapper": 0.33626407384872437, "rewards_raw/format_reward": 5.0, "step": 1221 }, { "Total_reward": -1.0005065576024208e-07, "advantage_std": 1.3559019565582275, "completion_length": 366.9107360839844, "epoch": 0.4219613259668508, "grad_norm": 5.548023198095307, "kl": 0.1005859375, "learning_rate": 3.1067927437560515e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.5912892818450928, "rewards_raw/accuracy_reward": 0.5741631984710693, "rewards_raw/clip_reward_wrapper": 0.3385161757469177, "rewards_raw/format_reward": 5.0, "step": 1222 }, { "Total_reward": 4.608716608345276e-07, "advantage_std": 1.6327189207077026, "completion_length": 373.51788330078125, "epoch": 0.42230662983425415, "grad_norm": 4.574437659943197, "kl": 0.099609375, "learning_rate": 3.1041614739353796e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8603745698928833, "rewards_raw/accuracy_reward": 0.5711552500724792, "rewards_raw/clip_reward_wrapper": 0.34223881363868713, "rewards_raw/format_reward": 5.0, "step": 1223 }, { "Total_reward": 1.0005065576024208e-07, "advantage_std": 1.3820637464523315, "completion_length": 366.9107360839844, "epoch": 0.42265193370165743, "grad_norm": 5.34372675112882, "kl": 0.10009765625, "learning_rate": 3.101529493137603e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 1.8623629808425903, "rewards_raw/accuracy_reward": 0.5392601490020752, "rewards_raw/clip_reward_wrapper": 0.33247479796409607, "rewards_raw/format_reward": 5.0, "step": 1224 }, { "Total_reward": 1.841357857301773e-07, "advantage_std": 1.824952244758606, "completion_length": 376.4464416503906, "epoch": 0.42299723756906077, "grad_norm": 6.347901522667193, "kl": 0.09814453125, "learning_rate": 3.098896804460036e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6832423806190491, "rewards_raw/accuracy_reward": 0.565135657787323, "rewards_raw/clip_reward_wrapper": 0.34823229908943176, "rewards_raw/format_reward": 4.901785850524902, "step": 1225 }, { "Total_reward": 1.9158636632710113e-07, "advantage_std": 1.5319796800613403, "completion_length": 375.8750305175781, "epoch": 0.4233425414364641, "grad_norm": 5.022040239405344, "kl": 0.0966796875, "learning_rate": 3.096263411000826e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7248802781105042, "rewards_raw/accuracy_reward": 0.580524742603302, "rewards_raw/clip_reward_wrapper": 0.3410542905330658, "rewards_raw/format_reward": 5.0, "step": 1226 }, { "Total_reward": 1.7615302283502388e-07, "advantage_std": 1.4358841180801392, "completion_length": 384.3214416503906, "epoch": 0.4236878453038674, "grad_norm": 5.611796567944744, "kl": 0.0986328125, "learning_rate": 3.093629315858952e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.1070377826690674, "rewards_raw/accuracy_reward": 0.5760809183120728, "rewards_raw/clip_reward_wrapper": 0.33549726009368896, "rewards_raw/format_reward": 5.0, "step": 1227 }, { "Total_reward": -4.491635934300575e-07, "advantage_std": 1.1734976768493652, "completion_length": 373.0000305175781, "epoch": 0.4240331491712707, "grad_norm": 2.966381298956562, "kl": 0.10302734375, "learning_rate": 3.090994522134213e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.9118162393569946, "rewards_raw/accuracy_reward": 0.5758698582649231, "rewards_raw/clip_reward_wrapper": 0.33160361647605896, "rewards_raw/format_reward": 5.0, "step": 1228 }, { "Total_reward": -4.363911543237009e-08, "advantage_std": 1.422362208366394, "completion_length": 358.51788330078125, "epoch": 0.42437845303867405, "grad_norm": 4.911548041166254, "kl": 0.10107421875, "learning_rate": 3.088359032927236e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8109229207038879, "rewards_raw/accuracy_reward": 0.55348801612854, "rewards_raw/clip_reward_wrapper": 0.34101828932762146, "rewards_raw/format_reward": 4.964285850524902, "step": 1229 }, { "Total_reward": -1.4922449054211029e-06, "advantage_std": 1.5269138813018799, "completion_length": 368.39288330078125, "epoch": 0.42472375690607733, "grad_norm": 4.1298730055251225, "kl": 0.1044921875, "learning_rate": 3.085722851339464e-07, "loss": 0.0042, "rewards_raw/CoT ICLG": 0.8497569561004639, "rewards_raw/accuracy_reward": 0.5554008483886719, "rewards_raw/clip_reward_wrapper": 0.3397034704685211, "rewards_raw/format_reward": 5.0, "step": 1230 }, { "Total_reward": -1.106943500417401e-06, "advantage_std": 1.5103191137313843, "completion_length": 374.8214416503906, "epoch": 0.42506906077348067, "grad_norm": 4.487684770547037, "kl": 0.09033203125, "learning_rate": 3.0830859804731545e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.8214551210403442, "rewards_raw/accuracy_reward": 0.5710360407829285, "rewards_raw/clip_reward_wrapper": 0.3567415177822113, "rewards_raw/format_reward": 5.0, "step": 1231 }, { "Total_reward": 6.556511493727157e-07, "advantage_std": 1.4699702262878418, "completion_length": 364.8035888671875, "epoch": 0.425414364640884, "grad_norm": 7.251902897959661, "kl": 0.0966796875, "learning_rate": 3.0804484234313764e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6499484181404114, "rewards_raw/accuracy_reward": 0.5667514204978943, "rewards_raw/clip_reward_wrapper": 0.3435710072517395, "rewards_raw/format_reward": 4.964285850524902, "step": 1232 }, { "Total_reward": 3.7891524584665603e-07, "advantage_std": 1.3890483379364014, "completion_length": 383.3571472167969, "epoch": 0.4257596685082873, "grad_norm": 4.203027996533753, "kl": 0.09716796875, "learning_rate": 3.077810183318008e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.0718042850494385, "rewards_raw/accuracy_reward": 0.581153154373169, "rewards_raw/clip_reward_wrapper": 0.3375484049320221, "rewards_raw/format_reward": 5.0, "step": 1233 }, { "Total_reward": 2.2564616131148796e-07, "advantage_std": 1.5776344537734985, "completion_length": 369.6964416503906, "epoch": 0.4261049723756906, "grad_norm": 4.759740764760706, "kl": 0.0947265625, "learning_rate": 3.075171263237728e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8318960070610046, "rewards_raw/accuracy_reward": 0.5690749883651733, "rewards_raw/clip_reward_wrapper": 0.3255276381969452, "rewards_raw/format_reward": 5.0, "step": 1234 }, { "Total_reward": -4.0552447444497375e-07, "advantage_std": 1.3373745679855347, "completion_length": 371.6785888671875, "epoch": 0.4264502762430939, "grad_norm": 10.355876333813788, "kl": 0.09716796875, "learning_rate": 3.0725316662960176e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.729256272315979, "rewards_raw/accuracy_reward": 0.5884218215942383, "rewards_raw/clip_reward_wrapper": 0.33806025981903076, "rewards_raw/format_reward": 5.0, "step": 1235 }, { "Total_reward": 1.4901161193847656e-08, "advantage_std": 1.2718626260757446, "completion_length": 362.7321472167969, "epoch": 0.42679558011049723, "grad_norm": 3.898876570470961, "kl": 0.1025390625, "learning_rate": 3.069891395599154e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.7752019762992859, "rewards_raw/accuracy_reward": 0.5874863862991333, "rewards_raw/clip_reward_wrapper": 0.3412780165672302, "rewards_raw/format_reward": 5.0, "step": 1236 }, { "Total_reward": 7.195132525339432e-07, "advantage_std": 1.6842619180679321, "completion_length": 374.96429443359375, "epoch": 0.42714088397790057, "grad_norm": 5.511887811787154, "kl": 0.09814453125, "learning_rate": 3.0672504542542073e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6897544264793396, "rewards_raw/accuracy_reward": 0.5656592845916748, "rewards_raw/clip_reward_wrapper": 0.3280887007713318, "rewards_raw/format_reward": 5.0, "step": 1237 }, { "Total_reward": -3.8530146184712066e-07, "advantage_std": 1.6048295497894287, "completion_length": 380.71429443359375, "epoch": 0.42748618784530384, "grad_norm": 6.50878222351186, "kl": 0.09326171875, "learning_rate": 3.0646088453690373e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 2.5938966274261475, "rewards_raw/accuracy_reward": 0.5455961227416992, "rewards_raw/clip_reward_wrapper": 0.34235507249832153, "rewards_raw/format_reward": 5.0, "step": 1238 }, { "Total_reward": 5.736947059631348e-07, "advantage_std": 1.5293455123901367, "completion_length": 383.9285888671875, "epoch": 0.4278314917127072, "grad_norm": 4.823031988291142, "kl": 0.09912109375, "learning_rate": 3.061966572052287e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7014171481132507, "rewards_raw/accuracy_reward": 0.5742270946502686, "rewards_raw/clip_reward_wrapper": 0.3239564895629883, "rewards_raw/format_reward": 5.0, "step": 1239 }, { "Total_reward": -3.193106223875475e-08, "advantage_std": 1.3269212245941162, "completion_length": 378.4464416503906, "epoch": 0.4281767955801105, "grad_norm": 3.723277822657508, "kl": 0.095703125, "learning_rate": 3.0593236374133846e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.808756947517395, "rewards_raw/accuracy_reward": 0.5721009373664856, "rewards_raw/clip_reward_wrapper": 0.33959171175956726, "rewards_raw/format_reward": 5.0, "step": 1240 }, { "Total_reward": 1.7455646172948036e-07, "advantage_std": 1.448020100593567, "completion_length": 378.5357360839844, "epoch": 0.4285220994475138, "grad_norm": 4.14864508025033, "kl": 0.09521484375, "learning_rate": 3.056680044562535e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6322941184043884, "rewards_raw/accuracy_reward": 0.5559723377227783, "rewards_raw/clip_reward_wrapper": 0.36173802614212036, "rewards_raw/format_reward": 5.0, "step": 1241 }, { "Total_reward": 6.279775277562294e-08, "advantage_std": 1.314889907836914, "completion_length": 378.14288330078125, "epoch": 0.42886740331491713, "grad_norm": 3.9244523328333103, "kl": 0.08984375, "learning_rate": 3.0540357966107177e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.879606306552887, "rewards_raw/accuracy_reward": 0.5851947665214539, "rewards_raw/clip_reward_wrapper": 0.32165583968162537, "rewards_raw/format_reward": 5.0, "step": 1242 }, { "Total_reward": 2.7034965910388564e-07, "advantage_std": 1.4933570623397827, "completion_length": 379.1964416503906, "epoch": 0.42921270718232046, "grad_norm": 4.6953565855535935, "kl": 0.09814453125, "learning_rate": 3.051390896669682e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6534435153007507, "rewards_raw/accuracy_reward": 0.5901212692260742, "rewards_raw/clip_reward_wrapper": 0.33225664496421814, "rewards_raw/format_reward": 5.0, "step": 1243 }, { "Total_reward": 1.2367963790893555e-06, "advantage_std": 1.3096742630004883, "completion_length": 376.5000305175781, "epoch": 0.42955801104972374, "grad_norm": 3.695213652193361, "kl": 0.10107421875, "learning_rate": 3.0487453478519476e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.6878622770309448, "rewards_raw/accuracy_reward": 0.5752221345901489, "rewards_raw/clip_reward_wrapper": 0.3397129774093628, "rewards_raw/format_reward": 5.0, "step": 1244 }, { "Total_reward": -1.6178404393940582e-07, "advantage_std": 1.412016749382019, "completion_length": 377.01788330078125, "epoch": 0.4299033149171271, "grad_norm": 5.241597879326339, "kl": 0.0947265625, "learning_rate": 3.046099153270795e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6591109037399292, "rewards_raw/accuracy_reward": 0.5725107192993164, "rewards_raw/clip_reward_wrapper": 0.3394952118396759, "rewards_raw/format_reward": 5.0, "step": 1245 }, { "Total_reward": 3.7252905826790084e-07, "advantage_std": 1.3564560413360596, "completion_length": 382.33929443359375, "epoch": 0.4302486187845304, "grad_norm": 3.586267375346527, "kl": 0.0986328125, "learning_rate": 3.0434523160402645e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.0031025409698486, "rewards_raw/accuracy_reward": 0.5838050246238708, "rewards_raw/clip_reward_wrapper": 0.3315722942352295, "rewards_raw/format_reward": 5.0, "step": 1246 }, { "Total_reward": -1.3315252544998657e-06, "advantage_std": 1.4340707063674927, "completion_length": 381.8750305175781, "epoch": 0.4305939226519337, "grad_norm": 6.990820752775306, "kl": 0.095703125, "learning_rate": 3.040804839275155e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6236686706542969, "rewards_raw/accuracy_reward": 0.5933035612106323, "rewards_raw/clip_reward_wrapper": 0.32680997252464294, "rewards_raw/format_reward": 5.0, "step": 1247 }, { "Total_reward": 3.469841942660423e-07, "advantage_std": 1.3991496562957764, "completion_length": 368.0000305175781, "epoch": 0.430939226519337, "grad_norm": 6.024379155647107, "kl": 0.0927734375, "learning_rate": 3.038156726091016e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7168230414390564, "rewards_raw/accuracy_reward": 0.5669558644294739, "rewards_raw/clip_reward_wrapper": 0.33690112829208374, "rewards_raw/format_reward": 5.0, "step": 1248 }, { "Total_reward": -8.834260256662674e-07, "advantage_std": 1.2722891569137573, "completion_length": 355.6250305175781, "epoch": 0.4312845303867403, "grad_norm": 4.228772063471913, "kl": 0.09619140625, "learning_rate": 3.0355079796041473e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8453601598739624, "rewards_raw/accuracy_reward": 0.5936868190765381, "rewards_raw/clip_reward_wrapper": 0.34086644649505615, "rewards_raw/format_reward": 5.0, "step": 1249 }, { "Total_reward": 6.897109301462478e-07, "advantage_std": 1.5364445447921753, "completion_length": 378.5000305175781, "epoch": 0.43162983425414364, "grad_norm": 4.804277386122175, "kl": 0.10009765625, "learning_rate": 3.032858602931592e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8539345264434814, "rewards_raw/accuracy_reward": 0.5763100385665894, "rewards_raw/clip_reward_wrapper": 0.3424986004829407, "rewards_raw/format_reward": 5.0, "step": 1250 }, { "Total_reward": 3.193106010712654e-07, "advantage_std": 1.4413310289382935, "completion_length": 384.2321472167969, "epoch": 0.431975138121547, "grad_norm": 10.566586417875731, "kl": 0.09716796875, "learning_rate": 3.030208599191136e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8032567501068115, "rewards_raw/accuracy_reward": 0.5919229984283447, "rewards_raw/clip_reward_wrapper": 0.32871389389038086, "rewards_raw/format_reward": 5.0, "step": 1251 }, { "Total_reward": 3.384692490726593e-07, "advantage_std": 1.5053157806396484, "completion_length": 386.08929443359375, "epoch": 0.43232044198895025, "grad_norm": 4.329122959621577, "kl": 0.08984375, "learning_rate": 3.0275579715013055e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.2401920557022095, "rewards_raw/accuracy_reward": 0.5985864996910095, "rewards_raw/clip_reward_wrapper": 0.33186957240104675, "rewards_raw/format_reward": 5.0, "step": 1252 }, { "Total_reward": -4.470348713425665e-08, "advantage_std": 1.3672845363616943, "completion_length": 382.89288330078125, "epoch": 0.4326657458563536, "grad_norm": 5.389180081253724, "kl": 0.09765625, "learning_rate": 3.024906722981356e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.49802809953689575, "rewards_raw/accuracy_reward": 0.5836024284362793, "rewards_raw/clip_reward_wrapper": 0.32474347949028015, "rewards_raw/format_reward": 5.0, "step": 1253 }, { "Total_reward": 1.8732889373040962e-07, "advantage_std": 1.4330012798309326, "completion_length": 385.89288330078125, "epoch": 0.4330110497237569, "grad_norm": 3.984026563592025, "kl": 0.0966796875, "learning_rate": 3.022254856751277e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7606714963912964, "rewards_raw/accuracy_reward": 0.5654544234275818, "rewards_raw/clip_reward_wrapper": 0.3389196991920471, "rewards_raw/format_reward": 5.0, "step": 1254 }, { "Total_reward": -4.2361875784990843e-07, "advantage_std": 1.2307379245758057, "completion_length": 375.2321472167969, "epoch": 0.4333563535911602, "grad_norm": 3.442658504541446, "kl": 0.0986328125, "learning_rate": 3.019602375931785e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 2.127293586730957, "rewards_raw/accuracy_reward": 0.5448786616325378, "rewards_raw/clip_reward_wrapper": 0.35535871982574463, "rewards_raw/format_reward": 5.0, "step": 1255 }, { "Total_reward": 2.4161170131264953e-07, "advantage_std": 1.449541449546814, "completion_length": 384.3571472167969, "epoch": 0.43370165745856354, "grad_norm": 4.948364637658487, "kl": 0.0947265625, "learning_rate": 3.0169492836443183e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.9596750736236572, "rewards_raw/accuracy_reward": 0.5680150389671326, "rewards_raw/clip_reward_wrapper": 0.3360831141471863, "rewards_raw/format_reward": 5.0, "step": 1256 }, { "Total_reward": 6.79067227338237e-07, "advantage_std": 1.4589512348175049, "completion_length": 380.4464416503906, "epoch": 0.4340469613259669, "grad_norm": 3.464431324090138, "kl": 0.09423828125, "learning_rate": 3.014295583011035e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.9420516490936279, "rewards_raw/accuracy_reward": 0.5933520793914795, "rewards_raw/clip_reward_wrapper": 0.32819077372550964, "rewards_raw/format_reward": 5.0, "step": 1257 }, { "Total_reward": -1.341104507446289e-07, "advantage_std": 1.4317243099212646, "completion_length": 391.6607360839844, "epoch": 0.43439226519337015, "grad_norm": 7.348952454635686, "kl": 0.09765625, "learning_rate": 3.011641277154811e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.5937982797622681, "rewards_raw/accuracy_reward": 0.5584465861320496, "rewards_raw/clip_reward_wrapper": 0.3432706594467163, "rewards_raw/format_reward": 5.0, "step": 1258 }, { "Total_reward": -1.4901161193847656e-07, "advantage_std": 1.6105153560638428, "completion_length": 389.5714416503906, "epoch": 0.4347375690607735, "grad_norm": 73.48242657837191, "kl": 0.10302734375, "learning_rate": 3.00898636919923e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 1.1222434043884277, "rewards_raw/accuracy_reward": 0.5813074111938477, "rewards_raw/clip_reward_wrapper": 0.32673022150993347, "rewards_raw/format_reward": 5.0, "step": 1259 }, { "Total_reward": -5.971108407720749e-07, "advantage_std": 1.3952276706695557, "completion_length": 384.71429443359375, "epoch": 0.4350828729281768, "grad_norm": 5.162107245057533, "kl": 0.09814453125, "learning_rate": 3.00633086226859e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7610628008842468, "rewards_raw/accuracy_reward": 0.5811748504638672, "rewards_raw/clip_reward_wrapper": 0.34929177165031433, "rewards_raw/format_reward": 5.0, "step": 1260 }, { "Total_reward": -6.471361757576233e-07, "advantage_std": 1.7360012531280518, "completion_length": 392.6250305175781, "epoch": 0.4354281767955801, "grad_norm": 9.213058547233514, "kl": 0.09326171875, "learning_rate": 3.0036747594878906e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.876410186290741, "rewards_raw/accuracy_reward": 0.5806135535240173, "rewards_raw/clip_reward_wrapper": 0.33522751927375793, "rewards_raw/format_reward": 5.0, "step": 1261 }, { "Total_reward": 4.874808610111359e-07, "advantage_std": 1.384685754776001, "completion_length": 383.3571472167969, "epoch": 0.43577348066298344, "grad_norm": 5.684544787721875, "kl": 0.09912109375, "learning_rate": 3.0010180639828313e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.6558020114898682, "rewards_raw/accuracy_reward": 0.5644174814224243, "rewards_raw/clip_reward_wrapper": 0.34466689825057983, "rewards_raw/format_reward": 5.0, "step": 1262 }, { "Total_reward": 4.3426243223620986e-07, "advantage_std": 1.398820400238037, "completion_length": 376.7321472167969, "epoch": 0.4361187845303867, "grad_norm": 7.228981438216654, "kl": 0.09423828125, "learning_rate": 2.998360778879813e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.9500673413276672, "rewards_raw/accuracy_reward": 0.5712900161743164, "rewards_raw/clip_reward_wrapper": 0.31607943773269653, "rewards_raw/format_reward": 5.0, "step": 1263 }, { "Total_reward": -3.0015198149158095e-07, "advantage_std": 1.5588369369506836, "completion_length": 386.6071472167969, "epoch": 0.43646408839779005, "grad_norm": 12.706316750493242, "kl": 0.09326171875, "learning_rate": 2.995702907305927e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.5715422034263611, "rewards_raw/accuracy_reward": 0.562281608581543, "rewards_raw/clip_reward_wrapper": 0.34693726897239685, "rewards_raw/format_reward": 5.0, "step": 1264 }, { "Total_reward": -1.4049666674509353e-07, "advantage_std": 1.4886648654937744, "completion_length": 378.58929443359375, "epoch": 0.4368093922651934, "grad_norm": 3.297158042371592, "kl": 0.091796875, "learning_rate": 2.9930444523889565e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.1450438499450684, "rewards_raw/accuracy_reward": 0.5734798908233643, "rewards_raw/clip_reward_wrapper": 0.35165244340896606, "rewards_raw/format_reward": 5.0, "step": 1265 }, { "Total_reward": 1.2293457984924316e-07, "advantage_std": 1.4485312700271606, "completion_length": 384.21429443359375, "epoch": 0.43715469613259667, "grad_norm": 5.497715956472397, "kl": 0.095703125, "learning_rate": 2.99038541725737e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8400958776473999, "rewards_raw/accuracy_reward": 0.5459092855453491, "rewards_raw/clip_reward_wrapper": 0.3471279442310333, "rewards_raw/format_reward": 5.0, "step": 1266 }, { "Total_reward": -7.940190585031814e-07, "advantage_std": 1.3280344009399414, "completion_length": 388.89288330078125, "epoch": 0.4375, "grad_norm": 4.167321991558884, "kl": 0.099609375, "learning_rate": 2.987725805040321e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.4703153073787689, "rewards_raw/accuracy_reward": 0.567659318447113, "rewards_raw/clip_reward_wrapper": 0.3388347327709198, "rewards_raw/format_reward": 5.0, "step": 1267 }, { "Total_reward": 3.0440943987741775e-07, "advantage_std": 1.4918692111968994, "completion_length": 389.1071472167969, "epoch": 0.43784530386740333, "grad_norm": 4.678963941111432, "kl": 0.10302734375, "learning_rate": 2.985065618867639e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.7587886452674866, "rewards_raw/accuracy_reward": 0.5784897208213806, "rewards_raw/clip_reward_wrapper": 0.3507782816886902, "rewards_raw/format_reward": 5.0, "step": 1268 }, { "Total_reward": -2.7460711748972244e-07, "advantage_std": 1.53684401512146, "completion_length": 384.89288330078125, "epoch": 0.4381906077348066, "grad_norm": 5.815174329731018, "kl": 0.09326171875, "learning_rate": 2.982404861869831e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.8071513772010803, "rewards_raw/accuracy_reward": 0.5686765909194946, "rewards_raw/clip_reward_wrapper": 0.3381958305835724, "rewards_raw/format_reward": 5.0, "step": 1269 }, { "Total_reward": 5.534717217869911e-08, "advantage_std": 1.503476619720459, "completion_length": 389.26788330078125, "epoch": 0.43853591160220995, "grad_norm": 7.077356708774065, "kl": 0.09326171875, "learning_rate": 2.9797435371780767e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.6938449740409851, "rewards_raw/accuracy_reward": 0.5796724557876587, "rewards_raw/clip_reward_wrapper": 0.33604100346565247, "rewards_raw/format_reward": 5.0, "step": 1270 }, { "Total_reward": 8.39786935102893e-07, "advantage_std": 1.4697474241256714, "completion_length": 387.01788330078125, "epoch": 0.4388812154696133, "grad_norm": 4.164463717268523, "kl": 0.0947265625, "learning_rate": 2.977081647924221e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.30332612991333, "rewards_raw/accuracy_reward": 0.5597069263458252, "rewards_raw/clip_reward_wrapper": 0.33802351355552673, "rewards_raw/format_reward": 5.0, "step": 1271 }, { "Total_reward": 3.6374800060912094e-07, "advantage_std": 1.5368375778198242, "completion_length": 385.7857360839844, "epoch": 0.43922651933701656, "grad_norm": 5.480818350986218, "kl": 0.09521484375, "learning_rate": 2.974419197240775e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.0104730129241943, "rewards_raw/accuracy_reward": 0.5840467810630798, "rewards_raw/clip_reward_wrapper": 0.3484514355659485, "rewards_raw/format_reward": 5.0, "step": 1272 }, { "Total_reward": 2.63963443103421e-07, "advantage_std": 1.3349472284317017, "completion_length": 397.2321472167969, "epoch": 0.4395718232044199, "grad_norm": 4.750933776820068, "kl": 0.09521484375, "learning_rate": 2.971756188260909e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.9146171808242798, "rewards_raw/accuracy_reward": 0.5684084892272949, "rewards_raw/clip_reward_wrapper": 0.3384949266910553, "rewards_raw/format_reward": 5.0, "step": 1273 }, { "Total_reward": -3.2143935868589324e-07, "advantage_std": 1.4091578722000122, "completion_length": 387.5714416503906, "epoch": 0.4399171270718232, "grad_norm": 3.1875843227525054, "kl": 0.095703125, "learning_rate": 2.9690926241184547e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8725610375404358, "rewards_raw/accuracy_reward": 0.5834453105926514, "rewards_raw/clip_reward_wrapper": 0.34483101963996887, "rewards_raw/format_reward": 5.0, "step": 1274 }, { "Total_reward": 3.2356808787881164e-07, "advantage_std": 1.5932456254959106, "completion_length": 390.4464416503906, "epoch": 0.4402624309392265, "grad_norm": 4.932236860544749, "kl": 0.09423828125, "learning_rate": 2.9664285079478907e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6675011515617371, "rewards_raw/accuracy_reward": 0.5722399950027466, "rewards_raw/clip_reward_wrapper": 0.3351723253726959, "rewards_raw/format_reward": 5.0, "step": 1275 }, { "Total_reward": 5.662441253662109e-07, "advantage_std": 1.4053746461868286, "completion_length": 392.3214416503906, "epoch": 0.44060773480662985, "grad_norm": 4.141758813280217, "kl": 0.09619140625, "learning_rate": 2.9637638428843496e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.5984599590301514, "rewards_raw/accuracy_reward": 0.5931992530822754, "rewards_raw/clip_reward_wrapper": 0.33134424686431885, "rewards_raw/format_reward": 5.0, "step": 1276 }, { "Total_reward": -2.7886460429726867e-07, "advantage_std": 1.2786879539489746, "completion_length": 390.7321472167969, "epoch": 0.4409530386740331, "grad_norm": 3.0281339172209334, "kl": 0.091796875, "learning_rate": 2.961098632063609e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9323439598083496, "rewards_raw/accuracy_reward": 0.574556827545166, "rewards_raw/clip_reward_wrapper": 0.3431761562824249, "rewards_raw/format_reward": 5.0, "step": 1277 }, { "Total_reward": -6.460717827394546e-07, "advantage_std": 1.354655385017395, "completion_length": 384.0535888671875, "epoch": 0.44129834254143646, "grad_norm": 9.461860805828104, "kl": 0.091796875, "learning_rate": 2.958432878622088e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7734073996543884, "rewards_raw/accuracy_reward": 0.5566062927246094, "rewards_raw/clip_reward_wrapper": 0.34458258748054504, "rewards_raw/format_reward": 5.0, "step": 1278 }, { "Total_reward": 1.0068928304463043e-06, "advantage_std": 1.4463847875595093, "completion_length": 396.9107360839844, "epoch": 0.4416436464088398, "grad_norm": 4.9794069446397335, "kl": 0.09326171875, "learning_rate": 2.9557665856968445e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9299758076667786, "rewards_raw/accuracy_reward": 0.5600625276565552, "rewards_raw/clip_reward_wrapper": 0.3338460326194763, "rewards_raw/format_reward": 5.0, "step": 1279 }, { "Total_reward": -5.790165573671402e-07, "advantage_std": 1.5624728202819824, "completion_length": 389.64288330078125, "epoch": 0.4419889502762431, "grad_norm": 4.0134499421135645, "kl": 0.091796875, "learning_rate": 2.953099756425572e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.2028719186782837, "rewards_raw/accuracy_reward": 0.5695277452468872, "rewards_raw/clip_reward_wrapper": 0.3407391905784607, "rewards_raw/format_reward": 5.0, "step": 1280 }, { "Total_reward": -9.792191946189632e-08, "advantage_std": 1.6993454694747925, "completion_length": 398.6250305175781, "epoch": 0.4423342541436464, "grad_norm": 23.74168987025808, "kl": 0.09130859375, "learning_rate": 2.950432393946594e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.8987935185432434, "rewards_raw/accuracy_reward": 0.5926560163497925, "rewards_raw/clip_reward_wrapper": 0.33674004673957825, "rewards_raw/format_reward": 5.0, "step": 1281 }, { "Total_reward": 2.4480479510202713e-07, "advantage_std": 1.2011724710464478, "completion_length": 387.6250305175781, "epoch": 0.44267955801104975, "grad_norm": 6.4165411772956435, "kl": 0.0927734375, "learning_rate": 2.947764501398863e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.3164329528808594, "rewards_raw/accuracy_reward": 0.6084980368614197, "rewards_raw/clip_reward_wrapper": 0.3359408676624298, "rewards_raw/format_reward": 5.0, "step": 1282 }, { "Total_reward": 5.8540280178931425e-08, "advantage_std": 1.4710981845855713, "completion_length": 388.8214416503906, "epoch": 0.443024861878453, "grad_norm": 15.544159633501854, "kl": 0.0966796875, "learning_rate": 2.945096081921955e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8378158807754517, "rewards_raw/accuracy_reward": 0.5820366144180298, "rewards_raw/clip_reward_wrapper": 0.337938517332077, "rewards_raw/format_reward": 5.0, "step": 1283 }, { "Total_reward": 2.6609217940176677e-08, "advantage_std": 1.3205647468566895, "completion_length": 391.9285888671875, "epoch": 0.44337016574585636, "grad_norm": 4.659987487113497, "kl": 0.087890625, "learning_rate": 2.942427138656064e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.7648159265518188, "rewards_raw/accuracy_reward": 0.576650857925415, "rewards_raw/clip_reward_wrapper": 0.33561083674430847, "rewards_raw/format_reward": 5.0, "step": 1284 }, { "Total_reward": 1.1708056035786285e-07, "advantage_std": 1.3820339441299438, "completion_length": 392.46429443359375, "epoch": 0.4437154696132597, "grad_norm": 4.171370841508726, "kl": 0.09423828125, "learning_rate": 2.9397576747420016e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.9546697735786438, "rewards_raw/accuracy_reward": 0.5744499564170837, "rewards_raw/clip_reward_wrapper": 0.35105904936790466, "rewards_raw/format_reward": 5.0, "step": 1285 }, { "Total_reward": 7.897616001173446e-07, "advantage_std": 1.6346198320388794, "completion_length": 387.8571472167969, "epoch": 0.444060773480663, "grad_norm": 6.376307741813114, "kl": 0.08984375, "learning_rate": 2.937087693321195e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.8285927176475525, "rewards_raw/accuracy_reward": 0.5857232809066772, "rewards_raw/clip_reward_wrapper": 0.32887306809425354, "rewards_raw/format_reward": 5.0, "step": 1286 }, { "Total_reward": 1.4368977474532585e-07, "advantage_std": 1.3649712800979614, "completion_length": 388.39288330078125, "epoch": 0.4444060773480663, "grad_norm": 3.264004573603249, "kl": 0.09716796875, "learning_rate": 2.934417197535675e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.313896656036377, "rewards_raw/accuracy_reward": 0.5534753799438477, "rewards_raw/clip_reward_wrapper": 0.3470205068588257, "rewards_raw/format_reward": 5.0, "step": 1287 }, { "Total_reward": -1.0643687176070671e-07, "advantage_std": 1.3850314617156982, "completion_length": 396.8035888671875, "epoch": 0.4447513812154696, "grad_norm": 8.383895262942424, "kl": 0.091796875, "learning_rate": 2.9317461905280816e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7390466928482056, "rewards_raw/accuracy_reward": 0.563965380191803, "rewards_raw/clip_reward_wrapper": 0.33734744787216187, "rewards_raw/format_reward": 5.0, "step": 1288 }, { "Total_reward": -4.470348713425665e-08, "advantage_std": 1.3114813566207886, "completion_length": 391.58929443359375, "epoch": 0.4450966850828729, "grad_norm": 3.8522684650761745, "kl": 0.09814453125, "learning_rate": 2.9290746754416564e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.890089750289917, "rewards_raw/accuracy_reward": 0.5586247444152832, "rewards_raw/clip_reward_wrapper": 0.33794885873794556, "rewards_raw/format_reward": 5.0, "step": 1289 }, { "Total_reward": 1.4475413934178505e-07, "advantage_std": 1.4793516397476196, "completion_length": 387.5000305175781, "epoch": 0.44544198895027626, "grad_norm": 4.049541952979315, "kl": 0.09326171875, "learning_rate": 2.9264026554202354e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.5503764152526855, "rewards_raw/accuracy_reward": 0.584466814994812, "rewards_raw/clip_reward_wrapper": 0.3492588698863983, "rewards_raw/format_reward": 5.0, "step": 1290 }, { "Total_reward": 3.448554650731239e-07, "advantage_std": 1.5433934926986694, "completion_length": 388.1071472167969, "epoch": 0.44578729281767954, "grad_norm": 17.51010163899703, "kl": 0.09521484375, "learning_rate": 2.9237301336082544e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.5792081356048584, "rewards_raw/accuracy_reward": 0.5779404640197754, "rewards_raw/clip_reward_wrapper": 0.33703526854515076, "rewards_raw/format_reward": 5.0, "step": 1291 }, { "Total_reward": -2.7460711748972244e-07, "advantage_std": 1.5644465684890747, "completion_length": 386.33929443359375, "epoch": 0.44613259668508287, "grad_norm": 5.250382907824017, "kl": 0.091796875, "learning_rate": 2.9210571131507327e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7664381265640259, "rewards_raw/accuracy_reward": 0.5721874237060547, "rewards_raw/clip_reward_wrapper": 0.34426936507225037, "rewards_raw/format_reward": 5.0, "step": 1292 }, { "Total_reward": -6.066901505619171e-07, "advantage_std": 1.5762375593185425, "completion_length": 395.76788330078125, "epoch": 0.4464779005524862, "grad_norm": 3.9872566770972364, "kl": 0.09130859375, "learning_rate": 2.9183835971932836e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7317690253257751, "rewards_raw/accuracy_reward": 0.5672707557678223, "rewards_raw/clip_reward_wrapper": 0.32670432329177856, "rewards_raw/format_reward": 5.0, "step": 1293 }, { "Total_reward": 3.8530146184712066e-07, "advantage_std": 1.538563847541809, "completion_length": 384.4107360839844, "epoch": 0.4468232044198895, "grad_norm": 5.465689395647552, "kl": 0.09619140625, "learning_rate": 2.915709588882099e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.0304327011108398, "rewards_raw/accuracy_reward": 0.5775831937789917, "rewards_raw/clip_reward_wrapper": 0.3389807939529419, "rewards_raw/format_reward": 5.0, "step": 1294 }, { "Total_reward": 6.726810397594818e-07, "advantage_std": 1.5420970916748047, "completion_length": 386.58929443359375, "epoch": 0.4471685082872928, "grad_norm": 5.0918470974572525, "kl": 0.0947265625, "learning_rate": 2.913035091363951e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6023990511894226, "rewards_raw/accuracy_reward": 0.5760130882263184, "rewards_raw/clip_reward_wrapper": 0.34357091784477234, "rewards_raw/format_reward": 5.0, "step": 1295 }, { "Total_reward": 8.089202196970291e-07, "advantage_std": 1.5562078952789307, "completion_length": 393.83929443359375, "epoch": 0.44751381215469616, "grad_norm": 5.147110001634294, "kl": 0.091796875, "learning_rate": 2.910360107786187e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7814924120903015, "rewards_raw/accuracy_reward": 0.5599321126937866, "rewards_raw/clip_reward_wrapper": 0.3377569913864136, "rewards_raw/format_reward": 5.0, "step": 1296 }, { "Total_reward": 4.385199190437561e-07, "advantage_std": 1.3266212940216064, "completion_length": 387.8750305175781, "epoch": 0.44785911602209943, "grad_norm": 5.741943681726448, "kl": 0.095703125, "learning_rate": 2.9076846412967306e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.1030833721160889, "rewards_raw/accuracy_reward": 0.5844317674636841, "rewards_raw/clip_reward_wrapper": 0.3371536433696747, "rewards_raw/format_reward": 5.0, "step": 1297 }, { "Total_reward": -5.194119125917496e-07, "advantage_std": 1.4130455255508423, "completion_length": 402.7321472167969, "epoch": 0.44820441988950277, "grad_norm": 5.35083487318525, "kl": 0.08935546875, "learning_rate": 2.9050086950440663e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.896199107170105, "rewards_raw/accuracy_reward": 0.5731405019760132, "rewards_raw/clip_reward_wrapper": 0.3368731439113617, "rewards_raw/format_reward": 5.0, "step": 1298 }, { "Total_reward": -1.341104507446289e-07, "advantage_std": 1.519577980041504, "completion_length": 379.4464416503906, "epoch": 0.4485497237569061, "grad_norm": 4.9976107583671645, "kl": 0.0947265625, "learning_rate": 2.902332272177249e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.575905442237854, "rewards_raw/accuracy_reward": 0.5756120681762695, "rewards_raw/clip_reward_wrapper": 0.33709782361984253, "rewards_raw/format_reward": 5.0, "step": 1299 }, { "Total_reward": 8.557524324714905e-07, "advantage_std": 1.561612844467163, "completion_length": 397.9464416503906, "epoch": 0.4488950276243094, "grad_norm": 5.127181228800505, "kl": 0.09130859375, "learning_rate": 2.899655375845891e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.0721728801727295, "rewards_raw/accuracy_reward": 0.5925693511962891, "rewards_raw/clip_reward_wrapper": 0.3437735140323639, "rewards_raw/format_reward": 5.0, "step": 1300 }, { "Total_reward": 5.321843445926788e-07, "advantage_std": 1.53816819190979, "completion_length": 385.71429443359375, "epoch": 0.4492403314917127, "grad_norm": 3.788644126527171, "kl": 0.0927734375, "learning_rate": 2.896978009200166e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.6800285577774048, "rewards_raw/accuracy_reward": 0.5998278856277466, "rewards_raw/clip_reward_wrapper": 0.33413735032081604, "rewards_raw/format_reward": 5.0, "step": 1301 }, { "Total_reward": 9.217433216690551e-07, "advantage_std": 1.549168586730957, "completion_length": 391.9821472167969, "epoch": 0.449585635359116, "grad_norm": 5.0240013602150375, "kl": 0.09228515625, "learning_rate": 2.8943001753907975e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7924755215644836, "rewards_raw/accuracy_reward": 0.5860341191291809, "rewards_raw/clip_reward_wrapper": 0.3340545892715454, "rewards_raw/format_reward": 5.0, "step": 1302 }, { "Total_reward": -3.448554650731239e-07, "advantage_std": 1.6290425062179565, "completion_length": 385.8750305175781, "epoch": 0.44993093922651933, "grad_norm": 4.267055003692244, "kl": 0.09326171875, "learning_rate": 2.891621877569058e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.0932023525238037, "rewards_raw/accuracy_reward": 0.6137999296188354, "rewards_raw/clip_reward_wrapper": 0.32030755281448364, "rewards_raw/format_reward": 5.0, "step": 1303 }, { "Total_reward": 2.575772271029564e-07, "advantage_std": 1.6190677881240845, "completion_length": 388.14288330078125, "epoch": 0.45027624309392267, "grad_norm": 5.222869610202741, "kl": 0.09326171875, "learning_rate": 2.8889431188867707e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.5460980534553528, "rewards_raw/accuracy_reward": 0.5711486339569092, "rewards_raw/clip_reward_wrapper": 0.3234105110168457, "rewards_raw/format_reward": 5.0, "step": 1304 }, { "Total_reward": -2.2564616131148796e-07, "advantage_std": 1.210829257965088, "completion_length": 391.9821472167969, "epoch": 0.45062154696132595, "grad_norm": 14.95048071474946, "kl": 0.099609375, "learning_rate": 2.886263902496297e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.5304531455039978, "rewards_raw/accuracy_reward": 0.5855082869529724, "rewards_raw/clip_reward_wrapper": 0.3429637551307678, "rewards_raw/format_reward": 5.0, "step": 1305 }, { "Total_reward": 8.280788392767136e-07, "advantage_std": 1.535406231880188, "completion_length": 398.9285888671875, "epoch": 0.4509668508287293, "grad_norm": 3.8690378021221434, "kl": 0.08984375, "learning_rate": 2.8835842315505374e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.8945273160934448, "rewards_raw/accuracy_reward": 0.5845628976821899, "rewards_raw/clip_reward_wrapper": 0.34024494886398315, "rewards_raw/format_reward": 5.0, "step": 1306 }, { "Total_reward": -3.5337041026650695e-07, "advantage_std": 1.5438611507415771, "completion_length": 389.1607360839844, "epoch": 0.4513121546961326, "grad_norm": 15.590373797269855, "kl": 0.091796875, "learning_rate": 2.8809041092029287e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.3904898166656494, "rewards_raw/accuracy_reward": 0.572770893573761, "rewards_raw/clip_reward_wrapper": 0.35314056277275085, "rewards_raw/format_reward": 5.0, "step": 1307 }, { "Total_reward": -1.0856560805905247e-07, "advantage_std": 1.3962422609329224, "completion_length": 395.96429443359375, "epoch": 0.4516574585635359, "grad_norm": 7.024337809548755, "kl": 0.0927734375, "learning_rate": 2.8782235386074393e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.8121278882026672, "rewards_raw/accuracy_reward": 0.567354679107666, "rewards_raw/clip_reward_wrapper": 0.3396317660808563, "rewards_raw/format_reward": 5.0, "step": 1308 }, { "Total_reward": 1.3128988030075561e-06, "advantage_std": 1.4237617254257202, "completion_length": 387.5000305175781, "epoch": 0.45200276243093923, "grad_norm": 5.652320776879836, "kl": 0.09765625, "learning_rate": 2.875542522918564e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6328896880149841, "rewards_raw/accuracy_reward": 0.5700333118438721, "rewards_raw/clip_reward_wrapper": 0.31379234790802, "rewards_raw/format_reward": 5.0, "step": 1309 }, { "Total_reward": 1.4688288274555816e-07, "advantage_std": 1.4059984683990479, "completion_length": 386.7500305175781, "epoch": 0.45234806629834257, "grad_norm": 3.7971958470690854, "kl": 0.08935546875, "learning_rate": 2.8728610652913194e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.4061863422393799, "rewards_raw/accuracy_reward": 0.5985534191131592, "rewards_raw/clip_reward_wrapper": 0.34799009561538696, "rewards_raw/format_reward": 5.0, "step": 1310 }, { "Total_reward": 4.257474817137563e-09, "advantage_std": 1.5611635446548462, "completion_length": 392.0000305175781, "epoch": 0.45269337016574585, "grad_norm": 6.119718139266419, "kl": 0.09423828125, "learning_rate": 2.870179168881247e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7396331429481506, "rewards_raw/accuracy_reward": 0.5924060940742493, "rewards_raw/clip_reward_wrapper": 0.3406253159046173, "rewards_raw/format_reward": 5.0, "step": 1311 }, { "Total_reward": -7.85504084888089e-07, "advantage_std": 1.428478717803955, "completion_length": 388.4464416503906, "epoch": 0.4530386740331492, "grad_norm": 13.14134672404037, "kl": 0.0947265625, "learning_rate": 2.8674968368444e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6970795392990112, "rewards_raw/accuracy_reward": 0.5842874050140381, "rewards_raw/clip_reward_wrapper": 0.3376748859882355, "rewards_raw/format_reward": 5.0, "step": 1312 }, { "Total_reward": 1.841357857301773e-07, "advantage_std": 1.4956389665603638, "completion_length": 395.4285888671875, "epoch": 0.45338397790055246, "grad_norm": 3.5357162445128356, "kl": 0.09033203125, "learning_rate": 2.864814072337348e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.1439716815948486, "rewards_raw/accuracy_reward": 0.5472300052642822, "rewards_raw/clip_reward_wrapper": 0.3422423005104065, "rewards_raw/format_reward": 5.0, "step": 1313 }, { "Total_reward": -1.8732889373040962e-07, "advantage_std": 1.2462172508239746, "completion_length": 391.76788330078125, "epoch": 0.4537292817679558, "grad_norm": 3.581974370930018, "kl": 0.09326171875, "learning_rate": 2.8621308785171666e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9629270434379578, "rewards_raw/accuracy_reward": 0.5629593133926392, "rewards_raw/clip_reward_wrapper": 0.34076160192489624, "rewards_raw/format_reward": 5.0, "step": 1314 }, { "Total_reward": 3.7891524584665603e-07, "advantage_std": 1.5508071184158325, "completion_length": 394.7857360839844, "epoch": 0.45407458563535913, "grad_norm": 7.374969891316951, "kl": 0.09423828125, "learning_rate": 2.859447258541437e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7929989099502563, "rewards_raw/accuracy_reward": 0.5716546177864075, "rewards_raw/clip_reward_wrapper": 0.3372277617454529, "rewards_raw/format_reward": 5.0, "step": 1315 }, { "Total_reward": 5.103647708892822e-07, "advantage_std": 1.442676305770874, "completion_length": 391.6785888671875, "epoch": 0.4544198895027624, "grad_norm": 3.7286970070242753, "kl": 0.095703125, "learning_rate": 2.8567632155682453e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7889034152030945, "rewards_raw/accuracy_reward": 0.5658814907073975, "rewards_raw/clip_reward_wrapper": 0.344212144613266, "rewards_raw/format_reward": 5.0, "step": 1316 }, { "Total_reward": 1.4049667242943542e-06, "advantage_std": 1.701640009880066, "completion_length": 391.4821472167969, "epoch": 0.45476519337016574, "grad_norm": 4.107810334603536, "kl": 0.0927734375, "learning_rate": 2.85407875275617e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.8440974354743958, "rewards_raw/accuracy_reward": 0.5812633037567139, "rewards_raw/clip_reward_wrapper": 0.3448309600353241, "rewards_raw/format_reward": 5.0, "step": 1317 }, { "Total_reward": -6.726810397594818e-07, "advantage_std": 1.6707229614257812, "completion_length": 391.6964416503906, "epoch": 0.4551104972375691, "grad_norm": 5.118811443010903, "kl": 0.0908203125, "learning_rate": 2.851393873264288e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.8341600894927979, "rewards_raw/accuracy_reward": 0.5609915256500244, "rewards_raw/clip_reward_wrapper": 0.3340131342411041, "rewards_raw/format_reward": 5.0, "step": 1318 }, { "Total_reward": 1.084059476852417e-06, "advantage_std": 1.4310901165008545, "completion_length": 389.7500305175781, "epoch": 0.45545580110497236, "grad_norm": 4.443025594982182, "kl": 0.0908203125, "learning_rate": 2.8487085802521635e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.4306377172470093, "rewards_raw/accuracy_reward": 0.5764690637588501, "rewards_raw/clip_reward_wrapper": 0.3312138617038727, "rewards_raw/format_reward": 5.0, "step": 1319 }, { "Total_reward": -4.76837158203125e-07, "advantage_std": 1.2453845739364624, "completion_length": 395.6964416503906, "epoch": 0.4558011049723757, "grad_norm": 3.357536159172411, "kl": 0.095703125, "learning_rate": 2.8460228768798507e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.151329755783081, "rewards_raw/accuracy_reward": 0.587094783782959, "rewards_raw/clip_reward_wrapper": 0.3601451516151428, "rewards_raw/format_reward": 5.0, "step": 1320 }, { "Total_reward": -1.7242773253656196e-07, "advantage_std": 1.4851528406143188, "completion_length": 381.4285888671875, "epoch": 0.456146408839779, "grad_norm": 7.5530477177756765, "kl": 0.09228515625, "learning_rate": 2.8433367663078835e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.1782581806182861, "rewards_raw/accuracy_reward": 0.6027810573577881, "rewards_raw/clip_reward_wrapper": 0.336813747882843, "rewards_raw/format_reward": 5.0, "step": 1321 }, { "Total_reward": -6.450074465647049e-07, "advantage_std": 1.411525845527649, "completion_length": 379.7500305175781, "epoch": 0.4564917127071823, "grad_norm": 3.808731297538868, "kl": 0.0966796875, "learning_rate": 2.840650251697277e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8320776224136353, "rewards_raw/accuracy_reward": 0.5713455677032471, "rewards_raw/clip_reward_wrapper": 0.33745691180229187, "rewards_raw/format_reward": 5.0, "step": 1322 }, { "Total_reward": -1.27724248955019e-08, "advantage_std": 1.5253455638885498, "completion_length": 381.8750305175781, "epoch": 0.45683701657458564, "grad_norm": 5.284771305129253, "kl": 0.09375, "learning_rate": 2.83796333620952e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6891161203384399, "rewards_raw/accuracy_reward": 0.5676941275596619, "rewards_raw/clip_reward_wrapper": 0.33692577481269836, "rewards_raw/format_reward": 5.0, "step": 1323 }, { "Total_reward": -2.9163703629819793e-07, "advantage_std": 1.2395180463790894, "completion_length": 396.33929443359375, "epoch": 0.457182320441989, "grad_norm": 7.673889957221177, "kl": 0.09912109375, "learning_rate": 2.8352760230065756e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8885300755500793, "rewards_raw/accuracy_reward": 0.5622724294662476, "rewards_raw/clip_reward_wrapper": 0.32074546813964844, "rewards_raw/format_reward": 5.0, "step": 1324 }, { "Total_reward": 4.789659442394623e-07, "advantage_std": 1.2473292350769043, "completion_length": 385.5714416503906, "epoch": 0.45752762430939226, "grad_norm": 5.365558357313517, "kl": 0.0927734375, "learning_rate": 2.8325883152508727e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.8067391514778137, "rewards_raw/accuracy_reward": 0.5874608755111694, "rewards_raw/clip_reward_wrapper": 0.338239848613739, "rewards_raw/format_reward": 5.0, "step": 1325 }, { "Total_reward": -2.4586915969848633e-07, "advantage_std": 1.4580575227737427, "completion_length": 382.2857360839844, "epoch": 0.4578729281767956, "grad_norm": 15.510023308541253, "kl": 0.099609375, "learning_rate": 2.829900216105305e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 1.0059765577316284, "rewards_raw/accuracy_reward": 0.5842748880386353, "rewards_raw/clip_reward_wrapper": 0.34535470604896545, "rewards_raw/format_reward": 5.0, "step": 1326 }, { "Total_reward": -3.6188535545989e-07, "advantage_std": 1.4157843589782715, "completion_length": 380.6071472167969, "epoch": 0.45821823204419887, "grad_norm": 3.427280775406377, "kl": 0.09521484375, "learning_rate": 2.827211728733228e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8724621534347534, "rewards_raw/accuracy_reward": 0.5987342596054077, "rewards_raw/clip_reward_wrapper": 0.3236384391784668, "rewards_raw/format_reward": 5.0, "step": 1327 }, { "Total_reward": 5.002532930120651e-07, "advantage_std": 1.4103368520736694, "completion_length": 381.7857360839844, "epoch": 0.4585635359116022, "grad_norm": 6.095713796651103, "kl": 0.09619140625, "learning_rate": 2.8245228562984515e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6594518423080444, "rewards_raw/accuracy_reward": 0.5666194558143616, "rewards_raw/clip_reward_wrapper": 0.3485912084579468, "rewards_raw/format_reward": 5.0, "step": 1328 }, { "Total_reward": 1.0537250005882015e-07, "advantage_std": 1.4621526002883911, "completion_length": 386.96429443359375, "epoch": 0.45890883977900554, "grad_norm": 3.992291620322163, "kl": 0.09765625, "learning_rate": 2.821833601965241e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.0062999725341797, "rewards_raw/accuracy_reward": 0.5705004930496216, "rewards_raw/clip_reward_wrapper": 0.34119197726249695, "rewards_raw/format_reward": 5.0, "step": 1329 }, { "Total_reward": 6.38621244775095e-09, "advantage_std": 1.4819587469100952, "completion_length": 394.1071472167969, "epoch": 0.4592541436464088, "grad_norm": 4.594094182199406, "kl": 0.09423828125, "learning_rate": 2.819143968898311e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.638373613357544, "rewards_raw/accuracy_reward": 0.5770317316055298, "rewards_raw/clip_reward_wrapper": 0.32573196291923523, "rewards_raw/format_reward": 5.0, "step": 1330 }, { "Total_reward": 2.341611207157257e-07, "advantage_std": 1.551051139831543, "completion_length": 384.1785888671875, "epoch": 0.45959944751381215, "grad_norm": 4.4033328649796255, "kl": 0.09765625, "learning_rate": 2.81645396026282e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9010583758354187, "rewards_raw/accuracy_reward": 0.6035224795341492, "rewards_raw/clip_reward_wrapper": 0.34113696217536926, "rewards_raw/format_reward": 5.0, "step": 1331 }, { "Total_reward": -7.216419817268616e-07, "advantage_std": 1.422280192375183, "completion_length": 380.3750305175781, "epoch": 0.4599447513812155, "grad_norm": 4.447411972352082, "kl": 0.09521484375, "learning_rate": 2.8137635792243705e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.1988013982772827, "rewards_raw/accuracy_reward": 0.5780359506607056, "rewards_raw/clip_reward_wrapper": 0.3447656035423279, "rewards_raw/format_reward": 5.0, "step": 1332 }, { "Total_reward": -1.1920928955078125e-06, "advantage_std": 1.5670305490493774, "completion_length": 380.71429443359375, "epoch": 0.46029005524861877, "grad_norm": 8.769493471252243, "kl": 0.095703125, "learning_rate": 2.8110728289490026e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8653118014335632, "rewards_raw/accuracy_reward": 0.5597251057624817, "rewards_raw/clip_reward_wrapper": 0.3348371684551239, "rewards_raw/format_reward": 5.0, "step": 1333 }, { "Total_reward": 1.1814493063866394e-06, "advantage_std": 1.4407958984375, "completion_length": 380.64288330078125, "epoch": 0.4606353591160221, "grad_norm": 6.637242194169691, "kl": 0.0966796875, "learning_rate": 2.80838171260319e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7009041905403137, "rewards_raw/accuracy_reward": 0.5734082460403442, "rewards_raw/clip_reward_wrapper": 0.3438189625740051, "rewards_raw/format_reward": 5.0, "step": 1334 }, { "Total_reward": -9.813479664444458e-07, "advantage_std": 1.689122200012207, "completion_length": 379.4107360839844, "epoch": 0.46098066298342544, "grad_norm": 3.9032221630624875, "kl": 0.09912109375, "learning_rate": 2.8056902333538384e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7767563462257385, "rewards_raw/accuracy_reward": 0.5980948805809021, "rewards_raw/clip_reward_wrapper": 0.3194766640663147, "rewards_raw/format_reward": 5.0, "step": 1335 }, { "Total_reward": 2.884439140871109e-07, "advantage_std": 1.7568763494491577, "completion_length": 395.9285888671875, "epoch": 0.4613259668508287, "grad_norm": 5.735882885947677, "kl": 0.08984375, "learning_rate": 2.80299839436828e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.7320917844772339, "rewards_raw/accuracy_reward": 0.558305025100708, "rewards_raw/clip_reward_wrapper": 0.331205278635025, "rewards_raw/format_reward": 5.0, "step": 1336 }, { "Total_reward": -1.6817025993987045e-07, "advantage_std": 1.4665182828903198, "completion_length": 401.1250305175781, "epoch": 0.46167127071823205, "grad_norm": 5.181009807875632, "kl": 0.09423828125, "learning_rate": 2.800306198814269e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.529328465461731, "rewards_raw/accuracy_reward": 0.5615395903587341, "rewards_raw/clip_reward_wrapper": 0.3405953049659729, "rewards_raw/format_reward": 4.964285850524902, "step": 1337 }, { "Total_reward": 1.2133803295455436e-07, "advantage_std": 1.599395751953125, "completion_length": 387.6250305175781, "epoch": 0.4620165745856354, "grad_norm": 5.003629819377791, "kl": 0.095703125, "learning_rate": 2.7976136498599827e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.2974705696105957, "rewards_raw/accuracy_reward": 0.5900030136108398, "rewards_raw/clip_reward_wrapper": 0.34349191188812256, "rewards_raw/format_reward": 5.0, "step": 1338 }, { "Total_reward": -3.150531426854286e-07, "advantage_std": 1.5629661083221436, "completion_length": 390.1250305175781, "epoch": 0.46236187845303867, "grad_norm": 9.613370568318313, "kl": 0.09912109375, "learning_rate": 2.794920750674012e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8106087446212769, "rewards_raw/accuracy_reward": 0.5811728239059448, "rewards_raw/clip_reward_wrapper": 0.3434372842311859, "rewards_raw/format_reward": 5.0, "step": 1339 }, { "Total_reward": 8.004052460819366e-07, "advantage_std": 1.3567308187484741, "completion_length": 398.1250305175781, "epoch": 0.462707182320442, "grad_norm": 3.5061599754270105, "kl": 0.09326171875, "learning_rate": 2.7922275044253586e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9340083599090576, "rewards_raw/accuracy_reward": 0.6012860536575317, "rewards_raw/clip_reward_wrapper": 0.3379215598106384, "rewards_raw/format_reward": 5.0, "step": 1340 }, { "Total_reward": -3.139887780889694e-07, "advantage_std": 1.2774468660354614, "completion_length": 394.71429443359375, "epoch": 0.4630524861878453, "grad_norm": 4.932535263385429, "kl": 0.0966796875, "learning_rate": 2.7895339142834346e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9898409843444824, "rewards_raw/accuracy_reward": 0.5646425485610962, "rewards_raw/clip_reward_wrapper": 0.34386447072029114, "rewards_raw/format_reward": 5.0, "step": 1341 }, { "Total_reward": -3.469841942660423e-07, "advantage_std": 1.5169202089309692, "completion_length": 371.1250305175781, "epoch": 0.4633977900552486, "grad_norm": 4.541610970009127, "kl": 0.099609375, "learning_rate": 2.7868399834180575e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7275532484054565, "rewards_raw/accuracy_reward": 0.5964436531066895, "rewards_raw/clip_reward_wrapper": 0.3321724534034729, "rewards_raw/format_reward": 5.0, "step": 1342 }, { "Total_reward": -2.618347139105026e-07, "advantage_std": 1.3179919719696045, "completion_length": 386.26788330078125, "epoch": 0.46374309392265195, "grad_norm": 6.137835608826501, "kl": 0.09423828125, "learning_rate": 2.784145714999443e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8086334466934204, "rewards_raw/accuracy_reward": 0.5795800685882568, "rewards_raw/clip_reward_wrapper": 0.3477146327495575, "rewards_raw/format_reward": 5.0, "step": 1343 }, { "Total_reward": -3.0440943987741775e-07, "advantage_std": 1.4602383375167847, "completion_length": 382.3750305175781, "epoch": 0.46408839779005523, "grad_norm": 3.7952860050882973, "kl": 0.09228515625, "learning_rate": 2.7814511121982076e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9214146137237549, "rewards_raw/accuracy_reward": 0.578080415725708, "rewards_raw/clip_reward_wrapper": 0.330401748418808, "rewards_raw/format_reward": 5.0, "step": 1344 }, { "Total_reward": -3.065381974920456e-07, "advantage_std": 1.4953255653381348, "completion_length": 392.5714416503906, "epoch": 0.46443370165745856, "grad_norm": 9.541960997809802, "kl": 0.0966796875, "learning_rate": 2.7787561781853567e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6402270793914795, "rewards_raw/accuracy_reward": 0.5763720273971558, "rewards_raw/clip_reward_wrapper": 0.3471170961856842, "rewards_raw/format_reward": 5.0, "step": 1345 }, { "Total_reward": -5.8540280178931425e-08, "advantage_std": 1.3860385417938232, "completion_length": 379.0357360839844, "epoch": 0.4647790055248619, "grad_norm": 8.749909425028774, "kl": 0.09619140625, "learning_rate": 2.7760609161322905e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8423891663551331, "rewards_raw/accuracy_reward": 0.570284366607666, "rewards_raw/clip_reward_wrapper": 0.3412551283836365, "rewards_raw/format_reward": 5.0, "step": 1346 }, { "Total_reward": 2.682209014892578e-07, "advantage_std": 1.6307930946350098, "completion_length": 388.89288330078125, "epoch": 0.4651243093922652, "grad_norm": 7.261659164544841, "kl": 0.09326171875, "learning_rate": 2.773365329210793e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.0136196613311768, "rewards_raw/accuracy_reward": 0.5746506452560425, "rewards_raw/clip_reward_wrapper": 0.32386934757232666, "rewards_raw/format_reward": 5.0, "step": 1347 }, { "Total_reward": -4.3639116142912826e-07, "advantage_std": 1.2774864435195923, "completion_length": 385.9821472167969, "epoch": 0.4654696132596685, "grad_norm": 4.775828682769505, "kl": 0.09228515625, "learning_rate": 2.770669420593028e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.8926154375076294, "rewards_raw/accuracy_reward": 0.549374520778656, "rewards_raw/clip_reward_wrapper": 0.3441309332847595, "rewards_raw/format_reward": 5.0, "step": 1348 }, { "Total_reward": 9.898629116378288e-08, "advantage_std": 1.5495741367340088, "completion_length": 382.5357360839844, "epoch": 0.46581491712707185, "grad_norm": 5.921470720019368, "kl": 0.09423828125, "learning_rate": 2.7679731934515423e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8981596827507019, "rewards_raw/accuracy_reward": 0.5805584192276001, "rewards_raw/clip_reward_wrapper": 0.3273073732852936, "rewards_raw/format_reward": 5.0, "step": 1349 }, { "Total_reward": -4.896095902040543e-07, "advantage_std": 1.52816641330719, "completion_length": 386.0714416503906, "epoch": 0.4661602209944751, "grad_norm": 4.6440351509613595, "kl": 0.09423828125, "learning_rate": 2.765276650959255e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8222786784172058, "rewards_raw/accuracy_reward": 0.5848234295845032, "rewards_raw/clip_reward_wrapper": 0.3316432535648346, "rewards_raw/format_reward": 5.0, "step": 1350 }, { "Total_reward": -7.769891681164154e-07, "advantage_std": 1.4853150844573975, "completion_length": 390.9464416503906, "epoch": 0.46650552486187846, "grad_norm": 5.736751700348555, "kl": 0.09765625, "learning_rate": 2.762579796289456e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.913637638092041, "rewards_raw/accuracy_reward": 0.6035453677177429, "rewards_raw/clip_reward_wrapper": 0.3244932293891907, "rewards_raw/format_reward": 5.0, "step": 1351 }, { "Total_reward": -3.405979782655777e-07, "advantage_std": 1.421463131904602, "completion_length": 383.39288330078125, "epoch": 0.46685082872928174, "grad_norm": 7.507516087250877, "kl": 0.09423828125, "learning_rate": 2.759882632615804e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8796296119689941, "rewards_raw/accuracy_reward": 0.5733073949813843, "rewards_raw/clip_reward_wrapper": 0.3340946137905121, "rewards_raw/format_reward": 5.0, "step": 1352 }, { "Total_reward": -3.5124168107358855e-07, "advantage_std": 1.4195594787597656, "completion_length": 386.8035888671875, "epoch": 0.4671961325966851, "grad_norm": 18.10577559595532, "kl": 0.0966796875, "learning_rate": 2.757185163112321e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9209995269775391, "rewards_raw/accuracy_reward": 0.5535539984703064, "rewards_raw/clip_reward_wrapper": 0.321132093667984, "rewards_raw/format_reward": 5.0, "step": 1353 }, { "Total_reward": 1.3602632407128112e-06, "advantage_std": 1.4260272979736328, "completion_length": 380.3750305175781, "epoch": 0.4675414364640884, "grad_norm": 7.96750600833413, "kl": 0.0927734375, "learning_rate": 2.7544873909533877e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.8937232494354248, "rewards_raw/accuracy_reward": 0.6217178106307983, "rewards_raw/clip_reward_wrapper": 0.3435071110725403, "rewards_raw/format_reward": 5.0, "step": 1354 }, { "Total_reward": 4.3000494542866363e-07, "advantage_std": 1.2846258878707886, "completion_length": 395.3035888671875, "epoch": 0.4678867403314917, "grad_norm": 3.130552503712777, "kl": 0.09521484375, "learning_rate": 2.7517893193137434e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.5079359412193298, "rewards_raw/accuracy_reward": 0.5750818252563477, "rewards_raw/clip_reward_wrapper": 0.34555137157440186, "rewards_raw/format_reward": 5.0, "step": 1355 }, { "Total_reward": 1.0771411780297058e-06, "advantage_std": 1.3499884605407715, "completion_length": 389.9464416503906, "epoch": 0.468232044198895, "grad_norm": 4.913705706200648, "kl": 0.095703125, "learning_rate": 2.7490909513684756e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.2499940395355225, "rewards_raw/accuracy_reward": 0.5883382558822632, "rewards_raw/clip_reward_wrapper": 0.3335004150867462, "rewards_raw/format_reward": 5.0, "step": 1356 }, { "Total_reward": 4.896095973094816e-08, "advantage_std": 1.3048335313796997, "completion_length": 379.1250305175781, "epoch": 0.46857734806629836, "grad_norm": 3.88143041076057, "kl": 0.09765625, "learning_rate": 2.746392290293026e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6868519186973572, "rewards_raw/accuracy_reward": 0.5907622575759888, "rewards_raw/clip_reward_wrapper": 0.34534791111946106, "rewards_raw/format_reward": 5.0, "step": 1357 }, { "Total_reward": -1.5965531119377374e-08, "advantage_std": 1.309969425201416, "completion_length": 397.58929443359375, "epoch": 0.46892265193370164, "grad_norm": 3.113772419556475, "kl": 0.09326171875, "learning_rate": 2.743693339263177e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7774614095687866, "rewards_raw/accuracy_reward": 0.5763532519340515, "rewards_raw/clip_reward_wrapper": 0.3395821750164032, "rewards_raw/format_reward": 5.0, "step": 1358 }, { "Total_reward": 1.6391278734317893e-07, "advantage_std": 1.4593675136566162, "completion_length": 392.3214416503906, "epoch": 0.469267955801105, "grad_norm": 3.6887842594930174, "kl": 0.09423828125, "learning_rate": 2.740994101455054e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.9600983262062073, "rewards_raw/accuracy_reward": 0.5667750835418701, "rewards_raw/clip_reward_wrapper": 0.3351231813430786, "rewards_raw/format_reward": 5.0, "step": 1359 }, { "Total_reward": -1.9371510973087425e-07, "advantage_std": 1.2737798690795898, "completion_length": 380.0000305175781, "epoch": 0.4696132596685083, "grad_norm": 4.476408148107535, "kl": 0.09375, "learning_rate": 2.7382945800451183e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7709051370620728, "rewards_raw/accuracy_reward": 0.5621787309646606, "rewards_raw/clip_reward_wrapper": 0.33144611120224, "rewards_raw/format_reward": 5.0, "step": 1360 }, { "Total_reward": 1.1920928955078125e-07, "advantage_std": 1.3886398077011108, "completion_length": 383.9107360839844, "epoch": 0.4699585635359116, "grad_norm": 3.7956887922512332, "kl": 0.0888671875, "learning_rate": 2.735594778210169e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 0.8614340424537659, "rewards_raw/accuracy_reward": 0.5596238374710083, "rewards_raw/clip_reward_wrapper": 0.3311118483543396, "rewards_raw/format_reward": 5.0, "step": 1361 }, { "Total_reward": 3.5549913945942535e-07, "advantage_std": 1.447005271911621, "completion_length": 387.3571472167969, "epoch": 0.4703038674033149, "grad_norm": 9.712097578470754, "kl": 0.09423828125, "learning_rate": 2.73289469912733e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.0537487268447876, "rewards_raw/accuracy_reward": 0.5591602921485901, "rewards_raw/clip_reward_wrapper": 0.3344659209251404, "rewards_raw/format_reward": 5.0, "step": 1362 }, { "Total_reward": 1.80942677729945e-07, "advantage_std": 1.4796768426895142, "completion_length": 386.89288330078125, "epoch": 0.47064917127071826, "grad_norm": 6.731147445161409, "kl": 0.09716796875, "learning_rate": 2.730194345974054e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9166715145111084, "rewards_raw/accuracy_reward": 0.5636265277862549, "rewards_raw/clip_reward_wrapper": 0.3347715735435486, "rewards_raw/format_reward": 5.0, "step": 1363 }, { "Total_reward": -6.631017299696396e-07, "advantage_std": 1.6181693077087402, "completion_length": 378.33929443359375, "epoch": 0.47099447513812154, "grad_norm": 6.6058966119541225, "kl": 0.09912109375, "learning_rate": 2.727493721928117e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7442878484725952, "rewards_raw/accuracy_reward": 0.5428058505058289, "rewards_raw/clip_reward_wrapper": 0.33877506852149963, "rewards_raw/format_reward": 5.0, "step": 1364 }, { "Total_reward": 0.0, "advantage_std": 1.5366458892822266, "completion_length": 386.14288330078125, "epoch": 0.4713397790055249, "grad_norm": 4.429868804544693, "kl": 0.09814453125, "learning_rate": 2.724792830167612e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.5130194425582886, "rewards_raw/accuracy_reward": 0.5721080303192139, "rewards_raw/clip_reward_wrapper": 0.3348620533943176, "rewards_raw/format_reward": 5.0, "step": 1365 }, { "Total_reward": 5.428280474006897e-07, "advantage_std": 1.3864054679870605, "completion_length": 390.6785888671875, "epoch": 0.47168508287292815, "grad_norm": 4.484356616983408, "kl": 0.09130859375, "learning_rate": 2.7220916738709493e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.8184059262275696, "rewards_raw/accuracy_reward": 0.5852479338645935, "rewards_raw/clip_reward_wrapper": 0.3378417193889618, "rewards_raw/format_reward": 5.0, "step": 1366 }, { "Total_reward": -1.3890011985040474e-07, "advantage_std": 1.681614637374878, "completion_length": 382.6964416503906, "epoch": 0.4720303867403315, "grad_norm": 5.60312631792356, "kl": 0.0966796875, "learning_rate": 2.719390256216847e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7049044370651245, "rewards_raw/accuracy_reward": 0.5678775906562805, "rewards_raw/clip_reward_wrapper": 0.32307425141334534, "rewards_raw/format_reward": 5.0, "step": 1367 }, { "Total_reward": 1.9371510973087425e-07, "advantage_std": 1.4982331991195679, "completion_length": 395.89288330078125, "epoch": 0.4723756906077348, "grad_norm": 7.156705727480536, "kl": 0.09765625, "learning_rate": 2.716688580384334e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7451789379119873, "rewards_raw/accuracy_reward": 0.5778433084487915, "rewards_raw/clip_reward_wrapper": 0.3515186607837677, "rewards_raw/format_reward": 5.0, "step": 1368 }, { "Total_reward": -3.916876778475853e-07, "advantage_std": 1.4940811395645142, "completion_length": 384.2321472167969, "epoch": 0.4727209944751381, "grad_norm": 3.739572906565051, "kl": 0.09228515625, "learning_rate": 2.7139866495527426e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.8960149884223938, "rewards_raw/accuracy_reward": 0.5437687039375305, "rewards_raw/clip_reward_wrapper": 0.34936127066612244, "rewards_raw/format_reward": 5.0, "step": 1369 }, { "Total_reward": 2.4054733671619033e-07, "advantage_std": 1.576417088508606, "completion_length": 388.51788330078125, "epoch": 0.47306629834254144, "grad_norm": 5.609676558795252, "kl": 0.09326171875, "learning_rate": 2.7112844669017033e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.0531278848648071, "rewards_raw/accuracy_reward": 0.5545705556869507, "rewards_raw/clip_reward_wrapper": 0.3457695245742798, "rewards_raw/format_reward": 5.0, "step": 1370 }, { "Total_reward": 4.3213370304329146e-07, "advantage_std": 1.3225966691970825, "completion_length": 382.58929443359375, "epoch": 0.47341160220994477, "grad_norm": 4.473724487209373, "kl": 0.0947265625, "learning_rate": 2.7085820356111446e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.9962693452835083, "rewards_raw/accuracy_reward": 0.5573371648788452, "rewards_raw/clip_reward_wrapper": 0.3455601930618286, "rewards_raw/format_reward": 5.0, "step": 1371 }, { "Total_reward": -7.982765168890182e-07, "advantage_std": 1.5364335775375366, "completion_length": 401.46429443359375, "epoch": 0.47375690607734805, "grad_norm": 6.003212320571272, "kl": 0.08935546875, "learning_rate": 2.705879358861287e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.1485295295715332, "rewards_raw/accuracy_reward": 0.5673326849937439, "rewards_raw/clip_reward_wrapper": 0.338741660118103, "rewards_raw/format_reward": 5.0, "step": 1372 }, { "Total_reward": 5.811452865600586e-07, "advantage_std": 1.5863527059555054, "completion_length": 394.2321472167969, "epoch": 0.4741022099447514, "grad_norm": 8.234159666479016, "kl": 0.09423828125, "learning_rate": 2.70317643983264e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8766905665397644, "rewards_raw/accuracy_reward": 0.5962668657302856, "rewards_raw/clip_reward_wrapper": 0.34364640712738037, "rewards_raw/format_reward": 5.0, "step": 1373 }, { "Total_reward": -2.1713121611810493e-07, "advantage_std": 1.3992263078689575, "completion_length": 395.0714416503906, "epoch": 0.4744475138121547, "grad_norm": 3.3855400438747636, "kl": 0.09619140625, "learning_rate": 2.7004732817059986e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.6961262226104736, "rewards_raw/accuracy_reward": 0.5589724779129028, "rewards_raw/clip_reward_wrapper": 0.3376811444759369, "rewards_raw/format_reward": 5.0, "step": 1374 }, { "Total_reward": 3.640140846528084e-07, "advantage_std": 1.433488368988037, "completion_length": 391.83929443359375, "epoch": 0.474792817679558, "grad_norm": 4.128225781369143, "kl": 0.0908203125, "learning_rate": 2.697769887662438e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.5732496976852417, "rewards_raw/accuracy_reward": 0.5562471151351929, "rewards_raw/clip_reward_wrapper": 0.332866907119751, "rewards_raw/format_reward": 5.0, "step": 1375 }, { "Total_reward": 1.6391278734317893e-07, "advantage_std": 1.730565071105957, "completion_length": 401.64288330078125, "epoch": 0.47513812154696133, "grad_norm": 4.7548075094193365, "kl": 0.09716796875, "learning_rate": 2.6950662608833127e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8106337785720825, "rewards_raw/accuracy_reward": 0.5583264827728271, "rewards_raw/clip_reward_wrapper": 0.3450295031070709, "rewards_raw/format_reward": 4.901785850524902, "step": 1376 }, { "Total_reward": 1.532690987460228e-07, "advantage_std": 1.418988823890686, "completion_length": 390.3214416503906, "epoch": 0.4754834254143646, "grad_norm": 6.161515313471936, "kl": 0.09423828125, "learning_rate": 2.692362404550249e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6713365912437439, "rewards_raw/accuracy_reward": 0.5755184888839722, "rewards_raw/clip_reward_wrapper": 0.3400768041610718, "rewards_raw/format_reward": 5.0, "step": 1377 }, { "Total_reward": -8.514949634275126e-09, "advantage_std": 1.5966112613677979, "completion_length": 388.01788330078125, "epoch": 0.47582872928176795, "grad_norm": 4.027470887964491, "kl": 0.0927734375, "learning_rate": 2.6896583218451456e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.0428532361984253, "rewards_raw/accuracy_reward": 0.5568269491195679, "rewards_raw/clip_reward_wrapper": 0.33766844868659973, "rewards_raw/format_reward": 5.0, "step": 1378 }, { "Total_reward": 5.023820222049835e-07, "advantage_std": 1.4945886135101318, "completion_length": 392.8571472167969, "epoch": 0.4761740331491713, "grad_norm": 4.727427802870639, "kl": 0.09423828125, "learning_rate": 2.686954015950165e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7600291967391968, "rewards_raw/accuracy_reward": 0.5910916328430176, "rewards_raw/clip_reward_wrapper": 0.33193129301071167, "rewards_raw/format_reward": 5.0, "step": 1379 }, { "Total_reward": 1.4688288274555816e-07, "advantage_std": 1.5269763469696045, "completion_length": 389.1964416503906, "epoch": 0.47651933701657456, "grad_norm": 3.7065242685880677, "kl": 0.095703125, "learning_rate": 2.6842494900477364e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8348944783210754, "rewards_raw/accuracy_reward": 0.5717301368713379, "rewards_raw/clip_reward_wrapper": 0.3395725190639496, "rewards_raw/format_reward": 5.0, "step": 1380 }, { "Total_reward": 2.1925995952187805e-07, "advantage_std": 1.6371142864227295, "completion_length": 396.08929443359375, "epoch": 0.4768646408839779, "grad_norm": 7.921125066898812, "kl": 0.09326171875, "learning_rate": 2.681544747320543e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7228987812995911, "rewards_raw/accuracy_reward": 0.5932125449180603, "rewards_raw/clip_reward_wrapper": 0.34636634588241577, "rewards_raw/format_reward": 5.0, "step": 1381 }, { "Total_reward": -5.747590847704487e-08, "advantage_std": 1.5660064220428467, "completion_length": 388.4107360839844, "epoch": 0.47720994475138123, "grad_norm": 4.265574488637397, "kl": 0.09326171875, "learning_rate": 2.6788397909515263e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9002820253372192, "rewards_raw/accuracy_reward": 0.5856038928031921, "rewards_raw/clip_reward_wrapper": 0.34272339940071106, "rewards_raw/format_reward": 5.0, "step": 1382 }, { "Total_reward": -7.66345493730114e-08, "advantage_std": 1.4543911218643188, "completion_length": 402.9107360839844, "epoch": 0.4775552486187845, "grad_norm": 3.958753635619299, "kl": 0.09228515625, "learning_rate": 2.676134624123879e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9254443049430847, "rewards_raw/accuracy_reward": 0.5808316469192505, "rewards_raw/clip_reward_wrapper": 0.3323005437850952, "rewards_raw/format_reward": 5.0, "step": 1383 }, { "Total_reward": -2.4054733671619033e-07, "advantage_std": 1.5446174144744873, "completion_length": 388.8571472167969, "epoch": 0.47790055248618785, "grad_norm": 4.246337278322362, "kl": 0.09765625, "learning_rate": 2.6734292500210396e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7128594517707825, "rewards_raw/accuracy_reward": 0.5612848997116089, "rewards_raw/clip_reward_wrapper": 0.33498066663742065, "rewards_raw/format_reward": 5.0, "step": 1384 }, { "Total_reward": -1.0696905405893631e-07, "advantage_std": 1.474094033241272, "completion_length": 394.8571472167969, "epoch": 0.4782458563535912, "grad_norm": 6.14941980115197, "kl": 0.09423828125, "learning_rate": 2.6707236718266936e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.9606476426124573, "rewards_raw/accuracy_reward": 0.579412579536438, "rewards_raw/clip_reward_wrapper": 0.34499552845954895, "rewards_raw/format_reward": 5.0, "step": 1385 }, { "Total_reward": -4.810946734323807e-07, "advantage_std": 1.4443113803863525, "completion_length": 397.7321472167969, "epoch": 0.47859116022099446, "grad_norm": 3.6773501590899342, "kl": 0.08837890625, "learning_rate": 2.668017892724763e-07, "loss": 0.0035, "rewards_raw/CoT ICLG": 1.4217289686203003, "rewards_raw/accuracy_reward": 0.5556129217147827, "rewards_raw/clip_reward_wrapper": 0.3580498993396759, "rewards_raw/format_reward": 5.0, "step": 1386 }, { "Total_reward": 1.841357857301773e-07, "advantage_std": 1.5449012517929077, "completion_length": 386.4107360839844, "epoch": 0.4789364640883978, "grad_norm": 8.799431471494406, "kl": 0.09423828125, "learning_rate": 2.665311915899409e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.0029685497283936, "rewards_raw/accuracy_reward": 0.5902578234672546, "rewards_raw/clip_reward_wrapper": 0.3452323079109192, "rewards_raw/format_reward": 5.0, "step": 1387 }, { "Total_reward": -3.6188535545989e-07, "advantage_std": 1.3677468299865723, "completion_length": 375.64288330078125, "epoch": 0.47928176795580113, "grad_norm": 3.425431892756235, "kl": 0.09619140625, "learning_rate": 2.662605744535025e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9025266766548157, "rewards_raw/accuracy_reward": 0.593695878982544, "rewards_raw/clip_reward_wrapper": 0.3406544625759125, "rewards_raw/format_reward": 5.0, "step": 1388 }, { "Total_reward": -2.2671054011880187e-07, "advantage_std": 1.479017972946167, "completion_length": 390.0357360839844, "epoch": 0.4796270718232044, "grad_norm": 8.288993910640936, "kl": 0.09716796875, "learning_rate": 2.6598993818162335e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6747514009475708, "rewards_raw/accuracy_reward": 0.5833251476287842, "rewards_raw/clip_reward_wrapper": 0.3346704840660095, "rewards_raw/format_reward": 5.0, "step": 1389 }, { "Total_reward": 6.380890340551559e-07, "advantage_std": 1.3826634883880615, "completion_length": 373.89288330078125, "epoch": 0.47997237569060774, "grad_norm": 5.580637677721994, "kl": 0.091796875, "learning_rate": 2.657192830927879e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.8350626230239868, "rewards_raw/accuracy_reward": 0.5902978777885437, "rewards_raw/clip_reward_wrapper": 0.3295704424381256, "rewards_raw/format_reward": 5.0, "step": 1390 }, { "Total_reward": -4.470348713425665e-08, "advantage_std": 1.4679433107376099, "completion_length": 383.64288330078125, "epoch": 0.480317679558011, "grad_norm": 9.776490730309481, "kl": 0.0986328125, "learning_rate": 2.654486095055033e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7983144521713257, "rewards_raw/accuracy_reward": 0.5781605839729309, "rewards_raw/clip_reward_wrapper": 0.33621108531951904, "rewards_raw/format_reward": 5.0, "step": 1391 }, { "Total_reward": 1.0324376944481628e-06, "advantage_std": 1.4478414058685303, "completion_length": 390.83929443359375, "epoch": 0.48066298342541436, "grad_norm": 7.260263522824711, "kl": 0.0947265625, "learning_rate": 2.6517791773829796e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6461189389228821, "rewards_raw/accuracy_reward": 0.5848804712295532, "rewards_raw/clip_reward_wrapper": 0.33366137742996216, "rewards_raw/format_reward": 5.0, "step": 1392 }, { "Total_reward": 5.300556153997604e-07, "advantage_std": 1.4086835384368896, "completion_length": 388.4821472167969, "epoch": 0.4810082872928177, "grad_norm": 5.546488119571828, "kl": 0.09765625, "learning_rate": 2.64907208109722e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6718696355819702, "rewards_raw/accuracy_reward": 0.5711225271224976, "rewards_raw/clip_reward_wrapper": 0.3311627507209778, "rewards_raw/format_reward": 5.0, "step": 1393 }, { "Total_reward": 2.341611207157257e-07, "advantage_std": 1.5367087125778198, "completion_length": 380.1964416503906, "epoch": 0.481353591160221, "grad_norm": 4.141614712812456, "kl": 0.09521484375, "learning_rate": 2.6463648093834644e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.3265891075134277, "rewards_raw/accuracy_reward": 0.5973008871078491, "rewards_raw/clip_reward_wrapper": 0.32776013016700745, "rewards_raw/format_reward": 5.0, "step": 1394 }, { "Total_reward": 3.193106010712654e-07, "advantage_std": 1.478081226348877, "completion_length": 388.3214416503906, "epoch": 0.4816988950276243, "grad_norm": 4.399871227756589, "kl": 0.09716796875, "learning_rate": 2.64365736542763e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7735193967819214, "rewards_raw/accuracy_reward": 0.5847257971763611, "rewards_raw/clip_reward_wrapper": 0.33305108547210693, "rewards_raw/format_reward": 5.0, "step": 1395 }, { "Total_reward": -1.2900148931294098e-06, "advantage_std": 1.4245110750198364, "completion_length": 386.4464416503906, "epoch": 0.48204419889502764, "grad_norm": 5.151550153016563, "kl": 0.09375, "learning_rate": 2.6409497524158356e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.1300709247589111, "rewards_raw/accuracy_reward": 0.6041809320449829, "rewards_raw/clip_reward_wrapper": 0.34947070479393005, "rewards_raw/format_reward": 5.0, "step": 1396 }, { "Total_reward": -3.469841942660423e-07, "advantage_std": 1.6174421310424805, "completion_length": 375.83929443359375, "epoch": 0.4823895027624309, "grad_norm": 10.266228517041826, "kl": 0.095703125, "learning_rate": 2.6382419735344003e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.674326479434967, "rewards_raw/accuracy_reward": 0.5794728398323059, "rewards_raw/clip_reward_wrapper": 0.33317744731903076, "rewards_raw/format_reward": 5.0, "step": 1397 }, { "Total_reward": -5.556004794016189e-07, "advantage_std": 1.508163332939148, "completion_length": 382.0000305175781, "epoch": 0.48273480662983426, "grad_norm": 3.5747667717662104, "kl": 0.0927734375, "learning_rate": 2.635534031969837e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.7415432929992676, "rewards_raw/accuracy_reward": 0.5819747447967529, "rewards_raw/clip_reward_wrapper": 0.3439566493034363, "rewards_raw/format_reward": 5.0, "step": 1398 }, { "Total_reward": 7.30156955341954e-07, "advantage_std": 1.3898134231567383, "completion_length": 384.1964416503906, "epoch": 0.4830801104972376, "grad_norm": 3.843134362612055, "kl": 0.0947265625, "learning_rate": 2.632825930908852e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7529028654098511, "rewards_raw/accuracy_reward": 0.5623365640640259, "rewards_raw/clip_reward_wrapper": 0.3418081998825073, "rewards_raw/format_reward": 5.0, "step": 1399 }, { "Total_reward": 8.84490418684436e-07, "advantage_std": 1.25021231174469, "completion_length": 370.5000305175781, "epoch": 0.48342541436464087, "grad_norm": 3.2222290135931293, "kl": 0.09521484375, "learning_rate": 2.630117673538338e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8435256481170654, "rewards_raw/accuracy_reward": 0.5815316438674927, "rewards_raw/clip_reward_wrapper": 0.33369767665863037, "rewards_raw/format_reward": 5.0, "step": 1400 }, { "Total_reward": -7.74860438923497e-07, "advantage_std": 1.4656182527542114, "completion_length": 378.0357360839844, "epoch": 0.4837707182320442, "grad_norm": 7.741571489149501, "kl": 0.09765625, "learning_rate": 2.6274092630453694e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.1577004194259644, "rewards_raw/accuracy_reward": 0.5431531071662903, "rewards_raw/clip_reward_wrapper": 0.339352011680603, "rewards_raw/format_reward": 5.0, "step": 1401 }, { "Total_reward": -7.620880069225677e-07, "advantage_std": 1.4708572626113892, "completion_length": 391.0357360839844, "epoch": 0.48411602209944754, "grad_norm": 6.378221367688454, "kl": 0.08935546875, "learning_rate": 2.6247007026172056e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.8997976779937744, "rewards_raw/accuracy_reward": 0.5643952488899231, "rewards_raw/clip_reward_wrapper": 0.3294502794742584, "rewards_raw/format_reward": 5.0, "step": 1402 }, { "Total_reward": -6.428787173717865e-07, "advantage_std": 1.380124807357788, "completion_length": 387.3750305175781, "epoch": 0.4844613259668508, "grad_norm": 3.762198422509986, "kl": 0.09814453125, "learning_rate": 2.62199199544128e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7769815325737, "rewards_raw/accuracy_reward": 0.5605753660202026, "rewards_raw/clip_reward_wrapper": 0.3442043662071228, "rewards_raw/format_reward": 5.0, "step": 1403 }, { "Total_reward": 1.309173569552513e-07, "advantage_std": 1.5433167219161987, "completion_length": 371.2500305175781, "epoch": 0.48480662983425415, "grad_norm": 5.170368427065442, "kl": 0.09716796875, "learning_rate": 2.6192831447051975e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7796944379806519, "rewards_raw/accuracy_reward": 0.5464221239089966, "rewards_raw/clip_reward_wrapper": 0.33830729126930237, "rewards_raw/format_reward": 5.0, "step": 1404 }, { "Total_reward": -4.6725784841328277e-07, "advantage_std": 1.2872577905654907, "completion_length": 379.4285888671875, "epoch": 0.48515193370165743, "grad_norm": 4.2174668080915305, "kl": 0.09619140625, "learning_rate": 2.6165741535967335e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7332542538642883, "rewards_raw/accuracy_reward": 0.5551673769950867, "rewards_raw/clip_reward_wrapper": 0.33656471967697144, "rewards_raw/format_reward": 5.0, "step": 1405 }, { "Total_reward": 8.089202196970291e-08, "advantage_std": 1.5598524808883667, "completion_length": 388.9285888671875, "epoch": 0.48549723756906077, "grad_norm": 4.715914610876484, "kl": 0.0947265625, "learning_rate": 2.6138650253038287e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.842715859413147, "rewards_raw/accuracy_reward": 0.5653505325317383, "rewards_raw/clip_reward_wrapper": 0.3388529121875763, "rewards_raw/format_reward": 5.0, "step": 1406 }, { "Total_reward": -3.5549913945942535e-07, "advantage_std": 1.4013901948928833, "completion_length": 377.5357360839844, "epoch": 0.4858425414364641, "grad_norm": 4.067552696540738, "kl": 0.09765625, "learning_rate": 2.611155763014584e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.766122579574585, "rewards_raw/accuracy_reward": 0.5849135518074036, "rewards_raw/clip_reward_wrapper": 0.33322498202323914, "rewards_raw/format_reward": 5.0, "step": 1407 }, { "Total_reward": 1.2346677635832748e-07, "advantage_std": 1.5847382545471191, "completion_length": 387.1250305175781, "epoch": 0.4861878453038674, "grad_norm": 13.834830358542707, "kl": 0.09228515625, "learning_rate": 2.6084463699172595e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.7895761728286743, "rewards_raw/accuracy_reward": 0.5648924708366394, "rewards_raw/clip_reward_wrapper": 0.3404236435890198, "rewards_raw/format_reward": 5.0, "step": 1408 }, { "Total_reward": -4.970601708009781e-07, "advantage_std": 1.6858965158462524, "completion_length": 378.2321472167969, "epoch": 0.4865331491712707, "grad_norm": 5.508589697034763, "kl": 0.09423828125, "learning_rate": 2.6057368492002663e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7825003266334534, "rewards_raw/accuracy_reward": 0.576056957244873, "rewards_raw/clip_reward_wrapper": 0.33230915665626526, "rewards_raw/format_reward": 5.0, "step": 1409 }, { "Total_reward": -6.189304144754715e-07, "advantage_std": 1.3928272724151611, "completion_length": 378.8750305175781, "epoch": 0.48687845303867405, "grad_norm": 4.200856171917844, "kl": 0.09521484375, "learning_rate": 2.603027204052171e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8206550478935242, "rewards_raw/accuracy_reward": 0.5884013175964355, "rewards_raw/clip_reward_wrapper": 0.3536824584007263, "rewards_raw/format_reward": 5.0, "step": 1410 }, { "Total_reward": 9.228076578438049e-07, "advantage_std": 1.6372904777526855, "completion_length": 403.0714416503906, "epoch": 0.48722375690607733, "grad_norm": 5.59463405798253, "kl": 0.0927734375, "learning_rate": 2.600317437661682e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.1355732679367065, "rewards_raw/accuracy_reward": 0.584345817565918, "rewards_raw/clip_reward_wrapper": 0.33658137917518616, "rewards_raw/format_reward": 5.0, "step": 1411 }, { "Total_reward": 3.193106010712654e-07, "advantage_std": 1.422092318534851, "completion_length": 385.2321472167969, "epoch": 0.48756906077348067, "grad_norm": 9.508219076765654, "kl": 0.0927734375, "learning_rate": 2.5976075532176524e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.6758329272270203, "rewards_raw/accuracy_reward": 0.5860198736190796, "rewards_raw/clip_reward_wrapper": 0.3307274878025055, "rewards_raw/format_reward": 5.0, "step": 1412 }, { "Total_reward": 8.36593812891806e-07, "advantage_std": 1.5234949588775635, "completion_length": 376.76788330078125, "epoch": 0.487914364640884, "grad_norm": 4.028049773995618, "kl": 0.0966796875, "learning_rate": 2.5948975539090715e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.6618713736534119, "rewards_raw/accuracy_reward": 0.5758650898933411, "rewards_raw/clip_reward_wrapper": 0.3406416177749634, "rewards_raw/format_reward": 5.0, "step": 1413 }, { "Total_reward": -4.76837158203125e-07, "advantage_std": 1.5555375814437866, "completion_length": 383.4107360839844, "epoch": 0.4882596685082873, "grad_norm": 4.396994130231363, "kl": 0.095703125, "learning_rate": 2.5921874429250696e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7244727611541748, "rewards_raw/accuracy_reward": 0.5692161917686462, "rewards_raw/clip_reward_wrapper": 0.3408936560153961, "rewards_raw/format_reward": 5.0, "step": 1414 }, { "Total_reward": 4.385199190437561e-07, "advantage_std": 1.5093859434127808, "completion_length": 391.0357360839844, "epoch": 0.4886049723756906, "grad_norm": 7.989171968697179, "kl": 0.09130859375, "learning_rate": 2.5894772234549015e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.2205915451049805, "rewards_raw/accuracy_reward": 0.5786058306694031, "rewards_raw/clip_reward_wrapper": 0.32730260491371155, "rewards_raw/format_reward": 5.0, "step": 1415 }, { "Total_reward": -4.704509706243698e-07, "advantage_std": 1.2288743257522583, "completion_length": 376.5714416503906, "epoch": 0.4889502762430939, "grad_norm": 4.268466804814682, "kl": 0.0966796875, "learning_rate": 2.586766898687955e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.1527782678604126, "rewards_raw/accuracy_reward": 0.5870816707611084, "rewards_raw/clip_reward_wrapper": 0.34024861454963684, "rewards_raw/format_reward": 5.0, "step": 1416 }, { "Total_reward": 6.162694603517593e-07, "advantage_std": 1.507804036140442, "completion_length": 393.7857360839844, "epoch": 0.48929558011049723, "grad_norm": 4.070353821139167, "kl": 0.09521484375, "learning_rate": 2.584056471813739e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8097110390663147, "rewards_raw/accuracy_reward": 0.5747601985931396, "rewards_raw/clip_reward_wrapper": 0.3296605348587036, "rewards_raw/format_reward": 5.0, "step": 1417 }, { "Total_reward": 5.747590847704487e-08, "advantage_std": 1.3539422750473022, "completion_length": 387.5000305175781, "epoch": 0.48964088397790057, "grad_norm": 3.8170793869885618, "kl": 0.0966796875, "learning_rate": 2.5813459460218826e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.0217528343200684, "rewards_raw/accuracy_reward": 0.5800594091415405, "rewards_raw/clip_reward_wrapper": 0.3352510631084442, "rewards_raw/format_reward": 5.0, "step": 1418 }, { "Total_reward": -1.1197158755749115e-06, "advantage_std": 1.6520799398422241, "completion_length": 381.5357360839844, "epoch": 0.48998618784530384, "grad_norm": 6.314642914067913, "kl": 0.09619140625, "learning_rate": 2.578635324502134e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.710936427116394, "rewards_raw/accuracy_reward": 0.5866801738739014, "rewards_raw/clip_reward_wrapper": 0.33381229639053345, "rewards_raw/format_reward": 5.0, "step": 1419 }, { "Total_reward": -6.024326921760803e-07, "advantage_std": 1.3007724285125732, "completion_length": 375.01788330078125, "epoch": 0.4903314917127072, "grad_norm": 2.965447338394891, "kl": 0.1005859375, "learning_rate": 2.57592461044435e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.9836928844451904, "rewards_raw/accuracy_reward": 0.5905179977416992, "rewards_raw/clip_reward_wrapper": 0.337110698223114, "rewards_raw/format_reward": 5.0, "step": 1420 }, { "Total_reward": -1.087252599063504e-06, "advantage_std": 1.2778325080871582, "completion_length": 392.2857360839844, "epoch": 0.4906767955801105, "grad_norm": 2.8875494162306907, "kl": 0.0927734375, "learning_rate": 2.5732138070385e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.6698800921440125, "rewards_raw/accuracy_reward": 0.5766693949699402, "rewards_raw/clip_reward_wrapper": 0.3249891400337219, "rewards_raw/format_reward": 5.0, "step": 1421 }, { "Total_reward": -3.597566262669716e-07, "advantage_std": 1.5743824243545532, "completion_length": 374.46429443359375, "epoch": 0.4910220994475138, "grad_norm": 4.776613037843931, "kl": 0.103515625, "learning_rate": 2.570502917474656e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.7321210503578186, "rewards_raw/accuracy_reward": 0.5855973362922668, "rewards_raw/clip_reward_wrapper": 0.3408949077129364, "rewards_raw/format_reward": 5.0, "step": 1422 }, { "Total_reward": 7.312212915167038e-07, "advantage_std": 1.5509129762649536, "completion_length": 387.4285888671875, "epoch": 0.49136740331491713, "grad_norm": 5.50175646848392, "kl": 0.09619140625, "learning_rate": 2.567791944942992e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.6147847175598145, "rewards_raw/accuracy_reward": 0.5716566443443298, "rewards_raw/clip_reward_wrapper": 0.3443187475204468, "rewards_raw/format_reward": 5.0, "step": 1423 }, { "Total_reward": 4.087175966560608e-07, "advantage_std": 1.698358416557312, "completion_length": 394.58929443359375, "epoch": 0.49171270718232046, "grad_norm": 5.789792003902216, "kl": 0.0908203125, "learning_rate": 2.56508089263378e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 1.6297500133514404, "rewards_raw/accuracy_reward": 0.5740936994552612, "rewards_raw/clip_reward_wrapper": 0.3413899540901184, "rewards_raw/format_reward": 5.0, "step": 1424 }, { "Total_reward": 5.992395699649933e-07, "advantage_std": 1.5733850002288818, "completion_length": 374.89288330078125, "epoch": 0.49205801104972374, "grad_norm": 4.221910395265328, "kl": 0.09716796875, "learning_rate": 2.562369763737387e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7119475603103638, "rewards_raw/accuracy_reward": 0.5849709510803223, "rewards_raw/clip_reward_wrapper": 0.34256091713905334, "rewards_raw/format_reward": 5.0, "step": 1425 }, { "Total_reward": -8.749110520511749e-07, "advantage_std": 1.5680993795394897, "completion_length": 386.83929443359375, "epoch": 0.4924033149171271, "grad_norm": 3.833248054725491, "kl": 0.09814453125, "learning_rate": 2.559658561444268e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.10184645652771, "rewards_raw/accuracy_reward": 0.5574817657470703, "rewards_raw/clip_reward_wrapper": 0.3383871018886566, "rewards_raw/format_reward": 5.0, "step": 1426 }, { "Total_reward": 1.2133803295455436e-07, "advantage_std": 1.5404473543167114, "completion_length": 397.4285888671875, "epoch": 0.4927486187845304, "grad_norm": 8.429281295730577, "kl": 0.0986328125, "learning_rate": 2.556947288944965e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8218700885772705, "rewards_raw/accuracy_reward": 0.5723780393600464, "rewards_raw/clip_reward_wrapper": 0.3364706337451935, "rewards_raw/format_reward": 5.0, "step": 1427 }, { "Total_reward": 5.673085183843796e-07, "advantage_std": 1.4108664989471436, "completion_length": 381.2321472167969, "epoch": 0.4930939226519337, "grad_norm": 3.6998119480325506, "kl": 0.099609375, "learning_rate": 2.554235949430104e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7937996983528137, "rewards_raw/accuracy_reward": 0.5878415107727051, "rewards_raw/clip_reward_wrapper": 0.3359336256980896, "rewards_raw/format_reward": 5.0, "step": 1428 }, { "Total_reward": 3.107956558778824e-07, "advantage_std": 1.7346254587173462, "completion_length": 386.6071472167969, "epoch": 0.493439226519337, "grad_norm": 8.995418782814577, "kl": 0.0947265625, "learning_rate": 2.5515245460903883e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.1701018810272217, "rewards_raw/accuracy_reward": 0.5839632749557495, "rewards_raw/clip_reward_wrapper": 0.3280177116394043, "rewards_raw/format_reward": 5.0, "step": 1429 }, { "Total_reward": 5.502786279976135e-07, "advantage_std": 1.4149800539016724, "completion_length": 394.7500305175781, "epoch": 0.4937845303867403, "grad_norm": 6.065261194731412, "kl": 0.09716796875, "learning_rate": 2.548813082116597e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7418599724769592, "rewards_raw/accuracy_reward": 0.5783302187919617, "rewards_raw/clip_reward_wrapper": 0.34727364778518677, "rewards_raw/format_reward": 5.0, "step": 1430 }, { "Total_reward": -7.557017767112484e-08, "advantage_std": 1.6486601829528809, "completion_length": 391.33929443359375, "epoch": 0.49412983425414364, "grad_norm": 6.71909962856744, "kl": 0.10009765625, "learning_rate": 2.5461015606995794e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 1.3838410377502441, "rewards_raw/accuracy_reward": 0.5954713821411133, "rewards_raw/clip_reward_wrapper": 0.3278883993625641, "rewards_raw/format_reward": 5.0, "step": 1431 }, { "Total_reward": 1.0239226639896515e-06, "advantage_std": 1.4622753858566284, "completion_length": 386.58929443359375, "epoch": 0.494475138121547, "grad_norm": 4.812991476317726, "kl": 0.099609375, "learning_rate": 2.543389985030255e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.9533836841583252, "rewards_raw/accuracy_reward": 0.5831606984138489, "rewards_raw/clip_reward_wrapper": 0.3575593829154968, "rewards_raw/format_reward": 5.0, "step": 1432 }, { "Total_reward": -2.682209014892578e-07, "advantage_std": 1.632008671760559, "completion_length": 386.14288330078125, "epoch": 0.49482044198895025, "grad_norm": 5.27453864944586, "kl": 0.10595703125, "learning_rate": 2.540678358299605e-07, "loss": 0.0042, "rewards_raw/CoT ICLG": 0.9048643112182617, "rewards_raw/accuracy_reward": 0.58796226978302, "rewards_raw/clip_reward_wrapper": 0.3328983783721924, "rewards_raw/format_reward": 5.0, "step": 1433 }, { "Total_reward": -3.491129234589607e-07, "advantage_std": 1.6184968948364258, "completion_length": 385.1250305175781, "epoch": 0.4951657458563536, "grad_norm": 5.352239316595702, "kl": 0.09375, "learning_rate": 2.5379666836986704e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.8259117007255554, "rewards_raw/accuracy_reward": 0.5685825347900391, "rewards_raw/clip_reward_wrapper": 0.3237302899360657, "rewards_raw/format_reward": 5.0, "step": 1434 }, { "Total_reward": -2.937657654911163e-07, "advantage_std": 1.611228585243225, "completion_length": 390.1071472167969, "epoch": 0.4955110497237569, "grad_norm": 15.402709559932664, "kl": 0.09326171875, "learning_rate": 2.5352549644185494e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9311500787734985, "rewards_raw/accuracy_reward": 0.5607922673225403, "rewards_raw/clip_reward_wrapper": 0.3419555723667145, "rewards_raw/format_reward": 4.964285850524902, "step": 1435 }, { "Total_reward": -7.471868457287201e-07, "advantage_std": 1.4735021591186523, "completion_length": 398.2500305175781, "epoch": 0.4958563535911602, "grad_norm": 5.011998991682961, "kl": 0.0908203125, "learning_rate": 2.5325432036503935e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.8609395623207092, "rewards_raw/accuracy_reward": 0.5620566606521606, "rewards_raw/clip_reward_wrapper": 0.3509470522403717, "rewards_raw/format_reward": 5.0, "step": 1436 }, { "Total_reward": -2.884439140871109e-07, "advantage_std": 1.3527648448944092, "completion_length": 376.8750305175781, "epoch": 0.49620165745856354, "grad_norm": 3.3161547742863906, "kl": 0.09033203125, "learning_rate": 2.529831404585401e-07, "loss": 0.0036, "rewards_raw/CoT ICLG": 0.8636061549186707, "rewards_raw/accuracy_reward": 0.6170417666435242, "rewards_raw/clip_reward_wrapper": 0.33935222029685974, "rewards_raw/format_reward": 5.0, "step": 1437 }, { "Total_reward": -5.779522211923904e-07, "advantage_std": 1.4126874208450317, "completion_length": 390.21429443359375, "epoch": 0.4965469613259669, "grad_norm": 3.2569691390210274, "kl": 0.0947265625, "learning_rate": 2.5271195704148185e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.2065714597702026, "rewards_raw/accuracy_reward": 0.597659707069397, "rewards_raw/clip_reward_wrapper": 0.3472095727920532, "rewards_raw/format_reward": 5.0, "step": 1438 }, { "Total_reward": -4.3426243223620986e-07, "advantage_std": 1.4447860717773438, "completion_length": 380.51788330078125, "epoch": 0.49689226519337015, "grad_norm": 6.481534892715683, "kl": 0.0947265625, "learning_rate": 2.5244077043299307e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.5855528712272644, "rewards_raw/accuracy_reward": 0.5756439566612244, "rewards_raw/clip_reward_wrapper": 0.3319206237792969, "rewards_raw/format_reward": 5.0, "step": 1439 }, { "Total_reward": 1.5922955753921997e-06, "advantage_std": 1.1748250722885132, "completion_length": 378.46429443359375, "epoch": 0.4972375690607735, "grad_norm": 3.3654336592119005, "kl": 0.09521484375, "learning_rate": 2.521695809522061e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.0649688243865967, "rewards_raw/accuracy_reward": 0.571343183517456, "rewards_raw/clip_reward_wrapper": 0.3269900679588318, "rewards_raw/format_reward": 5.0, "step": 1440 }, { "Total_reward": 4.6725784841328277e-07, "advantage_std": 1.376003384590149, "completion_length": 381.58929443359375, "epoch": 0.4975828729281768, "grad_norm": 3.756018425433902, "kl": 0.1025390625, "learning_rate": 2.518983889182567e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.6385988593101501, "rewards_raw/accuracy_reward": 0.5642992258071899, "rewards_raw/clip_reward_wrapper": 0.3386222720146179, "rewards_raw/format_reward": 5.0, "step": 1441 }, { "Total_reward": -6.556511493727157e-07, "advantage_std": 1.6027393341064453, "completion_length": 389.5714416503906, "epoch": 0.4979281767955801, "grad_norm": 4.06834600856355, "kl": 0.09716796875, "learning_rate": 2.5162719465028363e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9174368381500244, "rewards_raw/accuracy_reward": 0.5758584141731262, "rewards_raw/clip_reward_wrapper": 0.3664412796497345, "rewards_raw/format_reward": 5.0, "step": 1442 }, { "Total_reward": -1.596553005356327e-07, "advantage_std": 1.4603538513183594, "completion_length": 401.1071472167969, "epoch": 0.49827348066298344, "grad_norm": 4.113508253231199, "kl": 0.0966796875, "learning_rate": 2.5135599846742813e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7568613290786743, "rewards_raw/accuracy_reward": 0.5829422473907471, "rewards_raw/clip_reward_wrapper": 0.3343200087547302, "rewards_raw/format_reward": 5.0, "step": 1443 }, { "Total_reward": -3.7997963886482466e-07, "advantage_std": 1.3190789222717285, "completion_length": 376.1785888671875, "epoch": 0.4986187845303867, "grad_norm": 4.712448405583287, "kl": 0.10107421875, "learning_rate": 2.5108480068883397e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8191953897476196, "rewards_raw/accuracy_reward": 0.568769097328186, "rewards_raw/clip_reward_wrapper": 0.348213791847229, "rewards_raw/format_reward": 5.0, "step": 1444 }, { "Total_reward": -3.0440943987741775e-07, "advantage_std": 1.5428924560546875, "completion_length": 383.6250305175781, "epoch": 0.49896408839779005, "grad_norm": 5.2389827866181164, "kl": 0.09765625, "learning_rate": 2.508136016336465e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9277024269104004, "rewards_raw/accuracy_reward": 0.5789950489997864, "rewards_raw/clip_reward_wrapper": 0.3421257734298706, "rewards_raw/format_reward": 5.0, "step": 1445 }, { "Total_reward": -7.344144137277908e-07, "advantage_std": 1.649749755859375, "completion_length": 392.21429443359375, "epoch": 0.4993093922651934, "grad_norm": 6.531208287658396, "kl": 0.095703125, "learning_rate": 2.505424016210129e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.6038404703140259, "rewards_raw/accuracy_reward": 0.5758102536201477, "rewards_raw/clip_reward_wrapper": 0.34435445070266724, "rewards_raw/format_reward": 5.0, "step": 1446 }, { "Total_reward": -1.6178404393940582e-07, "advantage_std": 1.34548020362854, "completion_length": 387.9821472167969, "epoch": 0.49965469613259667, "grad_norm": 3.2409798054284416, "kl": 0.10009765625, "learning_rate": 2.5027120097008104e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8544610738754272, "rewards_raw/accuracy_reward": 0.5745611190795898, "rewards_raw/clip_reward_wrapper": 0.3234385550022125, "rewards_raw/format_reward": 5.0, "step": 1447 }, { "Total_reward": 2.1500247271433182e-07, "advantage_std": 1.6560612916946411, "completion_length": 392.9285888671875, "epoch": 0.5, "grad_norm": 7.549139197948304, "kl": 0.10009765625, "learning_rate": 2.5e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7043477296829224, "rewards_raw/accuracy_reward": 0.5662786364555359, "rewards_raw/clip_reward_wrapper": 0.3486778438091278, "rewards_raw/format_reward": 5.0, "step": 1448 }, { "Total_reward": -3.405979782655777e-07, "advantage_std": 1.5004713535308838, "completion_length": 369.7857360839844, "epoch": 0.5003453038674033, "grad_norm": 5.733857591857502, "kl": 0.10107421875, "learning_rate": 2.49728799029919e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.9264684915542603, "rewards_raw/accuracy_reward": 0.5632349848747253, "rewards_raw/clip_reward_wrapper": 0.32419121265411377, "rewards_raw/format_reward": 5.0, "step": 1449 }, { "Total_reward": -1.1569687785595306e-06, "advantage_std": 1.1073139905929565, "completion_length": 383.8035888671875, "epoch": 0.5006906077348067, "grad_norm": 2.8201607320705517, "kl": 0.10009765625, "learning_rate": 2.4945759837898714e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.9494460225105286, "rewards_raw/accuracy_reward": 0.5866327285766602, "rewards_raw/clip_reward_wrapper": 0.32054397463798523, "rewards_raw/format_reward": 5.0, "step": 1450 }, { "Total_reward": 4.917383193969727e-07, "advantage_std": 1.6388102769851685, "completion_length": 381.58929443359375, "epoch": 0.50103591160221, "grad_norm": 5.301433262416194, "kl": 0.1005859375, "learning_rate": 2.491863983663535e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 1.1731995344161987, "rewards_raw/accuracy_reward": 0.5987117290496826, "rewards_raw/clip_reward_wrapper": 0.32456350326538086, "rewards_raw/format_reward": 5.0, "step": 1451 }, { "Total_reward": -2.384185791015625e-07, "advantage_std": 1.6216667890548706, "completion_length": 382.8750305175781, "epoch": 0.5013812154696132, "grad_norm": 4.2657982250171145, "kl": 0.09619140625, "learning_rate": 2.4891519931116606e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7799550890922546, "rewards_raw/accuracy_reward": 0.570393979549408, "rewards_raw/clip_reward_wrapper": 0.33930379152297974, "rewards_raw/format_reward": 5.0, "step": 1452 }, { "Total_reward": -1.309173569552513e-07, "advantage_std": 1.6067211627960205, "completion_length": 402.26788330078125, "epoch": 0.5017265193370166, "grad_norm": 4.827026035308721, "kl": 0.09619140625, "learning_rate": 2.4864400153257185e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.5331209897994995, "rewards_raw/accuracy_reward": 0.5718077421188354, "rewards_raw/clip_reward_wrapper": 0.3541997969150543, "rewards_raw/format_reward": 5.0, "step": 1453 }, { "Total_reward": -2.937657654911163e-07, "advantage_std": 1.5331752300262451, "completion_length": 374.7857360839844, "epoch": 0.5020718232044199, "grad_norm": 4.207751595226441, "kl": 0.1005859375, "learning_rate": 2.483728053497164e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 1.4850132465362549, "rewards_raw/accuracy_reward": 0.5908195376396179, "rewards_raw/clip_reward_wrapper": 0.33147966861724854, "rewards_raw/format_reward": 5.0, "step": 1454 }, { "Total_reward": -3.0387727179004287e-07, "advantage_std": 1.3825767040252686, "completion_length": 381.14288330078125, "epoch": 0.5024171270718232, "grad_norm": 3.577691151774109, "kl": 0.09765625, "learning_rate": 2.481016110817433e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 2.363328695297241, "rewards_raw/accuracy_reward": 0.5678959488868713, "rewards_raw/clip_reward_wrapper": 0.33974435925483704, "rewards_raw/format_reward": 5.0, "step": 1455 }, { "Total_reward": 1.553978279389412e-07, "advantage_std": 1.3732978105545044, "completion_length": 372.1964416503906, "epoch": 0.5027624309392266, "grad_norm": 3.500474157554274, "kl": 0.09716796875, "learning_rate": 2.4783041904779383e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.2637550830841064, "rewards_raw/accuracy_reward": 0.5551690459251404, "rewards_raw/clip_reward_wrapper": 0.34005826711654663, "rewards_raw/format_reward": 5.0, "step": 1456 }, { "Total_reward": -3.384692490726593e-07, "advantage_std": 1.4651904106140137, "completion_length": 375.21429443359375, "epoch": 0.5031077348066298, "grad_norm": 3.936337682766922, "kl": 0.09521484375, "learning_rate": 2.475592295670069e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.854560136795044, "rewards_raw/accuracy_reward": 0.5607773065567017, "rewards_raw/clip_reward_wrapper": 0.3361368775367737, "rewards_raw/format_reward": 5.0, "step": 1457 }, { "Total_reward": 2.5459698917984497e-06, "advantage_std": 1.3736733198165894, "completion_length": 390.6071472167969, "epoch": 0.5034530386740331, "grad_norm": 3.9587250074134177, "kl": 0.091796875, "learning_rate": 2.4728804295851813e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9580224752426147, "rewards_raw/accuracy_reward": 0.587287187576294, "rewards_raw/clip_reward_wrapper": 0.3431095480918884, "rewards_raw/format_reward": 5.0, "step": 1458 }, { "Total_reward": 1.5007599074579048e-07, "advantage_std": 1.3596495389938354, "completion_length": 391.0357360839844, "epoch": 0.5037983425414365, "grad_norm": 5.2651682838689435, "kl": 0.095703125, "learning_rate": 2.4701685954145976e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.2436330318450928, "rewards_raw/accuracy_reward": 0.5686664581298828, "rewards_raw/clip_reward_wrapper": 0.3185670077800751, "rewards_raw/format_reward": 5.0, "step": 1459 }, { "Total_reward": 6.85453471760411e-07, "advantage_std": 1.4020169973373413, "completion_length": 369.4821472167969, "epoch": 0.5041436464088398, "grad_norm": 5.878341879601319, "kl": 0.10205078125, "learning_rate": 2.467456796349607e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 0.9059459567070007, "rewards_raw/accuracy_reward": 0.5633392333984375, "rewards_raw/clip_reward_wrapper": 0.3311808705329895, "rewards_raw/format_reward": 5.0, "step": 1460 }, { "Total_reward": -3.6028879435434646e-07, "advantage_std": 1.3585301637649536, "completion_length": 384.89288330078125, "epoch": 0.5044889502762431, "grad_norm": 9.332892513792176, "kl": 0.09814453125, "learning_rate": 2.464745035581451e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9190171957015991, "rewards_raw/accuracy_reward": 0.5775415301322937, "rewards_raw/clip_reward_wrapper": 0.3168168067932129, "rewards_raw/format_reward": 5.0, "step": 1461 }, { "Total_reward": 1.5114035534224968e-07, "advantage_std": 1.5959374904632568, "completion_length": 376.9821472167969, "epoch": 0.5048342541436464, "grad_norm": 3.6123352121200325, "kl": 0.09765625, "learning_rate": 2.462033316301331e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.0621894598007202, "rewards_raw/accuracy_reward": 0.5787113904953003, "rewards_raw/clip_reward_wrapper": 0.35668736696243286, "rewards_raw/format_reward": 5.0, "step": 1462 }, { "Total_reward": 9.749617220222717e-07, "advantage_std": 1.4550583362579346, "completion_length": 382.9107360839844, "epoch": 0.5051795580110497, "grad_norm": 4.437869439660199, "kl": 0.10009765625, "learning_rate": 2.4593216417003953e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 1.7092431783676147, "rewards_raw/accuracy_reward": 0.5844104290008545, "rewards_raw/clip_reward_wrapper": 0.34678956866264343, "rewards_raw/format_reward": 5.0, "step": 1463 }, { "Total_reward": 7.993409099071869e-07, "advantage_std": 1.517919659614563, "completion_length": 378.14288330078125, "epoch": 0.505524861878453, "grad_norm": 4.262236610706209, "kl": 0.09521484375, "learning_rate": 2.4566100149697453e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.109644889831543, "rewards_raw/accuracy_reward": 0.6036688089370728, "rewards_raw/clip_reward_wrapper": 0.35335931181907654, "rewards_raw/format_reward": 5.0, "step": 1464 }, { "Total_reward": 3.5549913945942535e-07, "advantage_std": 1.3914296627044678, "completion_length": 372.1250305175781, "epoch": 0.5058701657458563, "grad_norm": 4.363102525089127, "kl": 0.1005859375, "learning_rate": 2.4538984393004203e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7762078642845154, "rewards_raw/accuracy_reward": 0.578759491443634, "rewards_raw/clip_reward_wrapper": 0.3383403718471527, "rewards_raw/format_reward": 5.0, "step": 1465 }, { "Total_reward": -1.1771917343139648e-06, "advantage_std": 1.4427419900894165, "completion_length": 374.21429443359375, "epoch": 0.5062154696132597, "grad_norm": 3.854363269292082, "kl": 0.09912109375, "learning_rate": 2.4511869178834026e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8343056440353394, "rewards_raw/accuracy_reward": 0.5724868774414062, "rewards_raw/clip_reward_wrapper": 0.3477107882499695, "rewards_raw/format_reward": 5.0, "step": 1466 }, { "Total_reward": -1.6391278734317893e-07, "advantage_std": 1.4991644620895386, "completion_length": 378.7500305175781, "epoch": 0.506560773480663, "grad_norm": 5.605616141833824, "kl": 0.10107421875, "learning_rate": 2.4484754539096115e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7930784821510315, "rewards_raw/accuracy_reward": 0.5757755041122437, "rewards_raw/clip_reward_wrapper": 0.3432028889656067, "rewards_raw/format_reward": 5.0, "step": 1467 }, { "Total_reward": 5.343130737855972e-07, "advantage_std": 1.5573142766952515, "completion_length": 369.8750305175781, "epoch": 0.5069060773480663, "grad_norm": 5.885227504532377, "kl": 0.09912109375, "learning_rate": 2.4457640505698967e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8682224154472351, "rewards_raw/accuracy_reward": 0.5545142292976379, "rewards_raw/clip_reward_wrapper": 0.3440708816051483, "rewards_raw/format_reward": 5.0, "step": 1468 }, { "Total_reward": 1.0005065576024208e-07, "advantage_std": 1.4448001384735107, "completion_length": 380.58929443359375, "epoch": 0.5072513812154696, "grad_norm": 5.133733485887053, "kl": 0.095703125, "learning_rate": 2.4430527110550345e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.8753615021705627, "rewards_raw/accuracy_reward": 0.598431408405304, "rewards_raw/clip_reward_wrapper": 0.32309451699256897, "rewards_raw/format_reward": 5.0, "step": 1469 }, { "Total_reward": 5.470855057865265e-07, "advantage_std": 1.5102444887161255, "completion_length": 388.3750305175781, "epoch": 0.5075966850828729, "grad_norm": 4.733641788067501, "kl": 0.1005859375, "learning_rate": 2.440341438555732e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.6151690483093262, "rewards_raw/accuracy_reward": 0.5619034767150879, "rewards_raw/clip_reward_wrapper": 0.3476284146308899, "rewards_raw/format_reward": 5.0, "step": 1470 }, { "Total_reward": -2.2351743211856956e-07, "advantage_std": 1.3294450044631958, "completion_length": 390.51788330078125, "epoch": 0.5079419889502762, "grad_norm": 3.684473267636298, "kl": 0.0986328125, "learning_rate": 2.437630236262614e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.039695143699646, "rewards_raw/accuracy_reward": 0.5957935452461243, "rewards_raw/clip_reward_wrapper": 0.33269202709198, "rewards_raw/format_reward": 5.0, "step": 1471 }, { "Total_reward": -2.55448497910038e-07, "advantage_std": 1.4493935108184814, "completion_length": 383.1607360839844, "epoch": 0.5082872928176796, "grad_norm": 4.716128136093061, "kl": 0.0986328125, "learning_rate": 2.43491910736622e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.9825854301452637, "rewards_raw/accuracy_reward": 0.5838916897773743, "rewards_raw/clip_reward_wrapper": 0.3363057076931, "rewards_raw/format_reward": 5.0, "step": 1472 }, { "Total_reward": -3.618853483544626e-08, "advantage_std": 1.5674774646759033, "completion_length": 381.8571472167969, "epoch": 0.5086325966850829, "grad_norm": 6.508867334192935, "kl": 0.09619140625, "learning_rate": 2.4322080550570084e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 2.164062023162842, "rewards_raw/accuracy_reward": 0.534011721611023, "rewards_raw/clip_reward_wrapper": 0.33122560381889343, "rewards_raw/format_reward": 5.0, "step": 1473 }, { "Total_reward": 5.093004347145325e-07, "advantage_std": 1.3593679666519165, "completion_length": 388.6785888671875, "epoch": 0.5089779005524862, "grad_norm": 7.187796906940697, "kl": 0.09716796875, "learning_rate": 2.429497082525345e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.7395402789115906, "rewards_raw/accuracy_reward": 0.5738299489021301, "rewards_raw/clip_reward_wrapper": 0.3451088070869446, "rewards_raw/format_reward": 5.0, "step": 1474 }, { "Total_reward": -1.7029898913278885e-07, "advantage_std": 1.5426915884017944, "completion_length": 395.26788330078125, "epoch": 0.5093232044198895, "grad_norm": 4.451453193574564, "kl": 0.09912109375, "learning_rate": 2.4267861929615e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8039190769195557, "rewards_raw/accuracy_reward": 0.5565918684005737, "rewards_raw/clip_reward_wrapper": 0.32929229736328125, "rewards_raw/format_reward": 5.0, "step": 1475 }, { "Total_reward": 4.938671054333099e-07, "advantage_std": 1.7195489406585693, "completion_length": 386.21429443359375, "epoch": 0.5096685082872928, "grad_norm": 4.6265544571727775, "kl": 0.10107421875, "learning_rate": 2.42407538955565e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8381402492523193, "rewards_raw/accuracy_reward": 0.5690159797668457, "rewards_raw/clip_reward_wrapper": 0.34584078192710876, "rewards_raw/format_reward": 5.0, "step": 1476 }, { "Total_reward": -2.852508202977333e-07, "advantage_std": 1.4599679708480835, "completion_length": 373.4107360839844, "epoch": 0.5100138121546961, "grad_norm": 3.987648452232754, "kl": 0.09423828125, "learning_rate": 2.421364675497866e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.5047914385795593, "rewards_raw/accuracy_reward": 0.5649449229240417, "rewards_raw/clip_reward_wrapper": 0.3427009880542755, "rewards_raw/format_reward": 5.0, "step": 1477 }, { "Total_reward": -1.4049666674509353e-07, "advantage_std": 1.7045674324035645, "completion_length": 381.39288330078125, "epoch": 0.5103591160220995, "grad_norm": 5.628237223100911, "kl": 0.09375, "learning_rate": 2.4186540539781166e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 1.270416021347046, "rewards_raw/accuracy_reward": 0.5968570113182068, "rewards_raw/clip_reward_wrapper": 0.3516162633895874, "rewards_raw/format_reward": 5.0, "step": 1478 }, { "Total_reward": -2.863151848941925e-07, "advantage_std": 1.5982542037963867, "completion_length": 374.76788330078125, "epoch": 0.5107044198895028, "grad_norm": 5.205439149290943, "kl": 0.10009765625, "learning_rate": 2.415943528186261e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8508612513542175, "rewards_raw/accuracy_reward": 0.5690685510635376, "rewards_raw/clip_reward_wrapper": 0.3380429446697235, "rewards_raw/format_reward": 5.0, "step": 1479 }, { "Total_reward": -7.642167361154861e-07, "advantage_std": 1.2893080711364746, "completion_length": 379.2500305175781, "epoch": 0.511049723756906, "grad_norm": 2.944129795139196, "kl": 0.09423828125, "learning_rate": 2.413233101312045e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 1.1370927095413208, "rewards_raw/accuracy_reward": 0.5779367685317993, "rewards_raw/clip_reward_wrapper": 0.3297813832759857, "rewards_raw/format_reward": 5.0, "step": 1480 }, { "Total_reward": -5.3218435880353354e-08, "advantage_std": 1.5029973983764648, "completion_length": 360.9285888671875, "epoch": 0.5113950276243094, "grad_norm": 5.174416437719546, "kl": 0.099609375, "learning_rate": 2.410522776545098e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 1.0597853660583496, "rewards_raw/accuracy_reward": 0.5616678595542908, "rewards_raw/clip_reward_wrapper": 0.32894739508628845, "rewards_raw/format_reward": 5.0, "step": 1481 }, { "Total_reward": 1.1048147143810638e-06, "advantage_std": 1.608521819114685, "completion_length": 383.71429443359375, "epoch": 0.5117403314917127, "grad_norm": 4.584773667968992, "kl": 0.09619140625, "learning_rate": 2.4078125570749307e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.2211756706237793, "rewards_raw/accuracy_reward": 0.5901548266410828, "rewards_raw/clip_reward_wrapper": 0.3439733684062958, "rewards_raw/format_reward": 5.0, "step": 1482 }, { "Total_reward": -6.003039629831619e-07, "advantage_std": 1.4717512130737305, "completion_length": 377.9821472167969, "epoch": 0.512085635359116, "grad_norm": 3.8314660021013784, "kl": 0.10546875, "learning_rate": 2.4051024460909283e-07, "loss": 0.0042, "rewards_raw/CoT ICLG": 0.967227578163147, "rewards_raw/accuracy_reward": 0.5894429087638855, "rewards_raw/clip_reward_wrapper": 0.33728691935539246, "rewards_raw/format_reward": 5.0, "step": 1483 }, { "Total_reward": 4.193612710423622e-07, "advantage_std": 1.3991847038269043, "completion_length": 373.7500305175781, "epoch": 0.5124309392265194, "grad_norm": 9.09914130560984, "kl": 0.09765625, "learning_rate": 2.402392446782348e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 1.6353858709335327, "rewards_raw/accuracy_reward": 0.5427839159965515, "rewards_raw/clip_reward_wrapper": 0.3417072892189026, "rewards_raw/format_reward": 5.0, "step": 1484 }, { "Total_reward": -8.600098908573273e-07, "advantage_std": 1.6863532066345215, "completion_length": 384.9464416503906, "epoch": 0.5127762430939227, "grad_norm": 22.93074127957748, "kl": 0.1005859375, "learning_rate": 2.399682562338318e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7823826670646667, "rewards_raw/accuracy_reward": 0.5902326107025146, "rewards_raw/clip_reward_wrapper": 0.31254684925079346, "rewards_raw/format_reward": 5.0, "step": 1485 }, { "Total_reward": 1.0643686820799303e-08, "advantage_std": 1.6539512872695923, "completion_length": 373.2321472167969, "epoch": 0.513121546961326, "grad_norm": 7.548982940386157, "kl": 0.09228515625, "learning_rate": 2.396972795947829e-07, "loss": 0.0037, "rewards_raw/CoT ICLG": 0.9938065409660339, "rewards_raw/accuracy_reward": 0.574382483959198, "rewards_raw/clip_reward_wrapper": 0.3309870660305023, "rewards_raw/format_reward": 5.0, "step": 1486 }, { "Total_reward": -1.4901161193847656e-07, "advantage_std": 1.4619932174682617, "completion_length": 379.58929443359375, "epoch": 0.5134668508287292, "grad_norm": 7.012856543387009, "kl": 0.10009765625, "learning_rate": 2.3942631507997335e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8474305868148804, "rewards_raw/accuracy_reward": 0.5825018882751465, "rewards_raw/clip_reward_wrapper": 0.33132296800613403, "rewards_raw/format_reward": 5.0, "step": 1487 }, { "Total_reward": 7.099339427441009e-07, "advantage_std": 1.5008430480957031, "completion_length": 377.5357360839844, "epoch": 0.5138121546961326, "grad_norm": 5.682703101857971, "kl": 0.10302734375, "learning_rate": 2.3915536300827413e-07, "loss": 0.0041, "rewards_raw/CoT ICLG": 1.5177502632141113, "rewards_raw/accuracy_reward": 0.577979564666748, "rewards_raw/clip_reward_wrapper": 0.3390918970108032, "rewards_raw/format_reward": 5.0, "step": 1488 }, { "Total_reward": 1.1708056035786285e-07, "advantage_std": 1.3398616313934326, "completion_length": 389.4107360839844, "epoch": 0.5141574585635359, "grad_norm": 3.9786066602035097, "kl": 0.0947265625, "learning_rate": 2.3888442369854165e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.791385293006897, "rewards_raw/accuracy_reward": 0.5578294992446899, "rewards_raw/clip_reward_wrapper": 0.34386003017425537, "rewards_raw/format_reward": 5.0, "step": 1489 }, { "Total_reward": -3.2995430387927627e-07, "advantage_std": 1.3224456310272217, "completion_length": 381.89288330078125, "epoch": 0.5145027624309392, "grad_norm": 4.144033368952183, "kl": 0.0966796875, "learning_rate": 2.386134974696172e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.899734377861023, "rewards_raw/accuracy_reward": 0.5836469531059265, "rewards_raw/clip_reward_wrapper": 0.3437250852584839, "rewards_raw/format_reward": 5.0, "step": 1490 }, { "Total_reward": 9.962491276382934e-07, "advantage_std": 1.4032973051071167, "completion_length": 379.7500305175781, "epoch": 0.5148480662983426, "grad_norm": 4.383283446182632, "kl": 0.10107421875, "learning_rate": 2.3834258464032663e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.6776096224784851, "rewards_raw/accuracy_reward": 0.5898058414459229, "rewards_raw/clip_reward_wrapper": 0.3442130982875824, "rewards_raw/format_reward": 5.0, "step": 1491 }, { "Total_reward": -5.087682666271576e-07, "advantage_std": 1.4385544061660767, "completion_length": 371.46429443359375, "epoch": 0.5151933701657458, "grad_norm": 5.238109429755534, "kl": 0.1044921875, "learning_rate": 2.3807168552948025e-07, "loss": 0.0042, "rewards_raw/CoT ICLG": 0.730512797832489, "rewards_raw/accuracy_reward": 0.5723482966423035, "rewards_raw/clip_reward_wrapper": 0.3320527970790863, "rewards_raw/format_reward": 5.0, "step": 1492 }, { "Total_reward": -1.4049666674509353e-07, "advantage_std": 1.6348538398742676, "completion_length": 385.08929443359375, "epoch": 0.5155386740331491, "grad_norm": 5.457096819035676, "kl": 0.099609375, "learning_rate": 2.3780080045587204e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.8508470058441162, "rewards_raw/accuracy_reward": 0.5816534757614136, "rewards_raw/clip_reward_wrapper": 0.34421107172966003, "rewards_raw/format_reward": 5.0, "step": 1493 }, { "Total_reward": 5.428280474006897e-07, "advantage_std": 1.445627212524414, "completion_length": 390.39288330078125, "epoch": 0.5158839779005525, "grad_norm": 3.734712650922696, "kl": 0.099609375, "learning_rate": 2.3752992973827936e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.7461221218109131, "rewards_raw/accuracy_reward": 0.5644435882568359, "rewards_raw/clip_reward_wrapper": 0.33487454056739807, "rewards_raw/format_reward": 5.0, "step": 1494 }, { "Total_reward": 6.386212021425308e-07, "advantage_std": 1.577759027481079, "completion_length": 388.0714416503906, "epoch": 0.5162292817679558, "grad_norm": 4.877323598544322, "kl": 0.09912109375, "learning_rate": 2.3725907369546304e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 1.3389220237731934, "rewards_raw/accuracy_reward": 0.5691794157028198, "rewards_raw/clip_reward_wrapper": 0.3310479521751404, "rewards_raw/format_reward": 5.0, "step": 1495 }, { "Total_reward": -5.151544542059128e-07, "advantage_std": 1.6370428800582886, "completion_length": 379.7321472167969, "epoch": 0.5165745856353591, "grad_norm": 7.629646118522028, "kl": 0.1005859375, "learning_rate": 2.369882326461663e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.774518609046936, "rewards_raw/accuracy_reward": 0.5733206272125244, "rewards_raw/clip_reward_wrapper": 0.32685643434524536, "rewards_raw/format_reward": 5.0, "step": 1496 }, { "Total_reward": -7.844397487133392e-07, "advantage_std": 1.32831609249115, "completion_length": 380.2321472167969, "epoch": 0.5169198895027625, "grad_norm": 4.419363715523715, "kl": 0.09423828125, "learning_rate": 2.3671740690911472e-07, "loss": 0.0038, "rewards_raw/CoT ICLG": 0.7207391262054443, "rewards_raw/accuracy_reward": 0.5955316424369812, "rewards_raw/clip_reward_wrapper": 0.33885014057159424, "rewards_raw/format_reward": 5.0, "step": 1497 }, { "Total_reward": -5.364418029785156e-07, "advantage_std": 1.3626419305801392, "completion_length": 382.6964416503906, "epoch": 0.5172651933701657, "grad_norm": 4.9340906069656745, "kl": 0.0966796875, "learning_rate": 2.3644659680301626e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8345583081245422, "rewards_raw/accuracy_reward": 0.5677875876426697, "rewards_raw/clip_reward_wrapper": 0.3437296450138092, "rewards_raw/format_reward": 5.0, "step": 1498 }, { "Total_reward": 5.69437247577298e-07, "advantage_std": 1.6105802059173584, "completion_length": 375.6607360839844, "epoch": 0.517610497237569, "grad_norm": 4.630827559252759, "kl": 0.09814453125, "learning_rate": 2.3617580264655998e-07, "loss": 0.0039, "rewards_raw/CoT ICLG": 0.8773176670074463, "rewards_raw/accuracy_reward": 0.5831530094146729, "rewards_raw/clip_reward_wrapper": 0.34860101342201233, "rewards_raw/format_reward": 5.0, "step": 1499 }, { "Total_reward": 9.366444686520481e-08, "advantage_std": 1.639905571937561, "completion_length": 367.3214416503906, "epoch": 0.5179558011049724, "grad_norm": 5.035336565147052, "kl": 0.10009765625, "learning_rate": 2.359050247584164e-07, "loss": 0.004, "rewards_raw/CoT ICLG": 0.727196455001831, "rewards_raw/accuracy_reward": 0.5771687626838684, "rewards_raw/clip_reward_wrapper": 0.3346591889858246, "rewards_raw/format_reward": 5.0, "step": 1500 } ], "logging_steps": 1.0, "max_steps": 2896, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }