| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 4.0, |
| "eval_steps": 500, |
| "global_step": 956, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "completion_length": 21.796875, |
| "epoch": 0.0041841004184100415, |
| "grad_norm": 4.395772933959961, |
| "kl": 0.0, |
| "learning_rate": 1e-06, |
| "loss": -0.0, |
| "reward": 0.21946728974580765, |
| "reward_std": 0.14253678917884827, |
| "reward_std_all": 0.17847749590873718, |
| "rewards-max/forensics_iou_reward": 0.6859379708766937, |
| "rewards/forensics_iou_reward": 0.21946728974580765, |
| "step": 1 |
| }, |
| { |
| "completion_length": 22.546875, |
| "epoch": 0.008368200836820083, |
| "grad_norm": 2.851386308670044, |
| "kl": 0.000640869140625, |
| "learning_rate": 9.989539748953975e-07, |
| "loss": 0.0, |
| "reward": 0.2688695639371872, |
| "reward_std": 0.2108561247587204, |
| "reward_std_all": 0.20689550042152405, |
| "rewards-max/forensics_iou_reward": 0.9177520871162415, |
| "rewards/forensics_iou_reward": 0.2688695639371872, |
| "step": 2 |
| }, |
| { |
| "completion_length": 21.625, |
| "epoch": 0.012552301255230125, |
| "grad_norm": 2.5842034816741943, |
| "kl": 0.001110076904296875, |
| "learning_rate": 9.97907949790795e-07, |
| "loss": 0.0, |
| "reward": 0.18310855329036713, |
| "reward_std": 0.14751482009887695, |
| "reward_std_all": 0.14676997065544128, |
| "rewards-max/forensics_iou_reward": 0.7220738232135773, |
| "rewards/forensics_iou_reward": 0.18310855329036713, |
| "step": 3 |
| }, |
| { |
| "completion_length": 20.765625, |
| "epoch": 0.016736401673640166, |
| "grad_norm": 2.9219748973846436, |
| "kl": 0.00229644775390625, |
| "learning_rate": 9.968619246861925e-07, |
| "loss": 0.0001, |
| "reward": 0.20954851806163788, |
| "reward_std": 0.15059591829776764, |
| "reward_std_all": 0.18946200609207153, |
| "rewards-max/forensics_iou_reward": 0.6417644619941711, |
| "rewards/forensics_iou_reward": 0.20954851806163788, |
| "step": 4 |
| }, |
| { |
| "completion_length": 19.8125, |
| "epoch": 0.02092050209205021, |
| "grad_norm": 2.415116310119629, |
| "kl": 0.0011138916015625, |
| "learning_rate": 9.9581589958159e-07, |
| "loss": 0.0, |
| "reward": 0.21889735758304596, |
| "reward_std": 0.16885166615247726, |
| "reward_std_all": 0.16105034947395325, |
| "rewards-max/forensics_iou_reward": 0.6920783817768097, |
| "rewards/forensics_iou_reward": 0.21889735758304596, |
| "step": 5 |
| }, |
| { |
| "completion_length": 21.5625, |
| "epoch": 0.02510460251046025, |
| "grad_norm": 2.5597503185272217, |
| "kl": 0.02889251708984375, |
| "learning_rate": 9.947698744769873e-07, |
| "loss": 0.0012, |
| "reward": 0.19026833400130272, |
| "reward_std": 0.08241001889109612, |
| "reward_std_all": 0.1071675755083561, |
| "rewards-max/forensics_iou_reward": 0.6202515959739685, |
| "rewards/forensics_iou_reward": 0.19026833400130272, |
| "step": 6 |
| }, |
| { |
| "completion_length": 21.609375, |
| "epoch": 0.029288702928870293, |
| "grad_norm": 2.819694757461548, |
| "kl": 0.0142822265625, |
| "learning_rate": 9.937238493723848e-07, |
| "loss": 0.0006, |
| "reward": 0.3526920974254608, |
| "reward_std": 0.19898279011249542, |
| "reward_std_all": 0.19259458780288696, |
| "rewards-max/forensics_iou_reward": 0.8765092492103577, |
| "rewards/forensics_iou_reward": 0.3526920974254608, |
| "step": 7 |
| }, |
| { |
| "completion_length": 20.796875, |
| "epoch": 0.03347280334728033, |
| "grad_norm": 2.777377128601074, |
| "kl": 0.0454559326171875, |
| "learning_rate": 9.926778242677823e-07, |
| "loss": 0.0018, |
| "reward": 0.23064624518156052, |
| "reward_std": 0.17860149592161179, |
| "reward_std_all": 0.1731613241136074, |
| "rewards-max/forensics_iou_reward": 0.6676066219806671, |
| "rewards/forensics_iou_reward": 0.23064624518156052, |
| "step": 8 |
| }, |
| { |
| "completion_length": 21.578125, |
| "epoch": 0.03765690376569038, |
| "grad_norm": 2.584784746170044, |
| "kl": 0.0461883544921875, |
| "learning_rate": 9.916317991631798e-07, |
| "loss": 0.0018, |
| "reward": 0.36824147403240204, |
| "reward_std": 0.16444721817970276, |
| "reward_std_all": 0.17691729217767715, |
| "rewards-max/forensics_iou_reward": 0.9479813575744629, |
| "rewards/forensics_iou_reward": 0.36824147403240204, |
| "step": 9 |
| }, |
| { |
| "completion_length": 21.3125, |
| "epoch": 0.04184100418410042, |
| "grad_norm": 3.162238121032715, |
| "kl": 0.011566162109375, |
| "learning_rate": 9.905857740585774e-07, |
| "loss": 0.0005, |
| "reward": 0.2835554927587509, |
| "reward_std": 0.12571347132325172, |
| "reward_std_all": 0.15929076075553894, |
| "rewards-max/forensics_iou_reward": 0.8779269456863403, |
| "rewards/forensics_iou_reward": 0.2835554927587509, |
| "step": 10 |
| }, |
| { |
| "completion_length": 20.921875, |
| "epoch": 0.04602510460251046, |
| "grad_norm": 2.6641573905944824, |
| "kl": 0.03582763671875, |
| "learning_rate": 9.895397489539749e-07, |
| "loss": 0.0014, |
| "reward": 0.29736173152923584, |
| "reward_std": 0.15365038812160492, |
| "reward_std_all": 0.16391146183013916, |
| "rewards-max/forensics_iou_reward": 0.7974602282047272, |
| "rewards/forensics_iou_reward": 0.29736173152923584, |
| "step": 11 |
| }, |
| { |
| "completion_length": 20.265625, |
| "epoch": 0.0502092050209205, |
| "grad_norm": 3.4662892818450928, |
| "kl": 0.0242919921875, |
| "learning_rate": 9.884937238493724e-07, |
| "loss": 0.001, |
| "reward": 0.20774534344673157, |
| "reward_std": 0.1296086087822914, |
| "reward_std_all": 0.1316971331834793, |
| "rewards-max/forensics_iou_reward": 0.6126596033573151, |
| "rewards/forensics_iou_reward": 0.20774534344673157, |
| "step": 12 |
| }, |
| { |
| "completion_length": 21.0, |
| "epoch": 0.05439330543933055, |
| "grad_norm": 2.159379243850708, |
| "kl": 0.02545166015625, |
| "learning_rate": 9.8744769874477e-07, |
| "loss": 0.001, |
| "reward": 0.3205161541700363, |
| "reward_std": 0.176072109490633, |
| "reward_std_all": 0.15544801950454712, |
| "rewards-max/forensics_iou_reward": 0.7289430499076843, |
| "rewards/forensics_iou_reward": 0.3205161541700363, |
| "step": 13 |
| }, |
| { |
| "completion_length": 23.03125, |
| "epoch": 0.058577405857740586, |
| "grad_norm": 2.7199196815490723, |
| "kl": 0.0386962890625, |
| "learning_rate": 9.864016736401674e-07, |
| "loss": 0.0015, |
| "reward": 0.31443214416503906, |
| "reward_std": 0.16946867853403091, |
| "reward_std_all": 0.1899677738547325, |
| "rewards-max/forensics_iou_reward": 0.7364738881587982, |
| "rewards/forensics_iou_reward": 0.31443214416503906, |
| "step": 14 |
| }, |
| { |
| "completion_length": 21.5, |
| "epoch": 0.06276150627615062, |
| "grad_norm": 2.84133243560791, |
| "kl": 0.02978515625, |
| "learning_rate": 9.85355648535565e-07, |
| "loss": 0.0012, |
| "reward": 0.31523580849170685, |
| "reward_std": 0.12477636709809303, |
| "reward_std_all": 0.13443702459335327, |
| "rewards-max/forensics_iou_reward": 0.7332382202148438, |
| "rewards/forensics_iou_reward": 0.31523580849170685, |
| "step": 15 |
| }, |
| { |
| "completion_length": 23.21875, |
| "epoch": 0.06694560669456066, |
| "grad_norm": 3.0025956630706787, |
| "kl": 0.046875, |
| "learning_rate": 9.843096234309622e-07, |
| "loss": 0.0019, |
| "reward": 0.3482504189014435, |
| "reward_std": 0.09377044811844826, |
| "reward_std_all": 0.10566473752260208, |
| "rewards-max/forensics_iou_reward": 0.7007788717746735, |
| "rewards/forensics_iou_reward": 0.3482504189014435, |
| "step": 16 |
| }, |
| { |
| "completion_length": 21.984375, |
| "epoch": 0.07112970711297072, |
| "grad_norm": 3.2505197525024414, |
| "kl": 0.05908203125, |
| "learning_rate": 9.832635983263597e-07, |
| "loss": 0.0024, |
| "reward": 0.364320769906044, |
| "reward_std": 0.11721588671207428, |
| "reward_std_all": 0.11256621032953262, |
| "rewards-max/forensics_iou_reward": 0.9551397860050201, |
| "rewards/forensics_iou_reward": 0.364320769906044, |
| "step": 17 |
| }, |
| { |
| "completion_length": 21.484375, |
| "epoch": 0.07531380753138076, |
| "grad_norm": 3.2008302211761475, |
| "kl": 0.0506591796875, |
| "learning_rate": 9.822175732217572e-07, |
| "loss": 0.002, |
| "reward": 0.42224231362342834, |
| "reward_std": 0.08364618942141533, |
| "reward_std_all": 0.11626619473099709, |
| "rewards-max/forensics_iou_reward": 0.9024738669395447, |
| "rewards/forensics_iou_reward": 0.42224231362342834, |
| "step": 18 |
| }, |
| { |
| "completion_length": 23.09375, |
| "epoch": 0.0794979079497908, |
| "grad_norm": 2.992966890335083, |
| "kl": 0.0635986328125, |
| "learning_rate": 9.811715481171548e-07, |
| "loss": 0.0025, |
| "reward": 0.2699483558535576, |
| "reward_std": 0.10788407176733017, |
| "reward_std_all": 0.10205096006393433, |
| "rewards-max/forensics_iou_reward": 0.6099143624305725, |
| "rewards/forensics_iou_reward": 0.2699483558535576, |
| "step": 19 |
| }, |
| { |
| "completion_length": 23.8125, |
| "epoch": 0.08368200836820083, |
| "grad_norm": 3.6800601482391357, |
| "kl": 0.0499267578125, |
| "learning_rate": 9.801255230125523e-07, |
| "loss": 0.002, |
| "reward": 0.3251265734434128, |
| "reward_std": 0.09559235721826553, |
| "reward_std_all": 0.11610584706068039, |
| "rewards-max/forensics_iou_reward": 0.65638267993927, |
| "rewards/forensics_iou_reward": 0.3251265734434128, |
| "step": 20 |
| }, |
| { |
| "completion_length": 24.8125, |
| "epoch": 0.08786610878661087, |
| "grad_norm": 2.774524688720703, |
| "kl": 0.06640625, |
| "learning_rate": 9.790794979079498e-07, |
| "loss": 0.0027, |
| "reward": 0.2928735166788101, |
| "reward_std": 0.10458493232727051, |
| "reward_std_all": 0.10073460265994072, |
| "rewards-max/forensics_iou_reward": 0.6058242917060852, |
| "rewards/forensics_iou_reward": 0.2928735166788101, |
| "step": 21 |
| }, |
| { |
| "completion_length": 22.15625, |
| "epoch": 0.09205020920502092, |
| "grad_norm": 2.3112471103668213, |
| "kl": 0.0576171875, |
| "learning_rate": 9.780334728033473e-07, |
| "loss": 0.0023, |
| "reward": 0.2796153724193573, |
| "reward_std": 0.11774182692170143, |
| "reward_std_all": 0.10996968299150467, |
| "rewards-max/forensics_iou_reward": 0.7668718695640564, |
| "rewards/forensics_iou_reward": 0.2796153724193573, |
| "step": 22 |
| }, |
| { |
| "completion_length": 23.53125, |
| "epoch": 0.09623430962343096, |
| "grad_norm": 2.2924158573150635, |
| "kl": 0.056396484375, |
| "learning_rate": 9.769874476987448e-07, |
| "loss": 0.0023, |
| "reward": 0.34478092193603516, |
| "reward_std": 0.12704849615693092, |
| "reward_std_all": 0.12607211619615555, |
| "rewards-max/forensics_iou_reward": 0.8132333755493164, |
| "rewards/forensics_iou_reward": 0.34478092193603516, |
| "step": 23 |
| }, |
| { |
| "completion_length": 20.296875, |
| "epoch": 0.100418410041841, |
| "grad_norm": 2.4806981086730957, |
| "kl": 0.07568359375, |
| "learning_rate": 9.759414225941423e-07, |
| "loss": 0.003, |
| "reward": 0.4131374955177307, |
| "reward_std": 0.1281701996922493, |
| "reward_std_all": 0.1283605843782425, |
| "rewards-max/forensics_iou_reward": 0.820204496383667, |
| "rewards/forensics_iou_reward": 0.4131374955177307, |
| "step": 24 |
| }, |
| { |
| "completion_length": 20.96875, |
| "epoch": 0.10460251046025104, |
| "grad_norm": 2.3692715167999268, |
| "kl": 0.065185546875, |
| "learning_rate": 9.748953974895398e-07, |
| "loss": 0.0026, |
| "reward": 0.34604915976524353, |
| "reward_std": 0.09736831113696098, |
| "reward_std_all": 0.10048972070217133, |
| "rewards-max/forensics_iou_reward": 0.8351175785064697, |
| "rewards/forensics_iou_reward": 0.34604915976524353, |
| "step": 25 |
| }, |
| { |
| "completion_length": 20.859375, |
| "epoch": 0.1087866108786611, |
| "grad_norm": 2.828036069869995, |
| "kl": 0.06591796875, |
| "learning_rate": 9.738493723849371e-07, |
| "loss": 0.0026, |
| "reward": 0.37672337889671326, |
| "reward_std": 0.08296138793230057, |
| "reward_std_all": 0.09191370010375977, |
| "rewards-max/forensics_iou_reward": 0.8441919386386871, |
| "rewards/forensics_iou_reward": 0.37672337889671326, |
| "step": 26 |
| }, |
| { |
| "completion_length": 20.578125, |
| "epoch": 0.11297071129707113, |
| "grad_norm": 2.2261273860931396, |
| "kl": 0.070556640625, |
| "learning_rate": 9.728033472803347e-07, |
| "loss": 0.0028, |
| "reward": 0.3446190357208252, |
| "reward_std": 0.10913161560893059, |
| "reward_std_all": 0.11396045982837677, |
| "rewards-max/forensics_iou_reward": 0.7387594878673553, |
| "rewards/forensics_iou_reward": 0.3446190357208252, |
| "step": 27 |
| }, |
| { |
| "completion_length": 20.609375, |
| "epoch": 0.11715481171548117, |
| "grad_norm": 1.9082707166671753, |
| "kl": 0.09716796875, |
| "learning_rate": 9.717573221757322e-07, |
| "loss": 0.0039, |
| "reward": 0.4827222526073456, |
| "reward_std": 0.0817931741476059, |
| "reward_std_all": 0.0946575291454792, |
| "rewards-max/forensics_iou_reward": 0.8777662515640259, |
| "rewards/forensics_iou_reward": 0.4827222526073456, |
| "step": 28 |
| }, |
| { |
| "completion_length": 19.78125, |
| "epoch": 0.12133891213389121, |
| "grad_norm": 2.401249647140503, |
| "kl": 0.0673828125, |
| "learning_rate": 9.707112970711297e-07, |
| "loss": 0.0027, |
| "reward": 0.2828565686941147, |
| "reward_std": 0.044190337881445885, |
| "reward_std_all": 0.07950519397854805, |
| "rewards-max/forensics_iou_reward": 0.6432891190052032, |
| "rewards/forensics_iou_reward": 0.2828565686941147, |
| "step": 29 |
| }, |
| { |
| "completion_length": 20.75, |
| "epoch": 0.12552301255230125, |
| "grad_norm": 2.450561285018921, |
| "kl": 0.066650390625, |
| "learning_rate": 9.696652719665272e-07, |
| "loss": 0.0027, |
| "reward": 0.3693504184484482, |
| "reward_std": 0.10801473259925842, |
| "reward_std_all": 0.10606877133250237, |
| "rewards-max/forensics_iou_reward": 0.825148344039917, |
| "rewards/forensics_iou_reward": 0.3693504184484482, |
| "step": 30 |
| }, |
| { |
| "completion_length": 22.21875, |
| "epoch": 0.1297071129707113, |
| "grad_norm": 2.882416009902954, |
| "kl": 0.0853271484375, |
| "learning_rate": 9.686192468619247e-07, |
| "loss": 0.0034, |
| "reward": 0.3569013774394989, |
| "reward_std": 0.1653996780514717, |
| "reward_std_all": 0.17341994494199753, |
| "rewards-max/forensics_iou_reward": 0.8739746809005737, |
| "rewards/forensics_iou_reward": 0.3569013774394989, |
| "step": 31 |
| }, |
| { |
| "completion_length": 21.375, |
| "epoch": 0.13389121338912133, |
| "grad_norm": 2.4836392402648926, |
| "kl": 0.055908203125, |
| "learning_rate": 9.675732217573222e-07, |
| "loss": 0.0022, |
| "reward": 0.3603311628103256, |
| "reward_std": 0.09996200911700726, |
| "reward_std_all": 0.11152058094739914, |
| "rewards-max/forensics_iou_reward": 0.834677666425705, |
| "rewards/forensics_iou_reward": 0.3603311628103256, |
| "step": 32 |
| }, |
| { |
| "completion_length": 20.484375, |
| "epoch": 0.13807531380753138, |
| "grad_norm": 2.270937442779541, |
| "kl": 0.064697265625, |
| "learning_rate": 9.665271966527197e-07, |
| "loss": 0.0026, |
| "reward": 0.2971302345395088, |
| "reward_std": 0.10628249496221542, |
| "reward_std_all": 0.11716560274362564, |
| "rewards-max/forensics_iou_reward": 0.6572755575180054, |
| "rewards/forensics_iou_reward": 0.2971302345395088, |
| "step": 33 |
| }, |
| { |
| "completion_length": 21.25, |
| "epoch": 0.14225941422594143, |
| "grad_norm": 2.567176342010498, |
| "kl": 0.0582275390625, |
| "learning_rate": 9.654811715481172e-07, |
| "loss": 0.0023, |
| "reward": 0.3027327135205269, |
| "reward_std": 0.1347971335053444, |
| "reward_std_all": 0.1368163786828518, |
| "rewards-max/forensics_iou_reward": 0.886000007390976, |
| "rewards/forensics_iou_reward": 0.3027327135205269, |
| "step": 34 |
| }, |
| { |
| "completion_length": 19.234375, |
| "epoch": 0.14644351464435146, |
| "grad_norm": 3.730844736099243, |
| "kl": 0.075927734375, |
| "learning_rate": 9.644351464435145e-07, |
| "loss": 0.003, |
| "reward": 0.31494393944740295, |
| "reward_std": 0.1450028419494629, |
| "reward_std_all": 0.1469845250248909, |
| "rewards-max/forensics_iou_reward": 0.8016183972358704, |
| "rewards/forensics_iou_reward": 0.31494393944740295, |
| "step": 35 |
| }, |
| { |
| "completion_length": 19.609375, |
| "epoch": 0.1506276150627615, |
| "grad_norm": 3.5598018169403076, |
| "kl": 0.186279296875, |
| "learning_rate": 9.63389121338912e-07, |
| "loss": 0.0075, |
| "reward": 0.3317878544330597, |
| "reward_std": 0.11536705493927002, |
| "reward_std_all": 0.11744736135005951, |
| "rewards-max/forensics_iou_reward": 0.8836632668972015, |
| "rewards/forensics_iou_reward": 0.3317878544330597, |
| "step": 36 |
| }, |
| { |
| "completion_length": 19.578125, |
| "epoch": 0.15481171548117154, |
| "grad_norm": 2.3598432540893555, |
| "kl": 0.056396484375, |
| "learning_rate": 9.623430962343096e-07, |
| "loss": 0.0023, |
| "reward": 0.3962555080652237, |
| "reward_std": 0.09060253947973251, |
| "reward_std_all": 0.09514525160193443, |
| "rewards-max/forensics_iou_reward": 0.8404367864131927, |
| "rewards/forensics_iou_reward": 0.3962555080652237, |
| "step": 37 |
| }, |
| { |
| "completion_length": 19.609375, |
| "epoch": 0.1589958158995816, |
| "grad_norm": 3.008000135421753, |
| "kl": 0.103271484375, |
| "learning_rate": 9.61297071129707e-07, |
| "loss": 0.0041, |
| "reward": 0.37336377799510956, |
| "reward_std": 0.130904670804739, |
| "reward_std_all": 0.1266414001584053, |
| "rewards-max/forensics_iou_reward": 0.9246934354305267, |
| "rewards/forensics_iou_reward": 0.37336377799510956, |
| "step": 38 |
| }, |
| { |
| "completion_length": 20.21875, |
| "epoch": 0.16317991631799164, |
| "grad_norm": 2.525811195373535, |
| "kl": 0.091796875, |
| "learning_rate": 9.602510460251046e-07, |
| "loss": 0.0037, |
| "reward": 0.3498770147562027, |
| "reward_std": 0.1527438536286354, |
| "reward_std_all": 0.13807721436023712, |
| "rewards-max/forensics_iou_reward": 0.8582137525081635, |
| "rewards/forensics_iou_reward": 0.3498770147562027, |
| "step": 39 |
| }, |
| { |
| "completion_length": 19.21875, |
| "epoch": 0.16736401673640167, |
| "grad_norm": 2.7715396881103516, |
| "kl": 0.0635986328125, |
| "learning_rate": 9.592050209205021e-07, |
| "loss": 0.0025, |
| "reward": 0.38453714549541473, |
| "reward_std": 0.16090993583202362, |
| "reward_std_all": 0.1515103280544281, |
| "rewards-max/forensics_iou_reward": 0.778287798166275, |
| "rewards/forensics_iou_reward": 0.38453714549541473, |
| "step": 40 |
| }, |
| { |
| "completion_length": 19.5, |
| "epoch": 0.17154811715481172, |
| "grad_norm": 2.373370885848999, |
| "kl": 0.114501953125, |
| "learning_rate": 9.581589958158996e-07, |
| "loss": 0.0046, |
| "reward": 0.2723907232284546, |
| "reward_std": 0.1315925046801567, |
| "reward_std_all": 0.12654701247811317, |
| "rewards-max/forensics_iou_reward": 0.8265625536441803, |
| "rewards/forensics_iou_reward": 0.2723907232284546, |
| "step": 41 |
| }, |
| { |
| "completion_length": 20.75, |
| "epoch": 0.17573221757322174, |
| "grad_norm": 2.9579358100891113, |
| "kl": 0.07958984375, |
| "learning_rate": 9.571129707112971e-07, |
| "loss": 0.0032, |
| "reward": 0.25504910200834274, |
| "reward_std": 0.09011245891451836, |
| "reward_std_all": 0.0924566201865673, |
| "rewards-max/forensics_iou_reward": 0.6225314438343048, |
| "rewards/forensics_iou_reward": 0.25504910200834274, |
| "step": 42 |
| }, |
| { |
| "completion_length": 19.40625, |
| "epoch": 0.1799163179916318, |
| "grad_norm": 2.4648993015289307, |
| "kl": 0.06787109375, |
| "learning_rate": 9.560669456066946e-07, |
| "loss": 0.0027, |
| "reward": 0.25183001160621643, |
| "reward_std": 0.09366191923618317, |
| "reward_std_all": 0.10404319688677788, |
| "rewards-max/forensics_iou_reward": 0.7853032648563385, |
| "rewards/forensics_iou_reward": 0.25183001160621643, |
| "step": 43 |
| }, |
| { |
| "completion_length": 19.28125, |
| "epoch": 0.18410041841004185, |
| "grad_norm": 3.1736159324645996, |
| "kl": 0.107666015625, |
| "learning_rate": 9.55020920502092e-07, |
| "loss": 0.0043, |
| "reward": 0.30916714668273926, |
| "reward_std": 0.11193849891424179, |
| "reward_std_all": 0.10381406545639038, |
| "rewards-max/forensics_iou_reward": 0.8192278742790222, |
| "rewards/forensics_iou_reward": 0.30916714668273926, |
| "step": 44 |
| }, |
| { |
| "completion_length": 19.359375, |
| "epoch": 0.18828451882845187, |
| "grad_norm": 2.730895757675171, |
| "kl": 0.13232421875, |
| "learning_rate": 9.539748953974895e-07, |
| "loss": 0.0053, |
| "reward": 0.3085031658411026, |
| "reward_std": 0.1475491151213646, |
| "reward_std_all": 0.15565253794193268, |
| "rewards-max/forensics_iou_reward": 0.7712320387363434, |
| "rewards/forensics_iou_reward": 0.3085031658411026, |
| "step": 45 |
| }, |
| { |
| "completion_length": 19.296875, |
| "epoch": 0.19246861924686193, |
| "grad_norm": 3.0114896297454834, |
| "kl": 0.0760498046875, |
| "learning_rate": 9.52928870292887e-07, |
| "loss": 0.003, |
| "reward": 0.29067324101924896, |
| "reward_std": 0.115788035094738, |
| "reward_std_all": 0.13166113197803497, |
| "rewards-max/forensics_iou_reward": 0.887848436832428, |
| "rewards/forensics_iou_reward": 0.29067324101924896, |
| "step": 46 |
| }, |
| { |
| "completion_length": 19.265625, |
| "epoch": 0.19665271966527198, |
| "grad_norm": 2.991511344909668, |
| "kl": 0.072265625, |
| "learning_rate": 9.518828451882844e-07, |
| "loss": 0.0029, |
| "reward": 0.2912721633911133, |
| "reward_std": 0.14736977964639664, |
| "reward_std_all": 0.14577487111091614, |
| "rewards-max/forensics_iou_reward": 0.9690859615802765, |
| "rewards/forensics_iou_reward": 0.2912721633911133, |
| "step": 47 |
| }, |
| { |
| "completion_length": 19.8125, |
| "epoch": 0.200836820083682, |
| "grad_norm": 2.750657320022583, |
| "kl": 0.0926513671875, |
| "learning_rate": 9.508368200836819e-07, |
| "loss": 0.0037, |
| "reward": 0.28115595132112503, |
| "reward_std": 0.16473307460546494, |
| "reward_std_all": 0.16610294580459595, |
| "rewards-max/forensics_iou_reward": 0.8267879784107208, |
| "rewards/forensics_iou_reward": 0.28115595132112503, |
| "step": 48 |
| }, |
| { |
| "completion_length": 19.328125, |
| "epoch": 0.20502092050209206, |
| "grad_norm": 2.219653844833374, |
| "kl": 0.0570068359375, |
| "learning_rate": 9.497907949790794e-07, |
| "loss": 0.0023, |
| "reward": 0.4043098986148834, |
| "reward_std": 0.12726186960935593, |
| "reward_std_all": 0.13409016281366348, |
| "rewards-max/forensics_iou_reward": 0.8549624383449554, |
| "rewards/forensics_iou_reward": 0.4043098986148834, |
| "step": 49 |
| }, |
| { |
| "completion_length": 21.40625, |
| "epoch": 0.20920502092050208, |
| "grad_norm": 2.948784828186035, |
| "kl": 0.07177734375, |
| "learning_rate": 9.487447698744769e-07, |
| "loss": 0.0029, |
| "reward": 0.28126825392246246, |
| "reward_std": 0.0894559808075428, |
| "reward_std_all": 0.08360825479030609, |
| "rewards-max/forensics_iou_reward": 0.7786678075790405, |
| "rewards/forensics_iou_reward": 0.28126825392246246, |
| "step": 50 |
| }, |
| { |
| "completion_length": 19.21875, |
| "epoch": 0.21338912133891214, |
| "grad_norm": 2.394662857055664, |
| "kl": 0.09716796875, |
| "learning_rate": 9.476987447698744e-07, |
| "loss": 0.0039, |
| "reward": 0.35037267208099365, |
| "reward_std": 0.1305595375597477, |
| "reward_std_all": 0.13744258135557175, |
| "rewards-max/forensics_iou_reward": 0.7979799211025238, |
| "rewards/forensics_iou_reward": 0.35037267208099365, |
| "step": 51 |
| }, |
| { |
| "completion_length": 19.5625, |
| "epoch": 0.2175732217573222, |
| "grad_norm": 2.545318365097046, |
| "kl": 0.078369140625, |
| "learning_rate": 9.466527196652719e-07, |
| "loss": 0.0031, |
| "reward": 0.34720033407211304, |
| "reward_std": 0.08847271278500557, |
| "reward_std_all": 0.08611230179667473, |
| "rewards-max/forensics_iou_reward": 0.8207633197307587, |
| "rewards/forensics_iou_reward": 0.34720033407211304, |
| "step": 52 |
| }, |
| { |
| "completion_length": 19.21875, |
| "epoch": 0.2217573221757322, |
| "grad_norm": 2.6037473678588867, |
| "kl": 0.0648193359375, |
| "learning_rate": 9.456066945606695e-07, |
| "loss": 0.0026, |
| "reward": 0.33708786964416504, |
| "reward_std": 0.07953795976936817, |
| "reward_std_all": 0.08402814716100693, |
| "rewards-max/forensics_iou_reward": 0.869371622800827, |
| "rewards/forensics_iou_reward": 0.33708786964416504, |
| "step": 53 |
| }, |
| { |
| "completion_length": 19.109375, |
| "epoch": 0.22594142259414227, |
| "grad_norm": 3.294013500213623, |
| "kl": 0.075927734375, |
| "learning_rate": 9.44560669456067e-07, |
| "loss": 0.003, |
| "reward": 0.4165303409099579, |
| "reward_std": 0.11899575591087341, |
| "reward_std_all": 0.12528621777892113, |
| "rewards-max/forensics_iou_reward": 0.9344273805618286, |
| "rewards/forensics_iou_reward": 0.4165303409099579, |
| "step": 54 |
| }, |
| { |
| "completion_length": 20.421875, |
| "epoch": 0.2301255230125523, |
| "grad_norm": 3.174968957901001, |
| "kl": 0.099365234375, |
| "learning_rate": 9.435146443514645e-07, |
| "loss": 0.004, |
| "reward": 0.4009455144405365, |
| "reward_std": 0.11858171597123146, |
| "reward_std_all": 0.12532257288694382, |
| "rewards-max/forensics_iou_reward": 0.9069027006626129, |
| "rewards/forensics_iou_reward": 0.4009455144405365, |
| "step": 55 |
| }, |
| { |
| "completion_length": 19.609375, |
| "epoch": 0.23430962343096234, |
| "grad_norm": 2.1260461807250977, |
| "kl": 0.063232421875, |
| "learning_rate": 9.424686192468618e-07, |
| "loss": 0.0025, |
| "reward": 0.2959258258342743, |
| "reward_std": 0.06705441698431969, |
| "reward_std_all": 0.07854872196912766, |
| "rewards-max/forensics_iou_reward": 0.7456857860088348, |
| "rewards/forensics_iou_reward": 0.2959258258342743, |
| "step": 56 |
| }, |
| { |
| "completion_length": 19.875, |
| "epoch": 0.2384937238493724, |
| "grad_norm": 2.6935272216796875, |
| "kl": 0.12646484375, |
| "learning_rate": 9.414225941422593e-07, |
| "loss": 0.0051, |
| "reward": 0.44270336627960205, |
| "reward_std": 0.11502772942185402, |
| "reward_std_all": 0.11535507813096046, |
| "rewards-max/forensics_iou_reward": 0.940418154001236, |
| "rewards/forensics_iou_reward": 0.44270336627960205, |
| "step": 57 |
| }, |
| { |
| "completion_length": 20.625, |
| "epoch": 0.24267782426778242, |
| "grad_norm": 2.2716195583343506, |
| "kl": 0.096923828125, |
| "learning_rate": 9.403765690376568e-07, |
| "loss": 0.0039, |
| "reward": 0.378344863653183, |
| "reward_std": 0.1348758563399315, |
| "reward_std_all": 0.12951554730534554, |
| "rewards-max/forensics_iou_reward": 0.8397015631198883, |
| "rewards/forensics_iou_reward": 0.378344863653183, |
| "step": 58 |
| }, |
| { |
| "completion_length": 20.0, |
| "epoch": 0.24686192468619247, |
| "grad_norm": 2.4576172828674316, |
| "kl": 0.08740234375, |
| "learning_rate": 9.393305439330543e-07, |
| "loss": 0.0035, |
| "reward": 0.42570555210113525, |
| "reward_std": 0.11899979412555695, |
| "reward_std_all": 0.11497319862246513, |
| "rewards-max/forensics_iou_reward": 0.7962461411952972, |
| "rewards/forensics_iou_reward": 0.42570555210113525, |
| "step": 59 |
| }, |
| { |
| "completion_length": 20.296875, |
| "epoch": 0.2510460251046025, |
| "grad_norm": 2.221569776535034, |
| "kl": 0.108642578125, |
| "learning_rate": 9.382845188284518e-07, |
| "loss": 0.0043, |
| "reward": 0.35505980253219604, |
| "reward_std": 0.08185502141714096, |
| "reward_std_all": 0.08982518687844276, |
| "rewards-max/forensics_iou_reward": 0.8527531623840332, |
| "rewards/forensics_iou_reward": 0.35505980253219604, |
| "step": 60 |
| }, |
| { |
| "completion_length": 20.046875, |
| "epoch": 0.25523012552301255, |
| "grad_norm": 2.470527172088623, |
| "kl": 0.0770263671875, |
| "learning_rate": 9.372384937238493e-07, |
| "loss": 0.0031, |
| "reward": 0.4753943979740143, |
| "reward_std": 0.1020512767136097, |
| "reward_std_all": 0.10160453617572784, |
| "rewards-max/forensics_iou_reward": 0.9512910544872284, |
| "rewards/forensics_iou_reward": 0.4753943979740143, |
| "step": 61 |
| }, |
| { |
| "completion_length": 19.34375, |
| "epoch": 0.2594142259414226, |
| "grad_norm": 3.371222496032715, |
| "kl": 0.105224609375, |
| "learning_rate": 9.361924686192469e-07, |
| "loss": 0.0042, |
| "reward": 0.23948630690574646, |
| "reward_std": 0.12841660901904106, |
| "reward_std_all": 0.13108167424798012, |
| "rewards-max/forensics_iou_reward": 0.7999031245708466, |
| "rewards/forensics_iou_reward": 0.23948630690574646, |
| "step": 62 |
| }, |
| { |
| "completion_length": 19.234375, |
| "epoch": 0.26359832635983266, |
| "grad_norm": 2.6274142265319824, |
| "kl": 0.13134765625, |
| "learning_rate": 9.351464435146444e-07, |
| "loss": 0.0053, |
| "reward": 0.32813283801078796, |
| "reward_std": 0.13567223772406578, |
| "reward_std_all": 0.13706394284963608, |
| "rewards-max/forensics_iou_reward": 0.7376300394535065, |
| "rewards/forensics_iou_reward": 0.32813283801078796, |
| "step": 63 |
| }, |
| { |
| "completion_length": 21.140625, |
| "epoch": 0.26778242677824265, |
| "grad_norm": 3.31463360786438, |
| "kl": 0.10302734375, |
| "learning_rate": 9.341004184100419e-07, |
| "loss": 0.0041, |
| "reward": 0.3406233936548233, |
| "reward_std": 0.1359885111451149, |
| "reward_std_all": 0.1265971027314663, |
| "rewards-max/forensics_iou_reward": 0.8298932611942291, |
| "rewards/forensics_iou_reward": 0.3406233936548233, |
| "step": 64 |
| }, |
| { |
| "completion_length": 19.59375, |
| "epoch": 0.2719665271966527, |
| "grad_norm": 2.784193277359009, |
| "kl": 0.091552734375, |
| "learning_rate": 9.330543933054393e-07, |
| "loss": 0.0037, |
| "reward": 0.273764431476593, |
| "reward_std": 0.10677914693951607, |
| "reward_std_all": 0.10885962098836899, |
| "rewards-max/forensics_iou_reward": 0.5733771324157715, |
| "rewards/forensics_iou_reward": 0.273764431476593, |
| "step": 65 |
| }, |
| { |
| "completion_length": 19.796875, |
| "epoch": 0.27615062761506276, |
| "grad_norm": 2.6789958477020264, |
| "kl": 0.07763671875, |
| "learning_rate": 9.320083682008367e-07, |
| "loss": 0.0031, |
| "reward": 0.3188006579875946, |
| "reward_std": 0.14416810870170593, |
| "reward_std_all": 0.13660161942243576, |
| "rewards-max/forensics_iou_reward": 0.9237780570983887, |
| "rewards/forensics_iou_reward": 0.3188006579875946, |
| "step": 66 |
| }, |
| { |
| "completion_length": 20.359375, |
| "epoch": 0.2803347280334728, |
| "grad_norm": 2.525080442428589, |
| "kl": 0.071533203125, |
| "learning_rate": 9.309623430962342e-07, |
| "loss": 0.0029, |
| "reward": 0.3998964726924896, |
| "reward_std": 0.13413967937231064, |
| "reward_std_all": 0.1225668266415596, |
| "rewards-max/forensics_iou_reward": 0.8995339572429657, |
| "rewards/forensics_iou_reward": 0.3998964726924896, |
| "step": 67 |
| }, |
| { |
| "completion_length": 20.140625, |
| "epoch": 0.28451882845188287, |
| "grad_norm": 2.8171348571777344, |
| "kl": 0.072021484375, |
| "learning_rate": 9.299163179916317e-07, |
| "loss": 0.0029, |
| "reward": 0.34137214720249176, |
| "reward_std": 0.12821029871702194, |
| "reward_std_all": 0.1329115629196167, |
| "rewards-max/forensics_iou_reward": 0.8854825794696808, |
| "rewards/forensics_iou_reward": 0.34137214720249176, |
| "step": 68 |
| }, |
| { |
| "completion_length": 20.890625, |
| "epoch": 0.28870292887029286, |
| "grad_norm": 2.8282008171081543, |
| "kl": 0.06884765625, |
| "learning_rate": 9.288702928870292e-07, |
| "loss": 0.0028, |
| "reward": 0.39931102097034454, |
| "reward_std": 0.11704904958605766, |
| "reward_std_all": 0.11266573145985603, |
| "rewards-max/forensics_iou_reward": 0.8276611864566803, |
| "rewards/forensics_iou_reward": 0.39931102097034454, |
| "step": 69 |
| }, |
| { |
| "completion_length": 21.03125, |
| "epoch": 0.2928870292887029, |
| "grad_norm": 2.4680423736572266, |
| "kl": 0.088623046875, |
| "learning_rate": 9.278242677824268e-07, |
| "loss": 0.0035, |
| "reward": 0.34425148367881775, |
| "reward_std": 0.13214874640107155, |
| "reward_std_all": 0.13352156430482864, |
| "rewards-max/forensics_iou_reward": 0.8497827351093292, |
| "rewards/forensics_iou_reward": 0.34425148367881775, |
| "step": 70 |
| }, |
| { |
| "completion_length": 19.65625, |
| "epoch": 0.29707112970711297, |
| "grad_norm": 2.667959451675415, |
| "kl": 0.073974609375, |
| "learning_rate": 9.267782426778243e-07, |
| "loss": 0.003, |
| "reward": 0.35793067514896393, |
| "reward_std": 0.08443070203065872, |
| "reward_std_all": 0.11166559159755707, |
| "rewards-max/forensics_iou_reward": 0.804478108882904, |
| "rewards/forensics_iou_reward": 0.35793067514896393, |
| "step": 71 |
| }, |
| { |
| "completion_length": 19.578125, |
| "epoch": 0.301255230125523, |
| "grad_norm": 2.226055860519409, |
| "kl": 0.11181640625, |
| "learning_rate": 9.257322175732218e-07, |
| "loss": 0.0045, |
| "reward": 0.44273293018341064, |
| "reward_std": 0.08787363767623901, |
| "reward_std_all": 0.11705489829182625, |
| "rewards-max/forensics_iou_reward": 0.9613729417324066, |
| "rewards/forensics_iou_reward": 0.44273293018341064, |
| "step": 72 |
| }, |
| { |
| "completion_length": 19.15625, |
| "epoch": 0.3054393305439331, |
| "grad_norm": 2.529412269592285, |
| "kl": 0.054443359375, |
| "learning_rate": 9.246861924686192e-07, |
| "loss": 0.0022, |
| "reward": 0.33504460752010345, |
| "reward_std": 0.059977466240525246, |
| "reward_std_all": 0.06329547055065632, |
| "rewards-max/forensics_iou_reward": 0.7646428048610687, |
| "rewards/forensics_iou_reward": 0.33504460752010345, |
| "step": 73 |
| }, |
| { |
| "completion_length": 19.53125, |
| "epoch": 0.30962343096234307, |
| "grad_norm": 3.034578561782837, |
| "kl": 0.07958984375, |
| "learning_rate": 9.236401673640167e-07, |
| "loss": 0.0032, |
| "reward": 0.3671463429927826, |
| "reward_std": 0.09568856656551361, |
| "reward_std_all": 0.12954410165548325, |
| "rewards-max/forensics_iou_reward": 0.8484982252120972, |
| "rewards/forensics_iou_reward": 0.3671463429927826, |
| "step": 74 |
| }, |
| { |
| "completion_length": 19.96875, |
| "epoch": 0.3138075313807531, |
| "grad_norm": 3.326838970184326, |
| "kl": 0.091064453125, |
| "learning_rate": 9.225941422594142e-07, |
| "loss": 0.0036, |
| "reward": 0.415808841586113, |
| "reward_std": 0.14227010309696198, |
| "reward_std_all": 0.13592712581157684, |
| "rewards-max/forensics_iou_reward": 0.9191083908081055, |
| "rewards/forensics_iou_reward": 0.415808841586113, |
| "step": 75 |
| }, |
| { |
| "completion_length": 20.296875, |
| "epoch": 0.3179916317991632, |
| "grad_norm": 3.080474376678467, |
| "kl": 0.0538330078125, |
| "learning_rate": 9.215481171548116e-07, |
| "loss": 0.0022, |
| "reward": 0.4475569576025009, |
| "reward_std": 0.08863737061619759, |
| "reward_std_all": 0.12203838303685188, |
| "rewards-max/forensics_iou_reward": 0.915166825056076, |
| "rewards/forensics_iou_reward": 0.4475569576025009, |
| "step": 76 |
| }, |
| { |
| "completion_length": 19.4375, |
| "epoch": 0.32217573221757323, |
| "grad_norm": 3.021817922592163, |
| "kl": 0.08251953125, |
| "learning_rate": 9.205020920502091e-07, |
| "loss": 0.0033, |
| "reward": 0.3679630607366562, |
| "reward_std": 0.09009172022342682, |
| "reward_std_all": 0.10132382810115814, |
| "rewards-max/forensics_iou_reward": 0.8390916883945465, |
| "rewards/forensics_iou_reward": 0.3679630607366562, |
| "step": 77 |
| }, |
| { |
| "completion_length": 20.015625, |
| "epoch": 0.3263598326359833, |
| "grad_norm": 2.912874937057495, |
| "kl": 0.055419921875, |
| "learning_rate": 9.194560669456066e-07, |
| "loss": 0.0022, |
| "reward": 0.3181372359395027, |
| "reward_std": 0.12133364751935005, |
| "reward_std_all": 0.11493167653679848, |
| "rewards-max/forensics_iou_reward": 0.7819515764713287, |
| "rewards/forensics_iou_reward": 0.3181372359395027, |
| "step": 78 |
| }, |
| { |
| "completion_length": 19.4375, |
| "epoch": 0.3305439330543933, |
| "grad_norm": 2.665440797805786, |
| "kl": 0.080810546875, |
| "learning_rate": 9.184100418410042e-07, |
| "loss": 0.0032, |
| "reward": 0.38295790553092957, |
| "reward_std": 0.09124147146940231, |
| "reward_std_all": 0.10306024178862572, |
| "rewards-max/forensics_iou_reward": 0.9144349992275238, |
| "rewards/forensics_iou_reward": 0.38295790553092957, |
| "step": 79 |
| }, |
| { |
| "completion_length": 19.28125, |
| "epoch": 0.33472803347280333, |
| "grad_norm": 2.7352681159973145, |
| "kl": 0.0986328125, |
| "learning_rate": 9.173640167364017e-07, |
| "loss": 0.0039, |
| "reward": 0.39238448441028595, |
| "reward_std": 0.08842286095023155, |
| "reward_std_all": 0.09062398970127106, |
| "rewards-max/forensics_iou_reward": 0.9154135286808014, |
| "rewards/forensics_iou_reward": 0.39238448441028595, |
| "step": 80 |
| }, |
| { |
| "completion_length": 21.78125, |
| "epoch": 0.3389121338912134, |
| "grad_norm": 3.549900531768799, |
| "kl": 0.080810546875, |
| "learning_rate": 9.163179916317991e-07, |
| "loss": 0.0032, |
| "reward": 0.32703058421611786, |
| "reward_std": 0.1047283485531807, |
| "reward_std_all": 0.13347823545336723, |
| "rewards-max/forensics_iou_reward": 0.8525737226009369, |
| "rewards/forensics_iou_reward": 0.32703058421611786, |
| "step": 81 |
| }, |
| { |
| "completion_length": 22.984375, |
| "epoch": 0.34309623430962344, |
| "grad_norm": 3.2751166820526123, |
| "kl": 0.06640625, |
| "learning_rate": 9.152719665271966e-07, |
| "loss": 0.0027, |
| "reward": 0.3434244245290756, |
| "reward_std": 0.1029249019920826, |
| "reward_std_all": 0.11033365875482559, |
| "rewards-max/forensics_iou_reward": 0.8636673986911774, |
| "rewards/forensics_iou_reward": 0.3434244245290756, |
| "step": 82 |
| }, |
| { |
| "completion_length": 21.15625, |
| "epoch": 0.3472803347280335, |
| "grad_norm": 2.55847430229187, |
| "kl": 0.07861328125, |
| "learning_rate": 9.142259414225941e-07, |
| "loss": 0.0031, |
| "reward": 0.41312554478645325, |
| "reward_std": 0.08213358372449875, |
| "reward_std_all": 0.10457370802760124, |
| "rewards-max/forensics_iou_reward": 0.8113734126091003, |
| "rewards/forensics_iou_reward": 0.41312554478645325, |
| "step": 83 |
| }, |
| { |
| "completion_length": 20.96875, |
| "epoch": 0.3514644351464435, |
| "grad_norm": 3.7230544090270996, |
| "kl": 0.06884765625, |
| "learning_rate": 9.131799163179916e-07, |
| "loss": 0.0028, |
| "reward": 0.3915516883134842, |
| "reward_std": 0.09000499173998833, |
| "reward_std_all": 0.09343128465116024, |
| "rewards-max/forensics_iou_reward": 0.8999916315078735, |
| "rewards/forensics_iou_reward": 0.3915516883134842, |
| "step": 84 |
| }, |
| { |
| "completion_length": 20.296875, |
| "epoch": 0.35564853556485354, |
| "grad_norm": 2.7443161010742188, |
| "kl": 0.089599609375, |
| "learning_rate": 9.121338912133891e-07, |
| "loss": 0.0036, |
| "reward": 0.40096913278102875, |
| "reward_std": 0.09830423071980476, |
| "reward_std_all": 0.0948156900703907, |
| "rewards-max/forensics_iou_reward": 0.9148097932338715, |
| "rewards/forensics_iou_reward": 0.40096913278102875, |
| "step": 85 |
| }, |
| { |
| "completion_length": 20.859375, |
| "epoch": 0.3598326359832636, |
| "grad_norm": 2.223555088043213, |
| "kl": 0.105712890625, |
| "learning_rate": 9.110878661087865e-07, |
| "loss": 0.0042, |
| "reward": 0.30872824788093567, |
| "reward_std": 0.10324261710047722, |
| "reward_std_all": 0.10402477532625198, |
| "rewards-max/forensics_iou_reward": 0.8124184906482697, |
| "rewards/forensics_iou_reward": 0.30872824788093567, |
| "step": 86 |
| }, |
| { |
| "completion_length": 23.453125, |
| "epoch": 0.36401673640167365, |
| "grad_norm": 2.9277331829071045, |
| "kl": 0.0849609375, |
| "learning_rate": 9.10041841004184e-07, |
| "loss": 0.0034, |
| "reward": 0.32724614441394806, |
| "reward_std": 0.08340810611844063, |
| "reward_std_all": 0.10105587542057037, |
| "rewards-max/forensics_iou_reward": 0.8239950239658356, |
| "rewards/forensics_iou_reward": 0.32724614441394806, |
| "step": 87 |
| }, |
| { |
| "completion_length": 20.109375, |
| "epoch": 0.3682008368200837, |
| "grad_norm": 3.613189697265625, |
| "kl": 0.122314453125, |
| "learning_rate": 9.089958158995816e-07, |
| "loss": 0.0049, |
| "reward": 0.36047327518463135, |
| "reward_std": 0.11231084167957306, |
| "reward_std_all": 0.11205203086137772, |
| "rewards-max/forensics_iou_reward": 0.837378054857254, |
| "rewards/forensics_iou_reward": 0.36047327518463135, |
| "step": 88 |
| }, |
| { |
| "completion_length": 20.625, |
| "epoch": 0.3723849372384937, |
| "grad_norm": 3.3257322311401367, |
| "kl": 0.15185546875, |
| "learning_rate": 9.07949790794979e-07, |
| "loss": 0.0061, |
| "reward": 0.36618292331695557, |
| "reward_std": 0.13651467859745026, |
| "reward_std_all": 0.1263057291507721, |
| "rewards-max/forensics_iou_reward": 0.8148468732833862, |
| "rewards/forensics_iou_reward": 0.36618292331695557, |
| "step": 89 |
| }, |
| { |
| "completion_length": 19.28125, |
| "epoch": 0.37656903765690375, |
| "grad_norm": 2.60185170173645, |
| "kl": 0.130126953125, |
| "learning_rate": 9.069037656903765e-07, |
| "loss": 0.0052, |
| "reward": 0.39389149844646454, |
| "reward_std": 0.07747480273246765, |
| "reward_std_all": 0.09455011785030365, |
| "rewards-max/forensics_iou_reward": 0.7952612936496735, |
| "rewards/forensics_iou_reward": 0.39389149844646454, |
| "step": 90 |
| }, |
| { |
| "completion_length": 20.203125, |
| "epoch": 0.3807531380753138, |
| "grad_norm": 2.2427661418914795, |
| "kl": 0.09765625, |
| "learning_rate": 9.05857740585774e-07, |
| "loss": 0.0039, |
| "reward": 0.35124099254608154, |
| "reward_std": 0.08736037090420723, |
| "reward_std_all": 0.09570389986038208, |
| "rewards-max/forensics_iou_reward": 0.8612352311611176, |
| "rewards/forensics_iou_reward": 0.35124099254608154, |
| "step": 91 |
| }, |
| { |
| "completion_length": 22.046875, |
| "epoch": 0.38493723849372385, |
| "grad_norm": 2.734799861907959, |
| "kl": 0.15771484375, |
| "learning_rate": 9.048117154811715e-07, |
| "loss": 0.0063, |
| "reward": 0.3720535337924957, |
| "reward_std": 0.11555660888552666, |
| "reward_std_all": 0.1138298325240612, |
| "rewards-max/forensics_iou_reward": 0.8700869083404541, |
| "rewards/forensics_iou_reward": 0.3720535337924957, |
| "step": 92 |
| }, |
| { |
| "completion_length": 19.859375, |
| "epoch": 0.3891213389121339, |
| "grad_norm": 2.7268614768981934, |
| "kl": 0.089599609375, |
| "learning_rate": 9.03765690376569e-07, |
| "loss": 0.0036, |
| "reward": 0.41126900911331177, |
| "reward_std": 0.08255963027477264, |
| "reward_std_all": 0.08067955821752548, |
| "rewards-max/forensics_iou_reward": 0.9313558638095856, |
| "rewards/forensics_iou_reward": 0.41126900911331177, |
| "step": 93 |
| }, |
| { |
| "completion_length": 20.28125, |
| "epoch": 0.39330543933054396, |
| "grad_norm": 5.414638519287109, |
| "kl": 0.101806640625, |
| "learning_rate": 9.027196652719665e-07, |
| "loss": 0.0041, |
| "reward": 0.3449578136205673, |
| "reward_std": 0.08652861788868904, |
| "reward_std_all": 0.0920800007879734, |
| "rewards-max/forensics_iou_reward": 0.8913556635379791, |
| "rewards/forensics_iou_reward": 0.3449578136205673, |
| "step": 94 |
| }, |
| { |
| "completion_length": 20.03125, |
| "epoch": 0.39748953974895396, |
| "grad_norm": 5.699835300445557, |
| "kl": 0.33837890625, |
| "learning_rate": 9.01673640167364e-07, |
| "loss": 0.0136, |
| "reward": 0.3798789083957672, |
| "reward_std": 0.09232894703745842, |
| "reward_std_all": 0.08536499366164207, |
| "rewards-max/forensics_iou_reward": 0.8009272813796997, |
| "rewards/forensics_iou_reward": 0.3798789083957672, |
| "step": 95 |
| }, |
| { |
| "completion_length": 19.921875, |
| "epoch": 0.401673640167364, |
| "grad_norm": 2.7815775871276855, |
| "kl": 0.116455078125, |
| "learning_rate": 9.006276150627615e-07, |
| "loss": 0.0047, |
| "reward": 0.3383260667324066, |
| "reward_std": 0.11406636983156204, |
| "reward_std_all": 0.11257808655500412, |
| "rewards-max/forensics_iou_reward": 0.7527330815792084, |
| "rewards/forensics_iou_reward": 0.3383260667324066, |
| "step": 96 |
| }, |
| { |
| "completion_length": 20.28125, |
| "epoch": 0.40585774058577406, |
| "grad_norm": 3.7988040447235107, |
| "kl": 0.117431640625, |
| "learning_rate": 8.99581589958159e-07, |
| "loss": 0.0047, |
| "reward": 0.3790224492549896, |
| "reward_std": 0.0819813571870327, |
| "reward_std_all": 0.09796615689992905, |
| "rewards-max/forensics_iou_reward": 0.8357142806053162, |
| "rewards/forensics_iou_reward": 0.3790224492549896, |
| "step": 97 |
| }, |
| { |
| "completion_length": 19.515625, |
| "epoch": 0.4100418410041841, |
| "grad_norm": 3.1299517154693604, |
| "kl": 0.105224609375, |
| "learning_rate": 8.985355648535564e-07, |
| "loss": 0.0042, |
| "reward": 0.4341437667608261, |
| "reward_std": 0.07933546230196953, |
| "reward_std_all": 0.08859321847558022, |
| "rewards-max/forensics_iou_reward": 0.8574042916297913, |
| "rewards/forensics_iou_reward": 0.4341437667608261, |
| "step": 98 |
| }, |
| { |
| "completion_length": 20.15625, |
| "epoch": 0.41422594142259417, |
| "grad_norm": 3.294926404953003, |
| "kl": 0.212890625, |
| "learning_rate": 8.974895397489539e-07, |
| "loss": 0.0085, |
| "reward": 0.5148323476314545, |
| "reward_std": 0.09883326292037964, |
| "reward_std_all": 0.1013098917901516, |
| "rewards-max/forensics_iou_reward": 0.8402080833911896, |
| "rewards/forensics_iou_reward": 0.5148323476314545, |
| "step": 99 |
| }, |
| { |
| "completion_length": 19.515625, |
| "epoch": 0.41841004184100417, |
| "grad_norm": 3.1453073024749756, |
| "kl": 0.08935546875, |
| "learning_rate": 8.964435146443514e-07, |
| "loss": 0.0036, |
| "reward": 0.24383875727653503, |
| "reward_std": 0.03849951829761267, |
| "reward_std_all": 0.03590627294033766, |
| "rewards-max/forensics_iou_reward": 0.709039181470871, |
| "rewards/forensics_iou_reward": 0.24383875727653503, |
| "step": 100 |
| }, |
| { |
| "completion_length": 20.515625, |
| "epoch": 0.4225941422594142, |
| "grad_norm": 7.50711727142334, |
| "kl": 0.326171875, |
| "learning_rate": 8.953974895397489e-07, |
| "loss": 0.013, |
| "reward": 0.2792418748140335, |
| "reward_std": 0.08006584271788597, |
| "reward_std_all": 0.08800283446907997, |
| "rewards-max/forensics_iou_reward": 0.852678656578064, |
| "rewards/forensics_iou_reward": 0.2792418748140335, |
| "step": 101 |
| }, |
| { |
| "completion_length": 20.25, |
| "epoch": 0.42677824267782427, |
| "grad_norm": 3.4809398651123047, |
| "kl": 0.103515625, |
| "learning_rate": 8.943514644351464e-07, |
| "loss": 0.0041, |
| "reward": 0.4323473423719406, |
| "reward_std": 0.10629947483539581, |
| "reward_std_all": 0.11184418201446533, |
| "rewards-max/forensics_iou_reward": 0.8456238508224487, |
| "rewards/forensics_iou_reward": 0.4323473423719406, |
| "step": 102 |
| }, |
| { |
| "completion_length": 22.234375, |
| "epoch": 0.4309623430962343, |
| "grad_norm": 3.4493749141693115, |
| "kl": 0.118408203125, |
| "learning_rate": 8.933054393305439e-07, |
| "loss": 0.0047, |
| "reward": 0.3589583486318588, |
| "reward_std": 0.09052383899688721, |
| "reward_std_all": 0.09473787248134613, |
| "rewards-max/forensics_iou_reward": 0.9031531512737274, |
| "rewards/forensics_iou_reward": 0.3589583486318588, |
| "step": 103 |
| }, |
| { |
| "completion_length": 19.484375, |
| "epoch": 0.4351464435146444, |
| "grad_norm": 4.468377113342285, |
| "kl": 0.11572265625, |
| "learning_rate": 8.922594142259414e-07, |
| "loss": 0.0046, |
| "reward": 0.3876475989818573, |
| "reward_std": 0.11103124171495438, |
| "reward_std_all": 0.10979057103395462, |
| "rewards-max/forensics_iou_reward": 0.8992989361286163, |
| "rewards/forensics_iou_reward": 0.3876475989818573, |
| "step": 104 |
| }, |
| { |
| "completion_length": 20.421875, |
| "epoch": 0.4393305439330544, |
| "grad_norm": 3.7107601165771484, |
| "kl": 0.395751953125, |
| "learning_rate": 8.91213389121339e-07, |
| "loss": 0.0157, |
| "reward": 0.4643871933221817, |
| "reward_std": 0.08340652287006378, |
| "reward_std_all": 0.08395945280790329, |
| "rewards-max/forensics_iou_reward": 0.8185876607894897, |
| "rewards/forensics_iou_reward": 0.4643871933221817, |
| "step": 105 |
| }, |
| { |
| "completion_length": 21.671875, |
| "epoch": 0.4435146443514644, |
| "grad_norm": 4.512100696563721, |
| "kl": 0.099609375, |
| "learning_rate": 8.901673640167363e-07, |
| "loss": 0.004, |
| "reward": 0.5202136933803558, |
| "reward_std": 0.11594149842858315, |
| "reward_std_all": 0.12369188293814659, |
| "rewards-max/forensics_iou_reward": 0.9191615581512451, |
| "rewards/forensics_iou_reward": 0.5202136933803558, |
| "step": 106 |
| }, |
| { |
| "completion_length": 19.234375, |
| "epoch": 0.4476987447698745, |
| "grad_norm": 3.095491409301758, |
| "kl": 0.203125, |
| "learning_rate": 8.891213389121338e-07, |
| "loss": 0.0081, |
| "reward": 0.5073534995317459, |
| "reward_std": 0.07173437252640724, |
| "reward_std_all": 0.07793982326984406, |
| "rewards-max/forensics_iou_reward": 0.9261411130428314, |
| "rewards/forensics_iou_reward": 0.5073534995317459, |
| "step": 107 |
| }, |
| { |
| "completion_length": 19.53125, |
| "epoch": 0.45188284518828453, |
| "grad_norm": 3.450329303741455, |
| "kl": 0.095458984375, |
| "learning_rate": 8.880753138075313e-07, |
| "loss": 0.0038, |
| "reward": 0.38241153955459595, |
| "reward_std": 0.13388069346547127, |
| "reward_std_all": 0.11786055192351341, |
| "rewards-max/forensics_iou_reward": 0.8842475116252899, |
| "rewards/forensics_iou_reward": 0.38241153955459595, |
| "step": 108 |
| }, |
| { |
| "completion_length": 19.484375, |
| "epoch": 0.4560669456066946, |
| "grad_norm": 3.028557777404785, |
| "kl": 0.17724609375, |
| "learning_rate": 8.870292887029288e-07, |
| "loss": 0.0071, |
| "reward": 0.4500555694103241, |
| "reward_std": 0.09997005388140678, |
| "reward_std_all": 0.13762378692626953, |
| "rewards-max/forensics_iou_reward": 0.8681769073009491, |
| "rewards/forensics_iou_reward": 0.4500555694103241, |
| "step": 109 |
| }, |
| { |
| "completion_length": 19.09375, |
| "epoch": 0.4602510460251046, |
| "grad_norm": 3.3317534923553467, |
| "kl": 0.0966796875, |
| "learning_rate": 8.859832635983263e-07, |
| "loss": 0.0039, |
| "reward": 0.44222207367420197, |
| "reward_std": 0.09584652632474899, |
| "reward_std_all": 0.10490647703409195, |
| "rewards-max/forensics_iou_reward": 0.8933943808078766, |
| "rewards/forensics_iou_reward": 0.44222207367420197, |
| "step": 110 |
| }, |
| { |
| "completion_length": 19.375, |
| "epoch": 0.46443514644351463, |
| "grad_norm": 3.5985233783721924, |
| "kl": 0.1396484375, |
| "learning_rate": 8.849372384937238e-07, |
| "loss": 0.0056, |
| "reward": 0.36289775371551514, |
| "reward_std": 0.12112593650817871, |
| "reward_std_all": 0.1244744136929512, |
| "rewards-max/forensics_iou_reward": 0.8595488667488098, |
| "rewards/forensics_iou_reward": 0.36289775371551514, |
| "step": 111 |
| }, |
| { |
| "completion_length": 19.671875, |
| "epoch": 0.4686192468619247, |
| "grad_norm": 3.6765687465667725, |
| "kl": 0.136474609375, |
| "learning_rate": 8.838912133891213e-07, |
| "loss": 0.0055, |
| "reward": 0.3740224540233612, |
| "reward_std": 0.09379702806472778, |
| "reward_std_all": 0.10415259376168251, |
| "rewards-max/forensics_iou_reward": 0.7020694613456726, |
| "rewards/forensics_iou_reward": 0.3740224540233612, |
| "step": 112 |
| }, |
| { |
| "completion_length": 19.5625, |
| "epoch": 0.47280334728033474, |
| "grad_norm": 2.9470667839050293, |
| "kl": 0.20556640625, |
| "learning_rate": 8.828451882845189e-07, |
| "loss": 0.0082, |
| "reward": 0.48620718717575073, |
| "reward_std": 0.10837670415639877, |
| "reward_std_all": 0.1013684943318367, |
| "rewards-max/forensics_iou_reward": 0.9473958909511566, |
| "rewards/forensics_iou_reward": 0.48620718717575073, |
| "step": 113 |
| }, |
| { |
| "completion_length": 19.453125, |
| "epoch": 0.4769874476987448, |
| "grad_norm": 2.965634822845459, |
| "kl": 0.085693359375, |
| "learning_rate": 8.817991631799164e-07, |
| "loss": 0.0034, |
| "reward": 0.37867571413517, |
| "reward_std": 0.14203988015651703, |
| "reward_std_all": 0.13018783181905746, |
| "rewards-max/forensics_iou_reward": 0.9169578850269318, |
| "rewards/forensics_iou_reward": 0.37867571413517, |
| "step": 114 |
| }, |
| { |
| "completion_length": 20.40625, |
| "epoch": 0.4811715481171548, |
| "grad_norm": 3.2831954956054688, |
| "kl": 0.088134765625, |
| "learning_rate": 8.807531380753138e-07, |
| "loss": 0.0035, |
| "reward": 0.31762202084064484, |
| "reward_std": 0.07030550017952919, |
| "reward_std_all": 0.07263406366109848, |
| "rewards-max/forensics_iou_reward": 0.8671137392520905, |
| "rewards/forensics_iou_reward": 0.31762202084064484, |
| "step": 115 |
| }, |
| { |
| "completion_length": 19.3125, |
| "epoch": 0.48535564853556484, |
| "grad_norm": 3.1284854412078857, |
| "kl": 0.0523681640625, |
| "learning_rate": 8.797071129707112e-07, |
| "loss": 0.0021, |
| "reward": 0.35003238916397095, |
| "reward_std": 0.08032157272100449, |
| "reward_std_all": 0.09245913848280907, |
| "rewards-max/forensics_iou_reward": 0.8894360661506653, |
| "rewards/forensics_iou_reward": 0.35003238916397095, |
| "step": 116 |
| }, |
| { |
| "completion_length": 19.65625, |
| "epoch": 0.4895397489539749, |
| "grad_norm": 3.020718812942505, |
| "kl": 0.10595703125, |
| "learning_rate": 8.786610878661087e-07, |
| "loss": 0.0042, |
| "reward": 0.47101977467536926, |
| "reward_std": 0.10414474084973335, |
| "reward_std_all": 0.12655479833483696, |
| "rewards-max/forensics_iou_reward": 0.8575131297111511, |
| "rewards/forensics_iou_reward": 0.47101977467536926, |
| "step": 117 |
| }, |
| { |
| "completion_length": 19.671875, |
| "epoch": 0.49372384937238495, |
| "grad_norm": 3.2021970748901367, |
| "kl": 0.083251953125, |
| "learning_rate": 8.776150627615062e-07, |
| "loss": 0.0033, |
| "reward": 0.4221174269914627, |
| "reward_std": 0.09793438389897346, |
| "reward_std_all": 0.1000857762992382, |
| "rewards-max/forensics_iou_reward": 0.845660388469696, |
| "rewards/forensics_iou_reward": 0.4221174269914627, |
| "step": 118 |
| }, |
| { |
| "completion_length": 19.59375, |
| "epoch": 0.497907949790795, |
| "grad_norm": 3.0210819244384766, |
| "kl": 0.095703125, |
| "learning_rate": 8.765690376569037e-07, |
| "loss": 0.0038, |
| "reward": 0.3967274874448776, |
| "reward_std": 0.1036505214869976, |
| "reward_std_all": 0.09531508013606071, |
| "rewards-max/forensics_iou_reward": 0.9246301651000977, |
| "rewards/forensics_iou_reward": 0.3967274874448776, |
| "step": 119 |
| }, |
| { |
| "completion_length": 20.0625, |
| "epoch": 0.502092050209205, |
| "grad_norm": 3.2286930084228516, |
| "kl": 0.093994140625, |
| "learning_rate": 8.755230125523012e-07, |
| "loss": 0.0038, |
| "reward": 0.49460113048553467, |
| "reward_std": 0.09497364982962608, |
| "reward_std_all": 0.11876396089792252, |
| "rewards-max/forensics_iou_reward": 0.9519096314907074, |
| "rewards/forensics_iou_reward": 0.49460113048553467, |
| "step": 120 |
| }, |
| { |
| "completion_length": 19.75, |
| "epoch": 0.5062761506276151, |
| "grad_norm": 3.058939218521118, |
| "kl": 0.116943359375, |
| "learning_rate": 8.744769874476987e-07, |
| "loss": 0.0047, |
| "reward": 0.5042795836925507, |
| "reward_std": 0.10138309001922607, |
| "reward_std_all": 0.10157063603401184, |
| "rewards-max/forensics_iou_reward": 0.8771281540393829, |
| "rewards/forensics_iou_reward": 0.5042795836925507, |
| "step": 121 |
| }, |
| { |
| "completion_length": 19.375, |
| "epoch": 0.5104602510460251, |
| "grad_norm": 4.456614017486572, |
| "kl": 0.085205078125, |
| "learning_rate": 8.734309623430963e-07, |
| "loss": 0.0034, |
| "reward": 0.2755053862929344, |
| "reward_std": 0.06991549581289291, |
| "reward_std_all": 0.0774189792573452, |
| "rewards-max/forensics_iou_reward": 0.8020074367523193, |
| "rewards/forensics_iou_reward": 0.2755053862929344, |
| "step": 122 |
| }, |
| { |
| "completion_length": 19.703125, |
| "epoch": 0.5146443514644351, |
| "grad_norm": 3.793651580810547, |
| "kl": 0.095703125, |
| "learning_rate": 8.723849372384937e-07, |
| "loss": 0.0038, |
| "reward": 0.45851555466651917, |
| "reward_std": 0.13897352665662766, |
| "reward_std_all": 0.14841579645872116, |
| "rewards-max/forensics_iou_reward": 0.8377237915992737, |
| "rewards/forensics_iou_reward": 0.45851555466651917, |
| "step": 123 |
| }, |
| { |
| "completion_length": 19.203125, |
| "epoch": 0.5188284518828452, |
| "grad_norm": 3.231222152709961, |
| "kl": 0.07763671875, |
| "learning_rate": 8.713389121338912e-07, |
| "loss": 0.0031, |
| "reward": 0.4655531346797943, |
| "reward_std": 0.057837625965476036, |
| "reward_std_all": 0.06488797441124916, |
| "rewards-max/forensics_iou_reward": 0.8647291660308838, |
| "rewards/forensics_iou_reward": 0.4655531346797943, |
| "step": 124 |
| }, |
| { |
| "completion_length": 19.390625, |
| "epoch": 0.5230125523012552, |
| "grad_norm": 2.835853338241577, |
| "kl": 0.0771484375, |
| "learning_rate": 8.702928870292887e-07, |
| "loss": 0.0031, |
| "reward": 0.444814071059227, |
| "reward_std": 0.07856665551662445, |
| "reward_std_all": 0.08515511080622673, |
| "rewards-max/forensics_iou_reward": 0.8278183043003082, |
| "rewards/forensics_iou_reward": 0.444814071059227, |
| "step": 125 |
| }, |
| { |
| "completion_length": 19.953125, |
| "epoch": 0.5271966527196653, |
| "grad_norm": 3.1240320205688477, |
| "kl": 0.10595703125, |
| "learning_rate": 8.692468619246861e-07, |
| "loss": 0.0042, |
| "reward": 0.4700623005628586, |
| "reward_std": 0.11620824784040451, |
| "reward_std_all": 0.10783607512712479, |
| "rewards-max/forensics_iou_reward": 0.7971042394638062, |
| "rewards/forensics_iou_reward": 0.4700623005628586, |
| "step": 126 |
| }, |
| { |
| "completion_length": 19.578125, |
| "epoch": 0.5313807531380753, |
| "grad_norm": 2.88809871673584, |
| "kl": 0.07275390625, |
| "learning_rate": 8.682008368200836e-07, |
| "loss": 0.0029, |
| "reward": 0.5000173449516296, |
| "reward_std": 0.09211757220327854, |
| "reward_std_all": 0.11641894653439522, |
| "rewards-max/forensics_iou_reward": 0.9161061942577362, |
| "rewards/forensics_iou_reward": 0.5000173449516296, |
| "step": 127 |
| }, |
| { |
| "completion_length": 19.359375, |
| "epoch": 0.5355648535564853, |
| "grad_norm": 3.9742684364318848, |
| "kl": 0.063232421875, |
| "learning_rate": 8.671548117154811e-07, |
| "loss": 0.0025, |
| "reward": 0.4309849590063095, |
| "reward_std": 0.1312236785888672, |
| "reward_std_all": 0.11824413388967514, |
| "rewards-max/forensics_iou_reward": 0.9694229066371918, |
| "rewards/forensics_iou_reward": 0.4309849590063095, |
| "step": 128 |
| }, |
| { |
| "completion_length": 19.671875, |
| "epoch": 0.5397489539748954, |
| "grad_norm": 2.6139347553253174, |
| "kl": 0.093505859375, |
| "learning_rate": 8.661087866108786e-07, |
| "loss": 0.0037, |
| "reward": 0.43996530771255493, |
| "reward_std": 0.0993850789964199, |
| "reward_std_all": 0.10320431739091873, |
| "rewards-max/forensics_iou_reward": 0.8734866082668304, |
| "rewards/forensics_iou_reward": 0.43996530771255493, |
| "step": 129 |
| }, |
| { |
| "completion_length": 19.484375, |
| "epoch": 0.5439330543933054, |
| "grad_norm": 2.94709849357605, |
| "kl": 0.09423828125, |
| "learning_rate": 8.650627615062761e-07, |
| "loss": 0.0038, |
| "reward": 0.37180034816265106, |
| "reward_std": 0.06329183839261532, |
| "reward_std_all": 0.0836777314543724, |
| "rewards-max/forensics_iou_reward": 0.8030148446559906, |
| "rewards/forensics_iou_reward": 0.37180034816265106, |
| "step": 130 |
| }, |
| { |
| "completion_length": 19.796875, |
| "epoch": 0.5481171548117155, |
| "grad_norm": 3.861294746398926, |
| "kl": 0.0810546875, |
| "learning_rate": 8.640167364016737e-07, |
| "loss": 0.0032, |
| "reward": 0.3577638119459152, |
| "reward_std": 0.10280131921172142, |
| "reward_std_all": 0.10108163952827454, |
| "rewards-max/forensics_iou_reward": 0.8529291450977325, |
| "rewards/forensics_iou_reward": 0.3577638119459152, |
| "step": 131 |
| }, |
| { |
| "completion_length": 19.421875, |
| "epoch": 0.5523012552301255, |
| "grad_norm": 4.360830307006836, |
| "kl": 0.2578125, |
| "learning_rate": 8.629707112970711e-07, |
| "loss": 0.0103, |
| "reward": 0.465874508023262, |
| "reward_std": 0.11536744236946106, |
| "reward_std_all": 0.1160125769674778, |
| "rewards-max/forensics_iou_reward": 0.9080999791622162, |
| "rewards/forensics_iou_reward": 0.465874508023262, |
| "step": 132 |
| }, |
| { |
| "completion_length": 20.1875, |
| "epoch": 0.5564853556485355, |
| "grad_norm": 3.497697591781616, |
| "kl": 0.12451171875, |
| "learning_rate": 8.619246861924686e-07, |
| "loss": 0.005, |
| "reward": 0.36955347657203674, |
| "reward_std": 0.12287119403481483, |
| "reward_std_all": 0.11252688989043236, |
| "rewards-max/forensics_iou_reward": 0.9658547043800354, |
| "rewards/forensics_iou_reward": 0.36955347657203674, |
| "step": 133 |
| }, |
| { |
| "completion_length": 19.296875, |
| "epoch": 0.5606694560669456, |
| "grad_norm": 3.34159255027771, |
| "kl": 0.115234375, |
| "learning_rate": 8.608786610878661e-07, |
| "loss": 0.0046, |
| "reward": 0.405206635594368, |
| "reward_std": 0.07379162311553955, |
| "reward_std_all": 0.08204292505979538, |
| "rewards-max/forensics_iou_reward": 0.878467857837677, |
| "rewards/forensics_iou_reward": 0.405206635594368, |
| "step": 134 |
| }, |
| { |
| "completion_length": 19.46875, |
| "epoch": 0.5648535564853556, |
| "grad_norm": 3.9466984272003174, |
| "kl": 0.107421875, |
| "learning_rate": 8.598326359832636e-07, |
| "loss": 0.0043, |
| "reward": 0.3789442777633667, |
| "reward_std": 0.10252273827791214, |
| "reward_std_all": 0.1159542165696621, |
| "rewards-max/forensics_iou_reward": 0.9528478384017944, |
| "rewards/forensics_iou_reward": 0.3789442777633667, |
| "step": 135 |
| }, |
| { |
| "completion_length": 19.9375, |
| "epoch": 0.5690376569037657, |
| "grad_norm": 3.6171462535858154, |
| "kl": 0.16357421875, |
| "learning_rate": 8.58786610878661e-07, |
| "loss": 0.0065, |
| "reward": 0.43513745069503784, |
| "reward_std": 0.10361237078905106, |
| "reward_std_all": 0.11934696510434151, |
| "rewards-max/forensics_iou_reward": 0.8739857971668243, |
| "rewards/forensics_iou_reward": 0.43513745069503784, |
| "step": 136 |
| }, |
| { |
| "completion_length": 19.828125, |
| "epoch": 0.5732217573221757, |
| "grad_norm": 2.9245331287384033, |
| "kl": 0.121337890625, |
| "learning_rate": 8.577405857740585e-07, |
| "loss": 0.0049, |
| "reward": 0.4440338611602783, |
| "reward_std": 0.09632230922579765, |
| "reward_std_all": 0.1070604957640171, |
| "rewards-max/forensics_iou_reward": 0.988241970539093, |
| "rewards/forensics_iou_reward": 0.4440338611602783, |
| "step": 137 |
| }, |
| { |
| "completion_length": 19.703125, |
| "epoch": 0.5774058577405857, |
| "grad_norm": 4.398103713989258, |
| "kl": 0.181396484375, |
| "learning_rate": 8.56694560669456e-07, |
| "loss": 0.0072, |
| "reward": 0.504012867808342, |
| "reward_std": 0.07526565343141556, |
| "reward_std_all": 0.09639449417591095, |
| "rewards-max/forensics_iou_reward": 0.9424620568752289, |
| "rewards/forensics_iou_reward": 0.504012867808342, |
| "step": 138 |
| }, |
| { |
| "completion_length": 19.921875, |
| "epoch": 0.5815899581589958, |
| "grad_norm": 4.242702484130859, |
| "kl": 0.1064453125, |
| "learning_rate": 8.556485355648536e-07, |
| "loss": 0.0043, |
| "reward": 0.48651713132858276, |
| "reward_std": 0.08922155201435089, |
| "reward_std_all": 0.11395234987139702, |
| "rewards-max/forensics_iou_reward": 0.8143109381198883, |
| "rewards/forensics_iou_reward": 0.48651713132858276, |
| "step": 139 |
| }, |
| { |
| "completion_length": 19.390625, |
| "epoch": 0.5857740585774058, |
| "grad_norm": 5.344940185546875, |
| "kl": 0.11181640625, |
| "learning_rate": 8.54602510460251e-07, |
| "loss": 0.0045, |
| "reward": 0.46207596361637115, |
| "reward_std": 0.15015489608049393, |
| "reward_std_all": 0.1621679961681366, |
| "rewards-max/forensics_iou_reward": 0.925374448299408, |
| "rewards/forensics_iou_reward": 0.46207596361637115, |
| "step": 140 |
| }, |
| { |
| "completion_length": 20.609375, |
| "epoch": 0.5899581589958159, |
| "grad_norm": 5.003582000732422, |
| "kl": 0.109375, |
| "learning_rate": 8.535564853556485e-07, |
| "loss": 0.0044, |
| "reward": 0.45406465232372284, |
| "reward_std": 0.1450478360056877, |
| "reward_std_all": 0.14355986565351486, |
| "rewards-max/forensics_iou_reward": 0.9101746678352356, |
| "rewards/forensics_iou_reward": 0.45406465232372284, |
| "step": 141 |
| }, |
| { |
| "completion_length": 19.640625, |
| "epoch": 0.5941422594142259, |
| "grad_norm": 2.920452833175659, |
| "kl": 0.10986328125, |
| "learning_rate": 8.52510460251046e-07, |
| "loss": 0.0044, |
| "reward": 0.40162593126296997, |
| "reward_std": 0.08534115925431252, |
| "reward_std_all": 0.0888940654695034, |
| "rewards-max/forensics_iou_reward": 0.8247211575508118, |
| "rewards/forensics_iou_reward": 0.40162593126296997, |
| "step": 142 |
| }, |
| { |
| "completion_length": 19.796875, |
| "epoch": 0.5983263598326359, |
| "grad_norm": 6.063002109527588, |
| "kl": 0.54296875, |
| "learning_rate": 8.514644351464435e-07, |
| "loss": 0.0217, |
| "reward": 0.40321558713912964, |
| "reward_std": 0.09156971797347069, |
| "reward_std_all": 0.09213004261255264, |
| "rewards-max/forensics_iou_reward": 0.9225988388061523, |
| "rewards/forensics_iou_reward": 0.40321558713912964, |
| "step": 143 |
| }, |
| { |
| "completion_length": 19.109375, |
| "epoch": 0.602510460251046, |
| "grad_norm": 2.9145774841308594, |
| "kl": 0.1162109375, |
| "learning_rate": 8.50418410041841e-07, |
| "loss": 0.0046, |
| "reward": 0.3839212954044342, |
| "reward_std": 0.09103080630302429, |
| "reward_std_all": 0.0912783183157444, |
| "rewards-max/forensics_iou_reward": 0.8270699977874756, |
| "rewards/forensics_iou_reward": 0.3839212954044342, |
| "step": 144 |
| }, |
| { |
| "completion_length": 19.09375, |
| "epoch": 0.606694560669456, |
| "grad_norm": 4.071663856506348, |
| "kl": 0.131591796875, |
| "learning_rate": 8.493723849372385e-07, |
| "loss": 0.0053, |
| "reward": 0.32679712772369385, |
| "reward_std": 0.07504269480705261, |
| "reward_std_all": 0.07347371242940426, |
| "rewards-max/forensics_iou_reward": 0.7542228996753693, |
| "rewards/forensics_iou_reward": 0.32679712772369385, |
| "step": 145 |
| }, |
| { |
| "completion_length": 19.953125, |
| "epoch": 0.6108786610878661, |
| "grad_norm": 3.060218572616577, |
| "kl": 0.11572265625, |
| "learning_rate": 8.483263598326359e-07, |
| "loss": 0.0046, |
| "reward": 0.46875227987766266, |
| "reward_std": 0.08610957115888596, |
| "reward_std_all": 0.09186892211437225, |
| "rewards-max/forensics_iou_reward": 0.8625626862049103, |
| "rewards/forensics_iou_reward": 0.46875227987766266, |
| "step": 146 |
| }, |
| { |
| "completion_length": 19.203125, |
| "epoch": 0.6150627615062761, |
| "grad_norm": 3.7650036811828613, |
| "kl": 0.110595703125, |
| "learning_rate": 8.472803347280334e-07, |
| "loss": 0.0044, |
| "reward": 0.4534465968608856, |
| "reward_std": 0.09456213563680649, |
| "reward_std_all": 0.11008131131529808, |
| "rewards-max/forensics_iou_reward": 0.8544843196868896, |
| "rewards/forensics_iou_reward": 0.4534465968608856, |
| "step": 147 |
| }, |
| { |
| "completion_length": 18.84375, |
| "epoch": 0.6192468619246861, |
| "grad_norm": 2.87581467628479, |
| "kl": 0.128662109375, |
| "learning_rate": 8.462343096234308e-07, |
| "loss": 0.0051, |
| "reward": 0.397551491856575, |
| "reward_std": 0.0728011392056942, |
| "reward_std_all": 0.08051854744553566, |
| "rewards-max/forensics_iou_reward": 0.789860337972641, |
| "rewards/forensics_iou_reward": 0.397551491856575, |
| "step": 148 |
| }, |
| { |
| "completion_length": 19.625, |
| "epoch": 0.6234309623430963, |
| "grad_norm": 3.0439162254333496, |
| "kl": 0.1708984375, |
| "learning_rate": 8.451882845188284e-07, |
| "loss": 0.0068, |
| "reward": 0.4488241970539093, |
| "reward_std": 0.08717101812362671, |
| "reward_std_all": 0.08530791476368904, |
| "rewards-max/forensics_iou_reward": 0.7922177016735077, |
| "rewards/forensics_iou_reward": 0.4488241970539093, |
| "step": 149 |
| }, |
| { |
| "completion_length": 20.703125, |
| "epoch": 0.6276150627615062, |
| "grad_norm": 4.299969673156738, |
| "kl": 0.098876953125, |
| "learning_rate": 8.441422594142259e-07, |
| "loss": 0.004, |
| "reward": 0.3098059892654419, |
| "reward_std": 0.10619377717375755, |
| "reward_std_all": 0.10220659524202347, |
| "rewards-max/forensics_iou_reward": 0.689363420009613, |
| "rewards/forensics_iou_reward": 0.3098059892654419, |
| "step": 150 |
| }, |
| { |
| "completion_length": 19.609375, |
| "epoch": 0.6317991631799164, |
| "grad_norm": 2.93393874168396, |
| "kl": 0.151123046875, |
| "learning_rate": 8.430962343096234e-07, |
| "loss": 0.006, |
| "reward": 0.39652320742607117, |
| "reward_std": 0.08243073523044586, |
| "reward_std_all": 0.1041157990694046, |
| "rewards-max/forensics_iou_reward": 0.8489525318145752, |
| "rewards/forensics_iou_reward": 0.39652320742607117, |
| "step": 151 |
| }, |
| { |
| "completion_length": 19.625, |
| "epoch": 0.6359832635983264, |
| "grad_norm": 3.7490646839141846, |
| "kl": 0.13427734375, |
| "learning_rate": 8.420502092050209e-07, |
| "loss": 0.0054, |
| "reward": 0.43073663115501404, |
| "reward_std": 0.131295308470726, |
| "reward_std_all": 0.13213466480374336, |
| "rewards-max/forensics_iou_reward": 0.959115207195282, |
| "rewards/forensics_iou_reward": 0.43073663115501404, |
| "step": 152 |
| }, |
| { |
| "completion_length": 19.078125, |
| "epoch": 0.6401673640167364, |
| "grad_norm": 3.608609437942505, |
| "kl": 0.21533203125, |
| "learning_rate": 8.410041841004184e-07, |
| "loss": 0.0086, |
| "reward": 0.45684805512428284, |
| "reward_std": 0.09734245017170906, |
| "reward_std_all": 0.10648743435740471, |
| "rewards-max/forensics_iou_reward": 0.9061942398548126, |
| "rewards/forensics_iou_reward": 0.45684805512428284, |
| "step": 153 |
| }, |
| { |
| "completion_length": 19.1875, |
| "epoch": 0.6443514644351465, |
| "grad_norm": 3.8147826194763184, |
| "kl": 0.124267578125, |
| "learning_rate": 8.399581589958159e-07, |
| "loss": 0.005, |
| "reward": 0.4843851774930954, |
| "reward_std": 0.09395655617117882, |
| "reward_std_all": 0.09964603558182716, |
| "rewards-max/forensics_iou_reward": 0.9696888029575348, |
| "rewards/forensics_iou_reward": 0.4843851774930954, |
| "step": 154 |
| }, |
| { |
| "completion_length": 19.84375, |
| "epoch": 0.6485355648535565, |
| "grad_norm": 3.3627121448516846, |
| "kl": 0.1181640625, |
| "learning_rate": 8.389121338912134e-07, |
| "loss": 0.0047, |
| "reward": 0.46811720728874207, |
| "reward_std": 0.09847088158130646, |
| "reward_std_all": 0.10547181963920593, |
| "rewards-max/forensics_iou_reward": 0.9768840670585632, |
| "rewards/forensics_iou_reward": 0.46811720728874207, |
| "step": 155 |
| }, |
| { |
| "completion_length": 19.1875, |
| "epoch": 0.6527196652719666, |
| "grad_norm": 4.216452598571777, |
| "kl": 0.199462890625, |
| "learning_rate": 8.378661087866107e-07, |
| "loss": 0.008, |
| "reward": 0.4159805178642273, |
| "reward_std": 0.09669799730181694, |
| "reward_std_all": 0.10638130083680153, |
| "rewards-max/forensics_iou_reward": 0.8358974158763885, |
| "rewards/forensics_iou_reward": 0.4159805178642273, |
| "step": 156 |
| }, |
| { |
| "completion_length": 19.546875, |
| "epoch": 0.6569037656903766, |
| "grad_norm": 3.018077850341797, |
| "kl": 0.08447265625, |
| "learning_rate": 8.368200836820083e-07, |
| "loss": 0.0034, |
| "reward": 0.35626713931560516, |
| "reward_std": 0.09045984596014023, |
| "reward_std_all": 0.11067421361804008, |
| "rewards-max/forensics_iou_reward": 0.9198250472545624, |
| "rewards/forensics_iou_reward": 0.35626713931560516, |
| "step": 157 |
| }, |
| { |
| "completion_length": 19.421875, |
| "epoch": 0.6610878661087866, |
| "grad_norm": 4.479642868041992, |
| "kl": 0.12841796875, |
| "learning_rate": 8.357740585774058e-07, |
| "loss": 0.0051, |
| "reward": 0.4608016461133957, |
| "reward_std": 0.11193059757351875, |
| "reward_std_all": 0.10676513984799385, |
| "rewards-max/forensics_iou_reward": 0.9040745496749878, |
| "rewards/forensics_iou_reward": 0.4608016461133957, |
| "step": 158 |
| }, |
| { |
| "completion_length": 19.53125, |
| "epoch": 0.6652719665271967, |
| "grad_norm": 3.2512550354003906, |
| "kl": 0.35107421875, |
| "learning_rate": 8.347280334728033e-07, |
| "loss": 0.014, |
| "reward": 0.4398491084575653, |
| "reward_std": 0.07465616799890995, |
| "reward_std_all": 0.08407201617956161, |
| "rewards-max/forensics_iou_reward": 0.9379812479019165, |
| "rewards/forensics_iou_reward": 0.4398491084575653, |
| "step": 159 |
| }, |
| { |
| "completion_length": 19.328125, |
| "epoch": 0.6694560669456067, |
| "grad_norm": 3.807313919067383, |
| "kl": 0.11962890625, |
| "learning_rate": 8.336820083682008e-07, |
| "loss": 0.0048, |
| "reward": 0.44581305980682373, |
| "reward_std": 0.12747827917337418, |
| "reward_std_all": 0.13117362186312675, |
| "rewards-max/forensics_iou_reward": 0.9532714486122131, |
| "rewards/forensics_iou_reward": 0.44581305980682373, |
| "step": 160 |
| }, |
| { |
| "completion_length": 19.375, |
| "epoch": 0.6736401673640168, |
| "grad_norm": 3.0651748180389404, |
| "kl": 0.114013671875, |
| "learning_rate": 8.326359832635983e-07, |
| "loss": 0.0046, |
| "reward": 0.39753736555576324, |
| "reward_std": 0.07395768910646439, |
| "reward_std_all": 0.07679146714508533, |
| "rewards-max/forensics_iou_reward": 0.8925474286079407, |
| "rewards/forensics_iou_reward": 0.39753736555576324, |
| "step": 161 |
| }, |
| { |
| "completion_length": 19.796875, |
| "epoch": 0.6778242677824268, |
| "grad_norm": 4.452110290527344, |
| "kl": 0.111328125, |
| "learning_rate": 8.315899581589958e-07, |
| "loss": 0.0045, |
| "reward": 0.4869672507047653, |
| "reward_std": 0.09619773924350739, |
| "reward_std_all": 0.10103520005941391, |
| "rewards-max/forensics_iou_reward": 0.9314430952072144, |
| "rewards/forensics_iou_reward": 0.4869672507047653, |
| "step": 162 |
| }, |
| { |
| "completion_length": 19.46875, |
| "epoch": 0.6820083682008368, |
| "grad_norm": 4.304245471954346, |
| "kl": 0.1240234375, |
| "learning_rate": 8.305439330543933e-07, |
| "loss": 0.005, |
| "reward": 0.47389912605285645, |
| "reward_std": 0.1164003424346447, |
| "reward_std_all": 0.11429973691701889, |
| "rewards-max/forensics_iou_reward": 0.9235328137874603, |
| "rewards/forensics_iou_reward": 0.47389912605285645, |
| "step": 163 |
| }, |
| { |
| "completion_length": 19.1875, |
| "epoch": 0.6861924686192469, |
| "grad_norm": 3.187364101409912, |
| "kl": 0.091552734375, |
| "learning_rate": 8.294979079497908e-07, |
| "loss": 0.0037, |
| "reward": 0.4217020571231842, |
| "reward_std": 0.09625725820660591, |
| "reward_std_all": 0.10101260617375374, |
| "rewards-max/forensics_iou_reward": 0.9370211660861969, |
| "rewards/forensics_iou_reward": 0.4217020571231842, |
| "step": 164 |
| }, |
| { |
| "completion_length": 19.25, |
| "epoch": 0.6903765690376569, |
| "grad_norm": 2.6599228382110596, |
| "kl": 0.10400390625, |
| "learning_rate": 8.284518828451882e-07, |
| "loss": 0.0042, |
| "reward": 0.5142388939857483, |
| "reward_std": 0.08278244361281395, |
| "reward_std_all": 0.08308707177639008, |
| "rewards-max/forensics_iou_reward": 0.9815221726894379, |
| "rewards/forensics_iou_reward": 0.5142388939857483, |
| "step": 165 |
| }, |
| { |
| "completion_length": 21.328125, |
| "epoch": 0.694560669456067, |
| "grad_norm": 4.130545139312744, |
| "kl": 0.116455078125, |
| "learning_rate": 8.274058577405857e-07, |
| "loss": 0.0047, |
| "reward": 0.45902274549007416, |
| "reward_std": 0.06000002846121788, |
| "reward_std_all": 0.06685727834701538, |
| "rewards-max/forensics_iou_reward": 0.9460041522979736, |
| "rewards/forensics_iou_reward": 0.45902274549007416, |
| "step": 166 |
| }, |
| { |
| "completion_length": 19.609375, |
| "epoch": 0.698744769874477, |
| "grad_norm": 3.67128586769104, |
| "kl": 0.119873046875, |
| "learning_rate": 8.263598326359832e-07, |
| "loss": 0.0048, |
| "reward": 0.46539318561553955, |
| "reward_std": 0.08581854030489922, |
| "reward_std_all": 0.13239726424217224, |
| "rewards-max/forensics_iou_reward": 0.9069556593894958, |
| "rewards/forensics_iou_reward": 0.46539318561553955, |
| "step": 167 |
| }, |
| { |
| "completion_length": 19.390625, |
| "epoch": 0.702928870292887, |
| "grad_norm": 4.322988510131836, |
| "kl": 0.1064453125, |
| "learning_rate": 8.253138075313807e-07, |
| "loss": 0.0043, |
| "reward": 0.5558145493268967, |
| "reward_std": 0.10705137811601162, |
| "reward_std_all": 0.10910160094499588, |
| "rewards-max/forensics_iou_reward": 0.9214181005954742, |
| "rewards/forensics_iou_reward": 0.5558145493268967, |
| "step": 168 |
| }, |
| { |
| "completion_length": 19.28125, |
| "epoch": 0.7071129707112971, |
| "grad_norm": 3.9688351154327393, |
| "kl": 0.135986328125, |
| "learning_rate": 8.242677824267782e-07, |
| "loss": 0.0054, |
| "reward": 0.3831772804260254, |
| "reward_std": 0.10711187869310379, |
| "reward_std_all": 0.10511849820613861, |
| "rewards-max/forensics_iou_reward": 0.9730923771858215, |
| "rewards/forensics_iou_reward": 0.3831772804260254, |
| "step": 169 |
| }, |
| { |
| "completion_length": 19.84375, |
| "epoch": 0.7112970711297071, |
| "grad_norm": 4.736095428466797, |
| "kl": 0.13134765625, |
| "learning_rate": 8.232217573221757e-07, |
| "loss": 0.0053, |
| "reward": 0.44863565266132355, |
| "reward_std": 0.12586169317364693, |
| "reward_std_all": 0.15705863758921623, |
| "rewards-max/forensics_iou_reward": 0.9180340468883514, |
| "rewards/forensics_iou_reward": 0.44863565266132355, |
| "step": 170 |
| }, |
| { |
| "completion_length": 19.3125, |
| "epoch": 0.7154811715481172, |
| "grad_norm": 6.58045768737793, |
| "kl": 0.2119140625, |
| "learning_rate": 8.221757322175732e-07, |
| "loss": 0.0085, |
| "reward": 0.42560215294361115, |
| "reward_std": 0.10597502440214157, |
| "reward_std_all": 0.09672548621892929, |
| "rewards-max/forensics_iou_reward": 0.8324175775051117, |
| "rewards/forensics_iou_reward": 0.42560215294361115, |
| "step": 171 |
| }, |
| { |
| "completion_length": 19.328125, |
| "epoch": 0.7196652719665272, |
| "grad_norm": 3.7561914920806885, |
| "kl": 0.16064453125, |
| "learning_rate": 8.211297071129707e-07, |
| "loss": 0.0064, |
| "reward": 0.33089227974414825, |
| "reward_std": 0.078520517796278, |
| "reward_std_all": 0.07939289882779121, |
| "rewards-max/forensics_iou_reward": 0.8038185834884644, |
| "rewards/forensics_iou_reward": 0.33089227974414825, |
| "step": 172 |
| }, |
| { |
| "completion_length": 19.453125, |
| "epoch": 0.7238493723849372, |
| "grad_norm": 3.4416420459747314, |
| "kl": 0.12353515625, |
| "learning_rate": 8.200836820083682e-07, |
| "loss": 0.0049, |
| "reward": 0.43944142758846283, |
| "reward_std": 0.09607770293951035, |
| "reward_std_all": 0.09662525728344917, |
| "rewards-max/forensics_iou_reward": 0.9037594795227051, |
| "rewards/forensics_iou_reward": 0.43944142758846283, |
| "step": 173 |
| }, |
| { |
| "completion_length": 20.203125, |
| "epoch": 0.7280334728033473, |
| "grad_norm": 3.3264060020446777, |
| "kl": 0.1435546875, |
| "learning_rate": 8.190376569037657e-07, |
| "loss": 0.0057, |
| "reward": 0.429453507065773, |
| "reward_std": 0.0731898881494999, |
| "reward_std_all": 0.0822603851556778, |
| "rewards-max/forensics_iou_reward": 0.8843379318714142, |
| "rewards/forensics_iou_reward": 0.429453507065773, |
| "step": 174 |
| }, |
| { |
| "completion_length": 19.359375, |
| "epoch": 0.7322175732217573, |
| "grad_norm": 3.361323356628418, |
| "kl": 0.17041015625, |
| "learning_rate": 8.179916317991632e-07, |
| "loss": 0.0068, |
| "reward": 0.43083280324935913, |
| "reward_std": 0.13078941404819489, |
| "reward_std_all": 0.12124279886484146, |
| "rewards-max/forensics_iou_reward": 0.8793549239635468, |
| "rewards/forensics_iou_reward": 0.43083280324935913, |
| "step": 175 |
| }, |
| { |
| "completion_length": 19.21875, |
| "epoch": 0.7364016736401674, |
| "grad_norm": 3.212597608566284, |
| "kl": 0.1611328125, |
| "learning_rate": 8.169456066945606e-07, |
| "loss": 0.0064, |
| "reward": 0.3690592646598816, |
| "reward_std": 0.07528423890471458, |
| "reward_std_all": 0.0938841961324215, |
| "rewards-max/forensics_iou_reward": 0.8952283263206482, |
| "rewards/forensics_iou_reward": 0.3690592646598816, |
| "step": 176 |
| }, |
| { |
| "completion_length": 19.59375, |
| "epoch": 0.7405857740585774, |
| "grad_norm": 3.862867832183838, |
| "kl": 0.114501953125, |
| "learning_rate": 8.158995815899581e-07, |
| "loss": 0.0046, |
| "reward": 0.4722036123275757, |
| "reward_std": 0.10675648599863052, |
| "reward_std_all": 0.12544601783156395, |
| "rewards-max/forensics_iou_reward": 0.8933042287826538, |
| "rewards/forensics_iou_reward": 0.4722036123275757, |
| "step": 177 |
| }, |
| { |
| "completion_length": 20.171875, |
| "epoch": 0.7447698744769874, |
| "grad_norm": 2.6577999591827393, |
| "kl": 0.125732421875, |
| "learning_rate": 8.148535564853556e-07, |
| "loss": 0.005, |
| "reward": 0.4456258714199066, |
| "reward_std": 0.08108730614185333, |
| "reward_std_all": 0.10605844482779503, |
| "rewards-max/forensics_iou_reward": 0.9217005670070648, |
| "rewards/forensics_iou_reward": 0.4456258714199066, |
| "step": 178 |
| }, |
| { |
| "completion_length": 19.546875, |
| "epoch": 0.7489539748953975, |
| "grad_norm": 4.6652960777282715, |
| "kl": 0.1533203125, |
| "learning_rate": 8.138075313807531e-07, |
| "loss": 0.0061, |
| "reward": 0.4894832372665405, |
| "reward_std": 0.09123208373785019, |
| "reward_std_all": 0.10484527423977852, |
| "rewards-max/forensics_iou_reward": 0.904832512140274, |
| "rewards/forensics_iou_reward": 0.4894832372665405, |
| "step": 179 |
| }, |
| { |
| "completion_length": 19.203125, |
| "epoch": 0.7531380753138075, |
| "grad_norm": 3.520124912261963, |
| "kl": 0.20703125, |
| "learning_rate": 8.127615062761506e-07, |
| "loss": 0.0083, |
| "reward": 0.5038755387067795, |
| "reward_std": 0.08072058483958244, |
| "reward_std_all": 0.09067218750715256, |
| "rewards-max/forensics_iou_reward": 0.9561113119125366, |
| "rewards/forensics_iou_reward": 0.5038755387067795, |
| "step": 180 |
| }, |
| { |
| "completion_length": 19.4375, |
| "epoch": 0.7573221757322176, |
| "grad_norm": 2.705890655517578, |
| "kl": 0.117919921875, |
| "learning_rate": 8.117154811715481e-07, |
| "loss": 0.0047, |
| "reward": 0.4885488897562027, |
| "reward_std": 0.09237921610474586, |
| "reward_std_all": 0.0977010726928711, |
| "rewards-max/forensics_iou_reward": 0.8496558666229248, |
| "rewards/forensics_iou_reward": 0.4885488897562027, |
| "step": 181 |
| }, |
| { |
| "completion_length": 19.40625, |
| "epoch": 0.7615062761506276, |
| "grad_norm": 3.0296149253845215, |
| "kl": 0.09521484375, |
| "learning_rate": 8.106694560669455e-07, |
| "loss": 0.0038, |
| "reward": 0.48690013587474823, |
| "reward_std": 0.08847735822200775, |
| "reward_std_all": 0.08682471513748169, |
| "rewards-max/forensics_iou_reward": 0.9470980763435364, |
| "rewards/forensics_iou_reward": 0.48690013587474823, |
| "step": 182 |
| }, |
| { |
| "completion_length": 19.203125, |
| "epoch": 0.7656903765690377, |
| "grad_norm": 3.6842033863067627, |
| "kl": 0.12060546875, |
| "learning_rate": 8.096234309623431e-07, |
| "loss": 0.0048, |
| "reward": 0.5287641286849976, |
| "reward_std": 0.07517959922552109, |
| "reward_std_all": 0.07659581303596497, |
| "rewards-max/forensics_iou_reward": 0.888996809720993, |
| "rewards/forensics_iou_reward": 0.5287641286849976, |
| "step": 183 |
| }, |
| { |
| "completion_length": 19.84375, |
| "epoch": 0.7698744769874477, |
| "grad_norm": 3.8027403354644775, |
| "kl": 0.110107421875, |
| "learning_rate": 8.085774058577406e-07, |
| "loss": 0.0044, |
| "reward": 0.3668026626110077, |
| "reward_std": 0.09535450115799904, |
| "reward_std_all": 0.09503190778195858, |
| "rewards-max/forensics_iou_reward": 0.9238791763782501, |
| "rewards/forensics_iou_reward": 0.3668026626110077, |
| "step": 184 |
| }, |
| { |
| "completion_length": 19.28125, |
| "epoch": 0.7740585774058577, |
| "grad_norm": 6.412266254425049, |
| "kl": 0.251220703125, |
| "learning_rate": 8.075313807531381e-07, |
| "loss": 0.01, |
| "reward": 0.35265645384788513, |
| "reward_std": 0.08917206898331642, |
| "reward_std_all": 0.08182520046830177, |
| "rewards-max/forensics_iou_reward": 0.7747553884983063, |
| "rewards/forensics_iou_reward": 0.35265645384788513, |
| "step": 185 |
| }, |
| { |
| "completion_length": 19.28125, |
| "epoch": 0.7782426778242678, |
| "grad_norm": 5.452586650848389, |
| "kl": 0.123046875, |
| "learning_rate": 8.064853556485355e-07, |
| "loss": 0.0049, |
| "reward": 0.567565530538559, |
| "reward_std": 0.06757232546806335, |
| "reward_std_all": 0.07006840035319328, |
| "rewards-max/forensics_iou_reward": 0.9261627793312073, |
| "rewards/forensics_iou_reward": 0.567565530538559, |
| "step": 186 |
| }, |
| { |
| "completion_length": 20.078125, |
| "epoch": 0.7824267782426778, |
| "grad_norm": 4.461852073669434, |
| "kl": 0.12548828125, |
| "learning_rate": 8.05439330543933e-07, |
| "loss": 0.005, |
| "reward": 0.4866507202386856, |
| "reward_std": 0.09606288373470306, |
| "reward_std_all": 0.09606216847896576, |
| "rewards-max/forensics_iou_reward": 0.979692131280899, |
| "rewards/forensics_iou_reward": 0.4866507202386856, |
| "step": 187 |
| }, |
| { |
| "completion_length": 19.859375, |
| "epoch": 0.7866108786610879, |
| "grad_norm": 3.5579569339752197, |
| "kl": 0.130126953125, |
| "learning_rate": 8.043933054393305e-07, |
| "loss": 0.0052, |
| "reward": 0.46571506559848785, |
| "reward_std": 0.11243653669953346, |
| "reward_std_all": 0.12881090492010117, |
| "rewards-max/forensics_iou_reward": 0.8931588232517242, |
| "rewards/forensics_iou_reward": 0.46571506559848785, |
| "step": 188 |
| }, |
| { |
| "completion_length": 19.25, |
| "epoch": 0.7907949790794979, |
| "grad_norm": 2.448174238204956, |
| "kl": 0.12646484375, |
| "learning_rate": 8.03347280334728e-07, |
| "loss": 0.005, |
| "reward": 0.4236939251422882, |
| "reward_std": 0.05612361803650856, |
| "reward_std_all": 0.05462037771940231, |
| "rewards-max/forensics_iou_reward": 0.7648522555828094, |
| "rewards/forensics_iou_reward": 0.4236939251422882, |
| "step": 189 |
| }, |
| { |
| "completion_length": 19.234375, |
| "epoch": 0.7949790794979079, |
| "grad_norm": 4.754932403564453, |
| "kl": 0.1123046875, |
| "learning_rate": 8.023012552301254e-07, |
| "loss": 0.0045, |
| "reward": 0.5438366532325745, |
| "reward_std": 0.12178238481283188, |
| "reward_std_all": 0.11641237139701843, |
| "rewards-max/forensics_iou_reward": 0.9577924907207489, |
| "rewards/forensics_iou_reward": 0.5438366532325745, |
| "step": 190 |
| }, |
| { |
| "completion_length": 19.671875, |
| "epoch": 0.799163179916318, |
| "grad_norm": 2.6470870971679688, |
| "kl": 0.09765625, |
| "learning_rate": 8.01255230125523e-07, |
| "loss": 0.0039, |
| "reward": 0.46999478340148926, |
| "reward_std": 0.0738739762455225, |
| "reward_std_all": 0.0885202419012785, |
| "rewards-max/forensics_iou_reward": 0.9186141490936279, |
| "rewards/forensics_iou_reward": 0.46999478340148926, |
| "step": 191 |
| }, |
| { |
| "completion_length": 19.609375, |
| "epoch": 0.803347280334728, |
| "grad_norm": 3.632298707962036, |
| "kl": 0.1376953125, |
| "learning_rate": 8.002092050209205e-07, |
| "loss": 0.0055, |
| "reward": 0.5190939903259277, |
| "reward_std": 0.08471780642867088, |
| "reward_std_all": 0.09417570754885674, |
| "rewards-max/forensics_iou_reward": 0.9627601802349091, |
| "rewards/forensics_iou_reward": 0.5190939903259277, |
| "step": 192 |
| }, |
| { |
| "completion_length": 18.90625, |
| "epoch": 0.8075313807531381, |
| "grad_norm": 4.025694847106934, |
| "kl": 0.108154296875, |
| "learning_rate": 7.99163179916318e-07, |
| "loss": 0.0043, |
| "reward": 0.3692931681871414, |
| "reward_std": 0.07481111586093903, |
| "reward_std_all": 0.0756373405456543, |
| "rewards-max/forensics_iou_reward": 0.7684451043605804, |
| "rewards/forensics_iou_reward": 0.3692931681871414, |
| "step": 193 |
| }, |
| { |
| "completion_length": 19.109375, |
| "epoch": 0.8117154811715481, |
| "grad_norm": 3.737961769104004, |
| "kl": 0.17236328125, |
| "learning_rate": 7.981171548117155e-07, |
| "loss": 0.0069, |
| "reward": 0.4900645911693573, |
| "reward_std": 0.07164029031991959, |
| "reward_std_all": 0.09565438702702522, |
| "rewards-max/forensics_iou_reward": 0.8797412514686584, |
| "rewards/forensics_iou_reward": 0.4900645911693573, |
| "step": 194 |
| }, |
| { |
| "completion_length": 19.921875, |
| "epoch": 0.8158995815899581, |
| "grad_norm": 4.509769439697266, |
| "kl": 0.1796875, |
| "learning_rate": 7.97071129707113e-07, |
| "loss": 0.0072, |
| "reward": 0.4554114043712616, |
| "reward_std": 0.0981542095541954, |
| "reward_std_all": 0.10787514597177505, |
| "rewards-max/forensics_iou_reward": 0.8603058755397797, |
| "rewards/forensics_iou_reward": 0.4554114043712616, |
| "step": 195 |
| }, |
| { |
| "completion_length": 19.0625, |
| "epoch": 0.8200836820083682, |
| "grad_norm": 4.15757942199707, |
| "kl": 0.107421875, |
| "learning_rate": 7.960251046025104e-07, |
| "loss": 0.0043, |
| "reward": 0.507477343082428, |
| "reward_std": 0.06812693923711777, |
| "reward_std_all": 0.07399035803973675, |
| "rewards-max/forensics_iou_reward": 0.8587698340415955, |
| "rewards/forensics_iou_reward": 0.507477343082428, |
| "step": 196 |
| }, |
| { |
| "completion_length": 19.75, |
| "epoch": 0.8242677824267782, |
| "grad_norm": 3.4394657611846924, |
| "kl": 0.10888671875, |
| "learning_rate": 7.949790794979079e-07, |
| "loss": 0.0044, |
| "reward": 0.5058480948209763, |
| "reward_std": 0.08507076278328896, |
| "reward_std_all": 0.08109893091022968, |
| "rewards-max/forensics_iou_reward": 0.9084384441375732, |
| "rewards/forensics_iou_reward": 0.5058480948209763, |
| "step": 197 |
| }, |
| { |
| "completion_length": 19.421875, |
| "epoch": 0.8284518828451883, |
| "grad_norm": 5.120692729949951, |
| "kl": 0.097900390625, |
| "learning_rate": 7.939330543933053e-07, |
| "loss": 0.0039, |
| "reward": 0.453654021024704, |
| "reward_std": 0.08674725890159607, |
| "reward_std_all": 0.09270575270056725, |
| "rewards-max/forensics_iou_reward": 0.959631085395813, |
| "rewards/forensics_iou_reward": 0.453654021024704, |
| "step": 198 |
| }, |
| { |
| "completion_length": 19.6875, |
| "epoch": 0.8326359832635983, |
| "grad_norm": 3.9832820892333984, |
| "kl": 0.12451171875, |
| "learning_rate": 7.928870292887028e-07, |
| "loss": 0.005, |
| "reward": 0.4518960565328598, |
| "reward_std": 0.10339684039354324, |
| "reward_std_all": 0.10343530029058456, |
| "rewards-max/forensics_iou_reward": 0.9289035797119141, |
| "rewards/forensics_iou_reward": 0.4518960565328598, |
| "step": 199 |
| }, |
| { |
| "completion_length": 19.1875, |
| "epoch": 0.8368200836820083, |
| "grad_norm": 3.0412886142730713, |
| "kl": 0.1064453125, |
| "learning_rate": 7.918410041841004e-07, |
| "loss": 0.0043, |
| "reward": 0.4698241949081421, |
| "reward_std": 0.052199916914105415, |
| "reward_std_all": 0.060096900910139084, |
| "rewards-max/forensics_iou_reward": 0.9221774339675903, |
| "rewards/forensics_iou_reward": 0.4698241949081421, |
| "step": 200 |
| }, |
| { |
| "completion_length": 19.734375, |
| "epoch": 0.8410041841004184, |
| "grad_norm": 4.385019302368164, |
| "kl": 0.1455078125, |
| "learning_rate": 7.907949790794979e-07, |
| "loss": 0.0058, |
| "reward": 0.4635029137134552, |
| "reward_std": 0.11141990125179291, |
| "reward_std_all": 0.10553781688213348, |
| "rewards-max/forensics_iou_reward": 0.9136933982372284, |
| "rewards/forensics_iou_reward": 0.4635029137134552, |
| "step": 201 |
| }, |
| { |
| "completion_length": 19.375, |
| "epoch": 0.8451882845188284, |
| "grad_norm": 4.945239067077637, |
| "kl": 0.13427734375, |
| "learning_rate": 7.897489539748954e-07, |
| "loss": 0.0054, |
| "reward": 0.37165024876594543, |
| "reward_std": 0.10391958430409431, |
| "reward_std_all": 0.1034877561032772, |
| "rewards-max/forensics_iou_reward": 0.8706302344799042, |
| "rewards/forensics_iou_reward": 0.37165024876594543, |
| "step": 202 |
| }, |
| { |
| "completion_length": 19.375, |
| "epoch": 0.8493723849372385, |
| "grad_norm": 5.460031032562256, |
| "kl": 0.109619140625, |
| "learning_rate": 7.887029288702929e-07, |
| "loss": 0.0044, |
| "reward": 0.4414226859807968, |
| "reward_std": 0.07793040946125984, |
| "reward_std_all": 0.07914308458566666, |
| "rewards-max/forensics_iou_reward": 0.8989056944847107, |
| "rewards/forensics_iou_reward": 0.4414226859807968, |
| "step": 203 |
| }, |
| { |
| "completion_length": 19.46875, |
| "epoch": 0.8535564853556485, |
| "grad_norm": 4.202703952789307, |
| "kl": 0.17529296875, |
| "learning_rate": 7.876569037656904e-07, |
| "loss": 0.007, |
| "reward": 0.5571984946727753, |
| "reward_std": 0.1038469560444355, |
| "reward_std_all": 0.10262010619044304, |
| "rewards-max/forensics_iou_reward": 0.8751490712165833, |
| "rewards/forensics_iou_reward": 0.5571984946727753, |
| "step": 204 |
| }, |
| { |
| "completion_length": 19.359375, |
| "epoch": 0.8577405857740585, |
| "grad_norm": 4.0510149002075195, |
| "kl": 0.10791015625, |
| "learning_rate": 7.866108786610879e-07, |
| "loss": 0.0043, |
| "reward": 0.5452224016189575, |
| "reward_std": 0.08235850185155869, |
| "reward_std_all": 0.09830805286765099, |
| "rewards-max/forensics_iou_reward": 0.8735800683498383, |
| "rewards/forensics_iou_reward": 0.5452224016189575, |
| "step": 205 |
| }, |
| { |
| "completion_length": 20.546875, |
| "epoch": 0.8619246861924686, |
| "grad_norm": 5.097855567932129, |
| "kl": 0.1533203125, |
| "learning_rate": 7.855648535564853e-07, |
| "loss": 0.0061, |
| "reward": 0.4121173620223999, |
| "reward_std": 0.09300042316317558, |
| "reward_std_all": 0.11073583364486694, |
| "rewards-max/forensics_iou_reward": 0.9159899950027466, |
| "rewards/forensics_iou_reward": 0.4121173620223999, |
| "step": 206 |
| }, |
| { |
| "completion_length": 19.40625, |
| "epoch": 0.8661087866108786, |
| "grad_norm": 3.5252840518951416, |
| "kl": 0.1416015625, |
| "learning_rate": 7.845188284518827e-07, |
| "loss": 0.0057, |
| "reward": 0.46690039336681366, |
| "reward_std": 0.07622779160737991, |
| "reward_std_all": 0.09298742562532425, |
| "rewards-max/forensics_iou_reward": 0.9200402200222015, |
| "rewards/forensics_iou_reward": 0.46690039336681366, |
| "step": 207 |
| }, |
| { |
| "completion_length": 19.84375, |
| "epoch": 0.8702928870292888, |
| "grad_norm": 3.9738967418670654, |
| "kl": 0.103271484375, |
| "learning_rate": 7.834728033472802e-07, |
| "loss": 0.0041, |
| "reward": 0.4544639587402344, |
| "reward_std": 0.08543133735656738, |
| "reward_std_all": 0.09383503720164299, |
| "rewards-max/forensics_iou_reward": 0.8520342707633972, |
| "rewards/forensics_iou_reward": 0.4544639587402344, |
| "step": 208 |
| }, |
| { |
| "completion_length": 19.390625, |
| "epoch": 0.8744769874476988, |
| "grad_norm": 4.336952209472656, |
| "kl": 0.154296875, |
| "learning_rate": 7.824267782426778e-07, |
| "loss": 0.0062, |
| "reward": 0.4684612601995468, |
| "reward_std": 0.07005300745368004, |
| "reward_std_all": 0.09486282244324684, |
| "rewards-max/forensics_iou_reward": 0.9236563742160797, |
| "rewards/forensics_iou_reward": 0.4684612601995468, |
| "step": 209 |
| }, |
| { |
| "completion_length": 19.640625, |
| "epoch": 0.8786610878661087, |
| "grad_norm": 4.315922260284424, |
| "kl": 0.173583984375, |
| "learning_rate": 7.813807531380753e-07, |
| "loss": 0.0069, |
| "reward": 0.5078110992908478, |
| "reward_std": 0.11215802282094955, |
| "reward_std_all": 0.12042583152651787, |
| "rewards-max/forensics_iou_reward": 0.9662429392337799, |
| "rewards/forensics_iou_reward": 0.5078110992908478, |
| "step": 210 |
| }, |
| { |
| "completion_length": 18.984375, |
| "epoch": 0.8828451882845189, |
| "grad_norm": 2.6506271362304688, |
| "kl": 0.166015625, |
| "learning_rate": 7.803347280334728e-07, |
| "loss": 0.0066, |
| "reward": 0.5009565353393555, |
| "reward_std": 0.13033586740493774, |
| "reward_std_all": 0.12069370225071907, |
| "rewards-max/forensics_iou_reward": 0.9357084333896637, |
| "rewards/forensics_iou_reward": 0.5009565353393555, |
| "step": 211 |
| }, |
| { |
| "completion_length": 19.78125, |
| "epoch": 0.8870292887029289, |
| "grad_norm": 3.6235146522521973, |
| "kl": 0.15283203125, |
| "learning_rate": 7.792887029288703e-07, |
| "loss": 0.0061, |
| "reward": 0.5100796520709991, |
| "reward_std": 0.1457316130399704, |
| "reward_std_all": 0.14634385704994202, |
| "rewards-max/forensics_iou_reward": 0.8742960095405579, |
| "rewards/forensics_iou_reward": 0.5100796520709991, |
| "step": 212 |
| }, |
| { |
| "completion_length": 19.3125, |
| "epoch": 0.891213389121339, |
| "grad_norm": 4.00847864151001, |
| "kl": 0.125, |
| "learning_rate": 7.782426778242678e-07, |
| "loss": 0.005, |
| "reward": 0.5076137781143188, |
| "reward_std": 0.12965474277734756, |
| "reward_std_all": 0.11932456120848656, |
| "rewards-max/forensics_iou_reward": 0.9503086507320404, |
| "rewards/forensics_iou_reward": 0.5076137781143188, |
| "step": 213 |
| }, |
| { |
| "completion_length": 19.28125, |
| "epoch": 0.895397489539749, |
| "grad_norm": 3.7214808464050293, |
| "kl": 0.09423828125, |
| "learning_rate": 7.771966527196653e-07, |
| "loss": 0.0038, |
| "reward": 0.5626892149448395, |
| "reward_std": 0.06869082525372505, |
| "reward_std_all": 0.07172994129359722, |
| "rewards-max/forensics_iou_reward": 0.9290034472942352, |
| "rewards/forensics_iou_reward": 0.5626892149448395, |
| "step": 214 |
| }, |
| { |
| "completion_length": 20.515625, |
| "epoch": 0.899581589958159, |
| "grad_norm": 4.428376197814941, |
| "kl": 0.098876953125, |
| "learning_rate": 7.761506276150628e-07, |
| "loss": 0.004, |
| "reward": 0.383433997631073, |
| "reward_std": 0.10150695592164993, |
| "reward_std_all": 0.09814665839076042, |
| "rewards-max/forensics_iou_reward": 0.775057315826416, |
| "rewards/forensics_iou_reward": 0.383433997631073, |
| "step": 215 |
| }, |
| { |
| "completion_length": 19.234375, |
| "epoch": 0.9037656903765691, |
| "grad_norm": 5.832391738891602, |
| "kl": 0.099609375, |
| "learning_rate": 7.751046025104601e-07, |
| "loss": 0.004, |
| "reward": 0.40005742758512497, |
| "reward_std": 0.09758277982473373, |
| "reward_std_all": 0.10497210919857025, |
| "rewards-max/forensics_iou_reward": 0.9545771777629852, |
| "rewards/forensics_iou_reward": 0.40005742758512497, |
| "step": 216 |
| }, |
| { |
| "completion_length": 19.515625, |
| "epoch": 0.9079497907949791, |
| "grad_norm": 3.3938169479370117, |
| "kl": 0.18310546875, |
| "learning_rate": 7.740585774058576e-07, |
| "loss": 0.0073, |
| "reward": 0.5183698385953903, |
| "reward_std": 0.08967471122741699, |
| "reward_std_all": 0.08101029694080353, |
| "rewards-max/forensics_iou_reward": 0.8853634595870972, |
| "rewards/forensics_iou_reward": 0.5183698385953903, |
| "step": 217 |
| }, |
| { |
| "completion_length": 19.1875, |
| "epoch": 0.9121338912133892, |
| "grad_norm": 3.2692060470581055, |
| "kl": 0.11328125, |
| "learning_rate": 7.730125523012552e-07, |
| "loss": 0.0045, |
| "reward": 0.45665158331394196, |
| "reward_std": 0.07327044755220413, |
| "reward_std_all": 0.10038675367832184, |
| "rewards-max/forensics_iou_reward": 0.8982656598091125, |
| "rewards/forensics_iou_reward": 0.45665158331394196, |
| "step": 218 |
| }, |
| { |
| "completion_length": 19.1875, |
| "epoch": 0.9163179916317992, |
| "grad_norm": 7.413144588470459, |
| "kl": 0.41015625, |
| "learning_rate": 7.719665271966527e-07, |
| "loss": 0.0164, |
| "reward": 0.47892123460769653, |
| "reward_std": 0.0960774514824152, |
| "reward_std_all": 0.11151707172393799, |
| "rewards-max/forensics_iou_reward": 0.8612197041511536, |
| "rewards/forensics_iou_reward": 0.47892123460769653, |
| "step": 219 |
| }, |
| { |
| "completion_length": 19.109375, |
| "epoch": 0.9205020920502092, |
| "grad_norm": 3.4468226432800293, |
| "kl": 0.157958984375, |
| "learning_rate": 7.709205020920502e-07, |
| "loss": 0.0063, |
| "reward": 0.5302162766456604, |
| "reward_std": 0.08519165962934494, |
| "reward_std_all": 0.08958959206938744, |
| "rewards-max/forensics_iou_reward": 0.9237902462482452, |
| "rewards/forensics_iou_reward": 0.5302162766456604, |
| "step": 220 |
| }, |
| { |
| "completion_length": 19.390625, |
| "epoch": 0.9246861924686193, |
| "grad_norm": 3.5220770835876465, |
| "kl": 0.17529296875, |
| "learning_rate": 7.698744769874477e-07, |
| "loss": 0.007, |
| "reward": 0.4875237047672272, |
| "reward_std": 0.07400843501091003, |
| "reward_std_all": 0.08449666202068329, |
| "rewards-max/forensics_iou_reward": 0.9147974252700806, |
| "rewards/forensics_iou_reward": 0.4875237047672272, |
| "step": 221 |
| }, |
| { |
| "completion_length": 19.5625, |
| "epoch": 0.9288702928870293, |
| "grad_norm": 2.8949317932128906, |
| "kl": 0.11865234375, |
| "learning_rate": 7.688284518828452e-07, |
| "loss": 0.0048, |
| "reward": 0.35168763995170593, |
| "reward_std": 0.09561984613537788, |
| "reward_std_all": 0.08818450197577477, |
| "rewards-max/forensics_iou_reward": 0.8142001032829285, |
| "rewards/forensics_iou_reward": 0.35168763995170593, |
| "step": 222 |
| }, |
| { |
| "completion_length": 19.5625, |
| "epoch": 0.9330543933054394, |
| "grad_norm": 5.375015735626221, |
| "kl": 0.103515625, |
| "learning_rate": 7.677824267782427e-07, |
| "loss": 0.0041, |
| "reward": 0.488818883895874, |
| "reward_std": 0.07589146867394447, |
| "reward_std_all": 0.0834256000816822, |
| "rewards-max/forensics_iou_reward": 0.9018561244010925, |
| "rewards/forensics_iou_reward": 0.488818883895874, |
| "step": 223 |
| }, |
| { |
| "completion_length": 19.625, |
| "epoch": 0.9372384937238494, |
| "grad_norm": 4.011237621307373, |
| "kl": 0.130859375, |
| "learning_rate": 7.667364016736401e-07, |
| "loss": 0.0052, |
| "reward": 0.55644091963768, |
| "reward_std": 0.08536511287093163, |
| "reward_std_all": 0.09722235053777695, |
| "rewards-max/forensics_iou_reward": 0.9515487253665924, |
| "rewards/forensics_iou_reward": 0.55644091963768, |
| "step": 224 |
| }, |
| { |
| "completion_length": 19.09375, |
| "epoch": 0.9414225941422594, |
| "grad_norm": 3.619231700897217, |
| "kl": 0.107666015625, |
| "learning_rate": 7.656903765690376e-07, |
| "loss": 0.0043, |
| "reward": 0.5295969694852829, |
| "reward_std": 0.07228957116603851, |
| "reward_std_all": 0.07246513664722443, |
| "rewards-max/forensics_iou_reward": 0.9085963070392609, |
| "rewards/forensics_iou_reward": 0.5295969694852829, |
| "step": 225 |
| }, |
| { |
| "completion_length": 19.671875, |
| "epoch": 0.9456066945606695, |
| "grad_norm": 4.4820404052734375, |
| "kl": 0.117919921875, |
| "learning_rate": 7.64644351464435e-07, |
| "loss": 0.0047, |
| "reward": 0.38877031207084656, |
| "reward_std": 0.09601970575749874, |
| "reward_std_all": 0.09021072834730148, |
| "rewards-max/forensics_iou_reward": 0.8566752672195435, |
| "rewards/forensics_iou_reward": 0.38877031207084656, |
| "step": 226 |
| }, |
| { |
| "completion_length": 20.0, |
| "epoch": 0.9497907949790795, |
| "grad_norm": 3.7813522815704346, |
| "kl": 0.16162109375, |
| "learning_rate": 7.635983263598326e-07, |
| "loss": 0.0065, |
| "reward": 0.39784522354602814, |
| "reward_std": 0.11890479922294617, |
| "reward_std_all": 0.12542488053441048, |
| "rewards-max/forensics_iou_reward": 0.9798436164855957, |
| "rewards/forensics_iou_reward": 0.39784522354602814, |
| "step": 227 |
| }, |
| { |
| "completion_length": 20.015625, |
| "epoch": 0.9539748953974896, |
| "grad_norm": 3.429664373397827, |
| "kl": 0.126953125, |
| "learning_rate": 7.625523012552301e-07, |
| "loss": 0.0051, |
| "reward": 0.387285977602005, |
| "reward_std": 0.07872448675334454, |
| "reward_std_all": 0.07041275128722191, |
| "rewards-max/forensics_iou_reward": 0.8419378697872162, |
| "rewards/forensics_iou_reward": 0.387285977602005, |
| "step": 228 |
| }, |
| { |
| "completion_length": 20.9375, |
| "epoch": 0.9581589958158996, |
| "grad_norm": 4.105513095855713, |
| "kl": 0.13232421875, |
| "learning_rate": 7.615062761506276e-07, |
| "loss": 0.0053, |
| "reward": 0.3224245607852936, |
| "reward_std": 0.06652445532381535, |
| "reward_std_all": 0.061441805213689804, |
| "rewards-max/forensics_iou_reward": 0.663961261510849, |
| "rewards/forensics_iou_reward": 0.3224245607852936, |
| "step": 229 |
| }, |
| { |
| "completion_length": 19.5, |
| "epoch": 0.9623430962343096, |
| "grad_norm": 5.106868267059326, |
| "kl": 0.283935546875, |
| "learning_rate": 7.604602510460251e-07, |
| "loss": 0.0114, |
| "reward": 0.4797614514827728, |
| "reward_std": 0.1029440127313137, |
| "reward_std_all": 0.09710858017206192, |
| "rewards-max/forensics_iou_reward": 0.9285043179988861, |
| "rewards/forensics_iou_reward": 0.4797614514827728, |
| "step": 230 |
| }, |
| { |
| "completion_length": 19.421875, |
| "epoch": 0.9665271966527197, |
| "grad_norm": 4.135436058044434, |
| "kl": 0.12255859375, |
| "learning_rate": 7.594142259414226e-07, |
| "loss": 0.0049, |
| "reward": 0.45683518052101135, |
| "reward_std": 0.11870399117469788, |
| "reward_std_all": 0.11190219596028328, |
| "rewards-max/forensics_iou_reward": 0.9431818127632141, |
| "rewards/forensics_iou_reward": 0.45683518052101135, |
| "step": 231 |
| }, |
| { |
| "completion_length": 20.03125, |
| "epoch": 0.9707112970711297, |
| "grad_norm": 4.8457932472229, |
| "kl": 0.121337890625, |
| "learning_rate": 7.5836820083682e-07, |
| "loss": 0.0048, |
| "reward": 0.4705752432346344, |
| "reward_std": 0.07796146720647812, |
| "reward_std_all": 0.09926865622401237, |
| "rewards-max/forensics_iou_reward": 0.7822460234165192, |
| "rewards/forensics_iou_reward": 0.4705752432346344, |
| "step": 232 |
| }, |
| { |
| "completion_length": 20.0, |
| "epoch": 0.9748953974895398, |
| "grad_norm": 4.366388320922852, |
| "kl": 0.202880859375, |
| "learning_rate": 7.573221757322175e-07, |
| "loss": 0.0081, |
| "reward": 0.3751462697982788, |
| "reward_std": 0.0675010597333312, |
| "reward_std_all": 0.08735569566488266, |
| "rewards-max/forensics_iou_reward": 0.9279069900512695, |
| "rewards/forensics_iou_reward": 0.3751462697982788, |
| "step": 233 |
| }, |
| { |
| "completion_length": 21.359375, |
| "epoch": 0.9790794979079498, |
| "grad_norm": 6.8591837882995605, |
| "kl": 0.12646484375, |
| "learning_rate": 7.56276150627615e-07, |
| "loss": 0.0051, |
| "reward": 0.3760474622249603, |
| "reward_std": 0.08604123815894127, |
| "reward_std_all": 0.08735349401831627, |
| "rewards-max/forensics_iou_reward": 0.8922379016876221, |
| "rewards/forensics_iou_reward": 0.3760474622249603, |
| "step": 234 |
| }, |
| { |
| "completion_length": 19.46875, |
| "epoch": 0.9832635983263598, |
| "grad_norm": 3.971794605255127, |
| "kl": 0.110107421875, |
| "learning_rate": 7.552301255230126e-07, |
| "loss": 0.0044, |
| "reward": 0.46572747826576233, |
| "reward_std": 0.0932077020406723, |
| "reward_std_all": 0.09293944016098976, |
| "rewards-max/forensics_iou_reward": 0.8411550521850586, |
| "rewards/forensics_iou_reward": 0.46572747826576233, |
| "step": 235 |
| }, |
| { |
| "completion_length": 19.84375, |
| "epoch": 0.9874476987447699, |
| "grad_norm": 3.5212998390197754, |
| "kl": 0.100830078125, |
| "learning_rate": 7.5418410041841e-07, |
| "loss": 0.004, |
| "reward": 0.5267468094825745, |
| "reward_std": 0.10203258693218231, |
| "reward_std_all": 0.10415469110012054, |
| "rewards-max/forensics_iou_reward": 0.8897419571876526, |
| "rewards/forensics_iou_reward": 0.5267468094825745, |
| "step": 236 |
| }, |
| { |
| "completion_length": 19.375, |
| "epoch": 0.9916317991631799, |
| "grad_norm": 3.9443790912628174, |
| "kl": 0.114013671875, |
| "learning_rate": 7.531380753138075e-07, |
| "loss": 0.0046, |
| "reward": 0.5477520823478699, |
| "reward_std": 0.07652450352907181, |
| "reward_std_all": 0.07794992253184319, |
| "rewards-max/forensics_iou_reward": 0.9488230347633362, |
| "rewards/forensics_iou_reward": 0.5477520823478699, |
| "step": 237 |
| }, |
| { |
| "completion_length": 19.59375, |
| "epoch": 0.99581589958159, |
| "grad_norm": 3.7105860710144043, |
| "kl": 0.193359375, |
| "learning_rate": 7.52092050209205e-07, |
| "loss": 0.0077, |
| "reward": 0.42799100279808044, |
| "reward_std": 0.0862287525087595, |
| "reward_std_all": 0.08234142139554024, |
| "rewards-max/forensics_iou_reward": 0.9250263869762421, |
| "rewards/forensics_iou_reward": 0.42799100279808044, |
| "step": 238 |
| }, |
| { |
| "completion_length": 22.500000953674316, |
| "epoch": 1.0, |
| "grad_norm": 3.6715152263641357, |
| "kl": 0.140869140625, |
| "learning_rate": 7.510460251046025e-07, |
| "loss": 0.0055, |
| "reward": 0.485652320086956, |
| "reward_std": 0.10382498055696487, |
| "reward_std_all": 0.10079137980937958, |
| "rewards-max/forensics_iou_reward": 0.7640198767185211, |
| "rewards/forensics_iou_reward": 0.485652320086956, |
| "step": 239 |
| }, |
| { |
| "completion_length": 19.125, |
| "epoch": 1.00418410041841, |
| "grad_norm": 3.3969123363494873, |
| "kl": 0.1103515625, |
| "learning_rate": 7.5e-07, |
| "loss": 0.0044, |
| "reward": 0.5730691254138947, |
| "reward_std": 0.0870748944580555, |
| "reward_std_all": 0.08860461786389351, |
| "rewards-max/forensics_iou_reward": 0.9361418187618256, |
| "rewards/forensics_iou_reward": 0.5730691254138947, |
| "step": 240 |
| }, |
| { |
| "completion_length": 20.0625, |
| "epoch": 1.00836820083682, |
| "grad_norm": 3.1755197048187256, |
| "kl": 0.182373046875, |
| "learning_rate": 7.489539748953974e-07, |
| "loss": 0.0073, |
| "reward": 0.6047759354114532, |
| "reward_std": 0.07101139612495899, |
| "reward_std_all": 0.07755609601736069, |
| "rewards-max/forensics_iou_reward": 0.9807969629764557, |
| "rewards/forensics_iou_reward": 0.6047759354114532, |
| "step": 241 |
| }, |
| { |
| "completion_length": 19.609375, |
| "epoch": 1.0125523012552302, |
| "grad_norm": 7.87276029586792, |
| "kl": 0.437255859375, |
| "learning_rate": 7.479079497907949e-07, |
| "loss": 0.0175, |
| "reward": 0.41604354977607727, |
| "reward_std": 0.10926571488380432, |
| "reward_std_all": 0.09463565051555634, |
| "rewards-max/forensics_iou_reward": 0.950844794511795, |
| "rewards/forensics_iou_reward": 0.41604354977607727, |
| "step": 242 |
| }, |
| { |
| "completion_length": 19.9375, |
| "epoch": 1.0167364016736402, |
| "grad_norm": 4.559387683868408, |
| "kl": 0.13818359375, |
| "learning_rate": 7.468619246861925e-07, |
| "loss": 0.0055, |
| "reward": 0.5062660872936249, |
| "reward_std": 0.10195399820804596, |
| "reward_std_all": 0.1035170629620552, |
| "rewards-max/forensics_iou_reward": 0.893865704536438, |
| "rewards/forensics_iou_reward": 0.5062660872936249, |
| "step": 243 |
| }, |
| { |
| "completion_length": 19.265625, |
| "epoch": 1.0209205020920502, |
| "grad_norm": 4.393025875091553, |
| "kl": 0.11279296875, |
| "learning_rate": 7.4581589958159e-07, |
| "loss": 0.0045, |
| "reward": 0.4847296178340912, |
| "reward_std": 0.0966670773923397, |
| "reward_std_all": 0.09863319620490074, |
| "rewards-max/forensics_iou_reward": 0.829387754201889, |
| "rewards/forensics_iou_reward": 0.4847296178340912, |
| "step": 244 |
| }, |
| { |
| "completion_length": 19.609375, |
| "epoch": 1.0251046025104602, |
| "grad_norm": 3.6986005306243896, |
| "kl": 0.12939453125, |
| "learning_rate": 7.447698744769874e-07, |
| "loss": 0.0052, |
| "reward": 0.489314466714859, |
| "reward_std": 0.09907027706503868, |
| "reward_std_all": 0.1307668462395668, |
| "rewards-max/forensics_iou_reward": 0.93586465716362, |
| "rewards/forensics_iou_reward": 0.489314466714859, |
| "step": 245 |
| }, |
| { |
| "completion_length": 19.171875, |
| "epoch": 1.0292887029288702, |
| "grad_norm": 3.1163225173950195, |
| "kl": 0.12841796875, |
| "learning_rate": 7.437238493723849e-07, |
| "loss": 0.0051, |
| "reward": 0.3849606513977051, |
| "reward_std": 0.05274685099720955, |
| "reward_std_all": 0.053189974278211594, |
| "rewards-max/forensics_iou_reward": 0.6751431226730347, |
| "rewards/forensics_iou_reward": 0.3849606513977051, |
| "step": 246 |
| }, |
| { |
| "completion_length": 19.40625, |
| "epoch": 1.0334728033472804, |
| "grad_norm": 4.052732944488525, |
| "kl": 0.1748046875, |
| "learning_rate": 7.426778242677824e-07, |
| "loss": 0.007, |
| "reward": 0.5021269470453262, |
| "reward_std": 0.0541265532374382, |
| "reward_std_all": 0.07933457754552364, |
| "rewards-max/forensics_iou_reward": 0.8942216038703918, |
| "rewards/forensics_iou_reward": 0.5021269470453262, |
| "step": 247 |
| }, |
| { |
| "completion_length": 19.421875, |
| "epoch": 1.0376569037656904, |
| "grad_norm": 6.938345432281494, |
| "kl": 0.1171875, |
| "learning_rate": 7.416317991631799e-07, |
| "loss": 0.0047, |
| "reward": 0.34200841188430786, |
| "reward_std": 0.06286361068487167, |
| "reward_std_all": 0.08510885387659073, |
| "rewards-max/forensics_iou_reward": 0.8326643705368042, |
| "rewards/forensics_iou_reward": 0.34200841188430786, |
| "step": 248 |
| }, |
| { |
| "completion_length": 19.84375, |
| "epoch": 1.0418410041841004, |
| "grad_norm": 4.036919593811035, |
| "kl": 0.17333984375, |
| "learning_rate": 7.405857740585773e-07, |
| "loss": 0.0069, |
| "reward": 0.38853590190410614, |
| "reward_std": 0.10279043763875961, |
| "reward_std_all": 0.1289011687040329, |
| "rewards-max/forensics_iou_reward": 0.8735236823558807, |
| "rewards/forensics_iou_reward": 0.38853590190410614, |
| "step": 249 |
| }, |
| { |
| "completion_length": 19.53125, |
| "epoch": 1.0460251046025104, |
| "grad_norm": 3.898483991622925, |
| "kl": 0.158447265625, |
| "learning_rate": 7.395397489539748e-07, |
| "loss": 0.0063, |
| "reward": 0.5797886550426483, |
| "reward_std": 0.0675528533756733, |
| "reward_std_all": 0.0684650968760252, |
| "rewards-max/forensics_iou_reward": 0.9093568027019501, |
| "rewards/forensics_iou_reward": 0.5797886550426483, |
| "step": 250 |
| }, |
| { |
| "completion_length": 19.484375, |
| "epoch": 1.0502092050209204, |
| "grad_norm": 3.244629383087158, |
| "kl": 0.122314453125, |
| "learning_rate": 7.384937238493723e-07, |
| "loss": 0.0049, |
| "reward": 0.4889642596244812, |
| "reward_std": 0.061405256390571594, |
| "reward_std_all": 0.06312352791428566, |
| "rewards-max/forensics_iou_reward": 0.9688572585582733, |
| "rewards/forensics_iou_reward": 0.4889642596244812, |
| "step": 251 |
| }, |
| { |
| "completion_length": 19.265625, |
| "epoch": 1.0543933054393306, |
| "grad_norm": 3.4402074813842773, |
| "kl": 0.216796875, |
| "learning_rate": 7.374476987447699e-07, |
| "loss": 0.0087, |
| "reward": 0.434797078371048, |
| "reward_std": 0.09922629967331886, |
| "reward_std_all": 0.09678014181554317, |
| "rewards-max/forensics_iou_reward": 0.9318065345287323, |
| "rewards/forensics_iou_reward": 0.434797078371048, |
| "step": 252 |
| }, |
| { |
| "completion_length": 20.4375, |
| "epoch": 1.0585774058577406, |
| "grad_norm": 3.5526716709136963, |
| "kl": 0.134521484375, |
| "learning_rate": 7.364016736401674e-07, |
| "loss": 0.0054, |
| "reward": 0.540473222732544, |
| "reward_std": 0.14411496371030807, |
| "reward_std_all": 0.14729442819952965, |
| "rewards-max/forensics_iou_reward": 0.8672812283039093, |
| "rewards/forensics_iou_reward": 0.540473222732544, |
| "step": 253 |
| }, |
| { |
| "completion_length": 19.1875, |
| "epoch": 1.0627615062761506, |
| "grad_norm": 5.157316207885742, |
| "kl": 0.176025390625, |
| "learning_rate": 7.353556485355649e-07, |
| "loss": 0.0071, |
| "reward": 0.41513098776340485, |
| "reward_std": 0.09457075223326683, |
| "reward_std_all": 0.11636590212583542, |
| "rewards-max/forensics_iou_reward": 0.8661000728607178, |
| "rewards/forensics_iou_reward": 0.41513098776340485, |
| "step": 254 |
| }, |
| { |
| "completion_length": 19.609375, |
| "epoch": 1.0669456066945606, |
| "grad_norm": 3.161313772201538, |
| "kl": 0.12158203125, |
| "learning_rate": 7.343096234309623e-07, |
| "loss": 0.0049, |
| "reward": 0.4977545440196991, |
| "reward_std": 0.09112825989723206, |
| "reward_std_all": 0.10403095930814743, |
| "rewards-max/forensics_iou_reward": 0.9665080606937408, |
| "rewards/forensics_iou_reward": 0.4977545440196991, |
| "step": 255 |
| }, |
| { |
| "completion_length": 19.671875, |
| "epoch": 1.0711297071129706, |
| "grad_norm": 3.1887972354888916, |
| "kl": 0.10595703125, |
| "learning_rate": 7.332635983263598e-07, |
| "loss": 0.0042, |
| "reward": 0.5331947654485703, |
| "reward_std": 0.06318146921694279, |
| "reward_std_all": 0.06297558546066284, |
| "rewards-max/forensics_iou_reward": 0.8631449639797211, |
| "rewards/forensics_iou_reward": 0.5331947654485703, |
| "step": 256 |
| }, |
| { |
| "completion_length": 19.390625, |
| "epoch": 1.0753138075313808, |
| "grad_norm": 4.732301235198975, |
| "kl": 0.1611328125, |
| "learning_rate": 7.322175732217572e-07, |
| "loss": 0.0064, |
| "reward": 0.5327515602111816, |
| "reward_std": 0.12614207342267036, |
| "reward_std_all": 0.1321936547756195, |
| "rewards-max/forensics_iou_reward": 0.9713643789291382, |
| "rewards/forensics_iou_reward": 0.5327515602111816, |
| "step": 257 |
| }, |
| { |
| "completion_length": 19.265625, |
| "epoch": 1.0794979079497908, |
| "grad_norm": 3.9643702507019043, |
| "kl": 0.1337890625, |
| "learning_rate": 7.311715481171547e-07, |
| "loss": 0.0053, |
| "reward": 0.3317105174064636, |
| "reward_std": 0.08065538480877876, |
| "reward_std_all": 0.08269575238227844, |
| "rewards-max/forensics_iou_reward": 0.7550534009933472, |
| "rewards/forensics_iou_reward": 0.3317105174064636, |
| "step": 258 |
| }, |
| { |
| "completion_length": 19.15625, |
| "epoch": 1.0836820083682008, |
| "grad_norm": 3.6440396308898926, |
| "kl": 0.166015625, |
| "learning_rate": 7.301255230125522e-07, |
| "loss": 0.0066, |
| "reward": 0.34747643768787384, |
| "reward_std": 0.053844066336750984, |
| "reward_std_all": 0.060705848038196564, |
| "rewards-max/forensics_iou_reward": 0.9196370542049408, |
| "rewards/forensics_iou_reward": 0.34747643768787384, |
| "step": 259 |
| }, |
| { |
| "completion_length": 19.265625, |
| "epoch": 1.0878661087866108, |
| "grad_norm": 3.967500686645508, |
| "kl": 0.109375, |
| "learning_rate": 7.290794979079497e-07, |
| "loss": 0.0044, |
| "reward": 0.49452000856399536, |
| "reward_std": 0.09612195193767548, |
| "reward_std_all": 0.0889585055410862, |
| "rewards-max/forensics_iou_reward": 0.8521506190299988, |
| "rewards/forensics_iou_reward": 0.49452000856399536, |
| "step": 260 |
| }, |
| { |
| "completion_length": 19.390625, |
| "epoch": 1.0920502092050208, |
| "grad_norm": 4.144653797149658, |
| "kl": 0.189453125, |
| "learning_rate": 7.280334728033473e-07, |
| "loss": 0.0076, |
| "reward": 0.4303743690252304, |
| "reward_std": 0.06637033820152283, |
| "reward_std_all": 0.07041610777378082, |
| "rewards-max/forensics_iou_reward": 0.8137482702732086, |
| "rewards/forensics_iou_reward": 0.4303743690252304, |
| "step": 261 |
| }, |
| { |
| "completion_length": 19.421875, |
| "epoch": 1.096234309623431, |
| "grad_norm": 4.792544364929199, |
| "kl": 0.1240234375, |
| "learning_rate": 7.269874476987448e-07, |
| "loss": 0.005, |
| "reward": 0.4400089383125305, |
| "reward_std": 0.06621075235307217, |
| "reward_std_all": 0.07948477938771248, |
| "rewards-max/forensics_iou_reward": 0.9126126170158386, |
| "rewards/forensics_iou_reward": 0.4400089383125305, |
| "step": 262 |
| }, |
| { |
| "completion_length": 19.390625, |
| "epoch": 1.100418410041841, |
| "grad_norm": 4.957928657531738, |
| "kl": 0.14794921875, |
| "learning_rate": 7.259414225941423e-07, |
| "loss": 0.0059, |
| "reward": 0.44663384556770325, |
| "reward_std": 0.11440910771489143, |
| "reward_std_all": 0.1050088182091713, |
| "rewards-max/forensics_iou_reward": 0.9628716111183167, |
| "rewards/forensics_iou_reward": 0.44663384556770325, |
| "step": 263 |
| }, |
| { |
| "completion_length": 19.203125, |
| "epoch": 1.104602510460251, |
| "grad_norm": 4.160280227661133, |
| "kl": 0.15380859375, |
| "learning_rate": 7.248953974895398e-07, |
| "loss": 0.0061, |
| "reward": 0.5557908415794373, |
| "reward_std": 0.06941795721650124, |
| "reward_std_all": 0.07475024834275246, |
| "rewards-max/forensics_iou_reward": 0.9698213934898376, |
| "rewards/forensics_iou_reward": 0.5557908415794373, |
| "step": 264 |
| }, |
| { |
| "completion_length": 20.140625, |
| "epoch": 1.108786610878661, |
| "grad_norm": 3.7501795291900635, |
| "kl": 0.16015625, |
| "learning_rate": 7.238493723849371e-07, |
| "loss": 0.0064, |
| "reward": 0.6004802584648132, |
| "reward_std": 0.08982115983963013, |
| "reward_std_all": 0.08638032525777817, |
| "rewards-max/forensics_iou_reward": 0.9250715970993042, |
| "rewards/forensics_iou_reward": 0.6004802584648132, |
| "step": 265 |
| }, |
| { |
| "completion_length": 19.28125, |
| "epoch": 1.112970711297071, |
| "grad_norm": 6.418414115905762, |
| "kl": 0.10693359375, |
| "learning_rate": 7.228033472803346e-07, |
| "loss": 0.0043, |
| "reward": 0.604328453540802, |
| "reward_std": 0.07672776281833649, |
| "reward_std_all": 0.08927887305617332, |
| "rewards-max/forensics_iou_reward": 0.9738442301750183, |
| "rewards/forensics_iou_reward": 0.604328453540802, |
| "step": 266 |
| }, |
| { |
| "completion_length": 19.0625, |
| "epoch": 1.1171548117154813, |
| "grad_norm": 4.440456390380859, |
| "kl": 0.2060546875, |
| "learning_rate": 7.217573221757321e-07, |
| "loss": 0.0082, |
| "reward": 0.506777286529541, |
| "reward_std": 0.07160871475934982, |
| "reward_std_all": 0.0797371156513691, |
| "rewards-max/forensics_iou_reward": 0.8987987041473389, |
| "rewards/forensics_iou_reward": 0.506777286529541, |
| "step": 267 |
| }, |
| { |
| "completion_length": 19.671875, |
| "epoch": 1.1213389121338913, |
| "grad_norm": 5.1917619705200195, |
| "kl": 0.1259765625, |
| "learning_rate": 7.207112970711296e-07, |
| "loss": 0.005, |
| "reward": 0.3953999727964401, |
| "reward_std": 0.09883745387196541, |
| "reward_std_all": 0.09491131827235222, |
| "rewards-max/forensics_iou_reward": 0.9324098229408264, |
| "rewards/forensics_iou_reward": 0.3953999727964401, |
| "step": 268 |
| }, |
| { |
| "completion_length": 19.28125, |
| "epoch": 1.1255230125523012, |
| "grad_norm": 3.7314672470092773, |
| "kl": 0.13232421875, |
| "learning_rate": 7.196652719665272e-07, |
| "loss": 0.0053, |
| "reward": 0.5400157272815704, |
| "reward_std": 0.06483389437198639, |
| "reward_std_all": 0.07881157845258713, |
| "rewards-max/forensics_iou_reward": 0.938482791185379, |
| "rewards/forensics_iou_reward": 0.5400157272815704, |
| "step": 269 |
| }, |
| { |
| "completion_length": 19.5, |
| "epoch": 1.1297071129707112, |
| "grad_norm": 4.526717185974121, |
| "kl": 0.16357421875, |
| "learning_rate": 7.186192468619247e-07, |
| "loss": 0.0065, |
| "reward": 0.4366273581981659, |
| "reward_std": 0.08973738923668861, |
| "reward_std_all": 0.08993882313370705, |
| "rewards-max/forensics_iou_reward": 0.8418366611003876, |
| "rewards/forensics_iou_reward": 0.4366273581981659, |
| "step": 270 |
| }, |
| { |
| "completion_length": 19.0625, |
| "epoch": 1.1338912133891212, |
| "grad_norm": 4.152467250823975, |
| "kl": 0.16552734375, |
| "learning_rate": 7.175732217573222e-07, |
| "loss": 0.0066, |
| "reward": 0.4923616051673889, |
| "reward_std": 0.07172397896647453, |
| "reward_std_all": 0.07224966585636139, |
| "rewards-max/forensics_iou_reward": 0.953899085521698, |
| "rewards/forensics_iou_reward": 0.4923616051673889, |
| "step": 271 |
| }, |
| { |
| "completion_length": 19.4375, |
| "epoch": 1.1380753138075315, |
| "grad_norm": 3.7838456630706787, |
| "kl": 0.294921875, |
| "learning_rate": 7.165271966527197e-07, |
| "loss": 0.0118, |
| "reward": 0.5315192937850952, |
| "reward_std": 0.06239709258079529, |
| "reward_std_all": 0.08593020588159561, |
| "rewards-max/forensics_iou_reward": 0.9679974615573883, |
| "rewards/forensics_iou_reward": 0.5315192937850952, |
| "step": 272 |
| }, |
| { |
| "completion_length": 19.171875, |
| "epoch": 1.1422594142259415, |
| "grad_norm": 4.7353692054748535, |
| "kl": 0.30810546875, |
| "learning_rate": 7.154811715481172e-07, |
| "loss": 0.0123, |
| "reward": 0.581318199634552, |
| "reward_std": 0.06915182247757912, |
| "reward_std_all": 0.06850926578044891, |
| "rewards-max/forensics_iou_reward": 0.9608898162841797, |
| "rewards/forensics_iou_reward": 0.581318199634552, |
| "step": 273 |
| }, |
| { |
| "completion_length": 19.296875, |
| "epoch": 1.1464435146443515, |
| "grad_norm": 4.180524826049805, |
| "kl": 0.1572265625, |
| "learning_rate": 7.144351464435146e-07, |
| "loss": 0.0063, |
| "reward": 0.5715260207653046, |
| "reward_std": 0.10091854259371758, |
| "reward_std_all": 0.11234454438090324, |
| "rewards-max/forensics_iou_reward": 0.9804540872573853, |
| "rewards/forensics_iou_reward": 0.5715260207653046, |
| "step": 274 |
| }, |
| { |
| "completion_length": 19.78125, |
| "epoch": 1.1506276150627615, |
| "grad_norm": 9.163252830505371, |
| "kl": 0.796630859375, |
| "learning_rate": 7.13389121338912e-07, |
| "loss": 0.0318, |
| "reward": 0.46052978932857513, |
| "reward_std": 0.08904868736863136, |
| "reward_std_all": 0.08323127403855324, |
| "rewards-max/forensics_iou_reward": 0.9592784643173218, |
| "rewards/forensics_iou_reward": 0.46052978932857513, |
| "step": 275 |
| }, |
| { |
| "completion_length": 20.484375, |
| "epoch": 1.1548117154811715, |
| "grad_norm": 7.024928569793701, |
| "kl": 0.380126953125, |
| "learning_rate": 7.123430962343095e-07, |
| "loss": 0.0152, |
| "reward": 0.46998392045497894, |
| "reward_std": 0.09470368921756744, |
| "reward_std_all": 0.10820166766643524, |
| "rewards-max/forensics_iou_reward": 0.9463438093662262, |
| "rewards/forensics_iou_reward": 0.46998392045497894, |
| "step": 276 |
| }, |
| { |
| "completion_length": 19.953125, |
| "epoch": 1.1589958158995817, |
| "grad_norm": 4.361979007720947, |
| "kl": 0.20068359375, |
| "learning_rate": 7.11297071129707e-07, |
| "loss": 0.008, |
| "reward": 0.5771593451499939, |
| "reward_std": 0.09857115522027016, |
| "reward_std_all": 0.09571848064661026, |
| "rewards-max/forensics_iou_reward": 0.9403930008411407, |
| "rewards/forensics_iou_reward": 0.5771593451499939, |
| "step": 277 |
| }, |
| { |
| "completion_length": 19.25, |
| "epoch": 1.1631799163179917, |
| "grad_norm": 3.719820737838745, |
| "kl": 0.130859375, |
| "learning_rate": 7.102510460251046e-07, |
| "loss": 0.0052, |
| "reward": 0.4673056900501251, |
| "reward_std": 0.07714764773845673, |
| "reward_std_all": 0.0754250343888998, |
| "rewards-max/forensics_iou_reward": 0.8882409036159515, |
| "rewards/forensics_iou_reward": 0.4673056900501251, |
| "step": 278 |
| }, |
| { |
| "completion_length": 19.609375, |
| "epoch": 1.1673640167364017, |
| "grad_norm": 9.10448932647705, |
| "kl": 0.61376953125, |
| "learning_rate": 7.092050209205021e-07, |
| "loss": 0.0245, |
| "reward": 0.4933328330516815, |
| "reward_std": 0.10388658940792084, |
| "reward_std_all": 0.11130309849977493, |
| "rewards-max/forensics_iou_reward": 0.9436051845550537, |
| "rewards/forensics_iou_reward": 0.4933328330516815, |
| "step": 279 |
| }, |
| { |
| "completion_length": 21.40625, |
| "epoch": 1.1715481171548117, |
| "grad_norm": 4.608645915985107, |
| "kl": 0.1484375, |
| "learning_rate": 7.081589958158996e-07, |
| "loss": 0.0059, |
| "reward": 0.46562659740448, |
| "reward_std": 0.09004714712500572, |
| "reward_std_all": 0.10490024462342262, |
| "rewards-max/forensics_iou_reward": 0.8243093490600586, |
| "rewards/forensics_iou_reward": 0.46562659740448, |
| "step": 280 |
| }, |
| { |
| "completion_length": 19.625, |
| "epoch": 1.1757322175732217, |
| "grad_norm": 8.533886909484863, |
| "kl": 0.102783203125, |
| "learning_rate": 7.071129707112971e-07, |
| "loss": 0.0041, |
| "reward": 0.39124928414821625, |
| "reward_std": 0.06848480924963951, |
| "reward_std_all": 0.1066628098487854, |
| "rewards-max/forensics_iou_reward": 0.8465715944766998, |
| "rewards/forensics_iou_reward": 0.39124928414821625, |
| "step": 281 |
| }, |
| { |
| "completion_length": 19.609375, |
| "epoch": 1.1799163179916319, |
| "grad_norm": 4.085464000701904, |
| "kl": 0.154541015625, |
| "learning_rate": 7.060669456066946e-07, |
| "loss": 0.0062, |
| "reward": 0.48119519650936127, |
| "reward_std": 0.043919725343585014, |
| "reward_std_all": 0.046584730967879295, |
| "rewards-max/forensics_iou_reward": 0.9110615849494934, |
| "rewards/forensics_iou_reward": 0.48119519650936127, |
| "step": 282 |
| }, |
| { |
| "completion_length": 19.984375, |
| "epoch": 1.1841004184100419, |
| "grad_norm": 3.438798189163208, |
| "kl": 0.103759765625, |
| "learning_rate": 7.05020920502092e-07, |
| "loss": 0.0041, |
| "reward": 0.5548899173736572, |
| "reward_std": 0.07672056928277016, |
| "reward_std_all": 0.08591114729642868, |
| "rewards-max/forensics_iou_reward": 0.9494361877441406, |
| "rewards/forensics_iou_reward": 0.5548899173736572, |
| "step": 283 |
| }, |
| { |
| "completion_length": 19.5625, |
| "epoch": 1.1882845188284519, |
| "grad_norm": 4.520486831665039, |
| "kl": 0.1318359375, |
| "learning_rate": 7.039748953974895e-07, |
| "loss": 0.0053, |
| "reward": 0.5017320811748505, |
| "reward_std": 0.109980758279562, |
| "reward_std_all": 0.10138886049389839, |
| "rewards-max/forensics_iou_reward": 0.9445116221904755, |
| "rewards/forensics_iou_reward": 0.5017320811748505, |
| "step": 284 |
| }, |
| { |
| "completion_length": 20.328125, |
| "epoch": 1.1924686192468619, |
| "grad_norm": 6.495961666107178, |
| "kl": 0.265625, |
| "learning_rate": 7.029288702928869e-07, |
| "loss": 0.0106, |
| "reward": 0.4700874090194702, |
| "reward_std": 0.12484436854720116, |
| "reward_std_all": 0.11596870422363281, |
| "rewards-max/forensics_iou_reward": 0.9531926214694977, |
| "rewards/forensics_iou_reward": 0.4700874090194702, |
| "step": 285 |
| }, |
| { |
| "completion_length": 20.40625, |
| "epoch": 1.196652719665272, |
| "grad_norm": 5.518257141113281, |
| "kl": 0.16015625, |
| "learning_rate": 7.018828451882844e-07, |
| "loss": 0.0064, |
| "reward": 0.509015291929245, |
| "reward_std": 0.09304318577051163, |
| "reward_std_all": 0.09771359711885452, |
| "rewards-max/forensics_iou_reward": 0.9735339283943176, |
| "rewards/forensics_iou_reward": 0.509015291929245, |
| "step": 286 |
| }, |
| { |
| "completion_length": 19.6875, |
| "epoch": 1.200836820083682, |
| "grad_norm": 3.685452938079834, |
| "kl": 0.222412109375, |
| "learning_rate": 7.00836820083682e-07, |
| "loss": 0.0089, |
| "reward": 0.32780690491199493, |
| "reward_std": 0.0633193850517273, |
| "reward_std_all": 0.08711555227637291, |
| "rewards-max/forensics_iou_reward": 0.8202893435955048, |
| "rewards/forensics_iou_reward": 0.32780690491199493, |
| "step": 287 |
| }, |
| { |
| "completion_length": 19.421875, |
| "epoch": 1.205020920502092, |
| "grad_norm": 3.949469804763794, |
| "kl": 0.1328125, |
| "learning_rate": 6.997907949790795e-07, |
| "loss": 0.0053, |
| "reward": 0.5391095876693726, |
| "reward_std": 0.06011952646076679, |
| "reward_std_all": 0.0673537626862526, |
| "rewards-max/forensics_iou_reward": 0.8388495445251465, |
| "rewards/forensics_iou_reward": 0.5391095876693726, |
| "step": 288 |
| }, |
| { |
| "completion_length": 20.046875, |
| "epoch": 1.209205020920502, |
| "grad_norm": 4.714727878570557, |
| "kl": 0.136962890625, |
| "learning_rate": 6.98744769874477e-07, |
| "loss": 0.0055, |
| "reward": 0.5856902599334717, |
| "reward_std": 0.11310236155986786, |
| "reward_std_all": 0.10774178430438042, |
| "rewards-max/forensics_iou_reward": 0.9251733720302582, |
| "rewards/forensics_iou_reward": 0.5856902599334717, |
| "step": 289 |
| }, |
| { |
| "completion_length": 19.546875, |
| "epoch": 1.213389121338912, |
| "grad_norm": 5.874948501586914, |
| "kl": 0.120849609375, |
| "learning_rate": 6.976987447698745e-07, |
| "loss": 0.0048, |
| "reward": 0.5195189118385315, |
| "reward_std": 0.097367312759161, |
| "reward_std_all": 0.09958435595035553, |
| "rewards-max/forensics_iou_reward": 0.9799368381500244, |
| "rewards/forensics_iou_reward": 0.5195189118385315, |
| "step": 290 |
| }, |
| { |
| "completion_length": 19.453125, |
| "epoch": 1.217573221757322, |
| "grad_norm": 4.603507041931152, |
| "kl": 0.14501953125, |
| "learning_rate": 6.966527196652719e-07, |
| "loss": 0.0058, |
| "reward": 0.5793585330247879, |
| "reward_std": 0.06282965838909149, |
| "reward_std_all": 0.07393693923950195, |
| "rewards-max/forensics_iou_reward": 0.9112018048763275, |
| "rewards/forensics_iou_reward": 0.5793585330247879, |
| "step": 291 |
| }, |
| { |
| "completion_length": 20.15625, |
| "epoch": 1.2217573221757323, |
| "grad_norm": 4.843042373657227, |
| "kl": 0.17041015625, |
| "learning_rate": 6.956066945606694e-07, |
| "loss": 0.0068, |
| "reward": 0.47801974415779114, |
| "reward_std": 0.07606982812285423, |
| "reward_std_all": 0.0797606036067009, |
| "rewards-max/forensics_iou_reward": 0.9060122072696686, |
| "rewards/forensics_iou_reward": 0.47801974415779114, |
| "step": 292 |
| }, |
| { |
| "completion_length": 20.078125, |
| "epoch": 1.2259414225941423, |
| "grad_norm": 4.124558448791504, |
| "kl": 0.154052734375, |
| "learning_rate": 6.945606694560669e-07, |
| "loss": 0.0061, |
| "reward": 0.41678573191165924, |
| "reward_std": 0.09621147811412811, |
| "reward_std_all": 0.09064915776252747, |
| "rewards-max/forensics_iou_reward": 0.9118311703205109, |
| "rewards/forensics_iou_reward": 0.41678573191165924, |
| "step": 293 |
| }, |
| { |
| "completion_length": 20.359375, |
| "epoch": 1.2301255230125523, |
| "grad_norm": 4.1593756675720215, |
| "kl": 0.18359375, |
| "learning_rate": 6.935146443514644e-07, |
| "loss": 0.0073, |
| "reward": 0.4397118240594864, |
| "reward_std": 0.13391435891389847, |
| "reward_std_all": 0.14914284273982048, |
| "rewards-max/forensics_iou_reward": 0.8942942917346954, |
| "rewards/forensics_iou_reward": 0.4397118240594864, |
| "step": 294 |
| }, |
| { |
| "completion_length": 21.015625, |
| "epoch": 1.2343096234309623, |
| "grad_norm": 4.179725646972656, |
| "kl": 0.119873046875, |
| "learning_rate": 6.924686192468618e-07, |
| "loss": 0.0048, |
| "reward": 0.41492536664009094, |
| "reward_std": 0.11565083637833595, |
| "reward_std_all": 0.10879667103290558, |
| "rewards-max/forensics_iou_reward": 0.9543779194355011, |
| "rewards/forensics_iou_reward": 0.41492536664009094, |
| "step": 295 |
| }, |
| { |
| "completion_length": 20.0625, |
| "epoch": 1.2384937238493725, |
| "grad_norm": 5.725615978240967, |
| "kl": 0.15869140625, |
| "learning_rate": 6.914225941422594e-07, |
| "loss": 0.0064, |
| "reward": 0.4997791349887848, |
| "reward_std": 0.10513106733560562, |
| "reward_std_all": 0.1163068450987339, |
| "rewards-max/forensics_iou_reward": 0.8960946500301361, |
| "rewards/forensics_iou_reward": 0.4997791349887848, |
| "step": 296 |
| }, |
| { |
| "completion_length": 19.734375, |
| "epoch": 1.2426778242677825, |
| "grad_norm": 4.0261688232421875, |
| "kl": 0.134765625, |
| "learning_rate": 6.903765690376569e-07, |
| "loss": 0.0054, |
| "reward": 0.4487559497356415, |
| "reward_std": 0.10949309170246124, |
| "reward_std_all": 0.10187441110610962, |
| "rewards-max/forensics_iou_reward": 0.9521964490413666, |
| "rewards/forensics_iou_reward": 0.4487559497356415, |
| "step": 297 |
| }, |
| { |
| "completion_length": 21.171875, |
| "epoch": 1.2468619246861925, |
| "grad_norm": 3.6353282928466797, |
| "kl": 0.1650390625, |
| "learning_rate": 6.893305439330544e-07, |
| "loss": 0.0066, |
| "reward": 0.5425652861595154, |
| "reward_std": 0.08609882742166519, |
| "reward_std_all": 0.08619127050042152, |
| "rewards-max/forensics_iou_reward": 0.9624183773994446, |
| "rewards/forensics_iou_reward": 0.5425652861595154, |
| "step": 298 |
| }, |
| { |
| "completion_length": 19.296875, |
| "epoch": 1.2510460251046025, |
| "grad_norm": 3.6845970153808594, |
| "kl": 0.15087890625, |
| "learning_rate": 6.882845188284518e-07, |
| "loss": 0.0061, |
| "reward": 0.5279238224029541, |
| "reward_std": 0.07530192099511623, |
| "reward_std_all": 0.07291378453373909, |
| "rewards-max/forensics_iou_reward": 0.9692776501178741, |
| "rewards/forensics_iou_reward": 0.5279238224029541, |
| "step": 299 |
| }, |
| { |
| "completion_length": 19.546875, |
| "epoch": 1.2552301255230125, |
| "grad_norm": 4.643213272094727, |
| "kl": 0.14404296875, |
| "learning_rate": 6.872384937238493e-07, |
| "loss": 0.0058, |
| "reward": 0.34218989312648773, |
| "reward_std": 0.06628420948982239, |
| "reward_std_all": 0.062257250770926476, |
| "rewards-max/forensics_iou_reward": 0.8189721405506134, |
| "rewards/forensics_iou_reward": 0.34218989312648773, |
| "step": 300 |
| }, |
| { |
| "completion_length": 20.484375, |
| "epoch": 1.2594142259414225, |
| "grad_norm": 4.762936115264893, |
| "kl": 0.15478515625, |
| "learning_rate": 6.861924686192468e-07, |
| "loss": 0.0062, |
| "reward": 0.440638467669487, |
| "reward_std": 0.05102928355336189, |
| "reward_std_all": 0.05344418063759804, |
| "rewards-max/forensics_iou_reward": 0.9248142242431641, |
| "rewards/forensics_iou_reward": 0.440638467669487, |
| "step": 301 |
| }, |
| { |
| "completion_length": 20.03125, |
| "epoch": 1.2635983263598327, |
| "grad_norm": 3.9339394569396973, |
| "kl": 0.155517578125, |
| "learning_rate": 6.851464435146443e-07, |
| "loss": 0.0062, |
| "reward": 0.5055930614471436, |
| "reward_std": 0.08883821964263916, |
| "reward_std_all": 0.08394773676991463, |
| "rewards-max/forensics_iou_reward": 0.9332347512245178, |
| "rewards/forensics_iou_reward": 0.5055930614471436, |
| "step": 302 |
| }, |
| { |
| "completion_length": 19.640625, |
| "epoch": 1.2677824267782427, |
| "grad_norm": 6.613922595977783, |
| "kl": 0.17578125, |
| "learning_rate": 6.841004184100418e-07, |
| "loss": 0.007, |
| "reward": 0.4303828179836273, |
| "reward_std": 0.06841477379202843, |
| "reward_std_all": 0.09177644178271294, |
| "rewards-max/forensics_iou_reward": 0.878265917301178, |
| "rewards/forensics_iou_reward": 0.4303828179836273, |
| "step": 303 |
| }, |
| { |
| "completion_length": 19.875, |
| "epoch": 1.2719665271966527, |
| "grad_norm": 4.6824493408203125, |
| "kl": 0.156494140625, |
| "learning_rate": 6.830543933054394e-07, |
| "loss": 0.0063, |
| "reward": 0.6343096494674683, |
| "reward_std": 0.09962952695786953, |
| "reward_std_all": 0.1011408343911171, |
| "rewards-max/forensics_iou_reward": 0.977516382932663, |
| "rewards/forensics_iou_reward": 0.6343096494674683, |
| "step": 304 |
| }, |
| { |
| "completion_length": 19.4375, |
| "epoch": 1.2761506276150627, |
| "grad_norm": 11.928289413452148, |
| "kl": 0.14404296875, |
| "learning_rate": 6.820083682008368e-07, |
| "loss": 0.0058, |
| "reward": 0.4608629047870636, |
| "reward_std": 0.09770157188177109, |
| "reward_std_all": 0.09527923166751862, |
| "rewards-max/forensics_iou_reward": 0.9382664561271667, |
| "rewards/forensics_iou_reward": 0.4608629047870636, |
| "step": 305 |
| }, |
| { |
| "completion_length": 19.421875, |
| "epoch": 1.280334728033473, |
| "grad_norm": 4.462669372558594, |
| "kl": 0.17724609375, |
| "learning_rate": 6.809623430962343e-07, |
| "loss": 0.0071, |
| "reward": 0.5190652161836624, |
| "reward_std": 0.09414513781666756, |
| "reward_std_all": 0.09607294946908951, |
| "rewards-max/forensics_iou_reward": 0.9285318553447723, |
| "rewards/forensics_iou_reward": 0.5190652161836624, |
| "step": 306 |
| }, |
| { |
| "completion_length": 21.203125, |
| "epoch": 1.284518828451883, |
| "grad_norm": 7.028843402862549, |
| "kl": 0.1435546875, |
| "learning_rate": 6.799163179916317e-07, |
| "loss": 0.0057, |
| "reward": 0.42961350083351135, |
| "reward_std": 0.07439631223678589, |
| "reward_std_all": 0.06680687330663204, |
| "rewards-max/forensics_iou_reward": 0.9405443072319031, |
| "rewards/forensics_iou_reward": 0.42961350083351135, |
| "step": 307 |
| }, |
| { |
| "completion_length": 19.375, |
| "epoch": 1.288702928870293, |
| "grad_norm": 4.880223274230957, |
| "kl": 0.15185546875, |
| "learning_rate": 6.788702928870292e-07, |
| "loss": 0.0061, |
| "reward": 0.5255900472402573, |
| "reward_std": 0.06599114090204239, |
| "reward_std_all": 0.06780590862035751, |
| "rewards-max/forensics_iou_reward": 0.9599361717700958, |
| "rewards/forensics_iou_reward": 0.5255900472402573, |
| "step": 308 |
| }, |
| { |
| "completion_length": 19.140625, |
| "epoch": 1.292887029288703, |
| "grad_norm": 4.139949321746826, |
| "kl": 0.16259765625, |
| "learning_rate": 6.778242677824267e-07, |
| "loss": 0.0065, |
| "reward": 0.5764064192771912, |
| "reward_std": 0.06562759354710579, |
| "reward_std_all": 0.11250854283571243, |
| "rewards-max/forensics_iou_reward": 0.9320664405822754, |
| "rewards/forensics_iou_reward": 0.5764064192771912, |
| "step": 309 |
| }, |
| { |
| "completion_length": 19.515625, |
| "epoch": 1.297071129707113, |
| "grad_norm": 10.341793060302734, |
| "kl": 0.1474609375, |
| "learning_rate": 6.767782426778242e-07, |
| "loss": 0.0059, |
| "reward": 0.49778494238853455, |
| "reward_std": 0.0645188707858324, |
| "reward_std_all": 0.07135814242064953, |
| "rewards-max/forensics_iou_reward": 0.9138185083866119, |
| "rewards/forensics_iou_reward": 0.49778494238853455, |
| "step": 310 |
| }, |
| { |
| "completion_length": 19.6875, |
| "epoch": 1.301255230125523, |
| "grad_norm": 4.118427753448486, |
| "kl": 0.2373046875, |
| "learning_rate": 6.757322175732217e-07, |
| "loss": 0.0095, |
| "reward": 0.49736881256103516, |
| "reward_std": 0.0700550377368927, |
| "reward_std_all": 0.07992056384682655, |
| "rewards-max/forensics_iou_reward": 0.9385134875774384, |
| "rewards/forensics_iou_reward": 0.49736881256103516, |
| "step": 311 |
| }, |
| { |
| "completion_length": 19.765625, |
| "epoch": 1.3054393305439331, |
| "grad_norm": 4.928702354431152, |
| "kl": 0.1435546875, |
| "learning_rate": 6.746861924686192e-07, |
| "loss": 0.0058, |
| "reward": 0.35063761472702026, |
| "reward_std": 0.056184420362114906, |
| "reward_std_all": 0.05835169181227684, |
| "rewards-max/forensics_iou_reward": 0.816249430179596, |
| "rewards/forensics_iou_reward": 0.35063761472702026, |
| "step": 312 |
| }, |
| { |
| "completion_length": 19.765625, |
| "epoch": 1.3096234309623431, |
| "grad_norm": 4.663575172424316, |
| "kl": 0.11767578125, |
| "learning_rate": 6.736401673640168e-07, |
| "loss": 0.0047, |
| "reward": 0.5292966067790985, |
| "reward_std": 0.04866921342909336, |
| "reward_std_all": 0.08889344334602356, |
| "rewards-max/forensics_iou_reward": 0.9751510322093964, |
| "rewards/forensics_iou_reward": 0.5292966067790985, |
| "step": 313 |
| }, |
| { |
| "completion_length": 19.578125, |
| "epoch": 1.3138075313807531, |
| "grad_norm": 4.055056095123291, |
| "kl": 0.23876953125, |
| "learning_rate": 6.725941422594143e-07, |
| "loss": 0.0095, |
| "reward": 0.6387411952018738, |
| "reward_std": 0.08358979225158691, |
| "reward_std_all": 0.0887058638036251, |
| "rewards-max/forensics_iou_reward": 0.9619737267494202, |
| "rewards/forensics_iou_reward": 0.6387411952018738, |
| "step": 314 |
| }, |
| { |
| "completion_length": 19.1875, |
| "epoch": 1.3179916317991631, |
| "grad_norm": 6.07841157913208, |
| "kl": 0.116455078125, |
| "learning_rate": 6.715481171548117e-07, |
| "loss": 0.0047, |
| "reward": 0.5720384120941162, |
| "reward_std": 0.11890611052513123, |
| "reward_std_all": 0.10850885137915611, |
| "rewards-max/forensics_iou_reward": 0.9184277653694153, |
| "rewards/forensics_iou_reward": 0.5720384120941162, |
| "step": 315 |
| }, |
| { |
| "completion_length": 19.578125, |
| "epoch": 1.3221757322175733, |
| "grad_norm": 5.774809837341309, |
| "kl": 0.14111328125, |
| "learning_rate": 6.705020920502091e-07, |
| "loss": 0.0056, |
| "reward": 0.5570617318153381, |
| "reward_std": 0.05527880601584911, |
| "reward_std_all": 0.08826571330428123, |
| "rewards-max/forensics_iou_reward": 0.9326714277267456, |
| "rewards/forensics_iou_reward": 0.5570617318153381, |
| "step": 316 |
| }, |
| { |
| "completion_length": 19.4375, |
| "epoch": 1.3263598326359833, |
| "grad_norm": 4.5846123695373535, |
| "kl": 0.2109375, |
| "learning_rate": 6.694560669456066e-07, |
| "loss": 0.0084, |
| "reward": 0.48017627000808716, |
| "reward_std": 0.066504230722785, |
| "reward_std_all": 0.06000334210693836, |
| "rewards-max/forensics_iou_reward": 0.9235905408859253, |
| "rewards/forensics_iou_reward": 0.48017627000808716, |
| "step": 317 |
| }, |
| { |
| "completion_length": 19.421875, |
| "epoch": 1.3305439330543933, |
| "grad_norm": 4.864208698272705, |
| "kl": 0.164794921875, |
| "learning_rate": 6.684100418410041e-07, |
| "loss": 0.0066, |
| "reward": 0.5391278713941574, |
| "reward_std": 0.07287703640758991, |
| "reward_std_all": 0.08548468723893166, |
| "rewards-max/forensics_iou_reward": 0.9113034605979919, |
| "rewards/forensics_iou_reward": 0.5391278713941574, |
| "step": 318 |
| }, |
| { |
| "completion_length": 19.140625, |
| "epoch": 1.3347280334728033, |
| "grad_norm": 4.003464698791504, |
| "kl": 0.2119140625, |
| "learning_rate": 6.673640167364016e-07, |
| "loss": 0.0085, |
| "reward": 0.3678853362798691, |
| "reward_std": 0.10467544570565224, |
| "reward_std_all": 0.09091810882091522, |
| "rewards-max/forensics_iou_reward": 0.8285920917987823, |
| "rewards/forensics_iou_reward": 0.3678853362798691, |
| "step": 319 |
| }, |
| { |
| "completion_length": 19.234375, |
| "epoch": 1.3389121338912133, |
| "grad_norm": 4.489568710327148, |
| "kl": 0.27880859375, |
| "learning_rate": 6.663179916317991e-07, |
| "loss": 0.0112, |
| "reward": 0.4599158316850662, |
| "reward_std": 0.09775039553642273, |
| "reward_std_all": 0.09325218200683594, |
| "rewards-max/forensics_iou_reward": 0.9705342650413513, |
| "rewards/forensics_iou_reward": 0.4599158316850662, |
| "step": 320 |
| }, |
| { |
| "completion_length": 19.125, |
| "epoch": 1.3430962343096233, |
| "grad_norm": 6.137599945068359, |
| "kl": 0.14111328125, |
| "learning_rate": 6.652719665271967e-07, |
| "loss": 0.0057, |
| "reward": 0.5504738092422485, |
| "reward_std": 0.05691562965512276, |
| "reward_std_all": 0.06265989691019058, |
| "rewards-max/forensics_iou_reward": 0.9268262982368469, |
| "rewards/forensics_iou_reward": 0.5504738092422485, |
| "step": 321 |
| }, |
| { |
| "completion_length": 20.234375, |
| "epoch": 1.3472803347280335, |
| "grad_norm": 6.6042375564575195, |
| "kl": 0.21337890625, |
| "learning_rate": 6.642259414225942e-07, |
| "loss": 0.0086, |
| "reward": 0.47787949442863464, |
| "reward_std": 0.07964887842535973, |
| "reward_std_all": 0.09048284217715263, |
| "rewards-max/forensics_iou_reward": 0.9811445772647858, |
| "rewards/forensics_iou_reward": 0.47787949442863464, |
| "step": 322 |
| }, |
| { |
| "completion_length": 20.171875, |
| "epoch": 1.3514644351464435, |
| "grad_norm": 4.599197864532471, |
| "kl": 0.251953125, |
| "learning_rate": 6.631799163179917e-07, |
| "loss": 0.0101, |
| "reward": 0.48552405834198, |
| "reward_std": 0.05156927555799484, |
| "reward_std_all": 0.08733965829014778, |
| "rewards-max/forensics_iou_reward": 0.8834354281425476, |
| "rewards/forensics_iou_reward": 0.48552405834198, |
| "step": 323 |
| }, |
| { |
| "completion_length": 19.0625, |
| "epoch": 1.3556485355648535, |
| "grad_norm": 4.998231410980225, |
| "kl": 0.1796875, |
| "learning_rate": 6.621338912133892e-07, |
| "loss": 0.0072, |
| "reward": 0.6206342279911041, |
| "reward_std": 0.06699616461992264, |
| "reward_std_all": 0.06522578559815884, |
| "rewards-max/forensics_iou_reward": 0.972344696521759, |
| "rewards/forensics_iou_reward": 0.6206342279911041, |
| "step": 324 |
| }, |
| { |
| "completion_length": 19.046875, |
| "epoch": 1.3598326359832635, |
| "grad_norm": 9.521533012390137, |
| "kl": 0.201171875, |
| "learning_rate": 6.610878661087865e-07, |
| "loss": 0.008, |
| "reward": 0.4431242346763611, |
| "reward_std": 0.10860015079379082, |
| "reward_std_all": 0.10822264105081558, |
| "rewards-max/forensics_iou_reward": 0.8620361089706421, |
| "rewards/forensics_iou_reward": 0.4431242346763611, |
| "step": 325 |
| }, |
| { |
| "completion_length": 19.171875, |
| "epoch": 1.3640167364016738, |
| "grad_norm": 3.9938812255859375, |
| "kl": 0.283203125, |
| "learning_rate": 6.60041841004184e-07, |
| "loss": 0.0113, |
| "reward": 0.5372316539287567, |
| "reward_std": 0.10633495636284351, |
| "reward_std_all": 0.11502226442098618, |
| "rewards-max/forensics_iou_reward": 0.8909313678741455, |
| "rewards/forensics_iou_reward": 0.5372316539287567, |
| "step": 326 |
| }, |
| { |
| "completion_length": 19.15625, |
| "epoch": 1.3682008368200838, |
| "grad_norm": 3.5651473999023438, |
| "kl": 0.21728515625, |
| "learning_rate": 6.589958158995815e-07, |
| "loss": 0.0087, |
| "reward": 0.38679589331150055, |
| "reward_std": 0.04416140913963318, |
| "reward_std_all": 0.04477040655910969, |
| "rewards-max/forensics_iou_reward": 0.9251245558261871, |
| "rewards/forensics_iou_reward": 0.38679589331150055, |
| "step": 327 |
| }, |
| { |
| "completion_length": 19.46875, |
| "epoch": 1.3723849372384938, |
| "grad_norm": 5.689239501953125, |
| "kl": 0.18701171875, |
| "learning_rate": 6.57949790794979e-07, |
| "loss": 0.0075, |
| "reward": 0.5248095989227295, |
| "reward_std": 0.06764385476708412, |
| "reward_std_all": 0.08102754130959511, |
| "rewards-max/forensics_iou_reward": 0.9146392345428467, |
| "rewards/forensics_iou_reward": 0.5248095989227295, |
| "step": 328 |
| }, |
| { |
| "completion_length": 19.140625, |
| "epoch": 1.3765690376569037, |
| "grad_norm": 4.0496134757995605, |
| "kl": 0.208984375, |
| "learning_rate": 6.569037656903765e-07, |
| "loss": 0.0084, |
| "reward": 0.47975242137908936, |
| "reward_std": 0.07626276463270187, |
| "reward_std_all": 0.06935299187898636, |
| "rewards-max/forensics_iou_reward": 0.8831672072410583, |
| "rewards/forensics_iou_reward": 0.47975242137908936, |
| "step": 329 |
| }, |
| { |
| "completion_length": 19.4375, |
| "epoch": 1.3807531380753137, |
| "grad_norm": 3.2790632247924805, |
| "kl": 0.2314453125, |
| "learning_rate": 6.558577405857741e-07, |
| "loss": 0.0092, |
| "reward": 0.567481130361557, |
| "reward_std": 0.07658080384135246, |
| "reward_std_all": 0.08164994418621063, |
| "rewards-max/forensics_iou_reward": 0.9892585873603821, |
| "rewards/forensics_iou_reward": 0.567481130361557, |
| "step": 330 |
| }, |
| { |
| "completion_length": 19.578125, |
| "epoch": 1.3849372384937237, |
| "grad_norm": 3.9261345863342285, |
| "kl": 0.1484375, |
| "learning_rate": 6.548117154811716e-07, |
| "loss": 0.0059, |
| "reward": 0.5135020315647125, |
| "reward_std": 0.07650046795606613, |
| "reward_std_all": 0.0711212083697319, |
| "rewards-max/forensics_iou_reward": 0.9127996563911438, |
| "rewards/forensics_iou_reward": 0.5135020315647125, |
| "step": 331 |
| }, |
| { |
| "completion_length": 19.265625, |
| "epoch": 1.389121338912134, |
| "grad_norm": 4.758991718292236, |
| "kl": 0.186279296875, |
| "learning_rate": 6.537656903765691e-07, |
| "loss": 0.0074, |
| "reward": 0.497610867023468, |
| "reward_std": 0.09507468342781067, |
| "reward_std_all": 0.09196377545595169, |
| "rewards-max/forensics_iou_reward": 0.8956109285354614, |
| "rewards/forensics_iou_reward": 0.497610867023468, |
| "step": 332 |
| }, |
| { |
| "completion_length": 19.109375, |
| "epoch": 1.393305439330544, |
| "grad_norm": 6.048153400421143, |
| "kl": 0.1416015625, |
| "learning_rate": 6.527196652719665e-07, |
| "loss": 0.0057, |
| "reward": 0.5170380771160126, |
| "reward_std": 0.07408995181322098, |
| "reward_std_all": 0.06867878884077072, |
| "rewards-max/forensics_iou_reward": 0.9956140220165253, |
| "rewards/forensics_iou_reward": 0.5170380771160126, |
| "step": 333 |
| }, |
| { |
| "completion_length": 19.390625, |
| "epoch": 1.397489539748954, |
| "grad_norm": 5.196692943572998, |
| "kl": 0.1318359375, |
| "learning_rate": 6.51673640167364e-07, |
| "loss": 0.0053, |
| "reward": 0.4952208995819092, |
| "reward_std": 0.07340321317315102, |
| "reward_std_all": 0.08437421545386314, |
| "rewards-max/forensics_iou_reward": 0.8929824233055115, |
| "rewards/forensics_iou_reward": 0.4952208995819092, |
| "step": 334 |
| }, |
| { |
| "completion_length": 19.46875, |
| "epoch": 1.401673640167364, |
| "grad_norm": 4.374422073364258, |
| "kl": 0.150390625, |
| "learning_rate": 6.506276150627614e-07, |
| "loss": 0.006, |
| "reward": 0.5015119761228561, |
| "reward_std": 0.060804907232522964, |
| "reward_std_all": 0.057092830538749695, |
| "rewards-max/forensics_iou_reward": 0.9677769541740417, |
| "rewards/forensics_iou_reward": 0.5015119761228561, |
| "step": 335 |
| }, |
| { |
| "completion_length": 18.984375, |
| "epoch": 1.4058577405857742, |
| "grad_norm": 3.826995372772217, |
| "kl": 0.3994140625, |
| "learning_rate": 6.495815899581589e-07, |
| "loss": 0.0159, |
| "reward": 0.397101566195488, |
| "reward_std": 0.09972214326262474, |
| "reward_std_all": 0.09706199914216995, |
| "rewards-max/forensics_iou_reward": 0.8857128322124481, |
| "rewards/forensics_iou_reward": 0.397101566195488, |
| "step": 336 |
| }, |
| { |
| "completion_length": 19.515625, |
| "epoch": 1.4100418410041842, |
| "grad_norm": 4.933713436126709, |
| "kl": 0.171875, |
| "learning_rate": 6.485355648535564e-07, |
| "loss": 0.0069, |
| "reward": 0.5148961842060089, |
| "reward_std": 0.07498819380998611, |
| "reward_std_all": 0.07639136910438538, |
| "rewards-max/forensics_iou_reward": 0.8525796234607697, |
| "rewards/forensics_iou_reward": 0.5148961842060089, |
| "step": 337 |
| }, |
| { |
| "completion_length": 19.03125, |
| "epoch": 1.4142259414225942, |
| "grad_norm": 6.237710475921631, |
| "kl": 0.17626953125, |
| "learning_rate": 6.47489539748954e-07, |
| "loss": 0.0071, |
| "reward": 0.4236949682235718, |
| "reward_std": 0.08015405759215355, |
| "reward_std_all": 0.10089456290006638, |
| "rewards-max/forensics_iou_reward": 0.9761997163295746, |
| "rewards/forensics_iou_reward": 0.4236949682235718, |
| "step": 338 |
| }, |
| { |
| "completion_length": 19.640625, |
| "epoch": 1.4184100418410042, |
| "grad_norm": 4.498462200164795, |
| "kl": 0.133544921875, |
| "learning_rate": 6.464435146443515e-07, |
| "loss": 0.0053, |
| "reward": 0.565748393535614, |
| "reward_std": 0.07096056081354618, |
| "reward_std_all": 0.07390783727169037, |
| "rewards-max/forensics_iou_reward": 0.9296853840351105, |
| "rewards/forensics_iou_reward": 0.565748393535614, |
| "step": 339 |
| }, |
| { |
| "completion_length": 19.140625, |
| "epoch": 1.4225941422594142, |
| "grad_norm": 5.70767068862915, |
| "kl": 0.1572265625, |
| "learning_rate": 6.45397489539749e-07, |
| "loss": 0.0063, |
| "reward": 0.531242847442627, |
| "reward_std": 0.0671406015753746, |
| "reward_std_all": 0.09800952672958374, |
| "rewards-max/forensics_iou_reward": 0.9504497945308685, |
| "rewards/forensics_iou_reward": 0.531242847442627, |
| "step": 340 |
| }, |
| { |
| "completion_length": 19.234375, |
| "epoch": 1.4267782426778242, |
| "grad_norm": 4.0726776123046875, |
| "kl": 0.16845703125, |
| "learning_rate": 6.443514644351464e-07, |
| "loss": 0.0067, |
| "reward": 0.5442341417074203, |
| "reward_std": 0.05336356721818447, |
| "reward_std_all": 0.0683199092745781, |
| "rewards-max/forensics_iou_reward": 0.9482499063014984, |
| "rewards/forensics_iou_reward": 0.5442341417074203, |
| "step": 341 |
| }, |
| { |
| "completion_length": 19.421875, |
| "epoch": 1.4309623430962344, |
| "grad_norm": 4.549582481384277, |
| "kl": 0.16162109375, |
| "learning_rate": 6.433054393305439e-07, |
| "loss": 0.0065, |
| "reward": 0.5654447972774506, |
| "reward_std": 0.06465485878288746, |
| "reward_std_all": 0.08136826753616333, |
| "rewards-max/forensics_iou_reward": 0.923422634601593, |
| "rewards/forensics_iou_reward": 0.5654447972774506, |
| "step": 342 |
| }, |
| { |
| "completion_length": 18.90625, |
| "epoch": 1.4351464435146444, |
| "grad_norm": 6.490696430206299, |
| "kl": 0.443359375, |
| "learning_rate": 6.422594142259414e-07, |
| "loss": 0.0177, |
| "reward": 0.4459875822067261, |
| "reward_std": 0.06395359709858894, |
| "reward_std_all": 0.07550401240587234, |
| "rewards-max/forensics_iou_reward": 0.8999251127243042, |
| "rewards/forensics_iou_reward": 0.4459875822067261, |
| "step": 343 |
| }, |
| { |
| "completion_length": 19.46875, |
| "epoch": 1.4393305439330544, |
| "grad_norm": 4.296226501464844, |
| "kl": 0.1650390625, |
| "learning_rate": 6.412133891213389e-07, |
| "loss": 0.0066, |
| "reward": 0.46489474177360535, |
| "reward_std": 0.10022584348917007, |
| "reward_std_all": 0.09843984991312027, |
| "rewards-max/forensics_iou_reward": 0.9412768185138702, |
| "rewards/forensics_iou_reward": 0.46489474177360535, |
| "step": 344 |
| }, |
| { |
| "completion_length": 19.546875, |
| "epoch": 1.4435146443514644, |
| "grad_norm": 4.058298110961914, |
| "kl": 0.18310546875, |
| "learning_rate": 6.401673640167363e-07, |
| "loss": 0.0073, |
| "reward": 0.5444051623344421, |
| "reward_std": 0.05015461519360542, |
| "reward_std_all": 0.061755089089274406, |
| "rewards-max/forensics_iou_reward": 0.9646562039852142, |
| "rewards/forensics_iou_reward": 0.5444051623344421, |
| "step": 345 |
| }, |
| { |
| "completion_length": 19.34375, |
| "epoch": 1.4476987447698746, |
| "grad_norm": 5.252729892730713, |
| "kl": 0.37255859375, |
| "learning_rate": 6.391213389121338e-07, |
| "loss": 0.0149, |
| "reward": 0.4215920567512512, |
| "reward_std": 0.11244288831949234, |
| "reward_std_all": 0.10838207602500916, |
| "rewards-max/forensics_iou_reward": 0.9646700620651245, |
| "rewards/forensics_iou_reward": 0.4215920567512512, |
| "step": 346 |
| }, |
| { |
| "completion_length": 19.578125, |
| "epoch": 1.4518828451882846, |
| "grad_norm": 4.1002278327941895, |
| "kl": 0.17822265625, |
| "learning_rate": 6.380753138075314e-07, |
| "loss": 0.0071, |
| "reward": 0.4210767596960068, |
| "reward_std": 0.08159087970852852, |
| "reward_std_all": 0.08480934426188469, |
| "rewards-max/forensics_iou_reward": 0.8525198996067047, |
| "rewards/forensics_iou_reward": 0.4210767596960068, |
| "step": 347 |
| }, |
| { |
| "completion_length": 19.484375, |
| "epoch": 1.4560669456066946, |
| "grad_norm": 4.436542510986328, |
| "kl": 0.16748046875, |
| "learning_rate": 6.370292887029289e-07, |
| "loss": 0.0067, |
| "reward": 0.5627847909927368, |
| "reward_std": 0.044540826231241226, |
| "reward_std_all": 0.051771607249975204, |
| "rewards-max/forensics_iou_reward": 0.9043974578380585, |
| "rewards/forensics_iou_reward": 0.5627847909927368, |
| "step": 348 |
| }, |
| { |
| "completion_length": 19.25, |
| "epoch": 1.4602510460251046, |
| "grad_norm": 4.296202659606934, |
| "kl": 0.17578125, |
| "learning_rate": 6.359832635983263e-07, |
| "loss": 0.007, |
| "reward": 0.5748299807310104, |
| "reward_std": 0.0498813483864069, |
| "reward_std_all": 0.05367204733192921, |
| "rewards-max/forensics_iou_reward": 0.9400851726531982, |
| "rewards/forensics_iou_reward": 0.5748299807310104, |
| "step": 349 |
| }, |
| { |
| "completion_length": 19.53125, |
| "epoch": 1.4644351464435146, |
| "grad_norm": 3.955711603164673, |
| "kl": 0.15869140625, |
| "learning_rate": 6.349372384937238e-07, |
| "loss": 0.0063, |
| "reward": 0.6830177903175354, |
| "reward_std": 0.08708501979708672, |
| "reward_std_all": 0.08397995308041573, |
| "rewards-max/forensics_iou_reward": 0.9916392862796783, |
| "rewards/forensics_iou_reward": 0.6830177903175354, |
| "step": 350 |
| }, |
| { |
| "completion_length": 19.90625, |
| "epoch": 1.4686192468619246, |
| "grad_norm": 4.795264720916748, |
| "kl": 0.135009765625, |
| "learning_rate": 6.338912133891213e-07, |
| "loss": 0.0054, |
| "reward": 0.6146026849746704, |
| "reward_std": 0.0600028932094574, |
| "reward_std_all": 0.07514502666890621, |
| "rewards-max/forensics_iou_reward": 0.9142223298549652, |
| "rewards/forensics_iou_reward": 0.6146026849746704, |
| "step": 351 |
| }, |
| { |
| "completion_length": 19.859375, |
| "epoch": 1.4728033472803348, |
| "grad_norm": 6.7698798179626465, |
| "kl": 0.14794921875, |
| "learning_rate": 6.328451882845188e-07, |
| "loss": 0.0059, |
| "reward": 0.41358858346939087, |
| "reward_std": 0.1024700254201889, |
| "reward_std_all": 0.1161775104701519, |
| "rewards-max/forensics_iou_reward": 0.8584177792072296, |
| "rewards/forensics_iou_reward": 0.41358858346939087, |
| "step": 352 |
| }, |
| { |
| "completion_length": 19.5625, |
| "epoch": 1.4769874476987448, |
| "grad_norm": 5.834930896759033, |
| "kl": 0.37744140625, |
| "learning_rate": 6.317991631799163e-07, |
| "loss": 0.0151, |
| "reward": 0.4691027104854584, |
| "reward_std": 0.08694008365273476, |
| "reward_std_all": 0.08591792732477188, |
| "rewards-max/forensics_iou_reward": 0.926049679517746, |
| "rewards/forensics_iou_reward": 0.4691027104854584, |
| "step": 353 |
| }, |
| { |
| "completion_length": 19.46875, |
| "epoch": 1.4811715481171548, |
| "grad_norm": 5.804318428039551, |
| "kl": 0.15087890625, |
| "learning_rate": 6.307531380753138e-07, |
| "loss": 0.006, |
| "reward": 0.5109061896800995, |
| "reward_std": 0.13730565458536148, |
| "reward_std_all": 0.11982409656047821, |
| "rewards-max/forensics_iou_reward": 0.8926171958446503, |
| "rewards/forensics_iou_reward": 0.5109061896800995, |
| "step": 354 |
| }, |
| { |
| "completion_length": 19.84375, |
| "epoch": 1.4853556485355648, |
| "grad_norm": 7.860546588897705, |
| "kl": 0.14208984375, |
| "learning_rate": 6.297071129707112e-07, |
| "loss": 0.0057, |
| "reward": 0.5185563266277313, |
| "reward_std": 0.04158092103898525, |
| "reward_std_all": 0.048967693001031876, |
| "rewards-max/forensics_iou_reward": 0.9236686825752258, |
| "rewards/forensics_iou_reward": 0.5185563266277313, |
| "step": 355 |
| }, |
| { |
| "completion_length": 19.171875, |
| "epoch": 1.489539748953975, |
| "grad_norm": 6.024447441101074, |
| "kl": 0.2119140625, |
| "learning_rate": 6.286610878661088e-07, |
| "loss": 0.0085, |
| "reward": 0.4542190134525299, |
| "reward_std": 0.13329127803444862, |
| "reward_std_all": 0.1272861324250698, |
| "rewards-max/forensics_iou_reward": 0.9707970917224884, |
| "rewards/forensics_iou_reward": 0.4542190134525299, |
| "step": 356 |
| }, |
| { |
| "completion_length": 19.609375, |
| "epoch": 1.493723849372385, |
| "grad_norm": 4.435934543609619, |
| "kl": 0.275390625, |
| "learning_rate": 6.276150627615063e-07, |
| "loss": 0.011, |
| "reward": 0.4781939536333084, |
| "reward_std": 0.07487213984131813, |
| "reward_std_all": 0.06946338713169098, |
| "rewards-max/forensics_iou_reward": 0.9199597537517548, |
| "rewards/forensics_iou_reward": 0.4781939536333084, |
| "step": 357 |
| }, |
| { |
| "completion_length": 19.625, |
| "epoch": 1.497907949790795, |
| "grad_norm": 6.2088212966918945, |
| "kl": 0.16552734375, |
| "learning_rate": 6.265690376569037e-07, |
| "loss": 0.0066, |
| "reward": 0.5885232985019684, |
| "reward_std": 0.07844593562185764, |
| "reward_std_all": 0.07863690704107285, |
| "rewards-max/forensics_iou_reward": 0.9441322088241577, |
| "rewards/forensics_iou_reward": 0.5885232985019684, |
| "step": 358 |
| }, |
| { |
| "completion_length": 19.421875, |
| "epoch": 1.502092050209205, |
| "grad_norm": 5.012471675872803, |
| "kl": 0.193359375, |
| "learning_rate": 6.255230125523012e-07, |
| "loss": 0.0077, |
| "reward": 0.5795209407806396, |
| "reward_std": 0.06444434076547623, |
| "reward_std_all": 0.06845706701278687, |
| "rewards-max/forensics_iou_reward": 0.9106078147888184, |
| "rewards/forensics_iou_reward": 0.5795209407806396, |
| "step": 359 |
| }, |
| { |
| "completion_length": 19.171875, |
| "epoch": 1.506276150627615, |
| "grad_norm": 3.490187168121338, |
| "kl": 0.123291015625, |
| "learning_rate": 6.244769874476987e-07, |
| "loss": 0.0049, |
| "reward": 0.4683820754289627, |
| "reward_std": 0.10052425414323807, |
| "reward_std_all": 0.08636096864938736, |
| "rewards-max/forensics_iou_reward": 0.87700355052948, |
| "rewards/forensics_iou_reward": 0.4683820754289627, |
| "step": 360 |
| }, |
| { |
| "completion_length": 19.171875, |
| "epoch": 1.510460251046025, |
| "grad_norm": 4.301212310791016, |
| "kl": 0.21826171875, |
| "learning_rate": 6.234309623430962e-07, |
| "loss": 0.0087, |
| "reward": 0.5452109575271606, |
| "reward_std": 0.08869610726833344, |
| "reward_std_all": 0.08301210403442383, |
| "rewards-max/forensics_iou_reward": 0.994269609451294, |
| "rewards/forensics_iou_reward": 0.5452109575271606, |
| "step": 361 |
| }, |
| { |
| "completion_length": 18.859375, |
| "epoch": 1.514644351464435, |
| "grad_norm": 3.834918737411499, |
| "kl": 0.22998046875, |
| "learning_rate": 6.223849372384937e-07, |
| "loss": 0.0092, |
| "reward": 0.5967915058135986, |
| "reward_std": 0.1094992458820343, |
| "reward_std_all": 0.10958115011453629, |
| "rewards-max/forensics_iou_reward": 0.9621379673480988, |
| "rewards/forensics_iou_reward": 0.5967915058135986, |
| "step": 362 |
| }, |
| { |
| "completion_length": 19.296875, |
| "epoch": 1.5188284518828452, |
| "grad_norm": 5.164206504821777, |
| "kl": 0.19140625, |
| "learning_rate": 6.213389121338912e-07, |
| "loss": 0.0077, |
| "reward": 0.5388934910297394, |
| "reward_std": 0.08951093256473541, |
| "reward_std_all": 0.09620886668562889, |
| "rewards-max/forensics_iou_reward": 0.9143722951412201, |
| "rewards/forensics_iou_reward": 0.5388934910297394, |
| "step": 363 |
| }, |
| { |
| "completion_length": 19.84375, |
| "epoch": 1.5230125523012552, |
| "grad_norm": 4.218179702758789, |
| "kl": 0.22216796875, |
| "learning_rate": 6.202928870292888e-07, |
| "loss": 0.0089, |
| "reward": 0.6187573671340942, |
| "reward_std": 0.06837162375450134, |
| "reward_std_all": 0.07344818860292435, |
| "rewards-max/forensics_iou_reward": 0.9803764820098877, |
| "rewards/forensics_iou_reward": 0.6187573671340942, |
| "step": 364 |
| }, |
| { |
| "completion_length": 19.421875, |
| "epoch": 1.5271966527196654, |
| "grad_norm": 4.365896224975586, |
| "kl": 0.1298828125, |
| "learning_rate": 6.192468619246862e-07, |
| "loss": 0.0052, |
| "reward": 0.5309001207351685, |
| "reward_std": 0.08519108593463898, |
| "reward_std_all": 0.09492339193820953, |
| "rewards-max/forensics_iou_reward": 0.8708083033561707, |
| "rewards/forensics_iou_reward": 0.5309001207351685, |
| "step": 365 |
| }, |
| { |
| "completion_length": 19.234375, |
| "epoch": 1.5313807531380754, |
| "grad_norm": 7.016507625579834, |
| "kl": 0.2060546875, |
| "learning_rate": 6.182008368200836e-07, |
| "loss": 0.0082, |
| "reward": 0.3747158646583557, |
| "reward_std": 0.09107528254389763, |
| "reward_std_all": 0.09752406924962997, |
| "rewards-max/forensics_iou_reward": 0.7618667483329773, |
| "rewards/forensics_iou_reward": 0.3747158646583557, |
| "step": 366 |
| }, |
| { |
| "completion_length": 19.40625, |
| "epoch": 1.5355648535564854, |
| "grad_norm": 5.469703197479248, |
| "kl": 0.14208984375, |
| "learning_rate": 6.171548117154811e-07, |
| "loss": 0.0057, |
| "reward": 0.5470594614744186, |
| "reward_std": 0.06595640257000923, |
| "reward_std_all": 0.06547241844236851, |
| "rewards-max/forensics_iou_reward": 0.8707418143749237, |
| "rewards/forensics_iou_reward": 0.5470594614744186, |
| "step": 367 |
| }, |
| { |
| "completion_length": 19.328125, |
| "epoch": 1.5397489539748954, |
| "grad_norm": 4.06907844543457, |
| "kl": 0.19677734375, |
| "learning_rate": 6.161087866108786e-07, |
| "loss": 0.0079, |
| "reward": 0.7192459106445312, |
| "reward_std": 0.07584414258599281, |
| "reward_std_all": 0.07522810623049736, |
| "rewards-max/forensics_iou_reward": 0.9620780944824219, |
| "rewards/forensics_iou_reward": 0.7192459106445312, |
| "step": 368 |
| }, |
| { |
| "completion_length": 19.171875, |
| "epoch": 1.5439330543933054, |
| "grad_norm": 7.177676677703857, |
| "kl": 0.19189453125, |
| "learning_rate": 6.150627615062761e-07, |
| "loss": 0.0077, |
| "reward": 0.49409107863903046, |
| "reward_std": 0.08497638627886772, |
| "reward_std_all": 0.0993495024740696, |
| "rewards-max/forensics_iou_reward": 0.9504870176315308, |
| "rewards/forensics_iou_reward": 0.49409107863903046, |
| "step": 369 |
| }, |
| { |
| "completion_length": 19.234375, |
| "epoch": 1.5481171548117154, |
| "grad_norm": 4.277737140655518, |
| "kl": 0.111328125, |
| "learning_rate": 6.140167364016736e-07, |
| "loss": 0.0044, |
| "reward": 0.6486115455627441, |
| "reward_std": 0.0730966255068779, |
| "reward_std_all": 0.07550078257918358, |
| "rewards-max/forensics_iou_reward": 0.9491937756538391, |
| "rewards/forensics_iou_reward": 0.6486115455627441, |
| "step": 370 |
| }, |
| { |
| "completion_length": 19.296875, |
| "epoch": 1.5523012552301254, |
| "grad_norm": 4.043759822845459, |
| "kl": 0.1455078125, |
| "learning_rate": 6.129707112970711e-07, |
| "loss": 0.0058, |
| "reward": 0.37147894501686096, |
| "reward_std": 0.05076821334660053, |
| "reward_std_all": 0.05512266233563423, |
| "rewards-max/forensics_iou_reward": 0.6716994643211365, |
| "rewards/forensics_iou_reward": 0.37147894501686096, |
| "step": 371 |
| }, |
| { |
| "completion_length": 19.203125, |
| "epoch": 1.5564853556485354, |
| "grad_norm": 4.81291389465332, |
| "kl": 0.248046875, |
| "learning_rate": 6.119246861924686e-07, |
| "loss": 0.0099, |
| "reward": 0.44241973757743835, |
| "reward_std": 0.03309840150177479, |
| "reward_std_all": 0.03857630863785744, |
| "rewards-max/forensics_iou_reward": 0.8938015699386597, |
| "rewards/forensics_iou_reward": 0.44241973757743835, |
| "step": 372 |
| }, |
| { |
| "completion_length": 19.359375, |
| "epoch": 1.5606694560669456, |
| "grad_norm": 4.151546478271484, |
| "kl": 0.107666015625, |
| "learning_rate": 6.108786610878662e-07, |
| "loss": 0.0043, |
| "reward": 0.5193558186292648, |
| "reward_std": 0.08574776910245419, |
| "reward_std_all": 0.07907344587147236, |
| "rewards-max/forensics_iou_reward": 0.9232493042945862, |
| "rewards/forensics_iou_reward": 0.5193558186292648, |
| "step": 373 |
| }, |
| { |
| "completion_length": 19.09375, |
| "epoch": 1.5648535564853556, |
| "grad_norm": 10.849448204040527, |
| "kl": 0.4521484375, |
| "learning_rate": 6.098326359832637e-07, |
| "loss": 0.0181, |
| "reward": 0.5255909562110901, |
| "reward_std": 0.05883513204753399, |
| "reward_std_all": 0.061402205377817154, |
| "rewards-max/forensics_iou_reward": 0.9111111164093018, |
| "rewards/forensics_iou_reward": 0.5255909562110901, |
| "step": 374 |
| }, |
| { |
| "completion_length": 19.4375, |
| "epoch": 1.5690376569037658, |
| "grad_norm": 4.188118934631348, |
| "kl": 0.15625, |
| "learning_rate": 6.08786610878661e-07, |
| "loss": 0.0062, |
| "reward": 0.47416889667510986, |
| "reward_std": 0.04921870864927769, |
| "reward_std_all": 0.08976158872246742, |
| "rewards-max/forensics_iou_reward": 0.9246721267700195, |
| "rewards/forensics_iou_reward": 0.47416889667510986, |
| "step": 375 |
| }, |
| { |
| "completion_length": 19.484375, |
| "epoch": 1.5732217573221758, |
| "grad_norm": 6.345853328704834, |
| "kl": 0.13671875, |
| "learning_rate": 6.077405857740585e-07, |
| "loss": 0.0055, |
| "reward": 0.4890053868293762, |
| "reward_std": 0.07627384923398495, |
| "reward_std_all": 0.08599007502198219, |
| "rewards-max/forensics_iou_reward": 0.9708681702613831, |
| "rewards/forensics_iou_reward": 0.4890053868293762, |
| "step": 376 |
| }, |
| { |
| "completion_length": 20.28125, |
| "epoch": 1.5774058577405858, |
| "grad_norm": 4.033615589141846, |
| "kl": 0.19921875, |
| "learning_rate": 6.06694560669456e-07, |
| "loss": 0.008, |
| "reward": 0.5171460807323456, |
| "reward_std": 0.08013943582773209, |
| "reward_std_all": 0.08236821740865707, |
| "rewards-max/forensics_iou_reward": 0.9855158627033234, |
| "rewards/forensics_iou_reward": 0.5171460807323456, |
| "step": 377 |
| }, |
| { |
| "completion_length": 18.953125, |
| "epoch": 1.5815899581589958, |
| "grad_norm": 4.67762565612793, |
| "kl": 0.28125, |
| "learning_rate": 6.056485355648535e-07, |
| "loss": 0.0112, |
| "reward": 0.4272504597902298, |
| "reward_std": 0.055965740233659744, |
| "reward_std_all": 0.06104012951254845, |
| "rewards-max/forensics_iou_reward": 0.9041326642036438, |
| "rewards/forensics_iou_reward": 0.4272504597902298, |
| "step": 378 |
| }, |
| { |
| "completion_length": 20.1875, |
| "epoch": 1.5857740585774058, |
| "grad_norm": 3.334036350250244, |
| "kl": 0.15869140625, |
| "learning_rate": 6.04602510460251e-07, |
| "loss": 0.0064, |
| "reward": 0.5186734199523926, |
| "reward_std": 0.10888182744383812, |
| "reward_std_all": 0.11107178777456284, |
| "rewards-max/forensics_iou_reward": 0.9394817650318146, |
| "rewards/forensics_iou_reward": 0.5186734199523926, |
| "step": 379 |
| }, |
| { |
| "completion_length": 19.0625, |
| "epoch": 1.5899581589958158, |
| "grad_norm": 4.46399450302124, |
| "kl": 0.1318359375, |
| "learning_rate": 6.035564853556485e-07, |
| "loss": 0.0053, |
| "reward": 0.5946562588214874, |
| "reward_std": 0.07152715139091015, |
| "reward_std_all": 0.08071786165237427, |
| "rewards-max/forensics_iou_reward": 0.9292271137237549, |
| "rewards/forensics_iou_reward": 0.5946562588214874, |
| "step": 380 |
| }, |
| { |
| "completion_length": 20.015625, |
| "epoch": 1.5941422594142258, |
| "grad_norm": 3.7152798175811768, |
| "kl": 0.19189453125, |
| "learning_rate": 6.02510460251046e-07, |
| "loss": 0.0077, |
| "reward": 0.413114532828331, |
| "reward_std": 0.09220397472381592, |
| "reward_std_all": 0.08543669432401657, |
| "rewards-max/forensics_iou_reward": 0.8240068852901459, |
| "rewards/forensics_iou_reward": 0.413114532828331, |
| "step": 381 |
| }, |
| { |
| "completion_length": 19.0, |
| "epoch": 1.5983263598326358, |
| "grad_norm": 5.6149115562438965, |
| "kl": 0.157958984375, |
| "learning_rate": 6.014644351464436e-07, |
| "loss": 0.0063, |
| "reward": 0.6115733683109283, |
| "reward_std": 0.07197243347764015, |
| "reward_std_all": 0.07646430283784866, |
| "rewards-max/forensics_iou_reward": 0.9350489974021912, |
| "rewards/forensics_iou_reward": 0.6115733683109283, |
| "step": 382 |
| }, |
| { |
| "completion_length": 19.78125, |
| "epoch": 1.602510460251046, |
| "grad_norm": 3.681842088699341, |
| "kl": 0.19384765625, |
| "learning_rate": 6.00418410041841e-07, |
| "loss": 0.0078, |
| "reward": 0.5020703375339508, |
| "reward_std": 0.08583015576004982, |
| "reward_std_all": 0.10546988621354103, |
| "rewards-max/forensics_iou_reward": 0.9494300186634064, |
| "rewards/forensics_iou_reward": 0.5020703375339508, |
| "step": 383 |
| }, |
| { |
| "completion_length": 20.28125, |
| "epoch": 1.606694560669456, |
| "grad_norm": 4.0985212326049805, |
| "kl": 0.150146484375, |
| "learning_rate": 5.993723849372385e-07, |
| "loss": 0.006, |
| "reward": 0.5043050050735474, |
| "reward_std": 0.04166047275066376, |
| "reward_std_all": 0.05515353940427303, |
| "rewards-max/forensics_iou_reward": 0.9633291959762573, |
| "rewards/forensics_iou_reward": 0.5043050050735474, |
| "step": 384 |
| }, |
| { |
| "completion_length": 19.125, |
| "epoch": 1.6108786610878663, |
| "grad_norm": 3.4688799381256104, |
| "kl": 0.21240234375, |
| "learning_rate": 5.983263598326359e-07, |
| "loss": 0.0085, |
| "reward": 0.41741763055324554, |
| "reward_std": 0.04102993756532669, |
| "reward_std_all": 0.05022419989109039, |
| "rewards-max/forensics_iou_reward": 0.8888924419879913, |
| "rewards/forensics_iou_reward": 0.41741763055324554, |
| "step": 385 |
| }, |
| { |
| "completion_length": 20.921875, |
| "epoch": 1.6150627615062763, |
| "grad_norm": 5.446503639221191, |
| "kl": 0.1953125, |
| "learning_rate": 5.972803347280334e-07, |
| "loss": 0.0078, |
| "reward": 0.47721922397613525, |
| "reward_std": 0.08717312291264534, |
| "reward_std_all": 0.08151419460773468, |
| "rewards-max/forensics_iou_reward": 0.949293702840805, |
| "rewards/forensics_iou_reward": 0.47721922397613525, |
| "step": 386 |
| }, |
| { |
| "completion_length": 19.40625, |
| "epoch": 1.6192468619246863, |
| "grad_norm": 6.539466857910156, |
| "kl": 0.21337890625, |
| "learning_rate": 5.962343096234309e-07, |
| "loss": 0.0086, |
| "reward": 0.42694835364818573, |
| "reward_std": 0.13745252043008804, |
| "reward_std_all": 0.12818101793527603, |
| "rewards-max/forensics_iou_reward": 0.9541799128055573, |
| "rewards/forensics_iou_reward": 0.42694835364818573, |
| "step": 387 |
| }, |
| { |
| "completion_length": 20.3125, |
| "epoch": 1.6234309623430963, |
| "grad_norm": 5.682896137237549, |
| "kl": 0.3974609375, |
| "learning_rate": 5.951882845188284e-07, |
| "loss": 0.0159, |
| "reward": 0.45483970642089844, |
| "reward_std": 0.14036745205521584, |
| "reward_std_all": 0.12291493266820908, |
| "rewards-max/forensics_iou_reward": 0.9508938491344452, |
| "rewards/forensics_iou_reward": 0.45483970642089844, |
| "step": 388 |
| }, |
| { |
| "completion_length": 18.90625, |
| "epoch": 1.6276150627615062, |
| "grad_norm": 4.382844924926758, |
| "kl": 0.25048828125, |
| "learning_rate": 5.941422594142259e-07, |
| "loss": 0.01, |
| "reward": 0.3685346096754074, |
| "reward_std": 0.1358637884259224, |
| "reward_std_all": 0.13688794523477554, |
| "rewards-max/forensics_iou_reward": 0.8994262516498566, |
| "rewards/forensics_iou_reward": 0.3685346096754074, |
| "step": 389 |
| }, |
| { |
| "completion_length": 20.953125, |
| "epoch": 1.6317991631799162, |
| "grad_norm": 3.7688651084899902, |
| "kl": 0.16650390625, |
| "learning_rate": 5.930962343096235e-07, |
| "loss": 0.0067, |
| "reward": 0.43243323266506195, |
| "reward_std": 0.09627699851989746, |
| "reward_std_all": 0.11035609245300293, |
| "rewards-max/forensics_iou_reward": 0.9105691015720367, |
| "rewards/forensics_iou_reward": 0.43243323266506195, |
| "step": 390 |
| }, |
| { |
| "completion_length": 20.3125, |
| "epoch": 1.6359832635983262, |
| "grad_norm": 4.489749431610107, |
| "kl": 0.1376953125, |
| "learning_rate": 5.92050209205021e-07, |
| "loss": 0.0055, |
| "reward": 0.5156011581420898, |
| "reward_std": 0.05428332835435867, |
| "reward_std_all": 0.07890135422348976, |
| "rewards-max/forensics_iou_reward": 0.993689775466919, |
| "rewards/forensics_iou_reward": 0.5156011581420898, |
| "step": 391 |
| }, |
| { |
| "completion_length": 20.046875, |
| "epoch": 1.6401673640167362, |
| "grad_norm": 4.517269134521484, |
| "kl": 0.1611328125, |
| "learning_rate": 5.910041841004184e-07, |
| "loss": 0.0065, |
| "reward": 0.4630165696144104, |
| "reward_std": 0.14552951604127884, |
| "reward_std_all": 0.1346471719443798, |
| "rewards-max/forensics_iou_reward": 0.8967857360839844, |
| "rewards/forensics_iou_reward": 0.4630165696144104, |
| "step": 392 |
| }, |
| { |
| "completion_length": 21.0, |
| "epoch": 1.6443514644351465, |
| "grad_norm": 5.3088059425354, |
| "kl": 0.15576171875, |
| "learning_rate": 5.899581589958159e-07, |
| "loss": 0.0062, |
| "reward": 0.48775817453861237, |
| "reward_std": 0.09653740748763084, |
| "reward_std_all": 0.09622464701533318, |
| "rewards-max/forensics_iou_reward": 0.9457276463508606, |
| "rewards/forensics_iou_reward": 0.48775817453861237, |
| "step": 393 |
| }, |
| { |
| "completion_length": 19.734375, |
| "epoch": 1.6485355648535565, |
| "grad_norm": 6.121370792388916, |
| "kl": 0.1796875, |
| "learning_rate": 5.889121338912134e-07, |
| "loss": 0.0072, |
| "reward": 0.5085625350475311, |
| "reward_std": 0.0767921730875969, |
| "reward_std_all": 0.0887894555926323, |
| "rewards-max/forensics_iou_reward": 0.9283216595649719, |
| "rewards/forensics_iou_reward": 0.5085625350475311, |
| "step": 394 |
| }, |
| { |
| "completion_length": 19.578125, |
| "epoch": 1.6527196652719667, |
| "grad_norm": 3.6060633659362793, |
| "kl": 0.251953125, |
| "learning_rate": 5.878661087866108e-07, |
| "loss": 0.0101, |
| "reward": 0.6185061037540436, |
| "reward_std": 0.09807170182466507, |
| "reward_std_all": 0.10343900695443153, |
| "rewards-max/forensics_iou_reward": 0.9838841557502747, |
| "rewards/forensics_iou_reward": 0.6185061037540436, |
| "step": 395 |
| }, |
| { |
| "completion_length": 20.15625, |
| "epoch": 1.6569037656903767, |
| "grad_norm": 5.463798999786377, |
| "kl": 0.18701171875, |
| "learning_rate": 5.868200836820083e-07, |
| "loss": 0.0075, |
| "reward": 0.46769632399082184, |
| "reward_std": 0.0917581059038639, |
| "reward_std_all": 0.10211072117090225, |
| "rewards-max/forensics_iou_reward": 0.9592859745025635, |
| "rewards/forensics_iou_reward": 0.46769632399082184, |
| "step": 396 |
| }, |
| { |
| "completion_length": 19.765625, |
| "epoch": 1.6610878661087867, |
| "grad_norm": 3.5978922843933105, |
| "kl": 0.16943359375, |
| "learning_rate": 5.857740585774058e-07, |
| "loss": 0.0068, |
| "reward": 0.5596846342086792, |
| "reward_std": 0.07199717313051224, |
| "reward_std_all": 0.08901523798704147, |
| "rewards-max/forensics_iou_reward": 0.9564960896968842, |
| "rewards/forensics_iou_reward": 0.5596846342086792, |
| "step": 397 |
| }, |
| { |
| "completion_length": 19.90625, |
| "epoch": 1.6652719665271967, |
| "grad_norm": 5.159090518951416, |
| "kl": 0.1640625, |
| "learning_rate": 5.847280334728033e-07, |
| "loss": 0.0066, |
| "reward": 0.5265991389751434, |
| "reward_std": 0.10854258760809898, |
| "reward_std_all": 0.12420428544282913, |
| "rewards-max/forensics_iou_reward": 0.9114103019237518, |
| "rewards/forensics_iou_reward": 0.5265991389751434, |
| "step": 398 |
| }, |
| { |
| "completion_length": 19.75, |
| "epoch": 1.6694560669456067, |
| "grad_norm": 6.371973991394043, |
| "kl": 0.1591796875, |
| "learning_rate": 5.836820083682009e-07, |
| "loss": 0.0064, |
| "reward": 0.4477727562189102, |
| "reward_std": 0.12045930325984955, |
| "reward_std_all": 0.11265352368354797, |
| "rewards-max/forensics_iou_reward": 0.8684536516666412, |
| "rewards/forensics_iou_reward": 0.4477727562189102, |
| "step": 399 |
| }, |
| { |
| "completion_length": 19.59375, |
| "epoch": 1.6736401673640167, |
| "grad_norm": 4.398616790771484, |
| "kl": 0.2216796875, |
| "learning_rate": 5.826359832635983e-07, |
| "loss": 0.0089, |
| "reward": 0.5547403693199158, |
| "reward_std": 0.10005110874772072, |
| "reward_std_all": 0.12511148676276207, |
| "rewards-max/forensics_iou_reward": 0.904662698507309, |
| "rewards/forensics_iou_reward": 0.5547403693199158, |
| "step": 400 |
| }, |
| { |
| "completion_length": 19.25, |
| "epoch": 1.6778242677824267, |
| "grad_norm": 5.929278373718262, |
| "kl": 0.20849609375, |
| "learning_rate": 5.815899581589958e-07, |
| "loss": 0.0083, |
| "reward": 0.48735523223876953, |
| "reward_std": 0.09474822878837585, |
| "reward_std_all": 0.10608566924929619, |
| "rewards-max/forensics_iou_reward": 0.9133330881595612, |
| "rewards/forensics_iou_reward": 0.48735523223876953, |
| "step": 401 |
| }, |
| { |
| "completion_length": 20.171875, |
| "epoch": 1.6820083682008367, |
| "grad_norm": 6.585119724273682, |
| "kl": 0.18701171875, |
| "learning_rate": 5.805439330543933e-07, |
| "loss": 0.0075, |
| "reward": 0.48149703443050385, |
| "reward_std": 0.11010453850030899, |
| "reward_std_all": 0.1031949520111084, |
| "rewards-max/forensics_iou_reward": 0.942840576171875, |
| "rewards/forensics_iou_reward": 0.48149703443050385, |
| "step": 402 |
| }, |
| { |
| "completion_length": 19.671875, |
| "epoch": 1.6861924686192469, |
| "grad_norm": 6.064491271972656, |
| "kl": 0.15380859375, |
| "learning_rate": 5.794979079497908e-07, |
| "loss": 0.0062, |
| "reward": 0.4616726487874985, |
| "reward_std": 0.07192808203399181, |
| "reward_std_all": 0.08324017748236656, |
| "rewards-max/forensics_iou_reward": 0.8341276049613953, |
| "rewards/forensics_iou_reward": 0.4616726487874985, |
| "step": 403 |
| }, |
| { |
| "completion_length": 19.734375, |
| "epoch": 1.6903765690376569, |
| "grad_norm": 4.485347270965576, |
| "kl": 0.156005859375, |
| "learning_rate": 5.784518828451883e-07, |
| "loss": 0.0062, |
| "reward": 0.48493340611457825, |
| "reward_std": 0.07724104262888432, |
| "reward_std_all": 0.07885554805397987, |
| "rewards-max/forensics_iou_reward": 0.9621839821338654, |
| "rewards/forensics_iou_reward": 0.48493340611457825, |
| "step": 404 |
| }, |
| { |
| "completion_length": 20.40625, |
| "epoch": 1.694560669456067, |
| "grad_norm": 3.4682765007019043, |
| "kl": 0.19482421875, |
| "learning_rate": 5.774058577405857e-07, |
| "loss": 0.0078, |
| "reward": 0.4137971103191376, |
| "reward_std": 0.12370221316814423, |
| "reward_std_all": 0.11199108883738518, |
| "rewards-max/forensics_iou_reward": 0.8555458784103394, |
| "rewards/forensics_iou_reward": 0.4137971103191376, |
| "step": 405 |
| }, |
| { |
| "completion_length": 19.5625, |
| "epoch": 1.698744769874477, |
| "grad_norm": 4.285614490509033, |
| "kl": 0.17919921875, |
| "learning_rate": 5.763598326359832e-07, |
| "loss": 0.0072, |
| "reward": 0.5543860197067261, |
| "reward_std": 0.10379164665937424, |
| "reward_std_all": 0.1102873831987381, |
| "rewards-max/forensics_iou_reward": 0.9652969241142273, |
| "rewards/forensics_iou_reward": 0.5543860197067261, |
| "step": 406 |
| }, |
| { |
| "completion_length": 19.40625, |
| "epoch": 1.702928870292887, |
| "grad_norm": 4.1446661949157715, |
| "kl": 0.170166015625, |
| "learning_rate": 5.753138075313807e-07, |
| "loss": 0.0068, |
| "reward": 0.505991980433464, |
| "reward_std": 0.10179097950458527, |
| "reward_std_all": 0.11613754555583, |
| "rewards-max/forensics_iou_reward": 0.9056744575500488, |
| "rewards/forensics_iou_reward": 0.505991980433464, |
| "step": 407 |
| }, |
| { |
| "completion_length": 21.125, |
| "epoch": 1.707112970711297, |
| "grad_norm": 4.684301853179932, |
| "kl": 0.16796875, |
| "learning_rate": 5.742677824267782e-07, |
| "loss": 0.0067, |
| "reward": 0.5528751313686371, |
| "reward_std": 0.09554673358798027, |
| "reward_std_all": 0.10504313558340073, |
| "rewards-max/forensics_iou_reward": 0.9530547559261322, |
| "rewards/forensics_iou_reward": 0.5528751313686371, |
| "step": 408 |
| }, |
| { |
| "completion_length": 19.6875, |
| "epoch": 1.711297071129707, |
| "grad_norm": 5.069687366485596, |
| "kl": 0.138916015625, |
| "learning_rate": 5.732217573221757e-07, |
| "loss": 0.0056, |
| "reward": 0.5410149991512299, |
| "reward_std": 0.1299414113163948, |
| "reward_std_all": 0.12162881717085838, |
| "rewards-max/forensics_iou_reward": 0.9694206416606903, |
| "rewards/forensics_iou_reward": 0.5410149991512299, |
| "step": 409 |
| }, |
| { |
| "completion_length": 20.359375, |
| "epoch": 1.715481171548117, |
| "grad_norm": 4.953649997711182, |
| "kl": 0.166015625, |
| "learning_rate": 5.721757322175732e-07, |
| "loss": 0.0067, |
| "reward": 0.5958585739135742, |
| "reward_std": 0.05722961574792862, |
| "reward_std_all": 0.08948003128170967, |
| "rewards-max/forensics_iou_reward": 0.9670274555683136, |
| "rewards/forensics_iou_reward": 0.5958585739135742, |
| "step": 410 |
| }, |
| { |
| "completion_length": 20.1875, |
| "epoch": 1.719665271966527, |
| "grad_norm": 3.6988766193389893, |
| "kl": 0.19970703125, |
| "learning_rate": 5.711297071129707e-07, |
| "loss": 0.008, |
| "reward": 0.6309919953346252, |
| "reward_std": 0.07825234159827232, |
| "reward_std_all": 0.0816936083137989, |
| "rewards-max/forensics_iou_reward": 0.9737966656684875, |
| "rewards/forensics_iou_reward": 0.6309919953346252, |
| "step": 411 |
| }, |
| { |
| "completion_length": 21.40625, |
| "epoch": 1.723849372384937, |
| "grad_norm": 3.8305625915527344, |
| "kl": 0.1064453125, |
| "learning_rate": 5.700836820083682e-07, |
| "loss": 0.0043, |
| "reward": 0.4411422312259674, |
| "reward_std": 0.07418307662010193, |
| "reward_std_all": 0.07034280896186829, |
| "rewards-max/forensics_iou_reward": 0.9151532351970673, |
| "rewards/forensics_iou_reward": 0.4411422312259674, |
| "step": 412 |
| }, |
| { |
| "completion_length": 19.90625, |
| "epoch": 1.7280334728033473, |
| "grad_norm": 4.6575422286987305, |
| "kl": 0.20361328125, |
| "learning_rate": 5.690376569037657e-07, |
| "loss": 0.0081, |
| "reward": 0.492710143327713, |
| "reward_std": 0.11105312407016754, |
| "reward_std_all": 0.12024195492267609, |
| "rewards-max/forensics_iou_reward": 0.8869089186191559, |
| "rewards/forensics_iou_reward": 0.492710143327713, |
| "step": 413 |
| }, |
| { |
| "completion_length": 19.6875, |
| "epoch": 1.7322175732217573, |
| "grad_norm": 10.472980499267578, |
| "kl": 0.125, |
| "learning_rate": 5.679916317991632e-07, |
| "loss": 0.005, |
| "reward": 0.5003973096609116, |
| "reward_std": 0.11823515221476555, |
| "reward_std_all": 0.10983562096953392, |
| "rewards-max/forensics_iou_reward": 0.8878052234649658, |
| "rewards/forensics_iou_reward": 0.5003973096609116, |
| "step": 414 |
| }, |
| { |
| "completion_length": 19.546875, |
| "epoch": 1.7364016736401675, |
| "grad_norm": 5.548470973968506, |
| "kl": 0.17041015625, |
| "learning_rate": 5.669456066945606e-07, |
| "loss": 0.0068, |
| "reward": 0.430903896689415, |
| "reward_std": 0.07239141687750816, |
| "reward_std_all": 0.09648589417338371, |
| "rewards-max/forensics_iou_reward": 0.9365093111991882, |
| "rewards/forensics_iou_reward": 0.430903896689415, |
| "step": 415 |
| }, |
| { |
| "completion_length": 19.09375, |
| "epoch": 1.7405857740585775, |
| "grad_norm": 4.072353839874268, |
| "kl": 0.21142578125, |
| "learning_rate": 5.65899581589958e-07, |
| "loss": 0.0085, |
| "reward": 0.42114242911338806, |
| "reward_std": 0.0675617940723896, |
| "reward_std_all": 0.06534971296787262, |
| "rewards-max/forensics_iou_reward": 0.8964828550815582, |
| "rewards/forensics_iou_reward": 0.42114242911338806, |
| "step": 416 |
| }, |
| { |
| "completion_length": 19.171875, |
| "epoch": 1.7447698744769875, |
| "grad_norm": 4.545744895935059, |
| "kl": 0.150390625, |
| "learning_rate": 5.648535564853556e-07, |
| "loss": 0.006, |
| "reward": 0.5025504529476166, |
| "reward_std": 0.10746229812502861, |
| "reward_std_all": 0.10996945947408676, |
| "rewards-max/forensics_iou_reward": 0.9491910338401794, |
| "rewards/forensics_iou_reward": 0.5025504529476166, |
| "step": 417 |
| }, |
| { |
| "completion_length": 19.65625, |
| "epoch": 1.7489539748953975, |
| "grad_norm": 5.19827938079834, |
| "kl": 0.16455078125, |
| "learning_rate": 5.638075313807531e-07, |
| "loss": 0.0066, |
| "reward": 0.45657214522361755, |
| "reward_std": 0.09719664976000786, |
| "reward_std_all": 0.10212066024541855, |
| "rewards-max/forensics_iou_reward": 0.8639761805534363, |
| "rewards/forensics_iou_reward": 0.45657214522361755, |
| "step": 418 |
| }, |
| { |
| "completion_length": 19.65625, |
| "epoch": 1.7531380753138075, |
| "grad_norm": 4.8027753829956055, |
| "kl": 0.1728515625, |
| "learning_rate": 5.627615062761506e-07, |
| "loss": 0.0069, |
| "reward": 0.5759839415550232, |
| "reward_std": 0.11569670960307121, |
| "reward_std_all": 0.11385639011859894, |
| "rewards-max/forensics_iou_reward": 0.9260198473930359, |
| "rewards/forensics_iou_reward": 0.5759839415550232, |
| "step": 419 |
| }, |
| { |
| "completion_length": 19.625, |
| "epoch": 1.7573221757322175, |
| "grad_norm": 6.631958484649658, |
| "kl": 0.12939453125, |
| "learning_rate": 5.617154811715481e-07, |
| "loss": 0.0052, |
| "reward": 0.527145728468895, |
| "reward_std": 0.08988422900438309, |
| "reward_std_all": 0.09420210495591164, |
| "rewards-max/forensics_iou_reward": 0.9709378778934479, |
| "rewards/forensics_iou_reward": 0.527145728468895, |
| "step": 420 |
| }, |
| { |
| "completion_length": 19.671875, |
| "epoch": 1.7615062761506275, |
| "grad_norm": 3.498980760574341, |
| "kl": 0.2080078125, |
| "learning_rate": 5.606694560669456e-07, |
| "loss": 0.0083, |
| "reward": 0.5127128958702087, |
| "reward_std": 0.10432028025388718, |
| "reward_std_all": 0.1218305379152298, |
| "rewards-max/forensics_iou_reward": 0.9128898978233337, |
| "rewards/forensics_iou_reward": 0.5127128958702087, |
| "step": 421 |
| }, |
| { |
| "completion_length": 19.4375, |
| "epoch": 1.7656903765690377, |
| "grad_norm": 5.117757320404053, |
| "kl": 0.129638671875, |
| "learning_rate": 5.596234309623431e-07, |
| "loss": 0.0052, |
| "reward": 0.4796284884214401, |
| "reward_std": 0.10320670157670975, |
| "reward_std_all": 0.10385248064994812, |
| "rewards-max/forensics_iou_reward": 0.9486288726329803, |
| "rewards/forensics_iou_reward": 0.4796284884214401, |
| "step": 422 |
| }, |
| { |
| "completion_length": 19.859375, |
| "epoch": 1.7698744769874477, |
| "grad_norm": 5.013268947601318, |
| "kl": 0.11962890625, |
| "learning_rate": 5.585774058577406e-07, |
| "loss": 0.0048, |
| "reward": 0.48053818941116333, |
| "reward_std": 0.06723284907639027, |
| "reward_std_all": 0.06909292191267014, |
| "rewards-max/forensics_iou_reward": 0.9316958785057068, |
| "rewards/forensics_iou_reward": 0.48053818941116333, |
| "step": 423 |
| }, |
| { |
| "completion_length": 20.140625, |
| "epoch": 1.7740585774058577, |
| "grad_norm": 2.962606191635132, |
| "kl": 0.139404296875, |
| "learning_rate": 5.575313807531381e-07, |
| "loss": 0.0056, |
| "reward": 0.4539559930562973, |
| "reward_std": 0.07208932563662529, |
| "reward_std_all": 0.08254885673522949, |
| "rewards-max/forensics_iou_reward": 0.8585858643054962, |
| "rewards/forensics_iou_reward": 0.4539559930562973, |
| "step": 424 |
| }, |
| { |
| "completion_length": 20.703125, |
| "epoch": 1.778242677824268, |
| "grad_norm": 5.822474479675293, |
| "kl": 0.16064453125, |
| "learning_rate": 5.564853556485354e-07, |
| "loss": 0.0064, |
| "reward": 0.477075919508934, |
| "reward_std": 0.08868123590946198, |
| "reward_std_all": 0.10670329257845879, |
| "rewards-max/forensics_iou_reward": 0.8314699828624725, |
| "rewards/forensics_iou_reward": 0.477075919508934, |
| "step": 425 |
| }, |
| { |
| "completion_length": 19.359375, |
| "epoch": 1.782426778242678, |
| "grad_norm": 6.627179145812988, |
| "kl": 0.14599609375, |
| "learning_rate": 5.55439330543933e-07, |
| "loss": 0.0058, |
| "reward": 0.5305325984954834, |
| "reward_std": 0.109255850315094, |
| "reward_std_all": 0.10698350518941879, |
| "rewards-max/forensics_iou_reward": 0.9648894667625427, |
| "rewards/forensics_iou_reward": 0.5305325984954834, |
| "step": 426 |
| }, |
| { |
| "completion_length": 20.15625, |
| "epoch": 1.786610878661088, |
| "grad_norm": 8.23585033416748, |
| "kl": 0.114013671875, |
| "learning_rate": 5.543933054393305e-07, |
| "loss": 0.0046, |
| "reward": 0.6042118072509766, |
| "reward_std": 0.10460289567708969, |
| "reward_std_all": 0.10301118344068527, |
| "rewards-max/forensics_iou_reward": 0.9431432783603668, |
| "rewards/forensics_iou_reward": 0.6042118072509766, |
| "step": 427 |
| }, |
| { |
| "completion_length": 19.671875, |
| "epoch": 1.790794979079498, |
| "grad_norm": 5.7764692306518555, |
| "kl": 0.18701171875, |
| "learning_rate": 5.53347280334728e-07, |
| "loss": 0.0075, |
| "reward": 0.5155880749225616, |
| "reward_std": 0.09435568749904633, |
| "reward_std_all": 0.08891613781452179, |
| "rewards-max/forensics_iou_reward": 0.8749986290931702, |
| "rewards/forensics_iou_reward": 0.5155880749225616, |
| "step": 428 |
| }, |
| { |
| "completion_length": 20.5, |
| "epoch": 1.794979079497908, |
| "grad_norm": 4.1525492668151855, |
| "kl": 0.11181640625, |
| "learning_rate": 5.523012552301255e-07, |
| "loss": 0.0045, |
| "reward": 0.5018722265958786, |
| "reward_std": 0.04866182059049606, |
| "reward_std_all": 0.053160129114985466, |
| "rewards-max/forensics_iou_reward": 0.9844605028629303, |
| "rewards/forensics_iou_reward": 0.5018722265958786, |
| "step": 429 |
| }, |
| { |
| "completion_length": 20.65625, |
| "epoch": 1.799163179916318, |
| "grad_norm": 4.67884635925293, |
| "kl": 0.14013671875, |
| "learning_rate": 5.51255230125523e-07, |
| "loss": 0.0056, |
| "reward": 0.5010780543088913, |
| "reward_std": 0.05311075039207935, |
| "reward_std_all": 0.05387173965573311, |
| "rewards-max/forensics_iou_reward": 0.9260249137878418, |
| "rewards/forensics_iou_reward": 0.5010780543088913, |
| "step": 430 |
| }, |
| { |
| "completion_length": 20.578125, |
| "epoch": 1.803347280334728, |
| "grad_norm": 3.7546582221984863, |
| "kl": 0.138671875, |
| "learning_rate": 5.502092050209205e-07, |
| "loss": 0.0055, |
| "reward": 0.5520842671394348, |
| "reward_std": 0.10345697030425072, |
| "reward_std_all": 0.09724579751491547, |
| "rewards-max/forensics_iou_reward": 0.9793152809143066, |
| "rewards/forensics_iou_reward": 0.5520842671394348, |
| "step": 431 |
| }, |
| { |
| "completion_length": 20.15625, |
| "epoch": 1.8075313807531381, |
| "grad_norm": 6.087531566619873, |
| "kl": 0.14892578125, |
| "learning_rate": 5.49163179916318e-07, |
| "loss": 0.0059, |
| "reward": 0.5814825892448425, |
| "reward_std": 0.08602441474795341, |
| "reward_std_all": 0.1037452295422554, |
| "rewards-max/forensics_iou_reward": 0.9302466511726379, |
| "rewards/forensics_iou_reward": 0.5814825892448425, |
| "step": 432 |
| }, |
| { |
| "completion_length": 19.1875, |
| "epoch": 1.8117154811715481, |
| "grad_norm": 3.6845169067382812, |
| "kl": 0.18115234375, |
| "learning_rate": 5.481171548117156e-07, |
| "loss": 0.0072, |
| "reward": 0.6072952449321747, |
| "reward_std": 0.08595096692442894, |
| "reward_std_all": 0.08511481434106827, |
| "rewards-max/forensics_iou_reward": 0.9923365414142609, |
| "rewards/forensics_iou_reward": 0.6072952449321747, |
| "step": 433 |
| }, |
| { |
| "completion_length": 21.03125, |
| "epoch": 1.8158995815899581, |
| "grad_norm": 4.97121524810791, |
| "kl": 0.123291015625, |
| "learning_rate": 5.47071129707113e-07, |
| "loss": 0.0049, |
| "reward": 0.3277793675661087, |
| "reward_std": 0.07830165326595306, |
| "reward_std_all": 0.08684118837118149, |
| "rewards-max/forensics_iou_reward": 0.7919858992099762, |
| "rewards/forensics_iou_reward": 0.3277793675661087, |
| "step": 434 |
| }, |
| { |
| "completion_length": 19.453125, |
| "epoch": 1.8200836820083683, |
| "grad_norm": 3.893399238586426, |
| "kl": 0.19921875, |
| "learning_rate": 5.460251046025104e-07, |
| "loss": 0.008, |
| "reward": 0.618508368730545, |
| "reward_std": 0.0949369091540575, |
| "reward_std_all": 0.08939652144908905, |
| "rewards-max/forensics_iou_reward": 0.957099199295044, |
| "rewards/forensics_iou_reward": 0.618508368730545, |
| "step": 435 |
| }, |
| { |
| "completion_length": 19.4375, |
| "epoch": 1.8242677824267783, |
| "grad_norm": 4.579590320587158, |
| "kl": 0.21044921875, |
| "learning_rate": 5.449790794979079e-07, |
| "loss": 0.0084, |
| "reward": 0.4763955771923065, |
| "reward_std": 0.12209358811378479, |
| "reward_std_all": 0.1086692214012146, |
| "rewards-max/forensics_iou_reward": 0.9084072709083557, |
| "rewards/forensics_iou_reward": 0.4763955771923065, |
| "step": 436 |
| }, |
| { |
| "completion_length": 19.484375, |
| "epoch": 1.8284518828451883, |
| "grad_norm": 6.542791366577148, |
| "kl": 0.22265625, |
| "learning_rate": 5.439330543933054e-07, |
| "loss": 0.0089, |
| "reward": 0.5601446032524109, |
| "reward_std": 0.06465688534080982, |
| "reward_std_all": 0.07307375222444534, |
| "rewards-max/forensics_iou_reward": 0.924301952123642, |
| "rewards/forensics_iou_reward": 0.5601446032524109, |
| "step": 437 |
| }, |
| { |
| "completion_length": 19.984375, |
| "epoch": 1.8326359832635983, |
| "grad_norm": 7.4926676750183105, |
| "kl": 0.1708984375, |
| "learning_rate": 5.428870292887029e-07, |
| "loss": 0.0068, |
| "reward": 0.4754808396100998, |
| "reward_std": 0.09520110487937927, |
| "reward_std_all": 0.09760208055377007, |
| "rewards-max/forensics_iou_reward": 0.8513513505458832, |
| "rewards/forensics_iou_reward": 0.4754808396100998, |
| "step": 438 |
| }, |
| { |
| "completion_length": 20.046875, |
| "epoch": 1.8368200836820083, |
| "grad_norm": 4.583054065704346, |
| "kl": 0.1943359375, |
| "learning_rate": 5.418410041841004e-07, |
| "loss": 0.0078, |
| "reward": 0.5839079022407532, |
| "reward_std": 0.09018901735544205, |
| "reward_std_all": 0.09376774728298187, |
| "rewards-max/forensics_iou_reward": 0.9457979500293732, |
| "rewards/forensics_iou_reward": 0.5839079022407532, |
| "step": 439 |
| }, |
| { |
| "completion_length": 20.453125, |
| "epoch": 1.8410041841004183, |
| "grad_norm": 5.596710205078125, |
| "kl": 0.17578125, |
| "learning_rate": 5.407949790794979e-07, |
| "loss": 0.007, |
| "reward": 0.5613153576850891, |
| "reward_std": 0.10394404828548431, |
| "reward_std_all": 0.10367034003138542, |
| "rewards-max/forensics_iou_reward": 0.9044498205184937, |
| "rewards/forensics_iou_reward": 0.5613153576850891, |
| "step": 440 |
| }, |
| { |
| "completion_length": 20.0625, |
| "epoch": 1.8451882845188283, |
| "grad_norm": 3.9240565299987793, |
| "kl": 0.1943359375, |
| "learning_rate": 5.397489539748954e-07, |
| "loss": 0.0078, |
| "reward": 0.4435904324054718, |
| "reward_std": 0.06354477629065514, |
| "reward_std_all": 0.09447980672121048, |
| "rewards-max/forensics_iou_reward": 0.9100687205791473, |
| "rewards/forensics_iou_reward": 0.4435904324054718, |
| "step": 441 |
| }, |
| { |
| "completion_length": 20.859375, |
| "epoch": 1.8493723849372385, |
| "grad_norm": 5.038595676422119, |
| "kl": 0.2578125, |
| "learning_rate": 5.387029288702928e-07, |
| "loss": 0.0103, |
| "reward": 0.4237994998693466, |
| "reward_std": 0.09105991199612617, |
| "reward_std_all": 0.09816602990031242, |
| "rewards-max/forensics_iou_reward": 0.9038813710212708, |
| "rewards/forensics_iou_reward": 0.4237994998693466, |
| "step": 442 |
| }, |
| { |
| "completion_length": 19.25, |
| "epoch": 1.8535564853556485, |
| "grad_norm": 6.525815963745117, |
| "kl": 0.23291015625, |
| "learning_rate": 5.376569037656904e-07, |
| "loss": 0.0093, |
| "reward": 0.5145128518342972, |
| "reward_std": 0.08492538332939148, |
| "reward_std_all": 0.08887452632188797, |
| "rewards-max/forensics_iou_reward": 0.8702918291091919, |
| "rewards/forensics_iou_reward": 0.5145128518342972, |
| "step": 443 |
| }, |
| { |
| "completion_length": 19.34375, |
| "epoch": 1.8577405857740585, |
| "grad_norm": 3.864978313446045, |
| "kl": 0.173828125, |
| "learning_rate": 5.366108786610879e-07, |
| "loss": 0.0069, |
| "reward": 0.4243559092283249, |
| "reward_std": 0.08535045385360718, |
| "reward_std_all": 0.08618669584393501, |
| "rewards-max/forensics_iou_reward": 0.869237869977951, |
| "rewards/forensics_iou_reward": 0.4243559092283249, |
| "step": 444 |
| }, |
| { |
| "completion_length": 19.65625, |
| "epoch": 1.8619246861924688, |
| "grad_norm": 4.5718092918396, |
| "kl": 0.1806640625, |
| "learning_rate": 5.355648535564853e-07, |
| "loss": 0.0072, |
| "reward": 0.46588413417339325, |
| "reward_std": 0.0661572739481926, |
| "reward_std_all": 0.06481154449284077, |
| "rewards-max/forensics_iou_reward": 0.832165390253067, |
| "rewards/forensics_iou_reward": 0.46588413417339325, |
| "step": 445 |
| }, |
| { |
| "completion_length": 20.3125, |
| "epoch": 1.8661087866108788, |
| "grad_norm": 4.4092116355896, |
| "kl": 0.3115234375, |
| "learning_rate": 5.345188284518828e-07, |
| "loss": 0.0125, |
| "reward": 0.4958435148000717, |
| "reward_std": 0.09374640882015228, |
| "reward_std_all": 0.09766118600964546, |
| "rewards-max/forensics_iou_reward": 0.8747387826442719, |
| "rewards/forensics_iou_reward": 0.4958435148000717, |
| "step": 446 |
| }, |
| { |
| "completion_length": 18.90625, |
| "epoch": 1.8702928870292888, |
| "grad_norm": 5.315426349639893, |
| "kl": 0.122314453125, |
| "learning_rate": 5.334728033472803e-07, |
| "loss": 0.0049, |
| "reward": 0.6324470937252045, |
| "reward_std": 0.07625401392579079, |
| "reward_std_all": 0.0786251351237297, |
| "rewards-max/forensics_iou_reward": 0.9781166911125183, |
| "rewards/forensics_iou_reward": 0.6324470937252045, |
| "step": 447 |
| }, |
| { |
| "completion_length": 19.453125, |
| "epoch": 1.8744769874476988, |
| "grad_norm": 6.8483452796936035, |
| "kl": 0.36181640625, |
| "learning_rate": 5.324267782426778e-07, |
| "loss": 0.0144, |
| "reward": 0.4500398486852646, |
| "reward_std": 0.05868213251233101, |
| "reward_std_all": 0.06488719396293163, |
| "rewards-max/forensics_iou_reward": 0.8825807571411133, |
| "rewards/forensics_iou_reward": 0.4500398486852646, |
| "step": 448 |
| }, |
| { |
| "completion_length": 20.171875, |
| "epoch": 1.8786610878661087, |
| "grad_norm": 6.127427577972412, |
| "kl": 0.2099609375, |
| "learning_rate": 5.313807531380753e-07, |
| "loss": 0.0084, |
| "reward": 0.5788978040218353, |
| "reward_std": 0.1052376851439476, |
| "reward_std_all": 0.10300666466355324, |
| "rewards-max/forensics_iou_reward": 0.9736189544200897, |
| "rewards/forensics_iou_reward": 0.5788978040218353, |
| "step": 449 |
| }, |
| { |
| "completion_length": 19.296875, |
| "epoch": 1.8828451882845187, |
| "grad_norm": 79.22445678710938, |
| "kl": 0.158447265625, |
| "learning_rate": 5.303347280334727e-07, |
| "loss": 0.0063, |
| "reward": 0.675910234451294, |
| "reward_std": 0.10510312393307686, |
| "reward_std_all": 0.11007806286215782, |
| "rewards-max/forensics_iou_reward": 0.9508083164691925, |
| "rewards/forensics_iou_reward": 0.675910234451294, |
| "step": 450 |
| }, |
| { |
| "completion_length": 19.65625, |
| "epoch": 1.8870292887029287, |
| "grad_norm": 7.9123969078063965, |
| "kl": 0.4599609375, |
| "learning_rate": 5.292887029288703e-07, |
| "loss": 0.0184, |
| "reward": 0.5231629759073257, |
| "reward_std": 0.07648961804807186, |
| "reward_std_all": 0.08600300177931786, |
| "rewards-max/forensics_iou_reward": 0.8868350386619568, |
| "rewards/forensics_iou_reward": 0.5231629759073257, |
| "step": 451 |
| }, |
| { |
| "completion_length": 19.171875, |
| "epoch": 1.891213389121339, |
| "grad_norm": 4.483290672302246, |
| "kl": 0.17041015625, |
| "learning_rate": 5.282426778242678e-07, |
| "loss": 0.0068, |
| "reward": 0.6574525833129883, |
| "reward_std": 0.052961643785238266, |
| "reward_std_all": 0.05506248585879803, |
| "rewards-max/forensics_iou_reward": 0.9625847339630127, |
| "rewards/forensics_iou_reward": 0.6574525833129883, |
| "step": 452 |
| }, |
| { |
| "completion_length": 19.65625, |
| "epoch": 1.895397489539749, |
| "grad_norm": 7.422362327575684, |
| "kl": 0.15576171875, |
| "learning_rate": 5.271966527196653e-07, |
| "loss": 0.0062, |
| "reward": 0.4870176315307617, |
| "reward_std": 0.08619559556245804, |
| "reward_std_all": 0.07951122522354126, |
| "rewards-max/forensics_iou_reward": 0.9311726987361908, |
| "rewards/forensics_iou_reward": 0.4870176315307617, |
| "step": 453 |
| }, |
| { |
| "completion_length": 19.765625, |
| "epoch": 1.899581589958159, |
| "grad_norm": 4.226583957672119, |
| "kl": 0.17822265625, |
| "learning_rate": 5.261506276150628e-07, |
| "loss": 0.0071, |
| "reward": 0.4994839280843735, |
| "reward_std": 0.07466132566332817, |
| "reward_std_all": 0.09247482568025589, |
| "rewards-max/forensics_iou_reward": 0.9196352958679199, |
| "rewards/forensics_iou_reward": 0.4994839280843735, |
| "step": 454 |
| }, |
| { |
| "completion_length": 20.515625, |
| "epoch": 1.9037656903765692, |
| "grad_norm": 3.5066587924957275, |
| "kl": 0.1669921875, |
| "learning_rate": 5.251046025104602e-07, |
| "loss": 0.0067, |
| "reward": 0.5382945090532303, |
| "reward_std": 0.05141758732497692, |
| "reward_std_all": 0.06645607948303223, |
| "rewards-max/forensics_iou_reward": 0.8567295074462891, |
| "rewards/forensics_iou_reward": 0.5382945090532303, |
| "step": 455 |
| }, |
| { |
| "completion_length": 19.71875, |
| "epoch": 1.9079497907949792, |
| "grad_norm": 4.823730945587158, |
| "kl": 0.2353515625, |
| "learning_rate": 5.240585774058577e-07, |
| "loss": 0.0094, |
| "reward": 0.5336980819702148, |
| "reward_std": 0.10316763445734978, |
| "reward_std_all": 0.10718689113855362, |
| "rewards-max/forensics_iou_reward": 0.9895323216915131, |
| "rewards/forensics_iou_reward": 0.5336980819702148, |
| "step": 456 |
| }, |
| { |
| "completion_length": 19.21875, |
| "epoch": 1.9121338912133892, |
| "grad_norm": 7.138619899749756, |
| "kl": 0.16162109375, |
| "learning_rate": 5.230125523012552e-07, |
| "loss": 0.0065, |
| "reward": 0.65382120013237, |
| "reward_std": 0.07626281306147575, |
| "reward_std_all": 0.07234302163124084, |
| "rewards-max/forensics_iou_reward": 0.9468156099319458, |
| "rewards/forensics_iou_reward": 0.65382120013237, |
| "step": 457 |
| }, |
| { |
| "completion_length": 19.828125, |
| "epoch": 1.9163179916317992, |
| "grad_norm": 4.627990245819092, |
| "kl": 0.166015625, |
| "learning_rate": 5.219665271966526e-07, |
| "loss": 0.0066, |
| "reward": 0.43155935406684875, |
| "reward_std": 0.03522980306297541, |
| "reward_std_all": 0.036843057721853256, |
| "rewards-max/forensics_iou_reward": 0.9244701862335205, |
| "rewards/forensics_iou_reward": 0.43155935406684875, |
| "step": 458 |
| }, |
| { |
| "completion_length": 19.75, |
| "epoch": 1.9205020920502092, |
| "grad_norm": 15.590165138244629, |
| "kl": 0.27294921875, |
| "learning_rate": 5.209205020920501e-07, |
| "loss": 0.0109, |
| "reward": 0.4926871061325073, |
| "reward_std": 0.06920561194419861, |
| "reward_std_all": 0.07951872423291206, |
| "rewards-max/forensics_iou_reward": 0.9749822616577148, |
| "rewards/forensics_iou_reward": 0.4926871061325073, |
| "step": 459 |
| }, |
| { |
| "completion_length": 19.796875, |
| "epoch": 1.9246861924686192, |
| "grad_norm": 5.917546272277832, |
| "kl": 0.3544921875, |
| "learning_rate": 5.198744769874477e-07, |
| "loss": 0.0142, |
| "reward": 0.6221752166748047, |
| "reward_std": 0.1018894650042057, |
| "reward_std_all": 0.1249014139175415, |
| "rewards-max/forensics_iou_reward": 0.9135355353355408, |
| "rewards/forensics_iou_reward": 0.6221752166748047, |
| "step": 460 |
| }, |
| { |
| "completion_length": 19.875, |
| "epoch": 1.9288702928870292, |
| "grad_norm": 5.902897834777832, |
| "kl": 0.2099609375, |
| "learning_rate": 5.188284518828452e-07, |
| "loss": 0.0084, |
| "reward": 0.5069512575864792, |
| "reward_std": 0.10795615240931511, |
| "reward_std_all": 0.10354864597320557, |
| "rewards-max/forensics_iou_reward": 0.9188887476921082, |
| "rewards/forensics_iou_reward": 0.5069512575864792, |
| "step": 461 |
| }, |
| { |
| "completion_length": 21.703125, |
| "epoch": 1.9330543933054394, |
| "grad_norm": 6.7324910163879395, |
| "kl": 0.185791015625, |
| "learning_rate": 5.177824267782427e-07, |
| "loss": 0.0074, |
| "reward": 0.5338177382946014, |
| "reward_std": 0.08925708383321762, |
| "reward_std_all": 0.08819524198770523, |
| "rewards-max/forensics_iou_reward": 0.9573945701122284, |
| "rewards/forensics_iou_reward": 0.5338177382946014, |
| "step": 462 |
| }, |
| { |
| "completion_length": 18.953125, |
| "epoch": 1.9372384937238494, |
| "grad_norm": 3.9738965034484863, |
| "kl": 0.21826171875, |
| "learning_rate": 5.167364016736402e-07, |
| "loss": 0.0087, |
| "reward": 0.5663582682609558, |
| "reward_std": 0.05300603434443474, |
| "reward_std_all": 0.07142189890146255, |
| "rewards-max/forensics_iou_reward": 0.9327169358730316, |
| "rewards/forensics_iou_reward": 0.5663582682609558, |
| "step": 463 |
| }, |
| { |
| "completion_length": 20.40625, |
| "epoch": 1.9414225941422594, |
| "grad_norm": 5.527751445770264, |
| "kl": 0.15283203125, |
| "learning_rate": 5.156903765690377e-07, |
| "loss": 0.0061, |
| "reward": 0.5156200230121613, |
| "reward_std": 0.09129686281085014, |
| "reward_std_all": 0.0816834531724453, |
| "rewards-max/forensics_iou_reward": 0.8899450302124023, |
| "rewards/forensics_iou_reward": 0.5156200230121613, |
| "step": 464 |
| }, |
| { |
| "completion_length": 20.625, |
| "epoch": 1.9456066945606696, |
| "grad_norm": 4.741024971008301, |
| "kl": 0.21728515625, |
| "learning_rate": 5.146443514644351e-07, |
| "loss": 0.0087, |
| "reward": 0.44812652468681335, |
| "reward_std": 0.07072041369974613, |
| "reward_std_all": 0.08950818702578545, |
| "rewards-max/forensics_iou_reward": 0.8726121187210083, |
| "rewards/forensics_iou_reward": 0.44812652468681335, |
| "step": 465 |
| }, |
| { |
| "completion_length": 22.328125, |
| "epoch": 1.9497907949790796, |
| "grad_norm": 21.789772033691406, |
| "kl": 0.20166015625, |
| "learning_rate": 5.135983263598326e-07, |
| "loss": 0.0081, |
| "reward": 0.3905325084924698, |
| "reward_std": 0.04735649190843105, |
| "reward_std_all": 0.081610556691885, |
| "rewards-max/forensics_iou_reward": 0.8429849147796631, |
| "rewards/forensics_iou_reward": 0.3905325084924698, |
| "step": 466 |
| }, |
| { |
| "completion_length": 19.65625, |
| "epoch": 1.9539748953974896, |
| "grad_norm": 7.522097110748291, |
| "kl": 0.22412109375, |
| "learning_rate": 5.1255230125523e-07, |
| "loss": 0.009, |
| "reward": 0.5770957618951797, |
| "reward_std": 0.0804952122271061, |
| "reward_std_all": 0.0916077271103859, |
| "rewards-max/forensics_iou_reward": 0.9220119118690491, |
| "rewards/forensics_iou_reward": 0.5770957618951797, |
| "step": 467 |
| }, |
| { |
| "completion_length": 19.71875, |
| "epoch": 1.9581589958158996, |
| "grad_norm": 10.683097839355469, |
| "kl": 0.18359375, |
| "learning_rate": 5.115062761506275e-07, |
| "loss": 0.0073, |
| "reward": 0.4775351285934448, |
| "reward_std": 0.06247048079967499, |
| "reward_std_all": 0.05965879000723362, |
| "rewards-max/forensics_iou_reward": 0.9208943843841553, |
| "rewards/forensics_iou_reward": 0.4775351285934448, |
| "step": 468 |
| }, |
| { |
| "completion_length": 19.734375, |
| "epoch": 1.9623430962343096, |
| "grad_norm": 3.7789814472198486, |
| "kl": 0.2890625, |
| "learning_rate": 5.104602510460251e-07, |
| "loss": 0.0116, |
| "reward": 0.44128891825675964, |
| "reward_std": 0.035598321817815304, |
| "reward_std_all": 0.035019898787140846, |
| "rewards-max/forensics_iou_reward": 0.9108887016773224, |
| "rewards/forensics_iou_reward": 0.44128891825675964, |
| "step": 469 |
| }, |
| { |
| "completion_length": 19.859375, |
| "epoch": 1.9665271966527196, |
| "grad_norm": 6.509633541107178, |
| "kl": 0.2119140625, |
| "learning_rate": 5.094142259414226e-07, |
| "loss": 0.0085, |
| "reward": 0.5391809046268463, |
| "reward_std": 0.08927823603153229, |
| "reward_std_all": 0.0851849839091301, |
| "rewards-max/forensics_iou_reward": 0.967615932226181, |
| "rewards/forensics_iou_reward": 0.5391809046268463, |
| "step": 470 |
| }, |
| { |
| "completion_length": 19.34375, |
| "epoch": 1.9707112970711296, |
| "grad_norm": 4.702049732208252, |
| "kl": 0.27490234375, |
| "learning_rate": 5.083682008368201e-07, |
| "loss": 0.011, |
| "reward": 0.5043386518955231, |
| "reward_std": 0.090434480458498, |
| "reward_std_all": 0.09430666267871857, |
| "rewards-max/forensics_iou_reward": 0.9741602540016174, |
| "rewards/forensics_iou_reward": 0.5043386518955231, |
| "step": 471 |
| }, |
| { |
| "completion_length": 20.03125, |
| "epoch": 1.9748953974895398, |
| "grad_norm": 4.621397495269775, |
| "kl": 0.19384765625, |
| "learning_rate": 5.073221757322176e-07, |
| "loss": 0.0078, |
| "reward": 0.54640032351017, |
| "reward_std": 0.10269700363278389, |
| "reward_std_all": 0.09956176206469536, |
| "rewards-max/forensics_iou_reward": 0.9406779706478119, |
| "rewards/forensics_iou_reward": 0.54640032351017, |
| "step": 472 |
| }, |
| { |
| "completion_length": 19.3125, |
| "epoch": 1.9790794979079498, |
| "grad_norm": 7.462576866149902, |
| "kl": 0.1875, |
| "learning_rate": 5.062761506276151e-07, |
| "loss": 0.0075, |
| "reward": 0.47034846246242523, |
| "reward_std": 0.07642216235399246, |
| "reward_std_all": 0.07201582938432693, |
| "rewards-max/forensics_iou_reward": 0.8035988807678223, |
| "rewards/forensics_iou_reward": 0.47034846246242523, |
| "step": 473 |
| }, |
| { |
| "completion_length": 19.734375, |
| "epoch": 1.9832635983263598, |
| "grad_norm": 5.855444431304932, |
| "kl": 0.208984375, |
| "learning_rate": 5.052301255230126e-07, |
| "loss": 0.0083, |
| "reward": 0.6221421658992767, |
| "reward_std": 0.04665267467498779, |
| "reward_std_all": 0.06453741528093815, |
| "rewards-max/forensics_iou_reward": 0.9673323035240173, |
| "rewards/forensics_iou_reward": 0.6221421658992767, |
| "step": 474 |
| }, |
| { |
| "completion_length": 19.078125, |
| "epoch": 1.98744769874477, |
| "grad_norm": 6.827437400817871, |
| "kl": 0.4296875, |
| "learning_rate": 5.041841004184099e-07, |
| "loss": 0.0172, |
| "reward": 0.4883962571620941, |
| "reward_std": 0.05689498595893383, |
| "reward_std_all": 0.06634487956762314, |
| "rewards-max/forensics_iou_reward": 0.890277773141861, |
| "rewards/forensics_iou_reward": 0.4883962571620941, |
| "step": 475 |
| }, |
| { |
| "completion_length": 19.625, |
| "epoch": 1.99163179916318, |
| "grad_norm": 5.620689392089844, |
| "kl": 0.169921875, |
| "learning_rate": 5.031380753138074e-07, |
| "loss": 0.0068, |
| "reward": 0.5354270190000534, |
| "reward_std": 0.06366065889596939, |
| "reward_std_all": 0.07271598465740681, |
| "rewards-max/forensics_iou_reward": 0.9392266273498535, |
| "rewards/forensics_iou_reward": 0.5354270190000534, |
| "step": 476 |
| }, |
| { |
| "completion_length": 20.265625, |
| "epoch": 1.99581589958159, |
| "grad_norm": 8.518739700317383, |
| "kl": 0.4345703125, |
| "learning_rate": 5.02092050209205e-07, |
| "loss": 0.0173, |
| "reward": 0.5658960938453674, |
| "reward_std": 0.10525535047054291, |
| "reward_std_all": 0.10457641258835793, |
| "rewards-max/forensics_iou_reward": 0.95247021317482, |
| "rewards/forensics_iou_reward": 0.5658960938453674, |
| "step": 477 |
| }, |
| { |
| "completion_length": 19.785715103149414, |
| "epoch": 2.0, |
| "grad_norm": 6.0066399574279785, |
| "kl": 0.21728515625, |
| "learning_rate": 5.010460251046025e-07, |
| "loss": 0.0084, |
| "reward": 0.4813845157623291, |
| "reward_std": 0.07351555116474628, |
| "reward_std_all": 0.11157144233584404, |
| "rewards-max/forensics_iou_reward": 0.7390499114990234, |
| "rewards/forensics_iou_reward": 0.4813845157623291, |
| "step": 478 |
| }, |
| { |
| "completion_length": 19.953125, |
| "epoch": 2.00418410041841, |
| "grad_norm": 7.4054107666015625, |
| "kl": 0.22021484375, |
| "learning_rate": 5e-07, |
| "loss": 0.0088, |
| "reward": 0.49648483097553253, |
| "reward_std": 0.0970894992351532, |
| "reward_std_all": 0.0988314300775528, |
| "rewards-max/forensics_iou_reward": 0.9311334490776062, |
| "rewards/forensics_iou_reward": 0.49648483097553253, |
| "step": 479 |
| }, |
| { |
| "completion_length": 19.21875, |
| "epoch": 2.00836820083682, |
| "grad_norm": 5.384705066680908, |
| "kl": 0.19140625, |
| "learning_rate": 4.989539748953975e-07, |
| "loss": 0.0077, |
| "reward": 0.4407500624656677, |
| "reward_std": 0.039003562182188034, |
| "reward_std_all": 0.03895074222236872, |
| "rewards-max/forensics_iou_reward": 0.9314384758472443, |
| "rewards/forensics_iou_reward": 0.4407500624656677, |
| "step": 480 |
| }, |
| { |
| "completion_length": 19.3125, |
| "epoch": 2.01255230125523, |
| "grad_norm": 5.256072998046875, |
| "kl": 0.19287109375, |
| "learning_rate": 4.97907949790795e-07, |
| "loss": 0.0077, |
| "reward": 0.5150367319583893, |
| "reward_std": 0.07181433215737343, |
| "reward_std_all": 0.10404174402356148, |
| "rewards-max/forensics_iou_reward": 0.9003103673458099, |
| "rewards/forensics_iou_reward": 0.5150367319583893, |
| "step": 481 |
| }, |
| { |
| "completion_length": 20.140625, |
| "epoch": 2.01673640167364, |
| "grad_norm": 7.194294452667236, |
| "kl": 0.25341796875, |
| "learning_rate": 4.968619246861924e-07, |
| "loss": 0.0102, |
| "reward": 0.5246390700340271, |
| "reward_std": 0.09972565621137619, |
| "reward_std_all": 0.0932142324745655, |
| "rewards-max/forensics_iou_reward": 0.845279723405838, |
| "rewards/forensics_iou_reward": 0.5246390700340271, |
| "step": 482 |
| }, |
| { |
| "completion_length": 19.59375, |
| "epoch": 2.02092050209205, |
| "grad_norm": 4.87518835067749, |
| "kl": 0.2353515625, |
| "learning_rate": 4.958158995815899e-07, |
| "loss": 0.0094, |
| "reward": 0.5818134844303131, |
| "reward_std": 0.11339414119720459, |
| "reward_std_all": 0.10557413846254349, |
| "rewards-max/forensics_iou_reward": 0.9463961124420166, |
| "rewards/forensics_iou_reward": 0.5818134844303131, |
| "step": 483 |
| }, |
| { |
| "completion_length": 19.640625, |
| "epoch": 2.0251046025104604, |
| "grad_norm": 5.255373954772949, |
| "kl": 0.1767578125, |
| "learning_rate": 4.947698744769874e-07, |
| "loss": 0.0071, |
| "reward": 0.6364378929138184, |
| "reward_std": 0.07031858712434769, |
| "reward_std_all": 0.09100918099284172, |
| "rewards-max/forensics_iou_reward": 0.9659194648265839, |
| "rewards/forensics_iou_reward": 0.6364378929138184, |
| "step": 484 |
| }, |
| { |
| "completion_length": 19.5625, |
| "epoch": 2.0292887029288704, |
| "grad_norm": 6.249447822570801, |
| "kl": 0.20654296875, |
| "learning_rate": 4.93723849372385e-07, |
| "loss": 0.0083, |
| "reward": 0.6101342439651489, |
| "reward_std": 0.09066084958612919, |
| "reward_std_all": 0.10523737780749798, |
| "rewards-max/forensics_iou_reward": 0.9766807556152344, |
| "rewards/forensics_iou_reward": 0.6101342439651489, |
| "step": 485 |
| }, |
| { |
| "completion_length": 20.078125, |
| "epoch": 2.0334728033472804, |
| "grad_norm": 3.63797664642334, |
| "kl": 0.158203125, |
| "learning_rate": 4.926778242677825e-07, |
| "loss": 0.0063, |
| "reward": 0.4895514100790024, |
| "reward_std": 0.09839782118797302, |
| "reward_std_all": 0.08919601142406464, |
| "rewards-max/forensics_iou_reward": 0.947941929101944, |
| "rewards/forensics_iou_reward": 0.4895514100790024, |
| "step": 486 |
| }, |
| { |
| "completion_length": 19.921875, |
| "epoch": 2.0376569037656904, |
| "grad_norm": 6.485995292663574, |
| "kl": 0.16943359375, |
| "learning_rate": 4.916317991631799e-07, |
| "loss": 0.0068, |
| "reward": 0.5010503381490707, |
| "reward_std": 0.1167222261428833, |
| "reward_std_all": 0.14019476994872093, |
| "rewards-max/forensics_iou_reward": 0.9292958378791809, |
| "rewards/forensics_iou_reward": 0.5010503381490707, |
| "step": 487 |
| }, |
| { |
| "completion_length": 20.28125, |
| "epoch": 2.0418410041841004, |
| "grad_norm": 6.767298698425293, |
| "kl": 0.13818359375, |
| "learning_rate": 4.905857740585774e-07, |
| "loss": 0.0055, |
| "reward": 0.5959267914295197, |
| "reward_std": 0.1421753168106079, |
| "reward_std_all": 0.13716311752796173, |
| "rewards-max/forensics_iou_reward": 0.9688799679279327, |
| "rewards/forensics_iou_reward": 0.5959267914295197, |
| "step": 488 |
| }, |
| { |
| "completion_length": 19.171875, |
| "epoch": 2.0460251046025104, |
| "grad_norm": 4.837405204772949, |
| "kl": 0.21923828125, |
| "learning_rate": 4.895397489539749e-07, |
| "loss": 0.0088, |
| "reward": 0.484863743185997, |
| "reward_std": 0.10420941561460495, |
| "reward_std_all": 0.1044587790966034, |
| "rewards-max/forensics_iou_reward": 0.9723261892795563, |
| "rewards/forensics_iou_reward": 0.484863743185997, |
| "step": 489 |
| }, |
| { |
| "completion_length": 19.28125, |
| "epoch": 2.0502092050209204, |
| "grad_norm": 5.957054138183594, |
| "kl": 0.1796875, |
| "learning_rate": 4.884937238493724e-07, |
| "loss": 0.0072, |
| "reward": 0.5012648701667786, |
| "reward_std": 0.11667663604021072, |
| "reward_std_all": 0.13978663086891174, |
| "rewards-max/forensics_iou_reward": 0.9621434211730957, |
| "rewards/forensics_iou_reward": 0.5012648701667786, |
| "step": 490 |
| }, |
| { |
| "completion_length": 19.921875, |
| "epoch": 2.0543933054393304, |
| "grad_norm": 4.816135883331299, |
| "kl": 0.2021484375, |
| "learning_rate": 4.874476987447699e-07, |
| "loss": 0.0081, |
| "reward": 0.4212348163127899, |
| "reward_std": 0.1255178079009056, |
| "reward_std_all": 0.11555412039160728, |
| "rewards-max/forensics_iou_reward": 0.916740745306015, |
| "rewards/forensics_iou_reward": 0.4212348163127899, |
| "step": 491 |
| }, |
| { |
| "completion_length": 19.171875, |
| "epoch": 2.0585774058577404, |
| "grad_norm": 4.661808967590332, |
| "kl": 0.23828125, |
| "learning_rate": 4.864016736401673e-07, |
| "loss": 0.0095, |
| "reward": 0.6372391879558563, |
| "reward_std": 0.06704341899603605, |
| "reward_std_all": 0.06259860843420029, |
| "rewards-max/forensics_iou_reward": 0.9525079727172852, |
| "rewards/forensics_iou_reward": 0.6372391879558563, |
| "step": 492 |
| }, |
| { |
| "completion_length": 19.578125, |
| "epoch": 2.062761506276151, |
| "grad_norm": 6.068720817565918, |
| "kl": 0.15869140625, |
| "learning_rate": 4.853556485355648e-07, |
| "loss": 0.0063, |
| "reward": 0.6280372738838196, |
| "reward_std": 0.09204418584704399, |
| "reward_std_all": 0.09616755694150925, |
| "rewards-max/forensics_iou_reward": 0.9471103847026825, |
| "rewards/forensics_iou_reward": 0.6280372738838196, |
| "step": 493 |
| }, |
| { |
| "completion_length": 19.296875, |
| "epoch": 2.066945606694561, |
| "grad_norm": 3.935426712036133, |
| "kl": 0.16357421875, |
| "learning_rate": 4.843096234309624e-07, |
| "loss": 0.0066, |
| "reward": 0.6129321902990341, |
| "reward_std": 0.05924936383962631, |
| "reward_std_all": 0.06786887347698212, |
| "rewards-max/forensics_iou_reward": 0.8950034081935883, |
| "rewards/forensics_iou_reward": 0.6129321902990341, |
| "step": 494 |
| }, |
| { |
| "completion_length": 19.6875, |
| "epoch": 2.071129707112971, |
| "grad_norm": 5.153486728668213, |
| "kl": 0.21728515625, |
| "learning_rate": 4.832635983263599e-07, |
| "loss": 0.0087, |
| "reward": 0.5422218888998032, |
| "reward_std": 0.09754285216331482, |
| "reward_std_all": 0.11047489568591118, |
| "rewards-max/forensics_iou_reward": 0.8841262757778168, |
| "rewards/forensics_iou_reward": 0.5422218888998032, |
| "step": 495 |
| }, |
| { |
| "completion_length": 19.578125, |
| "epoch": 2.075313807531381, |
| "grad_norm": 3.886359453201294, |
| "kl": 0.17138671875, |
| "learning_rate": 4.822175732217573e-07, |
| "loss": 0.0068, |
| "reward": 0.536567211151123, |
| "reward_std": 0.061208467930555344, |
| "reward_std_all": 0.061973996460437775, |
| "rewards-max/forensics_iou_reward": 0.9299744963645935, |
| "rewards/forensics_iou_reward": 0.536567211151123, |
| "step": 496 |
| }, |
| { |
| "completion_length": 20.625, |
| "epoch": 2.079497907949791, |
| "grad_norm": 4.43943452835083, |
| "kl": 0.203125, |
| "learning_rate": 4.811715481171548e-07, |
| "loss": 0.0081, |
| "reward": 0.5088772773742676, |
| "reward_std": 0.11179663240909576, |
| "reward_std_all": 0.1071784496307373, |
| "rewards-max/forensics_iou_reward": 0.9371592700481415, |
| "rewards/forensics_iou_reward": 0.5088772773742676, |
| "step": 497 |
| }, |
| { |
| "completion_length": 19.546875, |
| "epoch": 2.083682008368201, |
| "grad_norm": 6.212147235870361, |
| "kl": 0.15478515625, |
| "learning_rate": 4.801255230125523e-07, |
| "loss": 0.0062, |
| "reward": 0.5716764628887177, |
| "reward_std": 0.04983414337038994, |
| "reward_std_all": 0.07129380106925964, |
| "rewards-max/forensics_iou_reward": 0.9698840081691742, |
| "rewards/forensics_iou_reward": 0.5716764628887177, |
| "step": 498 |
| }, |
| { |
| "completion_length": 19.625, |
| "epoch": 2.087866108786611, |
| "grad_norm": 5.487447261810303, |
| "kl": 0.189453125, |
| "learning_rate": 4.790794979079498e-07, |
| "loss": 0.0076, |
| "reward": 0.5654282867908478, |
| "reward_std": 0.10885344818234444, |
| "reward_std_all": 0.1125742606818676, |
| "rewards-max/forensics_iou_reward": 0.9219416081905365, |
| "rewards/forensics_iou_reward": 0.5654282867908478, |
| "step": 499 |
| }, |
| { |
| "completion_length": 20.15625, |
| "epoch": 2.092050209205021, |
| "grad_norm": 4.153016567230225, |
| "kl": 0.16845703125, |
| "learning_rate": 4.780334728033473e-07, |
| "loss": 0.0067, |
| "reward": 0.5702103078365326, |
| "reward_std": 0.09131700545549393, |
| "reward_std_all": 0.08898906037211418, |
| "rewards-max/forensics_iou_reward": 0.9307754039764404, |
| "rewards/forensics_iou_reward": 0.5702103078365326, |
| "step": 500 |
| }, |
| { |
| "completion_length": 19.640625, |
| "epoch": 2.096234309623431, |
| "grad_norm": 5.327526092529297, |
| "kl": 0.19091796875, |
| "learning_rate": 4.769874476987447e-07, |
| "loss": 0.0077, |
| "reward": 0.5315576195716858, |
| "reward_std": 0.10667691752314568, |
| "reward_std_all": 0.10939309373497963, |
| "rewards-max/forensics_iou_reward": 0.9797586500644684, |
| "rewards/forensics_iou_reward": 0.5315576195716858, |
| "step": 501 |
| }, |
| { |
| "completion_length": 20.03125, |
| "epoch": 2.100418410041841, |
| "grad_norm": 3.169680595397949, |
| "kl": 0.21533203125, |
| "learning_rate": 4.759414225941422e-07, |
| "loss": 0.0086, |
| "reward": 0.45668143033981323, |
| "reward_std": 0.08175645396113396, |
| "reward_std_all": 0.10177013278007507, |
| "rewards-max/forensics_iou_reward": 0.926235169172287, |
| "rewards/forensics_iou_reward": 0.45668143033981323, |
| "step": 502 |
| }, |
| { |
| "completion_length": 19.546875, |
| "epoch": 2.104602510460251, |
| "grad_norm": 4.616641998291016, |
| "kl": 0.15869140625, |
| "learning_rate": 4.748953974895397e-07, |
| "loss": 0.0063, |
| "reward": 0.4937085807323456, |
| "reward_std": 0.0846181008964777, |
| "reward_std_all": 0.08158099465072155, |
| "rewards-max/forensics_iou_reward": 0.8888888955116272, |
| "rewards/forensics_iou_reward": 0.4937085807323456, |
| "step": 503 |
| }, |
| { |
| "completion_length": 20.703125, |
| "epoch": 2.1087866108786613, |
| "grad_norm": 4.244184970855713, |
| "kl": 0.1650390625, |
| "learning_rate": 4.738493723849372e-07, |
| "loss": 0.0066, |
| "reward": 0.49944929778575897, |
| "reward_std": 0.10103286430239677, |
| "reward_std_all": 0.09714224934577942, |
| "rewards-max/forensics_iou_reward": 0.8984405100345612, |
| "rewards/forensics_iou_reward": 0.49944929778575897, |
| "step": 504 |
| }, |
| { |
| "completion_length": 20.734375, |
| "epoch": 2.1129707112970713, |
| "grad_norm": 8.503737449645996, |
| "kl": 0.19775390625, |
| "learning_rate": 4.7280334728033473e-07, |
| "loss": 0.0079, |
| "reward": 0.5103727877140045, |
| "reward_std": 0.08923926204442978, |
| "reward_std_all": 0.09329403564333916, |
| "rewards-max/forensics_iou_reward": 0.9674548506736755, |
| "rewards/forensics_iou_reward": 0.5103727877140045, |
| "step": 505 |
| }, |
| { |
| "completion_length": 20.171875, |
| "epoch": 2.1171548117154813, |
| "grad_norm": 5.154767036437988, |
| "kl": 0.1669921875, |
| "learning_rate": 4.7175732217573224e-07, |
| "loss": 0.0067, |
| "reward": 0.4700935333967209, |
| "reward_std": 0.10507720336318016, |
| "reward_std_all": 0.10247362777590752, |
| "rewards-max/forensics_iou_reward": 0.9040189981460571, |
| "rewards/forensics_iou_reward": 0.4700935333967209, |
| "step": 506 |
| }, |
| { |
| "completion_length": 20.03125, |
| "epoch": 2.1213389121338913, |
| "grad_norm": 5.967700004577637, |
| "kl": 0.17236328125, |
| "learning_rate": 4.7071129707112965e-07, |
| "loss": 0.0069, |
| "reward": 0.5908835530281067, |
| "reward_std": 0.06890885159373283, |
| "reward_std_all": 0.06752956658601761, |
| "rewards-max/forensics_iou_reward": 0.9556677043437958, |
| "rewards/forensics_iou_reward": 0.5908835530281067, |
| "step": 507 |
| }, |
| { |
| "completion_length": 19.375, |
| "epoch": 2.1255230125523012, |
| "grad_norm": 4.7859039306640625, |
| "kl": 0.17431640625, |
| "learning_rate": 4.6966527196652716e-07, |
| "loss": 0.007, |
| "reward": 0.4140782803297043, |
| "reward_std": 0.04379237722605467, |
| "reward_std_all": 0.048889375291764736, |
| "rewards-max/forensics_iou_reward": 0.9802871644496918, |
| "rewards/forensics_iou_reward": 0.4140782803297043, |
| "step": 508 |
| }, |
| { |
| "completion_length": 20.671875, |
| "epoch": 2.1297071129707112, |
| "grad_norm": 4.394736289978027, |
| "kl": 0.19287109375, |
| "learning_rate": 4.686192468619247e-07, |
| "loss": 0.0077, |
| "reward": 0.5443461239337921, |
| "reward_std": 0.10664764419198036, |
| "reward_std_all": 0.11402054131031036, |
| "rewards-max/forensics_iou_reward": 0.9669299125671387, |
| "rewards/forensics_iou_reward": 0.5443461239337921, |
| "step": 509 |
| }, |
| { |
| "completion_length": 19.984375, |
| "epoch": 2.1338912133891212, |
| "grad_norm": 5.022916316986084, |
| "kl": 0.31005859375, |
| "learning_rate": 4.675732217573222e-07, |
| "loss": 0.0124, |
| "reward": 0.40788911283016205, |
| "reward_std": 0.08747006952762604, |
| "reward_std_all": 0.09275783970952034, |
| "rewards-max/forensics_iou_reward": 0.9604755640029907, |
| "rewards/forensics_iou_reward": 0.40788911283016205, |
| "step": 510 |
| }, |
| { |
| "completion_length": 20.234375, |
| "epoch": 2.1380753138075312, |
| "grad_norm": 5.40936279296875, |
| "kl": 0.1708984375, |
| "learning_rate": 4.6652719665271965e-07, |
| "loss": 0.0068, |
| "reward": 0.5948018431663513, |
| "reward_std": 0.07044266164302826, |
| "reward_std_all": 0.09408826194703579, |
| "rewards-max/forensics_iou_reward": 0.9700064361095428, |
| "rewards/forensics_iou_reward": 0.5948018431663513, |
| "step": 511 |
| }, |
| { |
| "completion_length": 20.453125, |
| "epoch": 2.1422594142259412, |
| "grad_norm": 5.885490417480469, |
| "kl": 0.16552734375, |
| "learning_rate": 4.654811715481171e-07, |
| "loss": 0.0066, |
| "reward": 0.5642273128032684, |
| "reward_std": 0.07912319153547287, |
| "reward_std_all": 0.07537656277418137, |
| "rewards-max/forensics_iou_reward": 0.8893807828426361, |
| "rewards/forensics_iou_reward": 0.5642273128032684, |
| "step": 512 |
| }, |
| { |
| "completion_length": 20.953125, |
| "epoch": 2.1464435146443517, |
| "grad_norm": 4.090031623840332, |
| "kl": 0.18408203125, |
| "learning_rate": 4.644351464435146e-07, |
| "loss": 0.0074, |
| "reward": 0.5186956226825714, |
| "reward_std": 0.1101512536406517, |
| "reward_std_all": 0.1011403277516365, |
| "rewards-max/forensics_iou_reward": 0.9490372240543365, |
| "rewards/forensics_iou_reward": 0.5186956226825714, |
| "step": 513 |
| }, |
| { |
| "completion_length": 20.6875, |
| "epoch": 2.1506276150627617, |
| "grad_norm": 6.2134809494018555, |
| "kl": 0.19384765625, |
| "learning_rate": 4.6338912133891213e-07, |
| "loss": 0.0078, |
| "reward": 0.5164034962654114, |
| "reward_std": 0.08516819402575493, |
| "reward_std_all": 0.1011127233505249, |
| "rewards-max/forensics_iou_reward": 0.9053635001182556, |
| "rewards/forensics_iou_reward": 0.5164034962654114, |
| "step": 514 |
| }, |
| { |
| "completion_length": 20.640625, |
| "epoch": 2.1548117154811717, |
| "grad_norm": 5.435362815856934, |
| "kl": 0.21533203125, |
| "learning_rate": 4.623430962343096e-07, |
| "loss": 0.0086, |
| "reward": 0.5236126780509949, |
| "reward_std": 0.08744000643491745, |
| "reward_std_all": 0.10212040692567825, |
| "rewards-max/forensics_iou_reward": 0.9524124264717102, |
| "rewards/forensics_iou_reward": 0.5236126780509949, |
| "step": 515 |
| }, |
| { |
| "completion_length": 20.046875, |
| "epoch": 2.1589958158995817, |
| "grad_norm": 5.509181022644043, |
| "kl": 0.1787109375, |
| "learning_rate": 4.612970711297071e-07, |
| "loss": 0.0071, |
| "reward": 0.4971808195114136, |
| "reward_std": 0.07195421680808067, |
| "reward_std_all": 0.07020186632871628, |
| "rewards-max/forensics_iou_reward": 0.9113529324531555, |
| "rewards/forensics_iou_reward": 0.4971808195114136, |
| "step": 516 |
| }, |
| { |
| "completion_length": 19.953125, |
| "epoch": 2.1631799163179917, |
| "grad_norm": 4.650904178619385, |
| "kl": 0.23779296875, |
| "learning_rate": 4.6025104602510457e-07, |
| "loss": 0.0095, |
| "reward": 0.6069491803646088, |
| "reward_std": 0.0454351045191288, |
| "reward_std_all": 0.05573543161153793, |
| "rewards-max/forensics_iou_reward": 0.9581573009490967, |
| "rewards/forensics_iou_reward": 0.6069491803646088, |
| "step": 517 |
| }, |
| { |
| "completion_length": 20.4375, |
| "epoch": 2.1673640167364017, |
| "grad_norm": 5.064952850341797, |
| "kl": 0.1640625, |
| "learning_rate": 4.592050209205021e-07, |
| "loss": 0.0066, |
| "reward": 0.45851990580558777, |
| "reward_std": 0.08521562814712524, |
| "reward_std_all": 0.07853604108095169, |
| "rewards-max/forensics_iou_reward": 0.9667752981185913, |
| "rewards/forensics_iou_reward": 0.45851990580558777, |
| "step": 518 |
| }, |
| { |
| "completion_length": 19.46875, |
| "epoch": 2.1715481171548117, |
| "grad_norm": 5.363155841827393, |
| "kl": 0.361328125, |
| "learning_rate": 4.5815899581589954e-07, |
| "loss": 0.0144, |
| "reward": 0.5817721784114838, |
| "reward_std": 0.09638197720050812, |
| "reward_std_all": 0.0942663662135601, |
| "rewards-max/forensics_iou_reward": 0.9500448405742645, |
| "rewards/forensics_iou_reward": 0.5817721784114838, |
| "step": 519 |
| }, |
| { |
| "completion_length": 19.671875, |
| "epoch": 2.1757322175732217, |
| "grad_norm": 6.890188217163086, |
| "kl": 0.1943359375, |
| "learning_rate": 4.5711297071129705e-07, |
| "loss": 0.0078, |
| "reward": 0.5440508723258972, |
| "reward_std": 0.058271896094083786, |
| "reward_std_all": 0.07388198003172874, |
| "rewards-max/forensics_iou_reward": 0.9002150595188141, |
| "rewards/forensics_iou_reward": 0.5440508723258972, |
| "step": 520 |
| }, |
| { |
| "completion_length": 21.46875, |
| "epoch": 2.1799163179916317, |
| "grad_norm": 7.415527820587158, |
| "kl": 0.20556640625, |
| "learning_rate": 4.5606694560669456e-07, |
| "loss": 0.0082, |
| "reward": 0.5688314735889435, |
| "reward_std": 0.10039288178086281, |
| "reward_std_all": 0.09232667461037636, |
| "rewards-max/forensics_iou_reward": 0.9369619488716125, |
| "rewards/forensics_iou_reward": 0.5688314735889435, |
| "step": 521 |
| }, |
| { |
| "completion_length": 19.375, |
| "epoch": 2.1841004184100417, |
| "grad_norm": 4.614388465881348, |
| "kl": 0.2060546875, |
| "learning_rate": 4.55020920502092e-07, |
| "loss": 0.0083, |
| "reward": 0.524406909942627, |
| "reward_std": 0.07018227875232697, |
| "reward_std_all": 0.07879088446497917, |
| "rewards-max/forensics_iou_reward": 0.9810819625854492, |
| "rewards/forensics_iou_reward": 0.524406909942627, |
| "step": 522 |
| }, |
| { |
| "completion_length": 19.5, |
| "epoch": 2.1882845188284517, |
| "grad_norm": 3.6549441814422607, |
| "kl": 0.16650390625, |
| "learning_rate": 4.539748953974895e-07, |
| "loss": 0.0066, |
| "reward": 0.4510363042354584, |
| "reward_std": 0.06112790107727051, |
| "reward_std_all": 0.05741841346025467, |
| "rewards-max/forensics_iou_reward": 0.9332653880119324, |
| "rewards/forensics_iou_reward": 0.4510363042354584, |
| "step": 523 |
| }, |
| { |
| "completion_length": 19.15625, |
| "epoch": 2.192468619246862, |
| "grad_norm": 8.278698921203613, |
| "kl": 0.16455078125, |
| "learning_rate": 4.52928870292887e-07, |
| "loss": 0.0066, |
| "reward": 0.4291091710329056, |
| "reward_std": 0.09496006369590759, |
| "reward_std_all": 0.09729510545730591, |
| "rewards-max/forensics_iou_reward": 0.8300193250179291, |
| "rewards/forensics_iou_reward": 0.4291091710329056, |
| "step": 524 |
| }, |
| { |
| "completion_length": 19.25, |
| "epoch": 2.196652719665272, |
| "grad_norm": 8.545059204101562, |
| "kl": 0.181640625, |
| "learning_rate": 4.518828451882845e-07, |
| "loss": 0.0073, |
| "reward": 0.4922116845846176, |
| "reward_std": 0.09667269885540009, |
| "reward_std_all": 0.09584015235304832, |
| "rewards-max/forensics_iou_reward": 0.9471213221549988, |
| "rewards/forensics_iou_reward": 0.4922116845846176, |
| "step": 525 |
| }, |
| { |
| "completion_length": 19.671875, |
| "epoch": 2.200836820083682, |
| "grad_norm": 5.5958757400512695, |
| "kl": 0.1865234375, |
| "learning_rate": 4.50836820083682e-07, |
| "loss": 0.0075, |
| "reward": 0.5196413397789001, |
| "reward_std": 0.12360649183392525, |
| "reward_std_all": 0.11906597018241882, |
| "rewards-max/forensics_iou_reward": 0.9621364772319794, |
| "rewards/forensics_iou_reward": 0.5196413397789001, |
| "step": 526 |
| }, |
| { |
| "completion_length": 21.125, |
| "epoch": 2.205020920502092, |
| "grad_norm": 4.749166965484619, |
| "kl": 0.16845703125, |
| "learning_rate": 4.497907949790795e-07, |
| "loss": 0.0067, |
| "reward": 0.5043592751026154, |
| "reward_std": 0.09659349173307419, |
| "reward_std_all": 0.0952676422894001, |
| "rewards-max/forensics_iou_reward": 0.8942901194095612, |
| "rewards/forensics_iou_reward": 0.5043592751026154, |
| "step": 527 |
| }, |
| { |
| "completion_length": 19.921875, |
| "epoch": 2.209205020920502, |
| "grad_norm": 5.5704665184021, |
| "kl": 0.1298828125, |
| "learning_rate": 4.4874476987447694e-07, |
| "loss": 0.0052, |
| "reward": 0.5313211381435394, |
| "reward_std": 0.05981877073645592, |
| "reward_std_all": 0.07464935630559921, |
| "rewards-max/forensics_iou_reward": 0.8879498839378357, |
| "rewards/forensics_iou_reward": 0.5313211381435394, |
| "step": 528 |
| }, |
| { |
| "completion_length": 19.375, |
| "epoch": 2.213389121338912, |
| "grad_norm": 4.722716808319092, |
| "kl": 0.18603515625, |
| "learning_rate": 4.4769874476987445e-07, |
| "loss": 0.0074, |
| "reward": 0.5350513905286789, |
| "reward_std": 0.09019450470805168, |
| "reward_std_all": 0.10116573423147202, |
| "rewards-max/forensics_iou_reward": 0.9083774387836456, |
| "rewards/forensics_iou_reward": 0.5350513905286789, |
| "step": 529 |
| }, |
| { |
| "completion_length": 19.78125, |
| "epoch": 2.217573221757322, |
| "grad_norm": 4.863975524902344, |
| "kl": 0.22802734375, |
| "learning_rate": 4.4665271966527197e-07, |
| "loss": 0.0091, |
| "reward": 0.7285765409469604, |
| "reward_std": 0.07370911911129951, |
| "reward_std_all": 0.09925348311662674, |
| "rewards-max/forensics_iou_reward": 0.9729277193546295, |
| "rewards/forensics_iou_reward": 0.7285765409469604, |
| "step": 530 |
| }, |
| { |
| "completion_length": 19.78125, |
| "epoch": 2.221757322175732, |
| "grad_norm": 10.69340991973877, |
| "kl": 0.3330078125, |
| "learning_rate": 4.456066945606695e-07, |
| "loss": 0.0133, |
| "reward": 0.5445694029331207, |
| "reward_std": 0.1112344041466713, |
| "reward_std_all": 0.1240912601351738, |
| "rewards-max/forensics_iou_reward": 0.9379256963729858, |
| "rewards/forensics_iou_reward": 0.5445694029331207, |
| "step": 531 |
| }, |
| { |
| "completion_length": 20.046875, |
| "epoch": 2.225941422594142, |
| "grad_norm": 12.364156723022461, |
| "kl": 0.349609375, |
| "learning_rate": 4.445606694560669e-07, |
| "loss": 0.014, |
| "reward": 0.5079004168510437, |
| "reward_std": 0.13574287667870522, |
| "reward_std_all": 0.14829208701848984, |
| "rewards-max/forensics_iou_reward": 0.9060115814208984, |
| "rewards/forensics_iou_reward": 0.5079004168510437, |
| "step": 532 |
| }, |
| { |
| "completion_length": 20.671875, |
| "epoch": 2.2301255230125525, |
| "grad_norm": 5.893726348876953, |
| "kl": 0.3173828125, |
| "learning_rate": 4.435146443514644e-07, |
| "loss": 0.0127, |
| "reward": 0.4965629428625107, |
| "reward_std": 0.07824276760220528, |
| "reward_std_all": 0.09886457771062851, |
| "rewards-max/forensics_iou_reward": 0.8996243476867676, |
| "rewards/forensics_iou_reward": 0.4965629428625107, |
| "step": 533 |
| }, |
| { |
| "completion_length": 19.8125, |
| "epoch": 2.2343096234309625, |
| "grad_norm": 5.368199348449707, |
| "kl": 0.166015625, |
| "learning_rate": 4.424686192468619e-07, |
| "loss": 0.0066, |
| "reward": 0.6054002344608307, |
| "reward_std": 0.1257016472518444, |
| "reward_std_all": 0.11614128574728966, |
| "rewards-max/forensics_iou_reward": 0.9907163083553314, |
| "rewards/forensics_iou_reward": 0.6054002344608307, |
| "step": 534 |
| }, |
| { |
| "completion_length": 19.5625, |
| "epoch": 2.2384937238493725, |
| "grad_norm": 4.53639554977417, |
| "kl": 0.21044921875, |
| "learning_rate": 4.414225941422594e-07, |
| "loss": 0.0084, |
| "reward": 0.554469108581543, |
| "reward_std": 0.06240956485271454, |
| "reward_std_all": 0.05811576172709465, |
| "rewards-max/forensics_iou_reward": 0.9660106897354126, |
| "rewards/forensics_iou_reward": 0.554469108581543, |
| "step": 535 |
| }, |
| { |
| "completion_length": 20.328125, |
| "epoch": 2.2426778242677825, |
| "grad_norm": 6.020181179046631, |
| "kl": 0.21435546875, |
| "learning_rate": 4.403765690376569e-07, |
| "loss": 0.0086, |
| "reward": 0.5519704818725586, |
| "reward_std": 0.10455659404397011, |
| "reward_std_all": 0.10115881823003292, |
| "rewards-max/forensics_iou_reward": 0.9046338200569153, |
| "rewards/forensics_iou_reward": 0.5519704818725586, |
| "step": 536 |
| }, |
| { |
| "completion_length": 19.203125, |
| "epoch": 2.2468619246861925, |
| "grad_norm": 8.938350677490234, |
| "kl": 0.2431640625, |
| "learning_rate": 4.3933054393305435e-07, |
| "loss": 0.0097, |
| "reward": 0.4772404134273529, |
| "reward_std": 0.07506000623106956, |
| "reward_std_all": 0.0723981037735939, |
| "rewards-max/forensics_iou_reward": 0.9662620723247528, |
| "rewards/forensics_iou_reward": 0.4772404134273529, |
| "step": 537 |
| }, |
| { |
| "completion_length": 19.421875, |
| "epoch": 2.2510460251046025, |
| "grad_norm": 4.132063865661621, |
| "kl": 0.1611328125, |
| "learning_rate": 4.3828451882845186e-07, |
| "loss": 0.0065, |
| "reward": 0.5062144547700882, |
| "reward_std": 0.0641314908862114, |
| "reward_std_all": 0.061176761984825134, |
| "rewards-max/forensics_iou_reward": 0.8606622517108917, |
| "rewards/forensics_iou_reward": 0.5062144547700882, |
| "step": 538 |
| }, |
| { |
| "completion_length": 19.515625, |
| "epoch": 2.2552301255230125, |
| "grad_norm": 5.7022786140441895, |
| "kl": 0.17822265625, |
| "learning_rate": 4.3723849372384937e-07, |
| "loss": 0.0071, |
| "reward": 0.46651895344257355, |
| "reward_std": 0.057323455810546875, |
| "reward_std_all": 0.0537995882332325, |
| "rewards-max/forensics_iou_reward": 0.8820410668849945, |
| "rewards/forensics_iou_reward": 0.46651895344257355, |
| "step": 539 |
| }, |
| { |
| "completion_length": 20.0, |
| "epoch": 2.2594142259414225, |
| "grad_norm": 7.805844306945801, |
| "kl": 0.1640625, |
| "learning_rate": 4.3619246861924683e-07, |
| "loss": 0.0066, |
| "reward": 0.4886464774608612, |
| "reward_std": 0.08105583488941193, |
| "reward_std_all": 0.08096091821789742, |
| "rewards-max/forensics_iou_reward": 0.8965218663215637, |
| "rewards/forensics_iou_reward": 0.4886464774608612, |
| "step": 540 |
| }, |
| { |
| "completion_length": 19.28125, |
| "epoch": 2.2635983263598325, |
| "grad_norm": 5.58073616027832, |
| "kl": 0.138671875, |
| "learning_rate": 4.3514644351464434e-07, |
| "loss": 0.0056, |
| "reward": 0.7029106020927429, |
| "reward_std": 0.07879981771111488, |
| "reward_std_all": 0.08032531291246414, |
| "rewards-max/forensics_iou_reward": 0.9630094170570374, |
| "rewards/forensics_iou_reward": 0.7029106020927429, |
| "step": 541 |
| }, |
| { |
| "completion_length": 20.265625, |
| "epoch": 2.2677824267782425, |
| "grad_norm": 7.051024913787842, |
| "kl": 0.22998046875, |
| "learning_rate": 4.341004184100418e-07, |
| "loss": 0.0092, |
| "reward": 0.5959344804286957, |
| "reward_std": 0.1010354682803154, |
| "reward_std_all": 0.106355469673872, |
| "rewards-max/forensics_iou_reward": 0.9646376073360443, |
| "rewards/forensics_iou_reward": 0.5959344804286957, |
| "step": 542 |
| }, |
| { |
| "completion_length": 19.359375, |
| "epoch": 2.2719665271966525, |
| "grad_norm": 5.057266712188721, |
| "kl": 0.185302734375, |
| "learning_rate": 4.330543933054393e-07, |
| "loss": 0.0074, |
| "reward": 0.5347660481929779, |
| "reward_std": 0.09883803874254227, |
| "reward_std_all": 0.11119697615504265, |
| "rewards-max/forensics_iou_reward": 0.9033333361148834, |
| "rewards/forensics_iou_reward": 0.5347660481929779, |
| "step": 543 |
| }, |
| { |
| "completion_length": 19.96875, |
| "epoch": 2.276150627615063, |
| "grad_norm": 5.9300923347473145, |
| "kl": 0.26025390625, |
| "learning_rate": 4.3200836820083683e-07, |
| "loss": 0.0104, |
| "reward": 0.5396451056003571, |
| "reward_std": 0.12523826956748962, |
| "reward_std_all": 0.12711922079324722, |
| "rewards-max/forensics_iou_reward": 0.9400582909584045, |
| "rewards/forensics_iou_reward": 0.5396451056003571, |
| "step": 544 |
| }, |
| { |
| "completion_length": 19.15625, |
| "epoch": 2.280334728033473, |
| "grad_norm": 4.908992290496826, |
| "kl": 0.388671875, |
| "learning_rate": 4.309623430962343e-07, |
| "loss": 0.0155, |
| "reward": 0.5250506103038788, |
| "reward_std": 0.0742667131125927, |
| "reward_std_all": 0.07684288918972015, |
| "rewards-max/forensics_iou_reward": 0.874378114938736, |
| "rewards/forensics_iou_reward": 0.5250506103038788, |
| "step": 545 |
| }, |
| { |
| "completion_length": 20.34375, |
| "epoch": 2.284518828451883, |
| "grad_norm": 7.321910381317139, |
| "kl": 0.2060546875, |
| "learning_rate": 4.299163179916318e-07, |
| "loss": 0.0082, |
| "reward": 0.5740767121315002, |
| "reward_std": 0.08427730947732925, |
| "reward_std_all": 0.0779549591243267, |
| "rewards-max/forensics_iou_reward": 0.987438291311264, |
| "rewards/forensics_iou_reward": 0.5740767121315002, |
| "step": 546 |
| }, |
| { |
| "completion_length": 19.453125, |
| "epoch": 2.288702928870293, |
| "grad_norm": 5.057223320007324, |
| "kl": 0.17822265625, |
| "learning_rate": 4.2887029288702926e-07, |
| "loss": 0.0071, |
| "reward": 0.41954943537712097, |
| "reward_std": 0.05392079800367355, |
| "reward_std_all": 0.06365283206105232, |
| "rewards-max/forensics_iou_reward": 0.9130881726741791, |
| "rewards/forensics_iou_reward": 0.41954943537712097, |
| "step": 547 |
| }, |
| { |
| "completion_length": 20.109375, |
| "epoch": 2.292887029288703, |
| "grad_norm": 7.302426338195801, |
| "kl": 0.22900390625, |
| "learning_rate": 4.278242677824268e-07, |
| "loss": 0.0092, |
| "reward": 0.5637824237346649, |
| "reward_std": 0.1312611922621727, |
| "reward_std_all": 0.12549810856580734, |
| "rewards-max/forensics_iou_reward": 0.9778820276260376, |
| "rewards/forensics_iou_reward": 0.5637824237346649, |
| "step": 548 |
| }, |
| { |
| "completion_length": 19.140625, |
| "epoch": 2.297071129707113, |
| "grad_norm": 4.207304954528809, |
| "kl": 0.150390625, |
| "learning_rate": 4.2677824267782424e-07, |
| "loss": 0.006, |
| "reward": 0.5332267582416534, |
| "reward_std": 0.06781882047653198, |
| "reward_std_all": 0.0673408955335617, |
| "rewards-max/forensics_iou_reward": 0.9540369808673859, |
| "rewards/forensics_iou_reward": 0.5332267582416534, |
| "step": 549 |
| }, |
| { |
| "completion_length": 20.15625, |
| "epoch": 2.301255230125523, |
| "grad_norm": 4.932676792144775, |
| "kl": 0.22509765625, |
| "learning_rate": 4.2573221757322175e-07, |
| "loss": 0.009, |
| "reward": 0.491050124168396, |
| "reward_std": 0.07163269072771072, |
| "reward_std_all": 0.09569187089800835, |
| "rewards-max/forensics_iou_reward": 0.8930862545967102, |
| "rewards/forensics_iou_reward": 0.491050124168396, |
| "step": 550 |
| }, |
| { |
| "completion_length": 21.078125, |
| "epoch": 2.305439330543933, |
| "grad_norm": 6.052196502685547, |
| "kl": 0.29736328125, |
| "learning_rate": 4.2468619246861926e-07, |
| "loss": 0.0119, |
| "reward": 0.42404037714004517, |
| "reward_std": 0.08012782409787178, |
| "reward_std_all": 0.08076569437980652, |
| "rewards-max/forensics_iou_reward": 0.8668258488178253, |
| "rewards/forensics_iou_reward": 0.42404037714004517, |
| "step": 551 |
| }, |
| { |
| "completion_length": 19.21875, |
| "epoch": 2.309623430962343, |
| "grad_norm": 5.190319061279297, |
| "kl": 0.14990234375, |
| "learning_rate": 4.236401673640167e-07, |
| "loss": 0.006, |
| "reward": 0.591890960931778, |
| "reward_std": 0.06736364588141441, |
| "reward_std_all": 0.07220363989472389, |
| "rewards-max/forensics_iou_reward": 0.8542392551898956, |
| "rewards/forensics_iou_reward": 0.591890960931778, |
| "step": 552 |
| }, |
| { |
| "completion_length": 19.046875, |
| "epoch": 2.3138075313807533, |
| "grad_norm": 8.711341857910156, |
| "kl": 0.583984375, |
| "learning_rate": 4.225941422594142e-07, |
| "loss": 0.0234, |
| "reward": 0.5485439300537109, |
| "reward_std": 0.12058955803513527, |
| "reward_std_all": 0.11244720220565796, |
| "rewards-max/forensics_iou_reward": 0.9029492735862732, |
| "rewards/forensics_iou_reward": 0.5485439300537109, |
| "step": 553 |
| }, |
| { |
| "completion_length": 19.546875, |
| "epoch": 2.3179916317991633, |
| "grad_norm": 3.852173328399658, |
| "kl": 0.2705078125, |
| "learning_rate": 4.215481171548117e-07, |
| "loss": 0.0108, |
| "reward": 0.4850228428840637, |
| "reward_std": 0.1038849987089634, |
| "reward_std_all": 0.10308244824409485, |
| "rewards-max/forensics_iou_reward": 0.9662357866764069, |
| "rewards/forensics_iou_reward": 0.4850228428840637, |
| "step": 554 |
| }, |
| { |
| "completion_length": 19.546875, |
| "epoch": 2.3221757322175733, |
| "grad_norm": 6.438078880310059, |
| "kl": 0.2685546875, |
| "learning_rate": 4.205020920502092e-07, |
| "loss": 0.0107, |
| "reward": 0.5921706557273865, |
| "reward_std": 0.05765519291162491, |
| "reward_std_all": 0.0929030328989029, |
| "rewards-max/forensics_iou_reward": 0.9207355976104736, |
| "rewards/forensics_iou_reward": 0.5921706557273865, |
| "step": 555 |
| }, |
| { |
| "completion_length": 18.984375, |
| "epoch": 2.3263598326359833, |
| "grad_norm": 5.862714767456055, |
| "kl": 0.16162109375, |
| "learning_rate": 4.194560669456067e-07, |
| "loss": 0.0065, |
| "reward": 0.46277791261672974, |
| "reward_std": 0.051128942519426346, |
| "reward_std_all": 0.06216021068394184, |
| "rewards-max/forensics_iou_reward": 0.8553819358348846, |
| "rewards/forensics_iou_reward": 0.46277791261672974, |
| "step": 556 |
| }, |
| { |
| "completion_length": 19.984375, |
| "epoch": 2.3305439330543933, |
| "grad_norm": 4.350245952606201, |
| "kl": 0.3134765625, |
| "learning_rate": 4.184100418410041e-07, |
| "loss": 0.0126, |
| "reward": 0.5396669507026672, |
| "reward_std": 0.09666163846850395, |
| "reward_std_all": 0.1167444959282875, |
| "rewards-max/forensics_iou_reward": 0.8937785029411316, |
| "rewards/forensics_iou_reward": 0.5396669507026672, |
| "step": 557 |
| }, |
| { |
| "completion_length": 19.515625, |
| "epoch": 2.3347280334728033, |
| "grad_norm": 4.817399024963379, |
| "kl": 0.2060546875, |
| "learning_rate": 4.1736401673640164e-07, |
| "loss": 0.0082, |
| "reward": 0.5893628001213074, |
| "reward_std": 0.09736980497837067, |
| "reward_std_all": 0.1095322035253048, |
| "rewards-max/forensics_iou_reward": 0.9477556943893433, |
| "rewards/forensics_iou_reward": 0.5893628001213074, |
| "step": 558 |
| }, |
| { |
| "completion_length": 19.015625, |
| "epoch": 2.3389121338912133, |
| "grad_norm": 7.236514091491699, |
| "kl": 0.1962890625, |
| "learning_rate": 4.1631799163179915e-07, |
| "loss": 0.0079, |
| "reward": 0.5703742206096649, |
| "reward_std": 0.11573859676718712, |
| "reward_std_all": 0.11678797006607056, |
| "rewards-max/forensics_iou_reward": 0.9658472239971161, |
| "rewards/forensics_iou_reward": 0.5703742206096649, |
| "step": 559 |
| }, |
| { |
| "completion_length": 19.546875, |
| "epoch": 2.3430962343096233, |
| "grad_norm": 5.176253318786621, |
| "kl": 0.21337890625, |
| "learning_rate": 4.1527196652719666e-07, |
| "loss": 0.0085, |
| "reward": 0.6529631912708282, |
| "reward_std": 0.08662533760070801, |
| "reward_std_all": 0.10672318190336227, |
| "rewards-max/forensics_iou_reward": 0.922568678855896, |
| "rewards/forensics_iou_reward": 0.6529631912708282, |
| "step": 560 |
| }, |
| { |
| "completion_length": 19.3125, |
| "epoch": 2.3472803347280333, |
| "grad_norm": 5.947503089904785, |
| "kl": 0.2041015625, |
| "learning_rate": 4.142259414225941e-07, |
| "loss": 0.0082, |
| "reward": 0.5977965295314789, |
| "reward_std": 0.06742040812969208, |
| "reward_std_all": 0.0698176808655262, |
| "rewards-max/forensics_iou_reward": 0.922806054353714, |
| "rewards/forensics_iou_reward": 0.5977965295314789, |
| "step": 561 |
| }, |
| { |
| "completion_length": 19.375, |
| "epoch": 2.3514644351464433, |
| "grad_norm": 5.338788986206055, |
| "kl": 0.19384765625, |
| "learning_rate": 4.131799163179916e-07, |
| "loss": 0.0078, |
| "reward": 0.5031887292861938, |
| "reward_std": 0.0788910947740078, |
| "reward_std_all": 0.08053633198142052, |
| "rewards-max/forensics_iou_reward": 0.9195989966392517, |
| "rewards/forensics_iou_reward": 0.5031887292861938, |
| "step": 562 |
| }, |
| { |
| "completion_length": 19.078125, |
| "epoch": 2.3556485355648533, |
| "grad_norm": 5.636781215667725, |
| "kl": 0.24951171875, |
| "learning_rate": 4.121338912133891e-07, |
| "loss": 0.01, |
| "reward": 0.579261988401413, |
| "reward_std": 0.0636401828378439, |
| "reward_std_all": 0.06801888719201088, |
| "rewards-max/forensics_iou_reward": 0.9404742121696472, |
| "rewards/forensics_iou_reward": 0.579261988401413, |
| "step": 563 |
| }, |
| { |
| "completion_length": 19.203125, |
| "epoch": 2.3598326359832638, |
| "grad_norm": 10.759238243103027, |
| "kl": 0.98193359375, |
| "learning_rate": 4.110878661087866e-07, |
| "loss": 0.0393, |
| "reward": 0.6214332580566406, |
| "reward_std": 0.08024240285158157, |
| "reward_std_all": 0.09206952527165413, |
| "rewards-max/forensics_iou_reward": 0.9427420794963837, |
| "rewards/forensics_iou_reward": 0.6214332580566406, |
| "step": 564 |
| }, |
| { |
| "completion_length": 19.125, |
| "epoch": 2.3640167364016738, |
| "grad_norm": 7.433464050292969, |
| "kl": 0.15234375, |
| "learning_rate": 4.100418410041841e-07, |
| "loss": 0.0061, |
| "reward": 0.5748427510261536, |
| "reward_std": 0.045866090804338455, |
| "reward_std_all": 0.05587188899517059, |
| "rewards-max/forensics_iou_reward": 0.9559752643108368, |
| "rewards/forensics_iou_reward": 0.5748427510261536, |
| "step": 565 |
| }, |
| { |
| "completion_length": 18.984375, |
| "epoch": 2.3682008368200838, |
| "grad_norm": 9.359265327453613, |
| "kl": 0.162109375, |
| "learning_rate": 4.089958158995816e-07, |
| "loss": 0.0065, |
| "reward": 0.5219358503818512, |
| "reward_std": 0.09495491161942482, |
| "reward_std_all": 0.08999038860201836, |
| "rewards-max/forensics_iou_reward": 0.9462085962295532, |
| "rewards/forensics_iou_reward": 0.5219358503818512, |
| "step": 566 |
| }, |
| { |
| "completion_length": 19.796875, |
| "epoch": 2.3723849372384938, |
| "grad_norm": 4.951280117034912, |
| "kl": 0.13818359375, |
| "learning_rate": 4.0794979079497904e-07, |
| "loss": 0.0055, |
| "reward": 0.5305206775665283, |
| "reward_std": 0.06366849690675735, |
| "reward_std_all": 0.08142794668674469, |
| "rewards-max/forensics_iou_reward": 0.9350000023841858, |
| "rewards/forensics_iou_reward": 0.5305206775665283, |
| "step": 567 |
| }, |
| { |
| "completion_length": 19.625, |
| "epoch": 2.3765690376569037, |
| "grad_norm": 7.493334770202637, |
| "kl": 0.21337890625, |
| "learning_rate": 4.0690376569037656e-07, |
| "loss": 0.0086, |
| "reward": 0.46354494988918304, |
| "reward_std": 0.0663374587893486, |
| "reward_std_all": 0.09686511009931564, |
| "rewards-max/forensics_iou_reward": 0.9259005188941956, |
| "rewards/forensics_iou_reward": 0.46354494988918304, |
| "step": 568 |
| }, |
| { |
| "completion_length": 19.453125, |
| "epoch": 2.3807531380753137, |
| "grad_norm": 3.8079380989074707, |
| "kl": 0.18603515625, |
| "learning_rate": 4.0585774058577407e-07, |
| "loss": 0.0074, |
| "reward": 0.5652368813753128, |
| "reward_std": 0.07630596682429314, |
| "reward_std_all": 0.07661058194935322, |
| "rewards-max/forensics_iou_reward": 0.9397499859333038, |
| "rewards/forensics_iou_reward": 0.5652368813753128, |
| "step": 569 |
| }, |
| { |
| "completion_length": 19.875, |
| "epoch": 2.3849372384937237, |
| "grad_norm": 4.7792134284973145, |
| "kl": 0.3203125, |
| "learning_rate": 4.0481171548117153e-07, |
| "loss": 0.0128, |
| "reward": 0.5847038924694061, |
| "reward_std": 0.06939688883721828, |
| "reward_std_all": 0.0721498541533947, |
| "rewards-max/forensics_iou_reward": 0.9794100224971771, |
| "rewards/forensics_iou_reward": 0.5847038924694061, |
| "step": 570 |
| }, |
| { |
| "completion_length": 19.171875, |
| "epoch": 2.3891213389121337, |
| "grad_norm": 4.084263801574707, |
| "kl": 0.21728515625, |
| "learning_rate": 4.0376569037656904e-07, |
| "loss": 0.0087, |
| "reward": 0.6357080042362213, |
| "reward_std": 0.08801263570785522, |
| "reward_std_all": 0.09403364732861519, |
| "rewards-max/forensics_iou_reward": 0.9610955119132996, |
| "rewards/forensics_iou_reward": 0.6357080042362213, |
| "step": 571 |
| }, |
| { |
| "completion_length": 19.703125, |
| "epoch": 2.393305439330544, |
| "grad_norm": 5.028131484985352, |
| "kl": 0.165283203125, |
| "learning_rate": 4.027196652719665e-07, |
| "loss": 0.0066, |
| "reward": 0.4809716194868088, |
| "reward_std": 0.11836396902799606, |
| "reward_std_all": 0.11403081193566322, |
| "rewards-max/forensics_iou_reward": 0.8597024381160736, |
| "rewards/forensics_iou_reward": 0.4809716194868088, |
| "step": 572 |
| }, |
| { |
| "completion_length": 21.296875, |
| "epoch": 2.397489539748954, |
| "grad_norm": 4.798707008361816, |
| "kl": 0.1708984375, |
| "learning_rate": 4.01673640167364e-07, |
| "loss": 0.0068, |
| "reward": 0.6182678043842316, |
| "reward_std": 0.09953453205525875, |
| "reward_std_all": 0.09588855318725109, |
| "rewards-max/forensics_iou_reward": 0.9770741760730743, |
| "rewards/forensics_iou_reward": 0.6182678043842316, |
| "step": 573 |
| }, |
| { |
| "completion_length": 19.375, |
| "epoch": 2.401673640167364, |
| "grad_norm": 4.199126720428467, |
| "kl": 0.2099609375, |
| "learning_rate": 4.006276150627615e-07, |
| "loss": 0.0084, |
| "reward": 0.4934489130973816, |
| "reward_std": 0.09643223509192467, |
| "reward_std_all": 0.09519483894109726, |
| "rewards-max/forensics_iou_reward": 0.9187264442443848, |
| "rewards/forensics_iou_reward": 0.4934489130973816, |
| "step": 574 |
| }, |
| { |
| "completion_length": 20.078125, |
| "epoch": 2.405857740585774, |
| "grad_norm": 5.1685099601745605, |
| "kl": 0.18896484375, |
| "learning_rate": 3.99581589958159e-07, |
| "loss": 0.0076, |
| "reward": 0.6064937114715576, |
| "reward_std": 0.04445246607065201, |
| "reward_std_all": 0.06001163274049759, |
| "rewards-max/forensics_iou_reward": 0.9384546279907227, |
| "rewards/forensics_iou_reward": 0.6064937114715576, |
| "step": 575 |
| }, |
| { |
| "completion_length": 19.90625, |
| "epoch": 2.410041841004184, |
| "grad_norm": 7.092048645019531, |
| "kl": 0.28125, |
| "learning_rate": 3.985355648535565e-07, |
| "loss": 0.0113, |
| "reward": 0.6132737696170807, |
| "reward_std": 0.10486981272697449, |
| "reward_std_all": 0.1286623552441597, |
| "rewards-max/forensics_iou_reward": 0.9571650326251984, |
| "rewards/forensics_iou_reward": 0.6132737696170807, |
| "step": 576 |
| }, |
| { |
| "completion_length": 19.78125, |
| "epoch": 2.414225941422594, |
| "grad_norm": 6.049554824829102, |
| "kl": 0.16650390625, |
| "learning_rate": 3.9748953974895396e-07, |
| "loss": 0.0066, |
| "reward": 0.552889347076416, |
| "reward_std": 0.07248893193900585, |
| "reward_std_all": 0.08051736280322075, |
| "rewards-max/forensics_iou_reward": 0.9280952215194702, |
| "rewards/forensics_iou_reward": 0.552889347076416, |
| "step": 577 |
| }, |
| { |
| "completion_length": 19.203125, |
| "epoch": 2.418410041841004, |
| "grad_norm": 4.903729438781738, |
| "kl": 0.2900390625, |
| "learning_rate": 3.964435146443514e-07, |
| "loss": 0.0116, |
| "reward": 0.5389442890882492, |
| "reward_std": 0.06987258046865463, |
| "reward_std_all": 0.06422711908817291, |
| "rewards-max/forensics_iou_reward": 0.9809986352920532, |
| "rewards/forensics_iou_reward": 0.5389442890882492, |
| "step": 578 |
| }, |
| { |
| "completion_length": 19.09375, |
| "epoch": 2.422594142259414, |
| "grad_norm": 5.29821252822876, |
| "kl": 0.203125, |
| "learning_rate": 3.9539748953974893e-07, |
| "loss": 0.0081, |
| "reward": 0.4856945872306824, |
| "reward_std": 0.06573658995330334, |
| "reward_std_all": 0.06089919991791248, |
| "rewards-max/forensics_iou_reward": 0.9701147675514221, |
| "rewards/forensics_iou_reward": 0.4856945872306824, |
| "step": 579 |
| }, |
| { |
| "completion_length": 19.296875, |
| "epoch": 2.426778242677824, |
| "grad_norm": 5.741705417633057, |
| "kl": 0.24462890625, |
| "learning_rate": 3.9435146443514645e-07, |
| "loss": 0.0098, |
| "reward": 0.5883362591266632, |
| "reward_std": 0.0554497092962265, |
| "reward_std_all": 0.06447143852710724, |
| "rewards-max/forensics_iou_reward": 0.8690283298492432, |
| "rewards/forensics_iou_reward": 0.5883362591266632, |
| "step": 580 |
| }, |
| { |
| "completion_length": 19.25, |
| "epoch": 2.430962343096234, |
| "grad_norm": 7.566201210021973, |
| "kl": 0.2509765625, |
| "learning_rate": 3.9330543933054396e-07, |
| "loss": 0.01, |
| "reward": 0.5218641459941864, |
| "reward_std": 0.07697742432355881, |
| "reward_std_all": 0.09135706722736359, |
| "rewards-max/forensics_iou_reward": 0.913695365190506, |
| "rewards/forensics_iou_reward": 0.5218641459941864, |
| "step": 581 |
| }, |
| { |
| "completion_length": 18.953125, |
| "epoch": 2.435146443514644, |
| "grad_norm": 6.362239837646484, |
| "kl": 0.203125, |
| "learning_rate": 3.9225941422594137e-07, |
| "loss": 0.0081, |
| "reward": 0.49829617142677307, |
| "reward_std": 0.07224717363715172, |
| "reward_std_all": 0.07960584759712219, |
| "rewards-max/forensics_iou_reward": 0.9357989728450775, |
| "rewards/forensics_iou_reward": 0.49829617142677307, |
| "step": 582 |
| }, |
| { |
| "completion_length": 19.65625, |
| "epoch": 2.439330543933054, |
| "grad_norm": 6.914460182189941, |
| "kl": 0.2041015625, |
| "learning_rate": 3.912133891213389e-07, |
| "loss": 0.0082, |
| "reward": 0.6048685312271118, |
| "reward_std": 0.10499535128474236, |
| "reward_std_all": 0.11317381635308266, |
| "rewards-max/forensics_iou_reward": 0.9467732906341553, |
| "rewards/forensics_iou_reward": 0.6048685312271118, |
| "step": 583 |
| }, |
| { |
| "completion_length": 19.0625, |
| "epoch": 2.4435146443514646, |
| "grad_norm": 7.685165882110596, |
| "kl": 0.18701171875, |
| "learning_rate": 3.901673640167364e-07, |
| "loss": 0.0075, |
| "reward": 0.5395180583000183, |
| "reward_std": 0.07461253181099892, |
| "reward_std_all": 0.07339861616492271, |
| "rewards-max/forensics_iou_reward": 0.9401874542236328, |
| "rewards/forensics_iou_reward": 0.5395180583000183, |
| "step": 584 |
| }, |
| { |
| "completion_length": 19.484375, |
| "epoch": 2.4476987447698746, |
| "grad_norm": 5.540282249450684, |
| "kl": 0.19140625, |
| "learning_rate": 3.891213389121339e-07, |
| "loss": 0.0077, |
| "reward": 0.5438709408044815, |
| "reward_std": 0.11623699963092804, |
| "reward_std_all": 0.10566140711307526, |
| "rewards-max/forensics_iou_reward": 0.8934685289859772, |
| "rewards/forensics_iou_reward": 0.5438709408044815, |
| "step": 585 |
| }, |
| { |
| "completion_length": 19.828125, |
| "epoch": 2.4518828451882846, |
| "grad_norm": 11.818536758422852, |
| "kl": 0.263671875, |
| "learning_rate": 3.880753138075314e-07, |
| "loss": 0.0106, |
| "reward": 0.494501531124115, |
| "reward_std": 0.09991594776511192, |
| "reward_std_all": 0.1117788776755333, |
| "rewards-max/forensics_iou_reward": 0.8739394247531891, |
| "rewards/forensics_iou_reward": 0.494501531124115, |
| "step": 586 |
| }, |
| { |
| "completion_length": 19.890625, |
| "epoch": 2.4560669456066946, |
| "grad_norm": 8.675321578979492, |
| "kl": 0.15087890625, |
| "learning_rate": 3.870292887029288e-07, |
| "loss": 0.006, |
| "reward": 0.398063063621521, |
| "reward_std": 0.04484573379158974, |
| "reward_std_all": 0.048002053052186966, |
| "rewards-max/forensics_iou_reward": 0.7655965983867645, |
| "rewards/forensics_iou_reward": 0.398063063621521, |
| "step": 587 |
| }, |
| { |
| "completion_length": 19.65625, |
| "epoch": 2.4602510460251046, |
| "grad_norm": 10.659119606018066, |
| "kl": 0.205078125, |
| "learning_rate": 3.8598326359832634e-07, |
| "loss": 0.0082, |
| "reward": 0.7564293742179871, |
| "reward_std": 0.03317179251462221, |
| "reward_std_all": 0.04828437976539135, |
| "rewards-max/forensics_iou_reward": 0.9827628433704376, |
| "rewards/forensics_iou_reward": 0.7564293742179871, |
| "step": 588 |
| }, |
| { |
| "completion_length": 19.546875, |
| "epoch": 2.4644351464435146, |
| "grad_norm": 5.758215427398682, |
| "kl": 0.1923828125, |
| "learning_rate": 3.8493723849372385e-07, |
| "loss": 0.0077, |
| "reward": 0.631194144487381, |
| "reward_std": 0.0933237262070179, |
| "reward_std_all": 0.08890309557318687, |
| "rewards-max/forensics_iou_reward": 0.9689936339855194, |
| "rewards/forensics_iou_reward": 0.631194144487381, |
| "step": 589 |
| }, |
| { |
| "completion_length": 19.890625, |
| "epoch": 2.4686192468619246, |
| "grad_norm": 6.116886138916016, |
| "kl": 0.1962890625, |
| "learning_rate": 3.8389121338912136e-07, |
| "loss": 0.0078, |
| "reward": 0.5647292584180832, |
| "reward_std": 0.11045624315738678, |
| "reward_std_all": 0.10718714445829391, |
| "rewards-max/forensics_iou_reward": 0.9315787851810455, |
| "rewards/forensics_iou_reward": 0.5647292584180832, |
| "step": 590 |
| }, |
| { |
| "completion_length": 19.4375, |
| "epoch": 2.4728033472803346, |
| "grad_norm": 7.739634037017822, |
| "kl": 0.21728515625, |
| "learning_rate": 3.828451882845188e-07, |
| "loss": 0.0087, |
| "reward": 0.5452326834201813, |
| "reward_std": 0.12476886808872223, |
| "reward_std_all": 0.11572656035423279, |
| "rewards-max/forensics_iou_reward": 0.9860484898090363, |
| "rewards/forensics_iou_reward": 0.5452326834201813, |
| "step": 591 |
| }, |
| { |
| "completion_length": 20.3125, |
| "epoch": 2.476987447698745, |
| "grad_norm": 6.133391380310059, |
| "kl": 0.20849609375, |
| "learning_rate": 3.817991631799163e-07, |
| "loss": 0.0083, |
| "reward": 0.5484471768140793, |
| "reward_std": 0.07737666741013527, |
| "reward_std_all": 0.07592707872390747, |
| "rewards-max/forensics_iou_reward": 0.8878957033157349, |
| "rewards/forensics_iou_reward": 0.5484471768140793, |
| "step": 592 |
| }, |
| { |
| "completion_length": 19.5625, |
| "epoch": 2.481171548117155, |
| "grad_norm": 9.604822158813477, |
| "kl": 0.21044921875, |
| "learning_rate": 3.807531380753138e-07, |
| "loss": 0.0084, |
| "reward": 0.5304426997900009, |
| "reward_std": 0.0658947266638279, |
| "reward_std_all": 0.06502418965101242, |
| "rewards-max/forensics_iou_reward": 0.9115228950977325, |
| "rewards/forensics_iou_reward": 0.5304426997900009, |
| "step": 593 |
| }, |
| { |
| "completion_length": 19.265625, |
| "epoch": 2.485355648535565, |
| "grad_norm": 5.718770503997803, |
| "kl": 0.349609375, |
| "learning_rate": 3.797071129707113e-07, |
| "loss": 0.014, |
| "reward": 0.566256046295166, |
| "reward_std": 0.11732513830065727, |
| "reward_std_all": 0.10900340229272842, |
| "rewards-max/forensics_iou_reward": 0.9388319849967957, |
| "rewards/forensics_iou_reward": 0.566256046295166, |
| "step": 594 |
| }, |
| { |
| "completion_length": 19.0, |
| "epoch": 2.489539748953975, |
| "grad_norm": 5.97997522354126, |
| "kl": 0.18017578125, |
| "learning_rate": 3.7866108786610877e-07, |
| "loss": 0.0072, |
| "reward": 0.6344431042671204, |
| "reward_std": 0.07935217209160328, |
| "reward_std_all": 0.07550067454576492, |
| "rewards-max/forensics_iou_reward": 0.9710057079792023, |
| "rewards/forensics_iou_reward": 0.6344431042671204, |
| "step": 595 |
| }, |
| { |
| "completion_length": 19.09375, |
| "epoch": 2.493723849372385, |
| "grad_norm": 7.556829452514648, |
| "kl": 0.162109375, |
| "learning_rate": 3.776150627615063e-07, |
| "loss": 0.0065, |
| "reward": 0.6136423647403717, |
| "reward_std": 0.08580887317657471, |
| "reward_std_all": 0.0857502929866314, |
| "rewards-max/forensics_iou_reward": 0.9418064951896667, |
| "rewards/forensics_iou_reward": 0.6136423647403717, |
| "step": 596 |
| }, |
| { |
| "completion_length": 19.25, |
| "epoch": 2.497907949790795, |
| "grad_norm": 4.734360218048096, |
| "kl": 0.2119140625, |
| "learning_rate": 3.7656903765690374e-07, |
| "loss": 0.0085, |
| "reward": 0.6116406619548798, |
| "reward_std": 0.07756396755576134, |
| "reward_std_all": 0.06983989104628563, |
| "rewards-max/forensics_iou_reward": 0.9062623679637909, |
| "rewards/forensics_iou_reward": 0.6116406619548798, |
| "step": 597 |
| }, |
| { |
| "completion_length": 19.984375, |
| "epoch": 2.502092050209205, |
| "grad_norm": 5.547091960906982, |
| "kl": 0.29052734375, |
| "learning_rate": 3.7552301255230125e-07, |
| "loss": 0.0116, |
| "reward": 0.5753181576728821, |
| "reward_std": 0.09623568132519722, |
| "reward_std_all": 0.10986270755529404, |
| "rewards-max/forensics_iou_reward": 0.9745136499404907, |
| "rewards/forensics_iou_reward": 0.5753181576728821, |
| "step": 598 |
| }, |
| { |
| "completion_length": 19.453125, |
| "epoch": 2.506276150627615, |
| "grad_norm": 4.319060802459717, |
| "kl": 0.25244140625, |
| "learning_rate": 3.744769874476987e-07, |
| "loss": 0.0101, |
| "reward": 0.584156408905983, |
| "reward_std": 0.03175510931760073, |
| "reward_std_all": 0.04181253910064697, |
| "rewards-max/forensics_iou_reward": 0.9595956802368164, |
| "rewards/forensics_iou_reward": 0.584156408905983, |
| "step": 599 |
| }, |
| { |
| "completion_length": 19.375, |
| "epoch": 2.510460251046025, |
| "grad_norm": 7.203248977661133, |
| "kl": 0.6640625, |
| "learning_rate": 3.734309623430962e-07, |
| "loss": 0.0265, |
| "reward": 0.6187019348144531, |
| "reward_std": 0.11368886008858681, |
| "reward_std_all": 0.10182010754942894, |
| "rewards-max/forensics_iou_reward": 0.9801035225391388, |
| "rewards/forensics_iou_reward": 0.6187019348144531, |
| "step": 600 |
| }, |
| { |
| "completion_length": 20.015625, |
| "epoch": 2.514644351464435, |
| "grad_norm": 4.240921497344971, |
| "kl": 0.314453125, |
| "learning_rate": 3.723849372384937e-07, |
| "loss": 0.0125, |
| "reward": 0.5147935450077057, |
| "reward_std": 0.08044003508985043, |
| "reward_std_all": 0.0692700631916523, |
| "rewards-max/forensics_iou_reward": 0.9416041076183319, |
| "rewards/forensics_iou_reward": 0.5147935450077057, |
| "step": 601 |
| }, |
| { |
| "completion_length": 19.953125, |
| "epoch": 2.518828451882845, |
| "grad_norm": 5.29409122467041, |
| "kl": 0.35498046875, |
| "learning_rate": 3.713389121338912e-07, |
| "loss": 0.0142, |
| "reward": 0.6291642040014267, |
| "reward_std": 0.09790630638599396, |
| "reward_std_all": 0.10622410476207733, |
| "rewards-max/forensics_iou_reward": 0.9593192636966705, |
| "rewards/forensics_iou_reward": 0.6291642040014267, |
| "step": 602 |
| }, |
| { |
| "completion_length": 19.453125, |
| "epoch": 2.523012552301255, |
| "grad_norm": 24.687711715698242, |
| "kl": 0.23291015625, |
| "learning_rate": 3.7029288702928866e-07, |
| "loss": 0.0093, |
| "reward": 0.5030756592750549, |
| "reward_std": 0.10066987574100494, |
| "reward_std_all": 0.11300428211688995, |
| "rewards-max/forensics_iou_reward": 0.9038685858249664, |
| "rewards/forensics_iou_reward": 0.5030756592750549, |
| "step": 603 |
| }, |
| { |
| "completion_length": 19.890625, |
| "epoch": 2.5271966527196654, |
| "grad_norm": 8.80394458770752, |
| "kl": 0.25, |
| "learning_rate": 3.6924686192468617e-07, |
| "loss": 0.01, |
| "reward": 0.5100356787443161, |
| "reward_std": 0.12009497359395027, |
| "reward_std_all": 0.10901231691241264, |
| "rewards-max/forensics_iou_reward": 0.9572427272796631, |
| "rewards/forensics_iou_reward": 0.5100356787443161, |
| "step": 604 |
| }, |
| { |
| "completion_length": 19.4375, |
| "epoch": 2.5313807531380754, |
| "grad_norm": 5.955735683441162, |
| "kl": 0.23046875, |
| "learning_rate": 3.682008368200837e-07, |
| "loss": 0.0092, |
| "reward": 0.578285425901413, |
| "reward_std": 0.10790298879146576, |
| "reward_std_all": 0.12178940325975418, |
| "rewards-max/forensics_iou_reward": 0.9803585708141327, |
| "rewards/forensics_iou_reward": 0.578285425901413, |
| "step": 605 |
| }, |
| { |
| "completion_length": 19.6875, |
| "epoch": 2.5355648535564854, |
| "grad_norm": 8.1458158493042, |
| "kl": 0.25048828125, |
| "learning_rate": 3.6715481171548114e-07, |
| "loss": 0.01, |
| "reward": 0.5361139476299286, |
| "reward_std": 0.08175165578722954, |
| "reward_std_all": 0.08958136290311813, |
| "rewards-max/forensics_iou_reward": 0.949641227722168, |
| "rewards/forensics_iou_reward": 0.5361139476299286, |
| "step": 606 |
| }, |
| { |
| "completion_length": 19.53125, |
| "epoch": 2.5397489539748954, |
| "grad_norm": 7.286552429199219, |
| "kl": 0.3037109375, |
| "learning_rate": 3.661087866108786e-07, |
| "loss": 0.0122, |
| "reward": 0.5954054594039917, |
| "reward_std": 0.11569694802165031, |
| "reward_std_all": 0.11194776743650436, |
| "rewards-max/forensics_iou_reward": 0.9733688533306122, |
| "rewards/forensics_iou_reward": 0.5954054594039917, |
| "step": 607 |
| }, |
| { |
| "completion_length": 19.53125, |
| "epoch": 2.5439330543933054, |
| "grad_norm": 5.057409763336182, |
| "kl": 0.2138671875, |
| "learning_rate": 3.650627615062761e-07, |
| "loss": 0.0085, |
| "reward": 0.5571536719799042, |
| "reward_std": 0.09953396767377853, |
| "reward_std_all": 0.10268870368599892, |
| "rewards-max/forensics_iou_reward": 0.9660535752773285, |
| "rewards/forensics_iou_reward": 0.5571536719799042, |
| "step": 608 |
| }, |
| { |
| "completion_length": 19.28125, |
| "epoch": 2.5481171548117154, |
| "grad_norm": 5.513503551483154, |
| "kl": 0.19140625, |
| "learning_rate": 3.6401673640167363e-07, |
| "loss": 0.0076, |
| "reward": 0.5509865283966064, |
| "reward_std": 0.11182945966720581, |
| "reward_std_all": 0.11796445772051811, |
| "rewards-max/forensics_iou_reward": 0.9434540867805481, |
| "rewards/forensics_iou_reward": 0.5509865283966064, |
| "step": 609 |
| }, |
| { |
| "completion_length": 19.734375, |
| "epoch": 2.5523012552301254, |
| "grad_norm": 4.425666809082031, |
| "kl": 0.23193359375, |
| "learning_rate": 3.6297071129707114e-07, |
| "loss": 0.0093, |
| "reward": 0.5985761880874634, |
| "reward_std": 0.10631951317191124, |
| "reward_std_all": 0.09879902377724648, |
| "rewards-max/forensics_iou_reward": 0.9071108102798462, |
| "rewards/forensics_iou_reward": 0.5985761880874634, |
| "step": 610 |
| }, |
| { |
| "completion_length": 21.453125, |
| "epoch": 2.5564853556485354, |
| "grad_norm": 5.19856071472168, |
| "kl": 0.3046875, |
| "learning_rate": 3.6192468619246855e-07, |
| "loss": 0.0122, |
| "reward": 0.4928654432296753, |
| "reward_std": 0.08077089488506317, |
| "reward_std_all": 0.09709593281149864, |
| "rewards-max/forensics_iou_reward": 0.9782181084156036, |
| "rewards/forensics_iou_reward": 0.4928654432296753, |
| "step": 611 |
| }, |
| { |
| "completion_length": 19.484375, |
| "epoch": 2.560669456066946, |
| "grad_norm": 6.386296272277832, |
| "kl": 0.16015625, |
| "learning_rate": 3.6087866108786606e-07, |
| "loss": 0.0064, |
| "reward": 0.48737478256225586, |
| "reward_std": 0.0445366520434618, |
| "reward_std_all": 0.07277926057577133, |
| "rewards-max/forensics_iou_reward": 0.8556776642799377, |
| "rewards/forensics_iou_reward": 0.48737478256225586, |
| "step": 612 |
| }, |
| { |
| "completion_length": 20.859375, |
| "epoch": 2.564853556485356, |
| "grad_norm": 8.966535568237305, |
| "kl": 0.21484375, |
| "learning_rate": 3.598326359832636e-07, |
| "loss": 0.0086, |
| "reward": 0.5809648633003235, |
| "reward_std": 0.0976930484175682, |
| "reward_std_all": 0.10474608838558197, |
| "rewards-max/forensics_iou_reward": 0.9759944677352905, |
| "rewards/forensics_iou_reward": 0.5809648633003235, |
| "step": 613 |
| }, |
| { |
| "completion_length": 19.640625, |
| "epoch": 2.569037656903766, |
| "grad_norm": 5.590890884399414, |
| "kl": 0.23046875, |
| "learning_rate": 3.587866108786611e-07, |
| "loss": 0.0092, |
| "reward": 0.4878552556037903, |
| "reward_std": 0.050933944061398506, |
| "reward_std_all": 0.06908272206783295, |
| "rewards-max/forensics_iou_reward": 0.9081104397773743, |
| "rewards/forensics_iou_reward": 0.4878552556037903, |
| "step": 614 |
| }, |
| { |
| "completion_length": 20.28125, |
| "epoch": 2.573221757322176, |
| "grad_norm": 6.279561519622803, |
| "kl": 0.21435546875, |
| "learning_rate": 3.577405857740586e-07, |
| "loss": 0.0086, |
| "reward": 0.5674928426742554, |
| "reward_std": 0.05234045535326004, |
| "reward_std_all": 0.06612042896449566, |
| "rewards-max/forensics_iou_reward": 0.9635722935199738, |
| "rewards/forensics_iou_reward": 0.5674928426742554, |
| "step": 615 |
| }, |
| { |
| "completion_length": 19.8125, |
| "epoch": 2.577405857740586, |
| "grad_norm": 6.332793712615967, |
| "kl": 0.19775390625, |
| "learning_rate": 3.56694560669456e-07, |
| "loss": 0.0079, |
| "reward": 0.5250449180603027, |
| "reward_std": 0.08133356645703316, |
| "reward_std_all": 0.09589929133653641, |
| "rewards-max/forensics_iou_reward": 0.8777198791503906, |
| "rewards/forensics_iou_reward": 0.5250449180603027, |
| "step": 616 |
| }, |
| { |
| "completion_length": 20.03125, |
| "epoch": 2.581589958158996, |
| "grad_norm": 5.82543420791626, |
| "kl": 0.2138671875, |
| "learning_rate": 3.556485355648535e-07, |
| "loss": 0.0086, |
| "reward": 0.6179991960525513, |
| "reward_std": 0.09307226911187172, |
| "reward_std_all": 0.09228916838765144, |
| "rewards-max/forensics_iou_reward": 0.972343236207962, |
| "rewards/forensics_iou_reward": 0.6179991960525513, |
| "step": 617 |
| }, |
| { |
| "completion_length": 19.5, |
| "epoch": 2.585774058577406, |
| "grad_norm": 6.845239639282227, |
| "kl": 0.16357421875, |
| "learning_rate": 3.5460251046025103e-07, |
| "loss": 0.0065, |
| "reward": 0.43660813570022583, |
| "reward_std": 0.05628419108688831, |
| "reward_std_all": 0.06807602010667324, |
| "rewards-max/forensics_iou_reward": 0.7829679846763611, |
| "rewards/forensics_iou_reward": 0.43660813570022583, |
| "step": 618 |
| }, |
| { |
| "completion_length": 19.375, |
| "epoch": 2.589958158995816, |
| "grad_norm": 6.069146156311035, |
| "kl": 0.173828125, |
| "learning_rate": 3.5355648535564855e-07, |
| "loss": 0.007, |
| "reward": 0.6715943813323975, |
| "reward_std": 0.05382055789232254, |
| "reward_std_all": 0.07976163737475872, |
| "rewards-max/forensics_iou_reward": 0.9507453441619873, |
| "rewards/forensics_iou_reward": 0.6715943813323975, |
| "step": 619 |
| }, |
| { |
| "completion_length": 19.890625, |
| "epoch": 2.594142259414226, |
| "grad_norm": 6.037130832672119, |
| "kl": 0.2197265625, |
| "learning_rate": 3.52510460251046e-07, |
| "loss": 0.0088, |
| "reward": 0.6751532554626465, |
| "reward_std": 0.08632032573223114, |
| "reward_std_all": 0.07719362527132034, |
| "rewards-max/forensics_iou_reward": 0.9860534071922302, |
| "rewards/forensics_iou_reward": 0.6751532554626465, |
| "step": 620 |
| }, |
| { |
| "completion_length": 21.484375, |
| "epoch": 2.598326359832636, |
| "grad_norm": 5.972627639770508, |
| "kl": 0.23291015625, |
| "learning_rate": 3.5146443514644347e-07, |
| "loss": 0.0093, |
| "reward": 0.5859701335430145, |
| "reward_std": 0.07541960291564465, |
| "reward_std_all": 0.0810646153986454, |
| "rewards-max/forensics_iou_reward": 0.9026914238929749, |
| "rewards/forensics_iou_reward": 0.5859701335430145, |
| "step": 621 |
| }, |
| { |
| "completion_length": 19.921875, |
| "epoch": 2.602510460251046, |
| "grad_norm": 6.7778143882751465, |
| "kl": 0.18359375, |
| "learning_rate": 3.50418410041841e-07, |
| "loss": 0.0073, |
| "reward": 0.6387756168842316, |
| "reward_std": 0.054233914241194725, |
| "reward_std_all": 0.05649173632264137, |
| "rewards-max/forensics_iou_reward": 0.9723440110683441, |
| "rewards/forensics_iou_reward": 0.6387756168842316, |
| "step": 622 |
| }, |
| { |
| "completion_length": 20.15625, |
| "epoch": 2.606694560669456, |
| "grad_norm": 6.44041109085083, |
| "kl": 0.174560546875, |
| "learning_rate": 3.493723849372385e-07, |
| "loss": 0.007, |
| "reward": 0.48030059039592743, |
| "reward_std": 0.07493693195283413, |
| "reward_std_all": 0.07959511131048203, |
| "rewards-max/forensics_iou_reward": 0.9366238117218018, |
| "rewards/forensics_iou_reward": 0.48030059039592743, |
| "step": 623 |
| }, |
| { |
| "completion_length": 19.703125, |
| "epoch": 2.6108786610878663, |
| "grad_norm": 5.1883158683776855, |
| "kl": 0.2724609375, |
| "learning_rate": 3.4832635983263595e-07, |
| "loss": 0.0109, |
| "reward": 0.4518117308616638, |
| "reward_std": 0.08375141024589539, |
| "reward_std_all": 0.08861787244677544, |
| "rewards-max/forensics_iou_reward": 0.9370011687278748, |
| "rewards/forensics_iou_reward": 0.4518117308616638, |
| "step": 624 |
| }, |
| { |
| "completion_length": 19.296875, |
| "epoch": 2.6150627615062763, |
| "grad_norm": 4.726305961608887, |
| "kl": 0.24609375, |
| "learning_rate": 3.4728033472803346e-07, |
| "loss": 0.0098, |
| "reward": 0.5301078259944916, |
| "reward_std": 0.06523517519235611, |
| "reward_std_all": 0.07846333086490631, |
| "rewards-max/forensics_iou_reward": 0.9786477088928223, |
| "rewards/forensics_iou_reward": 0.5301078259944916, |
| "step": 625 |
| }, |
| { |
| "completion_length": 19.6875, |
| "epoch": 2.6192468619246863, |
| "grad_norm": 5.599084854125977, |
| "kl": 0.30322265625, |
| "learning_rate": 3.462343096234309e-07, |
| "loss": 0.0122, |
| "reward": 0.3785698711872101, |
| "reward_std": 0.052331551909446716, |
| "reward_std_all": 0.050726963207125664, |
| "rewards-max/forensics_iou_reward": 0.9230089783668518, |
| "rewards/forensics_iou_reward": 0.3785698711872101, |
| "step": 626 |
| }, |
| { |
| "completion_length": 22.09375, |
| "epoch": 2.6234309623430963, |
| "grad_norm": 7.455549716949463, |
| "kl": 0.25537109375, |
| "learning_rate": 3.4518828451882844e-07, |
| "loss": 0.0102, |
| "reward": 0.5211606919765472, |
| "reward_std": 0.06223337724804878, |
| "reward_std_all": 0.06341508589684963, |
| "rewards-max/forensics_iou_reward": 0.8330819606781006, |
| "rewards/forensics_iou_reward": 0.5211606919765472, |
| "step": 627 |
| }, |
| { |
| "completion_length": 21.96875, |
| "epoch": 2.6276150627615062, |
| "grad_norm": 5.703839302062988, |
| "kl": 0.19140625, |
| "learning_rate": 3.441422594142259e-07, |
| "loss": 0.0077, |
| "reward": 0.6169096529483795, |
| "reward_std": 0.1178915984928608, |
| "reward_std_all": 0.11197110265493393, |
| "rewards-max/forensics_iou_reward": 0.968899667263031, |
| "rewards/forensics_iou_reward": 0.6169096529483795, |
| "step": 628 |
| }, |
| { |
| "completion_length": 21.734375, |
| "epoch": 2.6317991631799162, |
| "grad_norm": 8.473784446716309, |
| "kl": 0.392578125, |
| "learning_rate": 3.430962343096234e-07, |
| "loss": 0.0158, |
| "reward": 0.592436283826828, |
| "reward_std": 0.11291155219078064, |
| "reward_std_all": 0.1083463802933693, |
| "rewards-max/forensics_iou_reward": 0.9578059315681458, |
| "rewards/forensics_iou_reward": 0.592436283826828, |
| "step": 629 |
| }, |
| { |
| "completion_length": 19.34375, |
| "epoch": 2.6359832635983262, |
| "grad_norm": 7.056461334228516, |
| "kl": 0.3544921875, |
| "learning_rate": 3.420502092050209e-07, |
| "loss": 0.0142, |
| "reward": 0.5383757948875427, |
| "reward_std": 0.11187298595905304, |
| "reward_std_all": 0.10301100090146065, |
| "rewards-max/forensics_iou_reward": 0.9604062736034393, |
| "rewards/forensics_iou_reward": 0.5383757948875427, |
| "step": 630 |
| }, |
| { |
| "completion_length": 19.5, |
| "epoch": 2.6401673640167362, |
| "grad_norm": 5.031503677368164, |
| "kl": 0.22119140625, |
| "learning_rate": 3.410041841004184e-07, |
| "loss": 0.0088, |
| "reward": 0.6642116904258728, |
| "reward_std": 0.07495866902172565, |
| "reward_std_all": 0.09966909512877464, |
| "rewards-max/forensics_iou_reward": 0.9334243834018707, |
| "rewards/forensics_iou_reward": 0.6642116904258728, |
| "step": 631 |
| }, |
| { |
| "completion_length": 20.65625, |
| "epoch": 2.6443514644351467, |
| "grad_norm": 5.169902324676514, |
| "kl": 0.21337890625, |
| "learning_rate": 3.3995815899581584e-07, |
| "loss": 0.0085, |
| "reward": 0.6001509130001068, |
| "reward_std": 0.0731675811111927, |
| "reward_std_all": 0.07654489576816559, |
| "rewards-max/forensics_iou_reward": 0.9648762345314026, |
| "rewards/forensics_iou_reward": 0.6001509130001068, |
| "step": 632 |
| }, |
| { |
| "completion_length": 20.71875, |
| "epoch": 2.6485355648535567, |
| "grad_norm": 8.470229148864746, |
| "kl": 0.19921875, |
| "learning_rate": 3.3891213389121336e-07, |
| "loss": 0.008, |
| "reward": 0.6332887709140778, |
| "reward_std": 0.07894948497414589, |
| "reward_std_all": 0.08281958475708961, |
| "rewards-max/forensics_iou_reward": 0.9594559073448181, |
| "rewards/forensics_iou_reward": 0.6332887709140778, |
| "step": 633 |
| }, |
| { |
| "completion_length": 20.53125, |
| "epoch": 2.6527196652719667, |
| "grad_norm": 5.282978057861328, |
| "kl": 0.24755859375, |
| "learning_rate": 3.3786610878661087e-07, |
| "loss": 0.0099, |
| "reward": 0.5599635541439056, |
| "reward_std": 0.12885113060474396, |
| "reward_std_all": 0.12160998210310936, |
| "rewards-max/forensics_iou_reward": 0.9695312976837158, |
| "rewards/forensics_iou_reward": 0.5599635541439056, |
| "step": 634 |
| }, |
| { |
| "completion_length": 19.359375, |
| "epoch": 2.6569037656903767, |
| "grad_norm": 8.925328254699707, |
| "kl": 0.36865234375, |
| "learning_rate": 3.368200836820084e-07, |
| "loss": 0.0147, |
| "reward": 0.6246571242809296, |
| "reward_std": 0.08896806836128235, |
| "reward_std_all": 0.10391237586736679, |
| "rewards-max/forensics_iou_reward": 0.9333495497703552, |
| "rewards/forensics_iou_reward": 0.6246571242809296, |
| "step": 635 |
| }, |
| { |
| "completion_length": 22.8125, |
| "epoch": 2.6610878661087867, |
| "grad_norm": 8.596258163452148, |
| "kl": 0.19580078125, |
| "learning_rate": 3.3577405857740584e-07, |
| "loss": 0.0078, |
| "reward": 0.5905107259750366, |
| "reward_std": 0.08217628300189972, |
| "reward_std_all": 0.09947672486305237, |
| "rewards-max/forensics_iou_reward": 0.9358455538749695, |
| "rewards/forensics_iou_reward": 0.5905107259750366, |
| "step": 636 |
| }, |
| { |
| "completion_length": 20.203125, |
| "epoch": 2.6652719665271967, |
| "grad_norm": 4.486676216125488, |
| "kl": 0.25830078125, |
| "learning_rate": 3.347280334728033e-07, |
| "loss": 0.0103, |
| "reward": 0.5812248587608337, |
| "reward_std": 0.07387968525290489, |
| "reward_std_all": 0.07637915574014187, |
| "rewards-max/forensics_iou_reward": 0.9121749699115753, |
| "rewards/forensics_iou_reward": 0.5812248587608337, |
| "step": 637 |
| }, |
| { |
| "completion_length": 20.6875, |
| "epoch": 2.6694560669456067, |
| "grad_norm": 5.938102722167969, |
| "kl": 0.2451171875, |
| "learning_rate": 3.336820083682008e-07, |
| "loss": 0.0098, |
| "reward": 0.5349158644676208, |
| "reward_std": 0.08295386284589767, |
| "reward_std_all": 0.07991372793912888, |
| "rewards-max/forensics_iou_reward": 0.9039276540279388, |
| "rewards/forensics_iou_reward": 0.5349158644676208, |
| "step": 638 |
| }, |
| { |
| "completion_length": 19.75, |
| "epoch": 2.6736401673640167, |
| "grad_norm": 5.297666549682617, |
| "kl": 0.20947265625, |
| "learning_rate": 3.3263598326359833e-07, |
| "loss": 0.0084, |
| "reward": 0.573955625295639, |
| "reward_std": 0.0680466741323471, |
| "reward_std_all": 0.06481539271771908, |
| "rewards-max/forensics_iou_reward": 0.9402119219303131, |
| "rewards/forensics_iou_reward": 0.573955625295639, |
| "step": 639 |
| }, |
| { |
| "completion_length": 22.34375, |
| "epoch": 2.6778242677824267, |
| "grad_norm": 4.352445602416992, |
| "kl": 0.2119140625, |
| "learning_rate": 3.3158995815899584e-07, |
| "loss": 0.0085, |
| "reward": 0.3837111294269562, |
| "reward_std": 0.0576246976852417, |
| "reward_std_all": 0.08278223127126694, |
| "rewards-max/forensics_iou_reward": 0.8678722381591797, |
| "rewards/forensics_iou_reward": 0.3837111294269562, |
| "step": 640 |
| }, |
| { |
| "completion_length": 19.6875, |
| "epoch": 2.6820083682008367, |
| "grad_norm": 6.416106224060059, |
| "kl": 0.29833984375, |
| "learning_rate": 3.3054393305439325e-07, |
| "loss": 0.0119, |
| "reward": 0.5369485020637512, |
| "reward_std": 0.059534622356295586, |
| "reward_std_all": 0.053477220237255096, |
| "rewards-max/forensics_iou_reward": 0.8906320333480835, |
| "rewards/forensics_iou_reward": 0.5369485020637512, |
| "step": 641 |
| }, |
| { |
| "completion_length": 19.5, |
| "epoch": 2.6861924686192467, |
| "grad_norm": 5.053671360015869, |
| "kl": 0.27392578125, |
| "learning_rate": 3.2949790794979076e-07, |
| "loss": 0.011, |
| "reward": 0.6032501757144928, |
| "reward_std": 0.056983707472682, |
| "reward_std_all": 0.07368767634034157, |
| "rewards-max/forensics_iou_reward": 0.9578810334205627, |
| "rewards/forensics_iou_reward": 0.6032501757144928, |
| "step": 642 |
| }, |
| { |
| "completion_length": 20.40625, |
| "epoch": 2.6903765690376567, |
| "grad_norm": 7.8649702072143555, |
| "kl": 0.294921875, |
| "learning_rate": 3.2845188284518827e-07, |
| "loss": 0.0118, |
| "reward": 0.43228377401828766, |
| "reward_std": 0.09267282858490944, |
| "reward_std_all": 0.09220236912369728, |
| "rewards-max/forensics_iou_reward": 0.9797789454460144, |
| "rewards/forensics_iou_reward": 0.43228377401828766, |
| "step": 643 |
| }, |
| { |
| "completion_length": 20.0625, |
| "epoch": 2.694560669456067, |
| "grad_norm": 6.046426296234131, |
| "kl": 0.23681640625, |
| "learning_rate": 3.274058577405858e-07, |
| "loss": 0.0095, |
| "reward": 0.6014449596405029, |
| "reward_std": 0.06822500936686993, |
| "reward_std_all": 0.06692889891564846, |
| "rewards-max/forensics_iou_reward": 0.9617104530334473, |
| "rewards/forensics_iou_reward": 0.6014449596405029, |
| "step": 644 |
| }, |
| { |
| "completion_length": 19.640625, |
| "epoch": 2.698744769874477, |
| "grad_norm": 4.97908353805542, |
| "kl": 0.23876953125, |
| "learning_rate": 3.2635983263598325e-07, |
| "loss": 0.0096, |
| "reward": 0.5544523149728775, |
| "reward_std": 0.04565433785319328, |
| "reward_std_all": 0.07108021341264248, |
| "rewards-max/forensics_iou_reward": 0.9737419486045837, |
| "rewards/forensics_iou_reward": 0.5544523149728775, |
| "step": 645 |
| }, |
| { |
| "completion_length": 20.65625, |
| "epoch": 2.702928870292887, |
| "grad_norm": 7.213099479675293, |
| "kl": 0.2734375, |
| "learning_rate": 3.253138075313807e-07, |
| "loss": 0.011, |
| "reward": 0.5503025054931641, |
| "reward_std": 0.09753555431962013, |
| "reward_std_all": 0.09287155792117119, |
| "rewards-max/forensics_iou_reward": 0.8723247647285461, |
| "rewards/forensics_iou_reward": 0.5503025054931641, |
| "step": 646 |
| }, |
| { |
| "completion_length": 20.546875, |
| "epoch": 2.707112970711297, |
| "grad_norm": 4.553412914276123, |
| "kl": 0.25146484375, |
| "learning_rate": 3.242677824267782e-07, |
| "loss": 0.01, |
| "reward": 0.5752085745334625, |
| "reward_std": 0.10776720196008682, |
| "reward_std_all": 0.10092145949602127, |
| "rewards-max/forensics_iou_reward": 0.9286017119884491, |
| "rewards/forensics_iou_reward": 0.5752085745334625, |
| "step": 647 |
| }, |
| { |
| "completion_length": 20.203125, |
| "epoch": 2.711297071129707, |
| "grad_norm": 5.949308395385742, |
| "kl": 0.18505859375, |
| "learning_rate": 3.2322175732217573e-07, |
| "loss": 0.0074, |
| "reward": 0.6004785597324371, |
| "reward_std": 0.12132970988750458, |
| "reward_std_all": 0.11610154062509537, |
| "rewards-max/forensics_iou_reward": 0.9174311757087708, |
| "rewards/forensics_iou_reward": 0.6004785597324371, |
| "step": 648 |
| }, |
| { |
| "completion_length": 20.703125, |
| "epoch": 2.715481171548117, |
| "grad_norm": 5.711065292358398, |
| "kl": 0.23486328125, |
| "learning_rate": 3.221757322175732e-07, |
| "loss": 0.0094, |
| "reward": 0.5220359414815903, |
| "reward_std": 0.07233402505517006, |
| "reward_std_all": 0.08614738844335079, |
| "rewards-max/forensics_iou_reward": 0.9124970436096191, |
| "rewards/forensics_iou_reward": 0.5220359414815903, |
| "step": 649 |
| }, |
| { |
| "completion_length": 20.109375, |
| "epoch": 2.719665271966527, |
| "grad_norm": 7.274899482727051, |
| "kl": 0.166015625, |
| "learning_rate": 3.211297071129707e-07, |
| "loss": 0.0066, |
| "reward": 0.540494978427887, |
| "reward_std": 0.05302170477807522, |
| "reward_std_all": 0.06744766794145107, |
| "rewards-max/forensics_iou_reward": 0.8668875098228455, |
| "rewards/forensics_iou_reward": 0.540494978427887, |
| "step": 650 |
| }, |
| { |
| "completion_length": 19.375, |
| "epoch": 2.723849372384937, |
| "grad_norm": 5.554447174072266, |
| "kl": 0.27392578125, |
| "learning_rate": 3.2008368200836816e-07, |
| "loss": 0.011, |
| "reward": 0.5888055562973022, |
| "reward_std": 0.10493575409054756, |
| "reward_std_all": 0.11282006651163101, |
| "rewards-max/forensics_iou_reward": 0.9211722314357758, |
| "rewards/forensics_iou_reward": 0.5888055562973022, |
| "step": 651 |
| }, |
| { |
| "completion_length": 20.71875, |
| "epoch": 2.7280334728033475, |
| "grad_norm": 5.964080333709717, |
| "kl": 0.18310546875, |
| "learning_rate": 3.190376569037657e-07, |
| "loss": 0.0073, |
| "reward": 0.5696921646595001, |
| "reward_std": 0.06792013347148895, |
| "reward_std_all": 0.09429286047816277, |
| "rewards-max/forensics_iou_reward": 0.9643504023551941, |
| "rewards/forensics_iou_reward": 0.5696921646595001, |
| "step": 652 |
| }, |
| { |
| "completion_length": 19.671875, |
| "epoch": 2.7322175732217575, |
| "grad_norm": 21.185270309448242, |
| "kl": 0.26806640625, |
| "learning_rate": 3.1799163179916314e-07, |
| "loss": 0.0107, |
| "reward": 0.49595655500888824, |
| "reward_std": 0.08709670603275299, |
| "reward_std_all": 0.09323317557573318, |
| "rewards-max/forensics_iou_reward": 0.9116002917289734, |
| "rewards/forensics_iou_reward": 0.49595655500888824, |
| "step": 653 |
| }, |
| { |
| "completion_length": 19.40625, |
| "epoch": 2.7364016736401675, |
| "grad_norm": 4.913635730743408, |
| "kl": 0.14794921875, |
| "learning_rate": 3.1694560669456065e-07, |
| "loss": 0.0059, |
| "reward": 0.5986993312835693, |
| "reward_std": 0.05463125556707382, |
| "reward_std_all": 0.05313395522534847, |
| "rewards-max/forensics_iou_reward": 0.9574324190616608, |
| "rewards/forensics_iou_reward": 0.5986993312835693, |
| "step": 654 |
| }, |
| { |
| "completion_length": 19.59375, |
| "epoch": 2.7405857740585775, |
| "grad_norm": 8.86190414428711, |
| "kl": 0.3349609375, |
| "learning_rate": 3.1589958158995816e-07, |
| "loss": 0.0134, |
| "reward": 0.456379696726799, |
| "reward_std": 0.06887935567647219, |
| "reward_std_all": 0.06980452314019203, |
| "rewards-max/forensics_iou_reward": 0.8775096237659454, |
| "rewards/forensics_iou_reward": 0.456379696726799, |
| "step": 655 |
| }, |
| { |
| "completion_length": 19.4375, |
| "epoch": 2.7447698744769875, |
| "grad_norm": 5.84404182434082, |
| "kl": 0.17724609375, |
| "learning_rate": 3.148535564853556e-07, |
| "loss": 0.0071, |
| "reward": 0.7281242311000824, |
| "reward_std": 0.049557825550436974, |
| "reward_std_all": 0.07158491387963295, |
| "rewards-max/forensics_iou_reward": 0.9907540082931519, |
| "rewards/forensics_iou_reward": 0.7281242311000824, |
| "step": 656 |
| }, |
| { |
| "completion_length": 21.078125, |
| "epoch": 2.7489539748953975, |
| "grad_norm": 7.228224277496338, |
| "kl": 0.3115234375, |
| "learning_rate": 3.1380753138075313e-07, |
| "loss": 0.0125, |
| "reward": 0.6472000777721405, |
| "reward_std": 0.08846910856664181, |
| "reward_std_all": 0.10649862512946129, |
| "rewards-max/forensics_iou_reward": 0.9718742370605469, |
| "rewards/forensics_iou_reward": 0.6472000777721405, |
| "step": 657 |
| }, |
| { |
| "completion_length": 20.203125, |
| "epoch": 2.7531380753138075, |
| "grad_norm": 5.411347389221191, |
| "kl": 0.2080078125, |
| "learning_rate": 3.127615062761506e-07, |
| "loss": 0.0083, |
| "reward": 0.5334767997264862, |
| "reward_std": 0.06390054151415825, |
| "reward_std_all": 0.08998306840658188, |
| "rewards-max/forensics_iou_reward": 0.9573336839675903, |
| "rewards/forensics_iou_reward": 0.5334767997264862, |
| "step": 658 |
| }, |
| { |
| "completion_length": 21.6875, |
| "epoch": 2.7573221757322175, |
| "grad_norm": 12.233942031860352, |
| "kl": 0.18603515625, |
| "learning_rate": 3.117154811715481e-07, |
| "loss": 0.0074, |
| "reward": 0.43835780024528503, |
| "reward_std": 0.08027282357215881, |
| "reward_std_all": 0.09498659148812294, |
| "rewards-max/forensics_iou_reward": 0.8390823602676392, |
| "rewards/forensics_iou_reward": 0.43835780024528503, |
| "step": 659 |
| }, |
| { |
| "completion_length": 20.40625, |
| "epoch": 2.7615062761506275, |
| "grad_norm": 6.1640143394470215, |
| "kl": 0.23193359375, |
| "learning_rate": 3.106694560669456e-07, |
| "loss": 0.0093, |
| "reward": 0.6681584715843201, |
| "reward_std": 0.07997581362724304, |
| "reward_std_all": 0.09044146537780762, |
| "rewards-max/forensics_iou_reward": 0.9813043475151062, |
| "rewards/forensics_iou_reward": 0.6681584715843201, |
| "step": 660 |
| }, |
| { |
| "completion_length": 20.109375, |
| "epoch": 2.7656903765690375, |
| "grad_norm": 5.485141277313232, |
| "kl": 0.19970703125, |
| "learning_rate": 3.096234309623431e-07, |
| "loss": 0.008, |
| "reward": 0.5852510929107666, |
| "reward_std": 0.08286839723587036, |
| "reward_std_all": 0.08854695037007332, |
| "rewards-max/forensics_iou_reward": 0.9563858807086945, |
| "rewards/forensics_iou_reward": 0.5852510929107666, |
| "step": 661 |
| }, |
| { |
| "completion_length": 20.078125, |
| "epoch": 2.7698744769874475, |
| "grad_norm": 12.608162879943848, |
| "kl": 0.22998046875, |
| "learning_rate": 3.0857740585774054e-07, |
| "loss": 0.0092, |
| "reward": 0.5177568048238754, |
| "reward_std": 0.0818827897310257, |
| "reward_std_all": 0.08941994234919548, |
| "rewards-max/forensics_iou_reward": 0.8453228175640106, |
| "rewards/forensics_iou_reward": 0.5177568048238754, |
| "step": 662 |
| }, |
| { |
| "completion_length": 19.671875, |
| "epoch": 2.7740585774058575, |
| "grad_norm": 5.3217082023620605, |
| "kl": 0.20947265625, |
| "learning_rate": 3.0753138075313805e-07, |
| "loss": 0.0084, |
| "reward": 0.6673120856285095, |
| "reward_std": 0.06620142050087452, |
| "reward_std_all": 0.07496319524943829, |
| "rewards-max/forensics_iou_reward": 0.9939972758293152, |
| "rewards/forensics_iou_reward": 0.6673120856285095, |
| "step": 663 |
| }, |
| { |
| "completion_length": 20.015625, |
| "epoch": 2.778242677824268, |
| "grad_norm": 5.935631275177002, |
| "kl": 0.2119140625, |
| "learning_rate": 3.0648535564853557e-07, |
| "loss": 0.0085, |
| "reward": 0.5571961104869843, |
| "reward_std": 0.08313331380486488, |
| "reward_std_all": 0.07852032780647278, |
| "rewards-max/forensics_iou_reward": 0.9504818320274353, |
| "rewards/forensics_iou_reward": 0.5571961104869843, |
| "step": 664 |
| }, |
| { |
| "completion_length": 19.015625, |
| "epoch": 2.782426778242678, |
| "grad_norm": 10.533642768859863, |
| "kl": 0.2373046875, |
| "learning_rate": 3.054393305439331e-07, |
| "loss": 0.0095, |
| "reward": 0.42205920815467834, |
| "reward_std": 0.06738501414656639, |
| "reward_std_all": 0.08672051131725311, |
| "rewards-max/forensics_iou_reward": 0.9267499446868896, |
| "rewards/forensics_iou_reward": 0.42205920815467834, |
| "step": 665 |
| }, |
| { |
| "completion_length": 20.71875, |
| "epoch": 2.786610878661088, |
| "grad_norm": 8.161139488220215, |
| "kl": 0.2119140625, |
| "learning_rate": 3.043933054393305e-07, |
| "loss": 0.0085, |
| "reward": 0.6449537873268127, |
| "reward_std": 0.08991029299795628, |
| "reward_std_all": 0.09348678216338158, |
| "rewards-max/forensics_iou_reward": 0.9587719440460205, |
| "rewards/forensics_iou_reward": 0.6449537873268127, |
| "step": 666 |
| }, |
| { |
| "completion_length": 20.734375, |
| "epoch": 2.790794979079498, |
| "grad_norm": 5.359894752502441, |
| "kl": 0.26318359375, |
| "learning_rate": 3.03347280334728e-07, |
| "loss": 0.0105, |
| "reward": 0.593361109495163, |
| "reward_std": 0.09200175851583481, |
| "reward_std_all": 0.08890000730752945, |
| "rewards-max/forensics_iou_reward": 0.9051029980182648, |
| "rewards/forensics_iou_reward": 0.593361109495163, |
| "step": 667 |
| }, |
| { |
| "completion_length": 19.359375, |
| "epoch": 2.794979079497908, |
| "grad_norm": 5.565791130065918, |
| "kl": 0.19140625, |
| "learning_rate": 3.023012552301255e-07, |
| "loss": 0.0077, |
| "reward": 0.5275854170322418, |
| "reward_std": 0.09295042976737022, |
| "reward_std_all": 0.0844392403960228, |
| "rewards-max/forensics_iou_reward": 0.9175116717815399, |
| "rewards/forensics_iou_reward": 0.5275854170322418, |
| "step": 668 |
| }, |
| { |
| "completion_length": 19.3125, |
| "epoch": 2.799163179916318, |
| "grad_norm": 5.2742133140563965, |
| "kl": 0.19970703125, |
| "learning_rate": 3.01255230125523e-07, |
| "loss": 0.008, |
| "reward": 0.47087225317955017, |
| "reward_std": 0.07533154636621475, |
| "reward_std_all": 0.0703699141740799, |
| "rewards-max/forensics_iou_reward": 0.9340579509735107, |
| "rewards/forensics_iou_reward": 0.47087225317955017, |
| "step": 669 |
| }, |
| { |
| "completion_length": 22.0625, |
| "epoch": 2.803347280334728, |
| "grad_norm": 6.1527628898620605, |
| "kl": 0.17724609375, |
| "learning_rate": 3.002092050209205e-07, |
| "loss": 0.0071, |
| "reward": 0.4998955875635147, |
| "reward_std": 0.04597029089927673, |
| "reward_std_all": 0.06215078756213188, |
| "rewards-max/forensics_iou_reward": 0.9901264011859894, |
| "rewards/forensics_iou_reward": 0.4998955875635147, |
| "step": 670 |
| }, |
| { |
| "completion_length": 19.46875, |
| "epoch": 2.8075313807531384, |
| "grad_norm": 8.102359771728516, |
| "kl": 0.2275390625, |
| "learning_rate": 2.9916317991631794e-07, |
| "loss": 0.0091, |
| "reward": 0.6175896525382996, |
| "reward_std": 0.11604422330856323, |
| "reward_std_all": 0.10359042510390282, |
| "rewards-max/forensics_iou_reward": 0.9432386755943298, |
| "rewards/forensics_iou_reward": 0.6175896525382996, |
| "step": 671 |
| }, |
| { |
| "completion_length": 20.28125, |
| "epoch": 2.8117154811715483, |
| "grad_norm": 5.811967372894287, |
| "kl": 0.23583984375, |
| "learning_rate": 2.9811715481171546e-07, |
| "loss": 0.0094, |
| "reward": 0.6478386223316193, |
| "reward_std": 0.08230705559253693, |
| "reward_std_all": 0.08535172790288925, |
| "rewards-max/forensics_iou_reward": 0.9471360445022583, |
| "rewards/forensics_iou_reward": 0.6478386223316193, |
| "step": 672 |
| }, |
| { |
| "completion_length": 21.046875, |
| "epoch": 2.8158995815899583, |
| "grad_norm": 6.4371724128723145, |
| "kl": 0.185546875, |
| "learning_rate": 2.9707112970711297e-07, |
| "loss": 0.0074, |
| "reward": 0.5785920768976212, |
| "reward_std": 0.06714348867535591, |
| "reward_std_all": 0.06520092487335205, |
| "rewards-max/forensics_iou_reward": 0.8579152226448059, |
| "rewards/forensics_iou_reward": 0.5785920768976212, |
| "step": 673 |
| }, |
| { |
| "completion_length": 19.75, |
| "epoch": 2.8200836820083683, |
| "grad_norm": 6.850490570068359, |
| "kl": 0.21435546875, |
| "learning_rate": 2.960251046025105e-07, |
| "loss": 0.0086, |
| "reward": 0.5989234745502472, |
| "reward_std": 0.06913204491138458, |
| "reward_std_all": 0.07098225504159927, |
| "rewards-max/forensics_iou_reward": 0.9338253736495972, |
| "rewards/forensics_iou_reward": 0.5989234745502472, |
| "step": 674 |
| }, |
| { |
| "completion_length": 20.015625, |
| "epoch": 2.8242677824267783, |
| "grad_norm": 7.99572229385376, |
| "kl": 0.1845703125, |
| "learning_rate": 2.9497907949790794e-07, |
| "loss": 0.0074, |
| "reward": 0.4790082275867462, |
| "reward_std": 0.0818212553858757, |
| "reward_std_all": 0.08222034201025963, |
| "rewards-max/forensics_iou_reward": 0.9137502014636993, |
| "rewards/forensics_iou_reward": 0.4790082275867462, |
| "step": 675 |
| }, |
| { |
| "completion_length": 20.34375, |
| "epoch": 2.8284518828451883, |
| "grad_norm": 422.0966491699219, |
| "kl": 12.64013671875, |
| "learning_rate": 2.939330543933054e-07, |
| "loss": 0.5059, |
| "reward": 0.6163082718849182, |
| "reward_std": 0.051357412710785866, |
| "reward_std_all": 0.04914236627519131, |
| "rewards-max/forensics_iou_reward": 0.9792978167533875, |
| "rewards/forensics_iou_reward": 0.6163082718849182, |
| "step": 676 |
| }, |
| { |
| "completion_length": 21.1875, |
| "epoch": 2.8326359832635983, |
| "grad_norm": 5.271423816680908, |
| "kl": 0.21044921875, |
| "learning_rate": 2.928870292887029e-07, |
| "loss": 0.0084, |
| "reward": 0.589468777179718, |
| "reward_std": 0.060648227110505104, |
| "reward_std_all": 0.0688716284930706, |
| "rewards-max/forensics_iou_reward": 0.8750134110450745, |
| "rewards/forensics_iou_reward": 0.589468777179718, |
| "step": 677 |
| }, |
| { |
| "completion_length": 21.59375, |
| "epoch": 2.8368200836820083, |
| "grad_norm": 4.440328598022461, |
| "kl": 0.29736328125, |
| "learning_rate": 2.9184100418410043e-07, |
| "loss": 0.0119, |
| "reward": 0.5041913688182831, |
| "reward_std": 0.11518813669681549, |
| "reward_std_all": 0.10563382878899574, |
| "rewards-max/forensics_iou_reward": 0.8983828127384186, |
| "rewards/forensics_iou_reward": 0.5041913688182831, |
| "step": 678 |
| }, |
| { |
| "completion_length": 21.125, |
| "epoch": 2.8410041841004183, |
| "grad_norm": 8.307473182678223, |
| "kl": 0.1953125, |
| "learning_rate": 2.907949790794979e-07, |
| "loss": 0.0078, |
| "reward": 0.525821715593338, |
| "reward_std": 0.07175155356526375, |
| "reward_std_all": 0.06736538745462894, |
| "rewards-max/forensics_iou_reward": 0.934030294418335, |
| "rewards/forensics_iou_reward": 0.525821715593338, |
| "step": 679 |
| }, |
| { |
| "completion_length": 19.625, |
| "epoch": 2.8451882845188283, |
| "grad_norm": 6.381728649139404, |
| "kl": 0.1640625, |
| "learning_rate": 2.897489539748954e-07, |
| "loss": 0.0066, |
| "reward": 0.48342567682266235, |
| "reward_std": 0.05489182658493519, |
| "reward_std_all": 0.05049131065607071, |
| "rewards-max/forensics_iou_reward": 0.9283588230609894, |
| "rewards/forensics_iou_reward": 0.48342567682266235, |
| "step": 680 |
| }, |
| { |
| "completion_length": 23.484375, |
| "epoch": 2.8493723849372383, |
| "grad_norm": 8.778007507324219, |
| "kl": 0.2138671875, |
| "learning_rate": 2.8870292887029286e-07, |
| "loss": 0.0085, |
| "reward": 0.46309715509414673, |
| "reward_std": 0.05991026386618614, |
| "reward_std_all": 0.08273226767778397, |
| "rewards-max/forensics_iou_reward": 0.7889388799667358, |
| "rewards/forensics_iou_reward": 0.46309715509414673, |
| "step": 681 |
| }, |
| { |
| "completion_length": 20.125, |
| "epoch": 2.8535564853556483, |
| "grad_norm": 8.000213623046875, |
| "kl": 0.2373046875, |
| "learning_rate": 2.8765690376569037e-07, |
| "loss": 0.0095, |
| "reward": 0.6101501286029816, |
| "reward_std": 0.062238194048404694, |
| "reward_std_all": 0.07403507828712463, |
| "rewards-max/forensics_iou_reward": 0.9192223846912384, |
| "rewards/forensics_iou_reward": 0.6101501286029816, |
| "step": 682 |
| }, |
| { |
| "completion_length": 20.84375, |
| "epoch": 2.8577405857740583, |
| "grad_norm": 7.916703701019287, |
| "kl": 0.236328125, |
| "learning_rate": 2.8661087866108783e-07, |
| "loss": 0.0094, |
| "reward": 0.5125181376934052, |
| "reward_std": 0.11119095236063004, |
| "reward_std_all": 0.10587369650602341, |
| "rewards-max/forensics_iou_reward": 0.8907282054424286, |
| "rewards/forensics_iou_reward": 0.5125181376934052, |
| "step": 683 |
| }, |
| { |
| "completion_length": 20.28125, |
| "epoch": 2.8619246861924688, |
| "grad_norm": 4.995837211608887, |
| "kl": 0.20166015625, |
| "learning_rate": 2.8556485355648535e-07, |
| "loss": 0.0081, |
| "reward": 0.5353900790214539, |
| "reward_std": 0.07919760048389435, |
| "reward_std_all": 0.07518276572227478, |
| "rewards-max/forensics_iou_reward": 0.8086346387863159, |
| "rewards/forensics_iou_reward": 0.5353900790214539, |
| "step": 684 |
| }, |
| { |
| "completion_length": 19.21875, |
| "epoch": 2.8661087866108788, |
| "grad_norm": 5.954885482788086, |
| "kl": 0.25146484375, |
| "learning_rate": 2.8451882845188286e-07, |
| "loss": 0.0101, |
| "reward": 0.5896471738815308, |
| "reward_std": 0.06876666098833084, |
| "reward_std_all": 0.08895953372120857, |
| "rewards-max/forensics_iou_reward": 0.9332710802555084, |
| "rewards/forensics_iou_reward": 0.5896471738815308, |
| "step": 685 |
| }, |
| { |
| "completion_length": 19.1875, |
| "epoch": 2.8702928870292888, |
| "grad_norm": 8.822208404541016, |
| "kl": 0.248046875, |
| "learning_rate": 2.834728033472803e-07, |
| "loss": 0.0099, |
| "reward": 0.6618748009204865, |
| "reward_std": 0.0462307333946228, |
| "reward_std_all": 0.08870934881269932, |
| "rewards-max/forensics_iou_reward": 0.9959482252597809, |
| "rewards/forensics_iou_reward": 0.6618748009204865, |
| "step": 686 |
| }, |
| { |
| "completion_length": 20.453125, |
| "epoch": 2.8744769874476988, |
| "grad_norm": 5.897029876708984, |
| "kl": 0.2353515625, |
| "learning_rate": 2.824267782426778e-07, |
| "loss": 0.0094, |
| "reward": 0.6293813288211823, |
| "reward_std": 0.09607967734336853, |
| "reward_std_all": 0.10579415783286095, |
| "rewards-max/forensics_iou_reward": 0.9639493227005005, |
| "rewards/forensics_iou_reward": 0.6293813288211823, |
| "step": 687 |
| }, |
| { |
| "completion_length": 19.5, |
| "epoch": 2.8786610878661087, |
| "grad_norm": 7.4676923751831055, |
| "kl": 0.16845703125, |
| "learning_rate": 2.813807531380753e-07, |
| "loss": 0.0067, |
| "reward": 0.6223922669887543, |
| "reward_std": 0.08854113519191742, |
| "reward_std_all": 0.08407371863722801, |
| "rewards-max/forensics_iou_reward": 0.9594480395317078, |
| "rewards/forensics_iou_reward": 0.6223922669887543, |
| "step": 688 |
| }, |
| { |
| "completion_length": 19.5, |
| "epoch": 2.8828451882845187, |
| "grad_norm": 5.8847880363464355, |
| "kl": 0.2060546875, |
| "learning_rate": 2.803347280334728e-07, |
| "loss": 0.0083, |
| "reward": 0.6327932775020599, |
| "reward_std": 0.05521741136908531, |
| "reward_std_all": 0.06168545037508011, |
| "rewards-max/forensics_iou_reward": 0.9490961730480194, |
| "rewards/forensics_iou_reward": 0.6327932775020599, |
| "step": 689 |
| }, |
| { |
| "completion_length": 21.109375, |
| "epoch": 2.8870292887029287, |
| "grad_norm": 8.098478317260742, |
| "kl": 0.20263671875, |
| "learning_rate": 2.792887029288703e-07, |
| "loss": 0.0081, |
| "reward": 0.3151053339242935, |
| "reward_std": 0.08788052946329117, |
| "reward_std_all": 0.08186617493629456, |
| "rewards-max/forensics_iou_reward": 0.8398086428642273, |
| "rewards/forensics_iou_reward": 0.3151053339242935, |
| "step": 690 |
| }, |
| { |
| "completion_length": 21.75, |
| "epoch": 2.891213389121339, |
| "grad_norm": 7.548036575317383, |
| "kl": 0.4521484375, |
| "learning_rate": 2.782426778242677e-07, |
| "loss": 0.0181, |
| "reward": 0.40625327825546265, |
| "reward_std": 0.10664207488298416, |
| "reward_std_all": 0.11430410668253899, |
| "rewards-max/forensics_iou_reward": 0.8703591227531433, |
| "rewards/forensics_iou_reward": 0.40625327825546265, |
| "step": 691 |
| }, |
| { |
| "completion_length": 20.078125, |
| "epoch": 2.895397489539749, |
| "grad_norm": 6.126254558563232, |
| "kl": 0.25048828125, |
| "learning_rate": 2.7719665271966524e-07, |
| "loss": 0.01, |
| "reward": 0.5833106637001038, |
| "reward_std": 0.13024403154850006, |
| "reward_std_all": 0.12614023685455322, |
| "rewards-max/forensics_iou_reward": 0.96182981133461, |
| "rewards/forensics_iou_reward": 0.5833106637001038, |
| "step": 692 |
| }, |
| { |
| "completion_length": 20.15625, |
| "epoch": 2.899581589958159, |
| "grad_norm": 6.280990123748779, |
| "kl": 0.21923828125, |
| "learning_rate": 2.7615062761506275e-07, |
| "loss": 0.0088, |
| "reward": 0.4284789562225342, |
| "reward_std": 0.06294021755456924, |
| "reward_std_all": 0.10236510261893272, |
| "rewards-max/forensics_iou_reward": 0.7550686001777649, |
| "rewards/forensics_iou_reward": 0.4284789562225342, |
| "step": 693 |
| }, |
| { |
| "completion_length": 20.171875, |
| "epoch": 2.903765690376569, |
| "grad_norm": 4.568995475769043, |
| "kl": 0.24267578125, |
| "learning_rate": 2.7510460251046026e-07, |
| "loss": 0.0097, |
| "reward": 0.5635810196399689, |
| "reward_std": 0.0630902424454689, |
| "reward_std_all": 0.0714602880179882, |
| "rewards-max/forensics_iou_reward": 0.8598558902740479, |
| "rewards/forensics_iou_reward": 0.5635810196399689, |
| "step": 694 |
| }, |
| { |
| "completion_length": 19.265625, |
| "epoch": 2.907949790794979, |
| "grad_norm": 6.661013126373291, |
| "kl": 0.16357421875, |
| "learning_rate": 2.740585774058578e-07, |
| "loss": 0.0065, |
| "reward": 0.6600413024425507, |
| "reward_std": 0.06785779073834419, |
| "reward_std_all": 0.07450610399246216, |
| "rewards-max/forensics_iou_reward": 0.9522274136543274, |
| "rewards/forensics_iou_reward": 0.6600413024425507, |
| "step": 695 |
| }, |
| { |
| "completion_length": 19.890625, |
| "epoch": 2.912133891213389, |
| "grad_norm": 5.345997333526611, |
| "kl": 0.275390625, |
| "learning_rate": 2.730125523012552e-07, |
| "loss": 0.011, |
| "reward": 0.6736137568950653, |
| "reward_std": 0.04793117754161358, |
| "reward_std_all": 0.051773495972156525, |
| "rewards-max/forensics_iou_reward": 0.90538090467453, |
| "rewards/forensics_iou_reward": 0.6736137568950653, |
| "step": 696 |
| }, |
| { |
| "completion_length": 20.46875, |
| "epoch": 2.916317991631799, |
| "grad_norm": 6.856446266174316, |
| "kl": 0.23828125, |
| "learning_rate": 2.719665271966527e-07, |
| "loss": 0.0095, |
| "reward": 0.5056113749742508, |
| "reward_std": 0.09016192331910133, |
| "reward_std_all": 0.08905795030295849, |
| "rewards-max/forensics_iou_reward": 0.8828627467155457, |
| "rewards/forensics_iou_reward": 0.5056113749742508, |
| "step": 697 |
| }, |
| { |
| "completion_length": 19.46875, |
| "epoch": 2.920502092050209, |
| "grad_norm": 5.050252914428711, |
| "kl": 0.2138671875, |
| "learning_rate": 2.709205020920502e-07, |
| "loss": 0.0086, |
| "reward": 0.5627539157867432, |
| "reward_std": 0.05442972108721733, |
| "reward_std_all": 0.07270777598023415, |
| "rewards-max/forensics_iou_reward": 0.9438105821609497, |
| "rewards/forensics_iou_reward": 0.5627539157867432, |
| "step": 698 |
| }, |
| { |
| "completion_length": 21.15625, |
| "epoch": 2.924686192468619, |
| "grad_norm": 5.244876861572266, |
| "kl": 0.2646484375, |
| "learning_rate": 2.698744769874477e-07, |
| "loss": 0.0106, |
| "reward": 0.6714411079883575, |
| "reward_std": 0.0996420681476593, |
| "reward_std_all": 0.0918592270463705, |
| "rewards-max/forensics_iou_reward": 0.9724774956703186, |
| "rewards/forensics_iou_reward": 0.6714411079883575, |
| "step": 699 |
| }, |
| { |
| "completion_length": 19.75, |
| "epoch": 2.928870292887029, |
| "grad_norm": 8.144543647766113, |
| "kl": 0.201171875, |
| "learning_rate": 2.688284518828452e-07, |
| "loss": 0.008, |
| "reward": 0.6315877139568329, |
| "reward_std": 0.07044698111712933, |
| "reward_std_all": 0.06991919875144958, |
| "rewards-max/forensics_iou_reward": 0.966690719127655, |
| "rewards/forensics_iou_reward": 0.6315877139568329, |
| "step": 700 |
| }, |
| { |
| "completion_length": 19.515625, |
| "epoch": 2.933054393305439, |
| "grad_norm": 5.622268199920654, |
| "kl": 0.1552734375, |
| "learning_rate": 2.6778242677824264e-07, |
| "loss": 0.0062, |
| "reward": 0.586273655295372, |
| "reward_std": 0.03434979077428579, |
| "reward_std_all": 0.03301286790519953, |
| "rewards-max/forensics_iou_reward": 0.9818909168243408, |
| "rewards/forensics_iou_reward": 0.586273655295372, |
| "step": 701 |
| }, |
| { |
| "completion_length": 20.203125, |
| "epoch": 2.937238493723849, |
| "grad_norm": 6.463517665863037, |
| "kl": 0.23046875, |
| "learning_rate": 2.6673640167364015e-07, |
| "loss": 0.0092, |
| "reward": 0.541838988661766, |
| "reward_std": 0.04659131541848183, |
| "reward_std_all": 0.06147036328911781, |
| "rewards-max/forensics_iou_reward": 0.9903284013271332, |
| "rewards/forensics_iou_reward": 0.541838988661766, |
| "step": 702 |
| }, |
| { |
| "completion_length": 19.265625, |
| "epoch": 2.941422594142259, |
| "grad_norm": 6.35270881652832, |
| "kl": 0.181640625, |
| "learning_rate": 2.6569037656903767e-07, |
| "loss": 0.0073, |
| "reward": 0.6123281717300415, |
| "reward_std": 0.049209171906113625, |
| "reward_std_all": 0.051315195858478546, |
| "rewards-max/forensics_iou_reward": 0.9755222797393799, |
| "rewards/forensics_iou_reward": 0.6123281717300415, |
| "step": 703 |
| }, |
| { |
| "completion_length": 20.171875, |
| "epoch": 2.9456066945606696, |
| "grad_norm": 7.183456897735596, |
| "kl": 0.232421875, |
| "learning_rate": 2.6464435146443513e-07, |
| "loss": 0.0093, |
| "reward": 0.5261377990245819, |
| "reward_std": 0.04160388931632042, |
| "reward_std_all": 0.05457376129925251, |
| "rewards-max/forensics_iou_reward": 0.847123771905899, |
| "rewards/forensics_iou_reward": 0.5261377990245819, |
| "step": 704 |
| }, |
| { |
| "completion_length": 19.90625, |
| "epoch": 2.9497907949790796, |
| "grad_norm": 6.203114032745361, |
| "kl": 0.1982421875, |
| "learning_rate": 2.6359832635983264e-07, |
| "loss": 0.0079, |
| "reward": 0.5785599499940872, |
| "reward_std": 0.04101710580289364, |
| "reward_std_all": 0.05564303696155548, |
| "rewards-max/forensics_iou_reward": 0.9486276209354401, |
| "rewards/forensics_iou_reward": 0.5785599499940872, |
| "step": 705 |
| }, |
| { |
| "completion_length": 20.890625, |
| "epoch": 2.9539748953974896, |
| "grad_norm": 5.0718889236450195, |
| "kl": 0.232421875, |
| "learning_rate": 2.625523012552301e-07, |
| "loss": 0.0093, |
| "reward": 0.5597297102212906, |
| "reward_std": 0.07738082483410835, |
| "reward_std_all": 0.0972357951104641, |
| "rewards-max/forensics_iou_reward": 0.9484373927116394, |
| "rewards/forensics_iou_reward": 0.5597297102212906, |
| "step": 706 |
| }, |
| { |
| "completion_length": 19.609375, |
| "epoch": 2.9581589958158996, |
| "grad_norm": 7.2950334548950195, |
| "kl": 0.240234375, |
| "learning_rate": 2.615062761506276e-07, |
| "loss": 0.0096, |
| "reward": 0.5771538317203522, |
| "reward_std": 0.12180263549089432, |
| "reward_std_all": 0.11337050795555115, |
| "rewards-max/forensics_iou_reward": 0.9401515126228333, |
| "rewards/forensics_iou_reward": 0.5771538317203522, |
| "step": 707 |
| }, |
| { |
| "completion_length": 19.40625, |
| "epoch": 2.9623430962343096, |
| "grad_norm": 6.421823978424072, |
| "kl": 0.15380859375, |
| "learning_rate": 2.6046025104602507e-07, |
| "loss": 0.0062, |
| "reward": 0.5665729641914368, |
| "reward_std": 0.05399802327156067, |
| "reward_std_all": 0.05936303921043873, |
| "rewards-max/forensics_iou_reward": 0.930281937122345, |
| "rewards/forensics_iou_reward": 0.5665729641914368, |
| "step": 708 |
| }, |
| { |
| "completion_length": 20.546875, |
| "epoch": 2.9665271966527196, |
| "grad_norm": 5.6574387550354, |
| "kl": 0.25390625, |
| "learning_rate": 2.594142259414226e-07, |
| "loss": 0.0102, |
| "reward": 0.6063274145126343, |
| "reward_std": 0.09772376529872417, |
| "reward_std_all": 0.09105821698904037, |
| "rewards-max/forensics_iou_reward": 0.9150562286376953, |
| "rewards/forensics_iou_reward": 0.6063274145126343, |
| "step": 709 |
| }, |
| { |
| "completion_length": 19.53125, |
| "epoch": 2.9707112970711296, |
| "grad_norm": 7.118496417999268, |
| "kl": 0.505859375, |
| "learning_rate": 2.583682008368201e-07, |
| "loss": 0.0202, |
| "reward": 0.5493392646312714, |
| "reward_std": 0.13396130502223969, |
| "reward_std_all": 0.1232045516371727, |
| "rewards-max/forensics_iou_reward": 0.9028353691101074, |
| "rewards/forensics_iou_reward": 0.5493392646312714, |
| "step": 710 |
| }, |
| { |
| "completion_length": 20.0625, |
| "epoch": 2.97489539748954, |
| "grad_norm": 10.442231178283691, |
| "kl": 0.22802734375, |
| "learning_rate": 2.5732217573221756e-07, |
| "loss": 0.0091, |
| "reward": 0.6169359087944031, |
| "reward_std": 0.10977315902709961, |
| "reward_std_all": 0.1001192070543766, |
| "rewards-max/forensics_iou_reward": 0.966961681842804, |
| "rewards/forensics_iou_reward": 0.6169359087944031, |
| "step": 711 |
| }, |
| { |
| "completion_length": 19.4375, |
| "epoch": 2.97907949790795, |
| "grad_norm": 4.323034763336182, |
| "kl": 0.1650390625, |
| "learning_rate": 2.56276150627615e-07, |
| "loss": 0.0066, |
| "reward": 0.6180989444255829, |
| "reward_std": 0.030333279632031918, |
| "reward_std_all": 0.04758835583925247, |
| "rewards-max/forensics_iou_reward": 0.8736709952354431, |
| "rewards/forensics_iou_reward": 0.6180989444255829, |
| "step": 712 |
| }, |
| { |
| "completion_length": 19.34375, |
| "epoch": 2.98326359832636, |
| "grad_norm": 5.337193965911865, |
| "kl": 0.2119140625, |
| "learning_rate": 2.5523012552301253e-07, |
| "loss": 0.0085, |
| "reward": 0.6633444726467133, |
| "reward_std": 0.05608426406979561, |
| "reward_std_all": 0.05427113547921181, |
| "rewards-max/forensics_iou_reward": 0.9070091247558594, |
| "rewards/forensics_iou_reward": 0.6633444726467133, |
| "step": 713 |
| }, |
| { |
| "completion_length": 20.3125, |
| "epoch": 2.98744769874477, |
| "grad_norm": 5.001827716827393, |
| "kl": 0.17919921875, |
| "learning_rate": 2.5418410041841004e-07, |
| "loss": 0.0072, |
| "reward": 0.5865767598152161, |
| "reward_std": 0.06411181390285492, |
| "reward_std_all": 0.061708204448223114, |
| "rewards-max/forensics_iou_reward": 0.9092646837234497, |
| "rewards/forensics_iou_reward": 0.5865767598152161, |
| "step": 714 |
| }, |
| { |
| "completion_length": 20.96875, |
| "epoch": 2.99163179916318, |
| "grad_norm": 5.499790668487549, |
| "kl": 0.2275390625, |
| "learning_rate": 2.5313807531380756e-07, |
| "loss": 0.0091, |
| "reward": 0.5140286684036255, |
| "reward_std": 0.08541734144091606, |
| "reward_std_all": 0.07756312564015388, |
| "rewards-max/forensics_iou_reward": 0.9924902319908142, |
| "rewards/forensics_iou_reward": 0.5140286684036255, |
| "step": 715 |
| }, |
| { |
| "completion_length": 19.921875, |
| "epoch": 2.99581589958159, |
| "grad_norm": 4.729665756225586, |
| "kl": 0.2607421875, |
| "learning_rate": 2.5209205020920496e-07, |
| "loss": 0.0104, |
| "reward": 0.5503662526607513, |
| "reward_std": 0.08145460486412048, |
| "reward_std_all": 0.0862368792295456, |
| "rewards-max/forensics_iou_reward": 0.969565361738205, |
| "rewards/forensics_iou_reward": 0.5503662526607513, |
| "step": 716 |
| }, |
| { |
| "completion_length": 18.85714340209961, |
| "epoch": 3.0, |
| "grad_norm": 6.892569541931152, |
| "kl": 0.1552734375, |
| "learning_rate": 2.510460251046025e-07, |
| "loss": 0.0063, |
| "reward": 0.48578645288944244, |
| "reward_std": 0.06054004468023777, |
| "reward_std_all": 0.04798308573663235, |
| "rewards-max/forensics_iou_reward": 0.6534557640552521, |
| "rewards/forensics_iou_reward": 0.48578645288944244, |
| "step": 717 |
| }, |
| { |
| "completion_length": 19.546875, |
| "epoch": 3.00418410041841, |
| "grad_norm": 9.432703971862793, |
| "kl": 0.16650390625, |
| "learning_rate": 2.5e-07, |
| "loss": 0.0067, |
| "reward": 0.5041728168725967, |
| "reward_std": 0.04918533377349377, |
| "reward_std_all": 0.04826069623231888, |
| "rewards-max/forensics_iou_reward": 0.8722682297229767, |
| "rewards/forensics_iou_reward": 0.5041728168725967, |
| "step": 718 |
| }, |
| { |
| "completion_length": 19.578125, |
| "epoch": 3.00836820083682, |
| "grad_norm": 6.124695777893066, |
| "kl": 0.189453125, |
| "learning_rate": 2.489539748953975e-07, |
| "loss": 0.0076, |
| "reward": 0.6247337162494659, |
| "reward_std": 0.0507295448333025, |
| "reward_std_all": 0.05302694998681545, |
| "rewards-max/forensics_iou_reward": 0.9461855888366699, |
| "rewards/forensics_iou_reward": 0.6247337162494659, |
| "step": 719 |
| }, |
| { |
| "completion_length": 19.625, |
| "epoch": 3.01255230125523, |
| "grad_norm": 4.311613082885742, |
| "kl": 0.1865234375, |
| "learning_rate": 2.4790794979079496e-07, |
| "loss": 0.0075, |
| "reward": 0.6857900321483612, |
| "reward_std": 0.06446470879018307, |
| "reward_std_all": 0.06289664097130299, |
| "rewards-max/forensics_iou_reward": 0.9620952308177948, |
| "rewards/forensics_iou_reward": 0.6857900321483612, |
| "step": 720 |
| }, |
| { |
| "completion_length": 19.4375, |
| "epoch": 3.01673640167364, |
| "grad_norm": 5.9747748374938965, |
| "kl": 0.1826171875, |
| "learning_rate": 2.468619246861925e-07, |
| "loss": 0.0073, |
| "reward": 0.500417172908783, |
| "reward_std": 0.05505803041160107, |
| "reward_std_all": 0.0809147022664547, |
| "rewards-max/forensics_iou_reward": 0.942897379398346, |
| "rewards/forensics_iou_reward": 0.500417172908783, |
| "step": 721 |
| }, |
| { |
| "completion_length": 19.6875, |
| "epoch": 3.02092050209205, |
| "grad_norm": 5.5459723472595215, |
| "kl": 0.14892578125, |
| "learning_rate": 2.4581589958158993e-07, |
| "loss": 0.006, |
| "reward": 0.6544876098632812, |
| "reward_std": 0.06657820753753185, |
| "reward_std_all": 0.06072540208697319, |
| "rewards-max/forensics_iou_reward": 0.9714261293411255, |
| "rewards/forensics_iou_reward": 0.6544876098632812, |
| "step": 722 |
| }, |
| { |
| "completion_length": 19.921875, |
| "epoch": 3.0251046025104604, |
| "grad_norm": 5.447535037994385, |
| "kl": 0.21875, |
| "learning_rate": 2.4476987447698745e-07, |
| "loss": 0.0088, |
| "reward": 0.6708250343799591, |
| "reward_std": 0.06824560277163982, |
| "reward_std_all": 0.06196371838450432, |
| "rewards-max/forensics_iou_reward": 0.9656511843204498, |
| "rewards/forensics_iou_reward": 0.6708250343799591, |
| "step": 723 |
| }, |
| { |
| "completion_length": 19.15625, |
| "epoch": 3.0292887029288704, |
| "grad_norm": 7.939253330230713, |
| "kl": 0.189453125, |
| "learning_rate": 2.4372384937238496e-07, |
| "loss": 0.0076, |
| "reward": 0.6067014336585999, |
| "reward_std": 0.08450144529342651, |
| "reward_std_all": 0.08161279931664467, |
| "rewards-max/forensics_iou_reward": 0.9805123209953308, |
| "rewards/forensics_iou_reward": 0.6067014336585999, |
| "step": 724 |
| }, |
| { |
| "completion_length": 20.40625, |
| "epoch": 3.0334728033472804, |
| "grad_norm": 6.778061866760254, |
| "kl": 0.2431640625, |
| "learning_rate": 2.426778242677824e-07, |
| "loss": 0.0097, |
| "reward": 0.6470783948898315, |
| "reward_std": 0.10905290395021439, |
| "reward_std_all": 0.10536146722733974, |
| "rewards-max/forensics_iou_reward": 0.976728081703186, |
| "rewards/forensics_iou_reward": 0.6470783948898315, |
| "step": 725 |
| }, |
| { |
| "completion_length": 19.578125, |
| "epoch": 3.0376569037656904, |
| "grad_norm": 7.11168098449707, |
| "kl": 0.185546875, |
| "learning_rate": 2.4163179916317993e-07, |
| "loss": 0.0074, |
| "reward": 0.46083569526672363, |
| "reward_std": 0.07048037648200989, |
| "reward_std_all": 0.06688093766570091, |
| "rewards-max/forensics_iou_reward": 0.9051274657249451, |
| "rewards/forensics_iou_reward": 0.46083569526672363, |
| "step": 726 |
| }, |
| { |
| "completion_length": 19.34375, |
| "epoch": 3.0418410041841004, |
| "grad_norm": 6.661019802093506, |
| "kl": 0.2138671875, |
| "learning_rate": 2.405857740585774e-07, |
| "loss": 0.0086, |
| "reward": 0.6501348912715912, |
| "reward_std": 0.05631633289158344, |
| "reward_std_all": 0.054699051193892956, |
| "rewards-max/forensics_iou_reward": 0.9094174206256866, |
| "rewards/forensics_iou_reward": 0.6501348912715912, |
| "step": 727 |
| }, |
| { |
| "completion_length": 19.796875, |
| "epoch": 3.0460251046025104, |
| "grad_norm": 8.001928329467773, |
| "kl": 0.2392578125, |
| "learning_rate": 2.395397489539749e-07, |
| "loss": 0.0096, |
| "reward": 0.5513501763343811, |
| "reward_std": 0.06276539154350758, |
| "reward_std_all": 0.05990402773022652, |
| "rewards-max/forensics_iou_reward": 0.9617409408092499, |
| "rewards/forensics_iou_reward": 0.5513501763343811, |
| "step": 728 |
| }, |
| { |
| "completion_length": 19.125, |
| "epoch": 3.0502092050209204, |
| "grad_norm": 6.64594841003418, |
| "kl": 0.30322265625, |
| "learning_rate": 2.3849372384937237e-07, |
| "loss": 0.0121, |
| "reward": 0.6041203737258911, |
| "reward_std": 0.0761927142739296, |
| "reward_std_all": 0.07135832495987415, |
| "rewards-max/forensics_iou_reward": 0.9545893371105194, |
| "rewards/forensics_iou_reward": 0.6041203737258911, |
| "step": 729 |
| }, |
| { |
| "completion_length": 19.28125, |
| "epoch": 3.0543933054393304, |
| "grad_norm": 6.240353584289551, |
| "kl": 0.19287109375, |
| "learning_rate": 2.3744769874476985e-07, |
| "loss": 0.0077, |
| "reward": 0.6299708187580109, |
| "reward_std": 0.05387216620147228, |
| "reward_std_all": 0.06635772250592709, |
| "rewards-max/forensics_iou_reward": 0.9326648712158203, |
| "rewards/forensics_iou_reward": 0.6299708187580109, |
| "step": 730 |
| }, |
| { |
| "completion_length": 19.25, |
| "epoch": 3.0585774058577404, |
| "grad_norm": 14.598549842834473, |
| "kl": 0.7294921875, |
| "learning_rate": 2.3640167364016736e-07, |
| "loss": 0.0292, |
| "reward": 0.6224877238273621, |
| "reward_std": 0.08576333150267601, |
| "reward_std_all": 0.07426349632441998, |
| "rewards-max/forensics_iou_reward": 0.9329075515270233, |
| "rewards/forensics_iou_reward": 0.6224877238273621, |
| "step": 731 |
| }, |
| { |
| "completion_length": 19.125, |
| "epoch": 3.062761506276151, |
| "grad_norm": 6.00515604019165, |
| "kl": 0.3408203125, |
| "learning_rate": 2.3535564853556482e-07, |
| "loss": 0.0137, |
| "reward": 0.5789096057415009, |
| "reward_std": 0.08553581684827805, |
| "reward_std_all": 0.08132453635334969, |
| "rewards-max/forensics_iou_reward": 0.953250378370285, |
| "rewards/forensics_iou_reward": 0.5789096057415009, |
| "step": 732 |
| }, |
| { |
| "completion_length": 19.421875, |
| "epoch": 3.066945606694561, |
| "grad_norm": 5.8809685707092285, |
| "kl": 0.32373046875, |
| "learning_rate": 2.3430962343096234e-07, |
| "loss": 0.0129, |
| "reward": 0.46148762106895447, |
| "reward_std": 0.08678127452731133, |
| "reward_std_all": 0.08195687457919121, |
| "rewards-max/forensics_iou_reward": 0.8564631342887878, |
| "rewards/forensics_iou_reward": 0.46148762106895447, |
| "step": 733 |
| }, |
| { |
| "completion_length": 20.71875, |
| "epoch": 3.071129707112971, |
| "grad_norm": 4.7215094566345215, |
| "kl": 0.2373046875, |
| "learning_rate": 2.3326359832635982e-07, |
| "loss": 0.0095, |
| "reward": 0.5948365330696106, |
| "reward_std": 0.05980649217963219, |
| "reward_std_all": 0.06365729495882988, |
| "rewards-max/forensics_iou_reward": 0.9534043967723846, |
| "rewards/forensics_iou_reward": 0.5948365330696106, |
| "step": 734 |
| }, |
| { |
| "completion_length": 20.171875, |
| "epoch": 3.075313807531381, |
| "grad_norm": 5.206642150878906, |
| "kl": 0.4736328125, |
| "learning_rate": 2.322175732217573e-07, |
| "loss": 0.019, |
| "reward": 0.5733465254306793, |
| "reward_std": 0.13134196028113365, |
| "reward_std_all": 0.13797365501523018, |
| "rewards-max/forensics_iou_reward": 0.9665870070457458, |
| "rewards/forensics_iou_reward": 0.5733465254306793, |
| "step": 735 |
| }, |
| { |
| "completion_length": 18.9375, |
| "epoch": 3.079497907949791, |
| "grad_norm": 5.787518501281738, |
| "kl": 0.24267578125, |
| "learning_rate": 2.311715481171548e-07, |
| "loss": 0.0097, |
| "reward": 0.6447945833206177, |
| "reward_std": 0.08838034607470036, |
| "reward_std_all": 0.10240528732538223, |
| "rewards-max/forensics_iou_reward": 0.9678078889846802, |
| "rewards/forensics_iou_reward": 0.6447945833206177, |
| "step": 736 |
| }, |
| { |
| "completion_length": 19.6875, |
| "epoch": 3.083682008368201, |
| "grad_norm": 6.015944957733154, |
| "kl": 0.2529296875, |
| "learning_rate": 2.3012552301255228e-07, |
| "loss": 0.0101, |
| "reward": 0.6181831955909729, |
| "reward_std": 0.05845150351524353, |
| "reward_std_all": 0.06853190809488297, |
| "rewards-max/forensics_iou_reward": 0.988664984703064, |
| "rewards/forensics_iou_reward": 0.6181831955909729, |
| "step": 737 |
| }, |
| { |
| "completion_length": 19.390625, |
| "epoch": 3.087866108786611, |
| "grad_norm": 6.854546546936035, |
| "kl": 0.27001953125, |
| "learning_rate": 2.2907949790794977e-07, |
| "loss": 0.0108, |
| "reward": 0.5101409703493118, |
| "reward_std": 0.11369482427835464, |
| "reward_std_all": 0.11132988147437572, |
| "rewards-max/forensics_iou_reward": 0.9271155893802643, |
| "rewards/forensics_iou_reward": 0.5101409703493118, |
| "step": 738 |
| }, |
| { |
| "completion_length": 20.21875, |
| "epoch": 3.092050209205021, |
| "grad_norm": 8.254032135009766, |
| "kl": 0.1806640625, |
| "learning_rate": 2.2803347280334728e-07, |
| "loss": 0.0072, |
| "reward": 0.6608828902244568, |
| "reward_std": 0.053975408896803856, |
| "reward_std_all": 0.08391302824020386, |
| "rewards-max/forensics_iou_reward": 0.9752649068832397, |
| "rewards/forensics_iou_reward": 0.6608828902244568, |
| "step": 739 |
| }, |
| { |
| "completion_length": 20.1875, |
| "epoch": 3.096234309623431, |
| "grad_norm": 8.865117073059082, |
| "kl": 0.21435546875, |
| "learning_rate": 2.2698744769874474e-07, |
| "loss": 0.0086, |
| "reward": 0.7051093876361847, |
| "reward_std": 0.0776971485465765, |
| "reward_std_all": 0.10377530753612518, |
| "rewards-max/forensics_iou_reward": 0.9794894158840179, |
| "rewards/forensics_iou_reward": 0.7051093876361847, |
| "step": 740 |
| }, |
| { |
| "completion_length": 20.109375, |
| "epoch": 3.100418410041841, |
| "grad_norm": 6.563053131103516, |
| "kl": 0.2451171875, |
| "learning_rate": 2.2594142259414225e-07, |
| "loss": 0.0098, |
| "reward": 0.6197292506694794, |
| "reward_std": 0.12540708854794502, |
| "reward_std_all": 0.1115255355834961, |
| "rewards-max/forensics_iou_reward": 0.9731316268444061, |
| "rewards/forensics_iou_reward": 0.6197292506694794, |
| "step": 741 |
| }, |
| { |
| "completion_length": 19.5625, |
| "epoch": 3.104602510460251, |
| "grad_norm": 5.298608303070068, |
| "kl": 0.2431640625, |
| "learning_rate": 2.2489539748953974e-07, |
| "loss": 0.0097, |
| "reward": 0.48430290818214417, |
| "reward_std": 0.09081798419356346, |
| "reward_std_all": 0.08588312938809395, |
| "rewards-max/forensics_iou_reward": 0.9550155401229858, |
| "rewards/forensics_iou_reward": 0.48430290818214417, |
| "step": 742 |
| }, |
| { |
| "completion_length": 23.828125, |
| "epoch": 3.1087866108786613, |
| "grad_norm": 5.9205322265625, |
| "kl": 0.2236328125, |
| "learning_rate": 2.2384937238493723e-07, |
| "loss": 0.009, |
| "reward": 0.4241692125797272, |
| "reward_std": 0.05805179476737976, |
| "reward_std_all": 0.06874298676848412, |
| "rewards-max/forensics_iou_reward": 0.9177270531654358, |
| "rewards/forensics_iou_reward": 0.4241692125797272, |
| "step": 743 |
| }, |
| { |
| "completion_length": 19.53125, |
| "epoch": 3.1129707112970713, |
| "grad_norm": 5.39417028427124, |
| "kl": 0.12841796875, |
| "learning_rate": 2.2280334728033474e-07, |
| "loss": 0.0051, |
| "reward": 0.5515531003475189, |
| "reward_std": 0.06569034233689308, |
| "reward_std_all": 0.06292043626308441, |
| "rewards-max/forensics_iou_reward": 0.9208036661148071, |
| "rewards/forensics_iou_reward": 0.5515531003475189, |
| "step": 744 |
| }, |
| { |
| "completion_length": 19.515625, |
| "epoch": 3.1171548117154813, |
| "grad_norm": 7.082902431488037, |
| "kl": 0.2568359375, |
| "learning_rate": 2.217573221757322e-07, |
| "loss": 0.0103, |
| "reward": 0.6232655346393585, |
| "reward_std": 0.07157501578330994, |
| "reward_std_all": 0.0820498839020729, |
| "rewards-max/forensics_iou_reward": 0.9711247980594635, |
| "rewards/forensics_iou_reward": 0.6232655346393585, |
| "step": 745 |
| }, |
| { |
| "completion_length": 20.265625, |
| "epoch": 3.1213389121338913, |
| "grad_norm": 5.061954975128174, |
| "kl": 0.17333984375, |
| "learning_rate": 2.207112970711297e-07, |
| "loss": 0.0069, |
| "reward": 0.41724516451358795, |
| "reward_std": 0.07453227788209915, |
| "reward_std_all": 0.07578664645552635, |
| "rewards-max/forensics_iou_reward": 0.9580851197242737, |
| "rewards/forensics_iou_reward": 0.41724516451358795, |
| "step": 746 |
| }, |
| { |
| "completion_length": 20.625, |
| "epoch": 3.1255230125523012, |
| "grad_norm": 7.1985554695129395, |
| "kl": 0.1845703125, |
| "learning_rate": 2.1966527196652717e-07, |
| "loss": 0.0074, |
| "reward": 0.5190598964691162, |
| "reward_std": 0.05211067199707031, |
| "reward_std_all": 0.0561315082013607, |
| "rewards-max/forensics_iou_reward": 0.9269001483917236, |
| "rewards/forensics_iou_reward": 0.5190598964691162, |
| "step": 747 |
| }, |
| { |
| "completion_length": 20.90625, |
| "epoch": 3.1297071129707112, |
| "grad_norm": 5.325139999389648, |
| "kl": 0.27001953125, |
| "learning_rate": 2.1861924686192469e-07, |
| "loss": 0.0108, |
| "reward": 0.5646078288555145, |
| "reward_std": 0.07697296515107155, |
| "reward_std_all": 0.07797013409435749, |
| "rewards-max/forensics_iou_reward": 0.9774366021156311, |
| "rewards/forensics_iou_reward": 0.5646078288555145, |
| "step": 748 |
| }, |
| { |
| "completion_length": 19.71875, |
| "epoch": 3.1338912133891212, |
| "grad_norm": 8.660388946533203, |
| "kl": 0.19091796875, |
| "learning_rate": 2.1757322175732217e-07, |
| "loss": 0.0077, |
| "reward": 0.5021370947360992, |
| "reward_std": 0.07036856189370155, |
| "reward_std_all": 0.07540809735655785, |
| "rewards-max/forensics_iou_reward": 0.9450238645076752, |
| "rewards/forensics_iou_reward": 0.5021370947360992, |
| "step": 749 |
| }, |
| { |
| "completion_length": 20.234375, |
| "epoch": 3.1380753138075312, |
| "grad_norm": 4.28463888168335, |
| "kl": 0.23681640625, |
| "learning_rate": 2.1652719665271966e-07, |
| "loss": 0.0095, |
| "reward": 0.5474894493818283, |
| "reward_std": 0.07573036104440689, |
| "reward_std_all": 0.07221834734082222, |
| "rewards-max/forensics_iou_reward": 0.9566619098186493, |
| "rewards/forensics_iou_reward": 0.5474894493818283, |
| "step": 750 |
| }, |
| { |
| "completion_length": 19.171875, |
| "epoch": 3.1422594142259412, |
| "grad_norm": 6.240118026733398, |
| "kl": 0.3798828125, |
| "learning_rate": 2.1548117154811714e-07, |
| "loss": 0.0152, |
| "reward": 0.6815042495727539, |
| "reward_std": 0.07093577086925507, |
| "reward_std_all": 0.0833585225045681, |
| "rewards-max/forensics_iou_reward": 0.9432314038276672, |
| "rewards/forensics_iou_reward": 0.6815042495727539, |
| "step": 751 |
| }, |
| { |
| "completion_length": 19.40625, |
| "epoch": 3.1464435146443517, |
| "grad_norm": 6.317922115325928, |
| "kl": 0.19140625, |
| "learning_rate": 2.1443514644351463e-07, |
| "loss": 0.0076, |
| "reward": 0.5662000775337219, |
| "reward_std": 0.09206547960639, |
| "reward_std_all": 0.0912390910089016, |
| "rewards-max/forensics_iou_reward": 0.9007905125617981, |
| "rewards/forensics_iou_reward": 0.5662000775337219, |
| "step": 752 |
| }, |
| { |
| "completion_length": 20.125, |
| "epoch": 3.1506276150627617, |
| "grad_norm": 8.388500213623047, |
| "kl": 0.21435546875, |
| "learning_rate": 2.1338912133891212e-07, |
| "loss": 0.0086, |
| "reward": 0.6336101591587067, |
| "reward_std": 0.06226171273738146, |
| "reward_std_all": 0.08354277163743973, |
| "rewards-max/forensics_iou_reward": 0.939220517873764, |
| "rewards/forensics_iou_reward": 0.6336101591587067, |
| "step": 753 |
| }, |
| { |
| "completion_length": 19.484375, |
| "epoch": 3.1548117154811717, |
| "grad_norm": 5.068207263946533, |
| "kl": 0.17236328125, |
| "learning_rate": 2.1234309623430963e-07, |
| "loss": 0.0069, |
| "reward": 0.5499742329120636, |
| "reward_std": 0.057756006717681885, |
| "reward_std_all": 0.0760653167963028, |
| "rewards-max/forensics_iou_reward": 0.8902051150798798, |
| "rewards/forensics_iou_reward": 0.5499742329120636, |
| "step": 754 |
| }, |
| { |
| "completion_length": 19.0625, |
| "epoch": 3.1589958158995817, |
| "grad_norm": 5.902076244354248, |
| "kl": 0.19970703125, |
| "learning_rate": 2.112970711297071e-07, |
| "loss": 0.008, |
| "reward": 0.6996037662029266, |
| "reward_std": 0.04380851052701473, |
| "reward_std_all": 0.06065096706151962, |
| "rewards-max/forensics_iou_reward": 0.9814370274543762, |
| "rewards/forensics_iou_reward": 0.6996037662029266, |
| "step": 755 |
| }, |
| { |
| "completion_length": 19.78125, |
| "epoch": 3.1631799163179917, |
| "grad_norm": 6.378020763397217, |
| "kl": 0.193359375, |
| "learning_rate": 2.102510460251046e-07, |
| "loss": 0.0078, |
| "reward": 0.47784510254859924, |
| "reward_std": 0.0630785021930933, |
| "reward_std_all": 0.06291162595152855, |
| "rewards-max/forensics_iou_reward": 0.9227272868156433, |
| "rewards/forensics_iou_reward": 0.47784510254859924, |
| "step": 756 |
| }, |
| { |
| "completion_length": 19.5625, |
| "epoch": 3.1673640167364017, |
| "grad_norm": 5.843949794769287, |
| "kl": 0.2666015625, |
| "learning_rate": 2.0920502092050206e-07, |
| "loss": 0.0106, |
| "reward": 0.625704288482666, |
| "reward_std": 0.046850116923451424, |
| "reward_std_all": 0.06419818103313446, |
| "rewards-max/forensics_iou_reward": 0.962742954492569, |
| "rewards/forensics_iou_reward": 0.625704288482666, |
| "step": 757 |
| }, |
| { |
| "completion_length": 21.234375, |
| "epoch": 3.1715481171548117, |
| "grad_norm": 5.885717391967773, |
| "kl": 0.18408203125, |
| "learning_rate": 2.0815899581589958e-07, |
| "loss": 0.0073, |
| "reward": 0.6060242056846619, |
| "reward_std": 0.06339172273874283, |
| "reward_std_all": 0.068434689193964, |
| "rewards-max/forensics_iou_reward": 0.9197919070720673, |
| "rewards/forensics_iou_reward": 0.6060242056846619, |
| "step": 758 |
| }, |
| { |
| "completion_length": 20.828125, |
| "epoch": 3.1757322175732217, |
| "grad_norm": 7.495584964752197, |
| "kl": 0.1708984375, |
| "learning_rate": 2.0711297071129706e-07, |
| "loss": 0.0068, |
| "reward": 0.5478712618350983, |
| "reward_std": 0.07583051174879074, |
| "reward_std_all": 0.08528545498847961, |
| "rewards-max/forensics_iou_reward": 0.8646120429039001, |
| "rewards/forensics_iou_reward": 0.5478712618350983, |
| "step": 759 |
| }, |
| { |
| "completion_length": 20.453125, |
| "epoch": 3.1799163179916317, |
| "grad_norm": 7.542618274688721, |
| "kl": 0.16650390625, |
| "learning_rate": 2.0606694560669455e-07, |
| "loss": 0.0067, |
| "reward": 0.49013711512088776, |
| "reward_std": 0.050641054287552834, |
| "reward_std_all": 0.056596141308546066, |
| "rewards-max/forensics_iou_reward": 0.9397144615650177, |
| "rewards/forensics_iou_reward": 0.49013711512088776, |
| "step": 760 |
| }, |
| { |
| "completion_length": 19.40625, |
| "epoch": 3.1841004184100417, |
| "grad_norm": 5.8988447189331055, |
| "kl": 0.17431640625, |
| "learning_rate": 2.0502092050209206e-07, |
| "loss": 0.007, |
| "reward": 0.5689688324928284, |
| "reward_std": 0.09813922271132469, |
| "reward_std_all": 0.09621361270546913, |
| "rewards-max/forensics_iou_reward": 0.9291139245033264, |
| "rewards/forensics_iou_reward": 0.5689688324928284, |
| "step": 761 |
| }, |
| { |
| "completion_length": 19.890625, |
| "epoch": 3.1882845188284517, |
| "grad_norm": 3.9559850692749023, |
| "kl": 0.31396484375, |
| "learning_rate": 2.0397489539748952e-07, |
| "loss": 0.0126, |
| "reward": 0.6617633700370789, |
| "reward_std": 0.05996755138039589, |
| "reward_std_all": 0.07125796377658844, |
| "rewards-max/forensics_iou_reward": 0.9746619760990143, |
| "rewards/forensics_iou_reward": 0.6617633700370789, |
| "step": 762 |
| }, |
| { |
| "completion_length": 19.78125, |
| "epoch": 3.192468619246862, |
| "grad_norm": 6.448940753936768, |
| "kl": 0.24951171875, |
| "learning_rate": 2.0292887029288703e-07, |
| "loss": 0.01, |
| "reward": 0.5114671289920807, |
| "reward_std": 0.08157967776060104, |
| "reward_std_all": 0.08636197447776794, |
| "rewards-max/forensics_iou_reward": 0.8741021454334259, |
| "rewards/forensics_iou_reward": 0.5114671289920807, |
| "step": 763 |
| }, |
| { |
| "completion_length": 19.84375, |
| "epoch": 3.196652719665272, |
| "grad_norm": 4.895190715789795, |
| "kl": 0.154296875, |
| "learning_rate": 2.0188284518828452e-07, |
| "loss": 0.0062, |
| "reward": 0.575258880853653, |
| "reward_std": 0.056580064818263054, |
| "reward_std_all": 0.060031804256141186, |
| "rewards-max/forensics_iou_reward": 0.935256838798523, |
| "rewards/forensics_iou_reward": 0.575258880853653, |
| "step": 764 |
| }, |
| { |
| "completion_length": 20.046875, |
| "epoch": 3.200836820083682, |
| "grad_norm": 7.55243444442749, |
| "kl": 0.16357421875, |
| "learning_rate": 2.00836820083682e-07, |
| "loss": 0.0065, |
| "reward": 0.5831246674060822, |
| "reward_std": 0.09551129676401615, |
| "reward_std_all": 0.10860168933868408, |
| "rewards-max/forensics_iou_reward": 0.901138573884964, |
| "rewards/forensics_iou_reward": 0.5831246674060822, |
| "step": 765 |
| }, |
| { |
| "completion_length": 19.78125, |
| "epoch": 3.205020920502092, |
| "grad_norm": 13.462621688842773, |
| "kl": 0.22412109375, |
| "learning_rate": 1.997907949790795e-07, |
| "loss": 0.009, |
| "reward": 0.5143369138240814, |
| "reward_std": 0.06122764013707638, |
| "reward_std_all": 0.07115611061453819, |
| "rewards-max/forensics_iou_reward": 0.924617350101471, |
| "rewards/forensics_iou_reward": 0.5143369138240814, |
| "step": 766 |
| }, |
| { |
| "completion_length": 21.765625, |
| "epoch": 3.209205020920502, |
| "grad_norm": 3.6902241706848145, |
| "kl": 0.28515625, |
| "learning_rate": 1.9874476987447698e-07, |
| "loss": 0.0114, |
| "reward": 0.580670639872551, |
| "reward_std": 0.09655306115746498, |
| "reward_std_all": 0.12055672332644463, |
| "rewards-max/forensics_iou_reward": 0.9180982112884521, |
| "rewards/forensics_iou_reward": 0.580670639872551, |
| "step": 767 |
| }, |
| { |
| "completion_length": 20.046875, |
| "epoch": 3.213389121338912, |
| "grad_norm": 6.679305076599121, |
| "kl": 0.1806640625, |
| "learning_rate": 1.9769874476987447e-07, |
| "loss": 0.0072, |
| "reward": 0.5538437366485596, |
| "reward_std": 0.08588430657982826, |
| "reward_std_all": 0.07989244908094406, |
| "rewards-max/forensics_iou_reward": 0.9211021959781647, |
| "rewards/forensics_iou_reward": 0.5538437366485596, |
| "step": 768 |
| }, |
| { |
| "completion_length": 19.609375, |
| "epoch": 3.217573221757322, |
| "grad_norm": 7.1081929206848145, |
| "kl": 0.228515625, |
| "learning_rate": 1.9665271966527198e-07, |
| "loss": 0.0091, |
| "reward": 0.6053208410739899, |
| "reward_std": 0.0948839783668518, |
| "reward_std_all": 0.11360898613929749, |
| "rewards-max/forensics_iou_reward": 0.9299145340919495, |
| "rewards/forensics_iou_reward": 0.6053208410739899, |
| "step": 769 |
| }, |
| { |
| "completion_length": 20.09375, |
| "epoch": 3.221757322175732, |
| "grad_norm": 8.462575912475586, |
| "kl": 0.29541015625, |
| "learning_rate": 1.9560669456066944e-07, |
| "loss": 0.0118, |
| "reward": 0.49723534286022186, |
| "reward_std": 0.09635042399168015, |
| "reward_std_all": 0.10140959918498993, |
| "rewards-max/forensics_iou_reward": 0.9223765432834625, |
| "rewards/forensics_iou_reward": 0.49723534286022186, |
| "step": 770 |
| }, |
| { |
| "completion_length": 19.15625, |
| "epoch": 3.225941422594142, |
| "grad_norm": 5.210278034210205, |
| "kl": 0.1943359375, |
| "learning_rate": 1.9456066945606695e-07, |
| "loss": 0.0078, |
| "reward": 0.6213760077953339, |
| "reward_std": 0.050861040130257607, |
| "reward_std_all": 0.0564655065536499, |
| "rewards-max/forensics_iou_reward": 0.9784799516201019, |
| "rewards/forensics_iou_reward": 0.6213760077953339, |
| "step": 771 |
| }, |
| { |
| "completion_length": 19.921875, |
| "epoch": 3.2301255230125525, |
| "grad_norm": 4.58713960647583, |
| "kl": 0.24609375, |
| "learning_rate": 1.935146443514644e-07, |
| "loss": 0.0098, |
| "reward": 0.6036842465400696, |
| "reward_std": 0.0627808291465044, |
| "reward_std_all": 0.08416811376810074, |
| "rewards-max/forensics_iou_reward": 0.9483174979686737, |
| "rewards/forensics_iou_reward": 0.6036842465400696, |
| "step": 772 |
| }, |
| { |
| "completion_length": 19.90625, |
| "epoch": 3.2343096234309625, |
| "grad_norm": 8.81950855255127, |
| "kl": 0.271484375, |
| "learning_rate": 1.9246861924686192e-07, |
| "loss": 0.0108, |
| "reward": 0.5732459425926208, |
| "reward_std": 0.09861810132861137, |
| "reward_std_all": 0.09359132498502731, |
| "rewards-max/forensics_iou_reward": 0.9367765486240387, |
| "rewards/forensics_iou_reward": 0.5732459425926208, |
| "step": 773 |
| }, |
| { |
| "completion_length": 20.109375, |
| "epoch": 3.2384937238493725, |
| "grad_norm": 7.234550952911377, |
| "kl": 0.3359375, |
| "learning_rate": 1.914225941422594e-07, |
| "loss": 0.0134, |
| "reward": 0.6085531413555145, |
| "reward_std": 0.13641641661524773, |
| "reward_std_all": 0.12945476919412613, |
| "rewards-max/forensics_iou_reward": 0.9317187666893005, |
| "rewards/forensics_iou_reward": 0.6085531413555145, |
| "step": 774 |
| }, |
| { |
| "completion_length": 19.671875, |
| "epoch": 3.2426778242677825, |
| "grad_norm": 6.274400234222412, |
| "kl": 0.265625, |
| "learning_rate": 1.903765690376569e-07, |
| "loss": 0.0106, |
| "reward": 0.6795805990695953, |
| "reward_std": 0.09082131460309029, |
| "reward_std_all": 0.09408511593937874, |
| "rewards-max/forensics_iou_reward": 0.958545058965683, |
| "rewards/forensics_iou_reward": 0.6795805990695953, |
| "step": 775 |
| }, |
| { |
| "completion_length": 19.953125, |
| "epoch": 3.2468619246861925, |
| "grad_norm": 6.451498985290527, |
| "kl": 0.18212890625, |
| "learning_rate": 1.8933054393305438e-07, |
| "loss": 0.0073, |
| "reward": 0.6673862338066101, |
| "reward_std": 0.05814921297132969, |
| "reward_std_all": 0.07815398834645748, |
| "rewards-max/forensics_iou_reward": 0.958583265542984, |
| "rewards/forensics_iou_reward": 0.6673862338066101, |
| "step": 776 |
| }, |
| { |
| "completion_length": 19.3125, |
| "epoch": 3.2510460251046025, |
| "grad_norm": 6.116027355194092, |
| "kl": 0.232421875, |
| "learning_rate": 1.8828451882845187e-07, |
| "loss": 0.0093, |
| "reward": 0.6377952098846436, |
| "reward_std": 0.11176620423793793, |
| "reward_std_all": 0.10424760729074478, |
| "rewards-max/forensics_iou_reward": 0.9140604138374329, |
| "rewards/forensics_iou_reward": 0.6377952098846436, |
| "step": 777 |
| }, |
| { |
| "completion_length": 19.46875, |
| "epoch": 3.2552301255230125, |
| "grad_norm": 5.6612629890441895, |
| "kl": 0.1884765625, |
| "learning_rate": 1.8723849372384936e-07, |
| "loss": 0.0075, |
| "reward": 0.5488036870956421, |
| "reward_std": 0.05492594093084335, |
| "reward_std_all": 0.09164521470665932, |
| "rewards-max/forensics_iou_reward": 0.9550567865371704, |
| "rewards/forensics_iou_reward": 0.5488036870956421, |
| "step": 778 |
| }, |
| { |
| "completion_length": 20.265625, |
| "epoch": 3.2594142259414225, |
| "grad_norm": 3.698397397994995, |
| "kl": 0.216796875, |
| "learning_rate": 1.8619246861924684e-07, |
| "loss": 0.0087, |
| "reward": 0.5638341009616852, |
| "reward_std": 0.06912058964371681, |
| "reward_std_all": 0.07494911551475525, |
| "rewards-max/forensics_iou_reward": 0.9686052799224854, |
| "rewards/forensics_iou_reward": 0.5638341009616852, |
| "step": 779 |
| }, |
| { |
| "completion_length": 19.6875, |
| "epoch": 3.2635983263598325, |
| "grad_norm": 5.7632246017456055, |
| "kl": 0.28515625, |
| "learning_rate": 1.8514644351464433e-07, |
| "loss": 0.0114, |
| "reward": 0.6926543116569519, |
| "reward_std": 0.09831669181585312, |
| "reward_std_all": 0.0918174497783184, |
| "rewards-max/forensics_iou_reward": 0.9374581277370453, |
| "rewards/forensics_iou_reward": 0.6926543116569519, |
| "step": 780 |
| }, |
| { |
| "completion_length": 19.390625, |
| "epoch": 3.2677824267782425, |
| "grad_norm": 9.020171165466309, |
| "kl": 0.20458984375, |
| "learning_rate": 1.8410041841004184e-07, |
| "loss": 0.0082, |
| "reward": 0.6960394978523254, |
| "reward_std": 0.04607067070901394, |
| "reward_std_all": 0.05040384270250797, |
| "rewards-max/forensics_iou_reward": 0.9784292578697205, |
| "rewards/forensics_iou_reward": 0.6960394978523254, |
| "step": 781 |
| }, |
| { |
| "completion_length": 19.625, |
| "epoch": 3.2719665271966525, |
| "grad_norm": 7.130446434020996, |
| "kl": 0.21533203125, |
| "learning_rate": 1.830543933054393e-07, |
| "loss": 0.0086, |
| "reward": 0.5420580357313156, |
| "reward_std": 0.08822111412882805, |
| "reward_std_all": 0.09873397275805473, |
| "rewards-max/forensics_iou_reward": 0.88941490650177, |
| "rewards/forensics_iou_reward": 0.5420580357313156, |
| "step": 782 |
| }, |
| { |
| "completion_length": 19.078125, |
| "epoch": 3.276150627615063, |
| "grad_norm": 7.807944297790527, |
| "kl": 0.4375, |
| "learning_rate": 1.8200836820083681e-07, |
| "loss": 0.0174, |
| "reward": 0.6004168093204498, |
| "reward_std": 0.061581581830978394, |
| "reward_std_all": 0.07332216948270798, |
| "rewards-max/forensics_iou_reward": 0.9458335638046265, |
| "rewards/forensics_iou_reward": 0.6004168093204498, |
| "step": 783 |
| }, |
| { |
| "completion_length": 20.828125, |
| "epoch": 3.280334728033473, |
| "grad_norm": 8.157979965209961, |
| "kl": 0.20556640625, |
| "learning_rate": 1.8096234309623427e-07, |
| "loss": 0.0082, |
| "reward": 0.36890245229005814, |
| "reward_std": 0.07284552976489067, |
| "reward_std_all": 0.0998285710811615, |
| "rewards-max/forensics_iou_reward": 0.8377096951007843, |
| "rewards/forensics_iou_reward": 0.36890245229005814, |
| "step": 784 |
| }, |
| { |
| "completion_length": 19.5, |
| "epoch": 3.284518828451883, |
| "grad_norm": 9.937553405761719, |
| "kl": 0.2607421875, |
| "learning_rate": 1.799163179916318e-07, |
| "loss": 0.0104, |
| "reward": 0.5136430859565735, |
| "reward_std": 0.06319335848093033, |
| "reward_std_all": 0.07050396502017975, |
| "rewards-max/forensics_iou_reward": 0.8727357387542725, |
| "rewards/forensics_iou_reward": 0.5136430859565735, |
| "step": 785 |
| }, |
| { |
| "completion_length": 20.75, |
| "epoch": 3.288702928870293, |
| "grad_norm": 5.872644901275635, |
| "kl": 0.1953125, |
| "learning_rate": 1.788702928870293e-07, |
| "loss": 0.0078, |
| "reward": 0.5362654328346252, |
| "reward_std": 0.06642713025212288, |
| "reward_std_all": 0.06555244885385036, |
| "rewards-max/forensics_iou_reward": 0.8146258294582367, |
| "rewards/forensics_iou_reward": 0.5362654328346252, |
| "step": 786 |
| }, |
| { |
| "completion_length": 19.75, |
| "epoch": 3.292887029288703, |
| "grad_norm": 6.639247417449951, |
| "kl": 0.431640625, |
| "learning_rate": 1.7782426778242676e-07, |
| "loss": 0.0173, |
| "reward": 0.7269488573074341, |
| "reward_std": 0.05773668549954891, |
| "reward_std_all": 0.08019185811281204, |
| "rewards-max/forensics_iou_reward": 0.9767164885997772, |
| "rewards/forensics_iou_reward": 0.7269488573074341, |
| "step": 787 |
| }, |
| { |
| "completion_length": 19.609375, |
| "epoch": 3.297071129707113, |
| "grad_norm": 4.4005126953125, |
| "kl": 0.27490234375, |
| "learning_rate": 1.7677824267782427e-07, |
| "loss": 0.011, |
| "reward": 0.6086003482341766, |
| "reward_std": 0.04641177225857973, |
| "reward_std_all": 0.05218944512307644, |
| "rewards-max/forensics_iou_reward": 0.9743832647800446, |
| "rewards/forensics_iou_reward": 0.6086003482341766, |
| "step": 788 |
| }, |
| { |
| "completion_length": 19.515625, |
| "epoch": 3.301255230125523, |
| "grad_norm": 5.5122809410095215, |
| "kl": 0.18603515625, |
| "learning_rate": 1.7573221757322173e-07, |
| "loss": 0.0074, |
| "reward": 0.5400965213775635, |
| "reward_std": 0.05298185348510742, |
| "reward_std_all": 0.060679489746689796, |
| "rewards-max/forensics_iou_reward": 0.9288766980171204, |
| "rewards/forensics_iou_reward": 0.5400965213775635, |
| "step": 789 |
| }, |
| { |
| "completion_length": 19.484375, |
| "epoch": 3.305439330543933, |
| "grad_norm": 6.994944095611572, |
| "kl": 0.41162109375, |
| "learning_rate": 1.7468619246861925e-07, |
| "loss": 0.0165, |
| "reward": 0.5904039442539215, |
| "reward_std": 0.07794393226504326, |
| "reward_std_all": 0.07246797159314156, |
| "rewards-max/forensics_iou_reward": 0.9643784463405609, |
| "rewards/forensics_iou_reward": 0.5904039442539215, |
| "step": 790 |
| }, |
| { |
| "completion_length": 19.828125, |
| "epoch": 3.309623430962343, |
| "grad_norm": 6.8759026527404785, |
| "kl": 0.177734375, |
| "learning_rate": 1.7364016736401673e-07, |
| "loss": 0.0071, |
| "reward": 0.6589472889900208, |
| "reward_std": 0.0837748572230339, |
| "reward_std_all": 0.10223507881164551, |
| "rewards-max/forensics_iou_reward": 0.9676106572151184, |
| "rewards/forensics_iou_reward": 0.6589472889900208, |
| "step": 791 |
| }, |
| { |
| "completion_length": 19.796875, |
| "epoch": 3.3138075313807533, |
| "grad_norm": 5.859192848205566, |
| "kl": 0.2802734375, |
| "learning_rate": 1.7259414225941422e-07, |
| "loss": 0.0112, |
| "reward": 0.604740709066391, |
| "reward_std": 0.11194107681512833, |
| "reward_std_all": 0.1266023889183998, |
| "rewards-max/forensics_iou_reward": 0.9610343873500824, |
| "rewards/forensics_iou_reward": 0.604740709066391, |
| "step": 792 |
| }, |
| { |
| "completion_length": 19.984375, |
| "epoch": 3.3179916317991633, |
| "grad_norm": 10.827493667602539, |
| "kl": 0.21533203125, |
| "learning_rate": 1.715481171548117e-07, |
| "loss": 0.0086, |
| "reward": 0.5900795161724091, |
| "reward_std": 0.09309807047247887, |
| "reward_std_all": 0.10895130783319473, |
| "rewards-max/forensics_iou_reward": 0.9572423696517944, |
| "rewards/forensics_iou_reward": 0.5900795161724091, |
| "step": 793 |
| }, |
| { |
| "completion_length": 19.921875, |
| "epoch": 3.3221757322175733, |
| "grad_norm": 10.595340728759766, |
| "kl": 0.28564453125, |
| "learning_rate": 1.705020920502092e-07, |
| "loss": 0.0114, |
| "reward": 0.4782000631093979, |
| "reward_std": 0.06281801871955395, |
| "reward_std_all": 0.07613565772771835, |
| "rewards-max/forensics_iou_reward": 0.9675405323505402, |
| "rewards/forensics_iou_reward": 0.4782000631093979, |
| "step": 794 |
| }, |
| { |
| "completion_length": 20.9375, |
| "epoch": 3.3263598326359833, |
| "grad_norm": 5.843415260314941, |
| "kl": 0.18408203125, |
| "learning_rate": 1.6945606694560668e-07, |
| "loss": 0.0074, |
| "reward": 0.5878386497497559, |
| "reward_std": 0.12111945822834969, |
| "reward_std_all": 0.11200403794646263, |
| "rewards-max/forensics_iou_reward": 0.9127533733844757, |
| "rewards/forensics_iou_reward": 0.5878386497497559, |
| "step": 795 |
| }, |
| { |
| "completion_length": 19.765625, |
| "epoch": 3.3305439330543933, |
| "grad_norm": 8.087431907653809, |
| "kl": 0.25439453125, |
| "learning_rate": 1.684100418410042e-07, |
| "loss": 0.0102, |
| "reward": 0.5958676636219025, |
| "reward_std": 0.05260419100522995, |
| "reward_std_all": 0.06102835014462471, |
| "rewards-max/forensics_iou_reward": 0.9715909063816071, |
| "rewards/forensics_iou_reward": 0.5958676636219025, |
| "step": 796 |
| }, |
| { |
| "completion_length": 21.09375, |
| "epoch": 3.3347280334728033, |
| "grad_norm": 5.164019584655762, |
| "kl": 0.2802734375, |
| "learning_rate": 1.6736401673640165e-07, |
| "loss": 0.0112, |
| "reward": 0.5280104577541351, |
| "reward_std": 0.10663007199764252, |
| "reward_std_all": 0.10885745286941528, |
| "rewards-max/forensics_iou_reward": 0.9191236793994904, |
| "rewards/forensics_iou_reward": 0.5280104577541351, |
| "step": 797 |
| }, |
| { |
| "completion_length": 19.40625, |
| "epoch": 3.3389121338912133, |
| "grad_norm": 6.230867862701416, |
| "kl": 0.20751953125, |
| "learning_rate": 1.6631799163179916e-07, |
| "loss": 0.0083, |
| "reward": 0.669348418712616, |
| "reward_std": 0.0599629245698452, |
| "reward_std_all": 0.06110947206616402, |
| "rewards-max/forensics_iou_reward": 0.9761953055858612, |
| "rewards/forensics_iou_reward": 0.669348418712616, |
| "step": 798 |
| }, |
| { |
| "completion_length": 19.703125, |
| "epoch": 3.3430962343096233, |
| "grad_norm": 4.95225715637207, |
| "kl": 0.3095703125, |
| "learning_rate": 1.6527196652719662e-07, |
| "loss": 0.0124, |
| "reward": 0.6424996256828308, |
| "reward_std": 0.06480151042342186, |
| "reward_std_all": 0.0732649713754654, |
| "rewards-max/forensics_iou_reward": 0.9453265070915222, |
| "rewards/forensics_iou_reward": 0.6424996256828308, |
| "step": 799 |
| }, |
| { |
| "completion_length": 19.328125, |
| "epoch": 3.3472803347280333, |
| "grad_norm": 7.308218002319336, |
| "kl": 0.18994140625, |
| "learning_rate": 1.6422594142259414e-07, |
| "loss": 0.0076, |
| "reward": 0.6986010372638702, |
| "reward_std": 0.05007253959774971, |
| "reward_std_all": 0.05918102338910103, |
| "rewards-max/forensics_iou_reward": 0.9527546465396881, |
| "rewards/forensics_iou_reward": 0.6986010372638702, |
| "step": 800 |
| }, |
| { |
| "completion_length": 20.453125, |
| "epoch": 3.3514644351464433, |
| "grad_norm": 5.716921329498291, |
| "kl": 0.1982421875, |
| "learning_rate": 1.6317991631799162e-07, |
| "loss": 0.0079, |
| "reward": 0.5048498809337616, |
| "reward_std": 0.07468540221452713, |
| "reward_std_all": 0.07274608872830868, |
| "rewards-max/forensics_iou_reward": 0.8830718994140625, |
| "rewards/forensics_iou_reward": 0.5048498809337616, |
| "step": 801 |
| }, |
| { |
| "completion_length": 19.828125, |
| "epoch": 3.3556485355648533, |
| "grad_norm": 5.8602142333984375, |
| "kl": 0.21826171875, |
| "learning_rate": 1.621338912133891e-07, |
| "loss": 0.0087, |
| "reward": 0.5508247315883636, |
| "reward_std": 0.06504769995808601, |
| "reward_std_all": 0.08421830460429192, |
| "rewards-max/forensics_iou_reward": 0.873252809047699, |
| "rewards/forensics_iou_reward": 0.5508247315883636, |
| "step": 802 |
| }, |
| { |
| "completion_length": 19.828125, |
| "epoch": 3.3598326359832638, |
| "grad_norm": 5.883810520172119, |
| "kl": 0.20556640625, |
| "learning_rate": 1.610878661087866e-07, |
| "loss": 0.0082, |
| "reward": 0.6528300642967224, |
| "reward_std": 0.1390136405825615, |
| "reward_std_all": 0.1253398433327675, |
| "rewards-max/forensics_iou_reward": 0.9797644913196564, |
| "rewards/forensics_iou_reward": 0.6528300642967224, |
| "step": 803 |
| }, |
| { |
| "completion_length": 19.96875, |
| "epoch": 3.3640167364016738, |
| "grad_norm": 8.028403282165527, |
| "kl": 0.2060546875, |
| "learning_rate": 1.6004184100418408e-07, |
| "loss": 0.0082, |
| "reward": 0.58676877617836, |
| "reward_std": 0.05862653627991676, |
| "reward_std_all": 0.06365111283957958, |
| "rewards-max/forensics_iou_reward": 0.9083631038665771, |
| "rewards/forensics_iou_reward": 0.58676877617836, |
| "step": 804 |
| }, |
| { |
| "completion_length": 19.796875, |
| "epoch": 3.3682008368200838, |
| "grad_norm": 8.229565620422363, |
| "kl": 0.2421875, |
| "learning_rate": 1.5899581589958157e-07, |
| "loss": 0.0097, |
| "reward": 0.5638560950756073, |
| "reward_std": 0.10876744240522385, |
| "reward_std_all": 0.10237228497862816, |
| "rewards-max/forensics_iou_reward": 0.9521458148956299, |
| "rewards/forensics_iou_reward": 0.5638560950756073, |
| "step": 805 |
| }, |
| { |
| "completion_length": 19.390625, |
| "epoch": 3.3723849372384938, |
| "grad_norm": 5.516202449798584, |
| "kl": 0.224609375, |
| "learning_rate": 1.5794979079497908e-07, |
| "loss": 0.009, |
| "reward": 0.7269829511642456, |
| "reward_std": 0.10115784034132957, |
| "reward_std_all": 0.0978621393442154, |
| "rewards-max/forensics_iou_reward": 0.9687389135360718, |
| "rewards/forensics_iou_reward": 0.7269829511642456, |
| "step": 806 |
| }, |
| { |
| "completion_length": 20.421875, |
| "epoch": 3.3765690376569037, |
| "grad_norm": 5.682495594024658, |
| "kl": 0.20556640625, |
| "learning_rate": 1.5690376569037657e-07, |
| "loss": 0.0082, |
| "reward": 0.5195904076099396, |
| "reward_std": 0.061871238052845, |
| "reward_std_all": 0.06607550010085106, |
| "rewards-max/forensics_iou_reward": 0.9670237600803375, |
| "rewards/forensics_iou_reward": 0.5195904076099396, |
| "step": 807 |
| }, |
| { |
| "completion_length": 19.703125, |
| "epoch": 3.3807531380753137, |
| "grad_norm": 6.317902565002441, |
| "kl": 0.22802734375, |
| "learning_rate": 1.5585774058577405e-07, |
| "loss": 0.0091, |
| "reward": 0.580836147069931, |
| "reward_std": 0.11532403528690338, |
| "reward_std_all": 0.10145687311887741, |
| "rewards-max/forensics_iou_reward": 0.9586507081985474, |
| "rewards/forensics_iou_reward": 0.580836147069931, |
| "step": 808 |
| }, |
| { |
| "completion_length": 20.203125, |
| "epoch": 3.3849372384937237, |
| "grad_norm": 7.126334190368652, |
| "kl": 0.310546875, |
| "learning_rate": 1.5481171548117154e-07, |
| "loss": 0.0125, |
| "reward": 0.5896505415439606, |
| "reward_std": 0.07469452172517776, |
| "reward_std_all": 0.09159021452069283, |
| "rewards-max/forensics_iou_reward": 0.976190447807312, |
| "rewards/forensics_iou_reward": 0.5896505415439606, |
| "step": 809 |
| }, |
| { |
| "completion_length": 19.75, |
| "epoch": 3.3891213389121337, |
| "grad_norm": 5.86687707901001, |
| "kl": 0.173828125, |
| "learning_rate": 1.5376569037656903e-07, |
| "loss": 0.007, |
| "reward": 0.49215126037597656, |
| "reward_std": 0.07327626273036003, |
| "reward_std_all": 0.07350489869713783, |
| "rewards-max/forensics_iou_reward": 0.9233162999153137, |
| "rewards/forensics_iou_reward": 0.49215126037597656, |
| "step": 810 |
| }, |
| { |
| "completion_length": 19.25, |
| "epoch": 3.393305439330544, |
| "grad_norm": 6.264730930328369, |
| "kl": 0.20947265625, |
| "learning_rate": 1.5271966527196654e-07, |
| "loss": 0.0084, |
| "reward": 0.635380893945694, |
| "reward_std": 0.06412963941693306, |
| "reward_std_all": 0.07750844210386276, |
| "rewards-max/forensics_iou_reward": 0.8985491991043091, |
| "rewards/forensics_iou_reward": 0.635380893945694, |
| "step": 811 |
| }, |
| { |
| "completion_length": 21.484375, |
| "epoch": 3.397489539748954, |
| "grad_norm": 4.493706226348877, |
| "kl": 0.19580078125, |
| "learning_rate": 1.51673640167364e-07, |
| "loss": 0.0078, |
| "reward": 0.4946751594543457, |
| "reward_std": 0.07930398732423782, |
| "reward_std_all": 0.07209059968590736, |
| "rewards-max/forensics_iou_reward": 0.9614990949630737, |
| "rewards/forensics_iou_reward": 0.4946751594543457, |
| "step": 812 |
| }, |
| { |
| "completion_length": 20.4375, |
| "epoch": 3.401673640167364, |
| "grad_norm": 11.346385955810547, |
| "kl": 0.15380859375, |
| "learning_rate": 1.506276150627615e-07, |
| "loss": 0.0062, |
| "reward": 0.43597227334976196, |
| "reward_std": 0.05052247643470764, |
| "reward_std_all": 0.049845386296510696, |
| "rewards-max/forensics_iou_reward": 0.8817493617534637, |
| "rewards/forensics_iou_reward": 0.43597227334976196, |
| "step": 813 |
| }, |
| { |
| "completion_length": 20.140625, |
| "epoch": 3.405857740585774, |
| "grad_norm": 5.964457035064697, |
| "kl": 0.2890625, |
| "learning_rate": 1.4958158995815897e-07, |
| "loss": 0.0115, |
| "reward": 0.5394172072410583, |
| "reward_std": 0.10693490877747536, |
| "reward_std_all": 0.11188631504774094, |
| "rewards-max/forensics_iou_reward": 0.9648913145065308, |
| "rewards/forensics_iou_reward": 0.5394172072410583, |
| "step": 814 |
| }, |
| { |
| "completion_length": 20.0, |
| "epoch": 3.410041841004184, |
| "grad_norm": 5.017680644989014, |
| "kl": 0.259765625, |
| "learning_rate": 1.4853556485355648e-07, |
| "loss": 0.0104, |
| "reward": 0.5285449922084808, |
| "reward_std": 0.07835233956575394, |
| "reward_std_all": 0.0958566889166832, |
| "rewards-max/forensics_iou_reward": 0.9117218852043152, |
| "rewards/forensics_iou_reward": 0.5285449922084808, |
| "step": 815 |
| }, |
| { |
| "completion_length": 19.984375, |
| "epoch": 3.414225941422594, |
| "grad_norm": 5.358748912811279, |
| "kl": 0.251953125, |
| "learning_rate": 1.4748953974895397e-07, |
| "loss": 0.0101, |
| "reward": 0.6792267560958862, |
| "reward_std": 0.12210533395409584, |
| "reward_std_all": 0.12140447273850441, |
| "rewards-max/forensics_iou_reward": 0.9575137495994568, |
| "rewards/forensics_iou_reward": 0.6792267560958862, |
| "step": 816 |
| }, |
| { |
| "completion_length": 19.53125, |
| "epoch": 3.418410041841004, |
| "grad_norm": 7.774308681488037, |
| "kl": 0.1962890625, |
| "learning_rate": 1.4644351464435146e-07, |
| "loss": 0.0079, |
| "reward": 0.6000303626060486, |
| "reward_std": 0.08625317364931107, |
| "reward_std_all": 0.08505029231309891, |
| "rewards-max/forensics_iou_reward": 0.969068706035614, |
| "rewards/forensics_iou_reward": 0.6000303626060486, |
| "step": 817 |
| }, |
| { |
| "completion_length": 20.234375, |
| "epoch": 3.422594142259414, |
| "grad_norm": 5.595709323883057, |
| "kl": 0.287109375, |
| "learning_rate": 1.4539748953974894e-07, |
| "loss": 0.0115, |
| "reward": 0.493372842669487, |
| "reward_std": 0.061027199029922485, |
| "reward_std_all": 0.08613515459001064, |
| "rewards-max/forensics_iou_reward": 0.9230876863002777, |
| "rewards/forensics_iou_reward": 0.493372842669487, |
| "step": 818 |
| }, |
| { |
| "completion_length": 19.171875, |
| "epoch": 3.426778242677824, |
| "grad_norm": 4.900079250335693, |
| "kl": 0.1533203125, |
| "learning_rate": 1.4435146443514643e-07, |
| "loss": 0.0061, |
| "reward": 0.5551385879516602, |
| "reward_std": 0.0639699175953865, |
| "reward_std_all": 0.06217091903090477, |
| "rewards-max/forensics_iou_reward": 0.9413949251174927, |
| "rewards/forensics_iou_reward": 0.5551385879516602, |
| "step": 819 |
| }, |
| { |
| "completion_length": 20.359375, |
| "epoch": 3.430962343096234, |
| "grad_norm": 6.638409614562988, |
| "kl": 0.19384765625, |
| "learning_rate": 1.4330543933054392e-07, |
| "loss": 0.0078, |
| "reward": 0.5727564692497253, |
| "reward_std": 0.05213290639221668, |
| "reward_std_all": 0.06792653724551201, |
| "rewards-max/forensics_iou_reward": 0.9181802570819855, |
| "rewards/forensics_iou_reward": 0.5727564692497253, |
| "step": 820 |
| }, |
| { |
| "completion_length": 20.0625, |
| "epoch": 3.435146443514644, |
| "grad_norm": 5.508879661560059, |
| "kl": 0.1923828125, |
| "learning_rate": 1.4225941422594143e-07, |
| "loss": 0.0077, |
| "reward": 0.590133547782898, |
| "reward_std": 0.09668285399675369, |
| "reward_std_all": 0.10349787026643753, |
| "rewards-max/forensics_iou_reward": 0.9066753089427948, |
| "rewards/forensics_iou_reward": 0.590133547782898, |
| "step": 821 |
| }, |
| { |
| "completion_length": 20.171875, |
| "epoch": 3.439330543933054, |
| "grad_norm": 5.571535587310791, |
| "kl": 0.189453125, |
| "learning_rate": 1.412133891213389e-07, |
| "loss": 0.0076, |
| "reward": 0.6260912716388702, |
| "reward_std": 0.06752615608274937, |
| "reward_std_all": 0.0902699064463377, |
| "rewards-max/forensics_iou_reward": 0.9485519528388977, |
| "rewards/forensics_iou_reward": 0.6260912716388702, |
| "step": 822 |
| }, |
| { |
| "completion_length": 19.546875, |
| "epoch": 3.4435146443514646, |
| "grad_norm": 4.31847620010376, |
| "kl": 0.2099609375, |
| "learning_rate": 1.401673640167364e-07, |
| "loss": 0.0084, |
| "reward": 0.6310233771800995, |
| "reward_std": 0.08592020161449909, |
| "reward_std_all": 0.08791209384799004, |
| "rewards-max/forensics_iou_reward": 0.9224305152893066, |
| "rewards/forensics_iou_reward": 0.6310233771800995, |
| "step": 823 |
| }, |
| { |
| "completion_length": 20.0, |
| "epoch": 3.4476987447698746, |
| "grad_norm": 5.69787073135376, |
| "kl": 0.193359375, |
| "learning_rate": 1.3912133891213386e-07, |
| "loss": 0.0077, |
| "reward": 0.4174574315547943, |
| "reward_std": 0.10731750354170799, |
| "reward_std_all": 0.099104143679142, |
| "rewards-max/forensics_iou_reward": 0.9319145381450653, |
| "rewards/forensics_iou_reward": 0.4174574315547943, |
| "step": 824 |
| }, |
| { |
| "completion_length": 19.609375, |
| "epoch": 3.4518828451882846, |
| "grad_norm": 7.046478748321533, |
| "kl": 0.2177734375, |
| "learning_rate": 1.3807531380753137e-07, |
| "loss": 0.0087, |
| "reward": 0.5358051061630249, |
| "reward_std": 0.05980382487177849, |
| "reward_std_all": 0.071236127987504, |
| "rewards-max/forensics_iou_reward": 0.8627466261386871, |
| "rewards/forensics_iou_reward": 0.5358051061630249, |
| "step": 825 |
| }, |
| { |
| "completion_length": 22.5, |
| "epoch": 3.4560669456066946, |
| "grad_norm": 9.721525192260742, |
| "kl": 0.15087890625, |
| "learning_rate": 1.370292887029289e-07, |
| "loss": 0.006, |
| "reward": 0.48971718549728394, |
| "reward_std": 0.12121559679508209, |
| "reward_std_all": 0.11374055594205856, |
| "rewards-max/forensics_iou_reward": 0.9333992600440979, |
| "rewards/forensics_iou_reward": 0.48971718549728394, |
| "step": 826 |
| }, |
| { |
| "completion_length": 20.109375, |
| "epoch": 3.4602510460251046, |
| "grad_norm": 6.249843597412109, |
| "kl": 0.2353515625, |
| "learning_rate": 1.3598326359832635e-07, |
| "loss": 0.0094, |
| "reward": 0.6223547160625458, |
| "reward_std": 0.073121078312397, |
| "reward_std_all": 0.06885498017072678, |
| "rewards-max/forensics_iou_reward": 0.981876790523529, |
| "rewards/forensics_iou_reward": 0.6223547160625458, |
| "step": 827 |
| }, |
| { |
| "completion_length": 20.59375, |
| "epoch": 3.4644351464435146, |
| "grad_norm": 8.584314346313477, |
| "kl": 0.20654296875, |
| "learning_rate": 1.3493723849372386e-07, |
| "loss": 0.0083, |
| "reward": 0.5867487788200378, |
| "reward_std": 0.09824050962924957, |
| "reward_std_all": 0.10136029124259949, |
| "rewards-max/forensics_iou_reward": 0.94468954205513, |
| "rewards/forensics_iou_reward": 0.5867487788200378, |
| "step": 828 |
| }, |
| { |
| "completion_length": 19.828125, |
| "epoch": 3.4686192468619246, |
| "grad_norm": 6.280603408813477, |
| "kl": 0.17578125, |
| "learning_rate": 1.3389121338912132e-07, |
| "loss": 0.007, |
| "reward": 0.501270204782486, |
| "reward_std": 0.09716101177036762, |
| "reward_std_all": 0.08886038884520531, |
| "rewards-max/forensics_iou_reward": 0.87962207198143, |
| "rewards/forensics_iou_reward": 0.501270204782486, |
| "step": 829 |
| }, |
| { |
| "completion_length": 20.25, |
| "epoch": 3.4728033472803346, |
| "grad_norm": 6.265865802764893, |
| "kl": 0.1767578125, |
| "learning_rate": 1.3284518828451883e-07, |
| "loss": 0.0071, |
| "reward": 0.704263299703598, |
| "reward_std": 0.058448679745197296, |
| "reward_std_all": 0.054094865918159485, |
| "rewards-max/forensics_iou_reward": 0.9962403178215027, |
| "rewards/forensics_iou_reward": 0.704263299703598, |
| "step": 830 |
| }, |
| { |
| "completion_length": 20.09375, |
| "epoch": 3.476987447698745, |
| "grad_norm": 6.474465370178223, |
| "kl": 0.1767578125, |
| "learning_rate": 1.3179916317991632e-07, |
| "loss": 0.0071, |
| "reward": 0.6648664772510529, |
| "reward_std": 0.07238938845694065, |
| "reward_std_all": 0.06528526544570923, |
| "rewards-max/forensics_iou_reward": 0.9635196030139923, |
| "rewards/forensics_iou_reward": 0.6648664772510529, |
| "step": 831 |
| }, |
| { |
| "completion_length": 19.359375, |
| "epoch": 3.481171548117155, |
| "grad_norm": 7.192984580993652, |
| "kl": 0.1884765625, |
| "learning_rate": 1.307531380753138e-07, |
| "loss": 0.0075, |
| "reward": 0.6077288389205933, |
| "reward_std": 0.06281134858727455, |
| "reward_std_all": 0.05822756141424179, |
| "rewards-max/forensics_iou_reward": 0.9137344360351562, |
| "rewards/forensics_iou_reward": 0.6077288389205933, |
| "step": 832 |
| }, |
| { |
| "completion_length": 20.0625, |
| "epoch": 3.485355648535565, |
| "grad_norm": 7.11025857925415, |
| "kl": 0.23046875, |
| "learning_rate": 1.297071129707113e-07, |
| "loss": 0.0092, |
| "reward": 0.5790211856365204, |
| "reward_std": 0.1183757409453392, |
| "reward_std_all": 0.10466152057051659, |
| "rewards-max/forensics_iou_reward": 0.9611729979515076, |
| "rewards/forensics_iou_reward": 0.5790211856365204, |
| "step": 833 |
| }, |
| { |
| "completion_length": 19.625, |
| "epoch": 3.489539748953975, |
| "grad_norm": 6.915976047515869, |
| "kl": 0.26220703125, |
| "learning_rate": 1.2866108786610878e-07, |
| "loss": 0.0105, |
| "reward": 0.5884430855512619, |
| "reward_std": 0.07404802739620209, |
| "reward_std_all": 0.07965733483433723, |
| "rewards-max/forensics_iou_reward": 0.9454655647277832, |
| "rewards/forensics_iou_reward": 0.5884430855512619, |
| "step": 834 |
| }, |
| { |
| "completion_length": 19.921875, |
| "epoch": 3.493723849372385, |
| "grad_norm": 7.268563747406006, |
| "kl": 0.1484375, |
| "learning_rate": 1.2761506276150627e-07, |
| "loss": 0.0059, |
| "reward": 0.6038404703140259, |
| "reward_std": 0.055999964475631714, |
| "reward_std_all": 0.052169427275657654, |
| "rewards-max/forensics_iou_reward": 0.9077577590942383, |
| "rewards/forensics_iou_reward": 0.6038404703140259, |
| "step": 835 |
| }, |
| { |
| "completion_length": 19.375, |
| "epoch": 3.497907949790795, |
| "grad_norm": 6.9536566734313965, |
| "kl": 0.19677734375, |
| "learning_rate": 1.2656903765690378e-07, |
| "loss": 0.0079, |
| "reward": 0.5803249478340149, |
| "reward_std": 0.09605956450104713, |
| "reward_std_all": 0.08681102842092514, |
| "rewards-max/forensics_iou_reward": 0.9314397573471069, |
| "rewards/forensics_iou_reward": 0.5803249478340149, |
| "step": 836 |
| }, |
| { |
| "completion_length": 19.375, |
| "epoch": 3.502092050209205, |
| "grad_norm": 7.110646724700928, |
| "kl": 0.29541015625, |
| "learning_rate": 1.2552301255230124e-07, |
| "loss": 0.0118, |
| "reward": 0.5735092163085938, |
| "reward_std": 0.051785191521048546, |
| "reward_std_all": 0.05044993758201599, |
| "rewards-max/forensics_iou_reward": 0.9209499955177307, |
| "rewards/forensics_iou_reward": 0.5735092163085938, |
| "step": 837 |
| }, |
| { |
| "completion_length": 19.65625, |
| "epoch": 3.506276150627615, |
| "grad_norm": 9.51911449432373, |
| "kl": 0.17724609375, |
| "learning_rate": 1.2447698744769875e-07, |
| "loss": 0.0071, |
| "reward": 0.5097635388374329, |
| "reward_std": 0.044786954298615456, |
| "reward_std_all": 0.045328862965106964, |
| "rewards-max/forensics_iou_reward": 0.9655624032020569, |
| "rewards/forensics_iou_reward": 0.5097635388374329, |
| "step": 838 |
| }, |
| { |
| "completion_length": 19.75, |
| "epoch": 3.510460251046025, |
| "grad_norm": 5.548820972442627, |
| "kl": 0.2021484375, |
| "learning_rate": 1.2343096234309624e-07, |
| "loss": 0.0081, |
| "reward": 0.6496655642986298, |
| "reward_std": 0.08098545297980309, |
| "reward_std_all": 0.07325200363993645, |
| "rewards-max/forensics_iou_reward": 0.942970871925354, |
| "rewards/forensics_iou_reward": 0.6496655642986298, |
| "step": 839 |
| }, |
| { |
| "completion_length": 19.125, |
| "epoch": 3.514644351464435, |
| "grad_norm": 14.049922943115234, |
| "kl": 0.22998046875, |
| "learning_rate": 1.2238493723849372e-07, |
| "loss": 0.0092, |
| "reward": 0.5946958363056183, |
| "reward_std": 0.05239917151629925, |
| "reward_std_all": 0.06902468018233776, |
| "rewards-max/forensics_iou_reward": 0.9533799588680267, |
| "rewards/forensics_iou_reward": 0.5946958363056183, |
| "step": 840 |
| }, |
| { |
| "completion_length": 20.3125, |
| "epoch": 3.518828451882845, |
| "grad_norm": 6.728308200836182, |
| "kl": 0.18896484375, |
| "learning_rate": 1.213389121338912e-07, |
| "loss": 0.0075, |
| "reward": 0.5776767134666443, |
| "reward_std": 0.08750787749886513, |
| "reward_std_all": 0.08885563910007477, |
| "rewards-max/forensics_iou_reward": 0.9419830739498138, |
| "rewards/forensics_iou_reward": 0.5776767134666443, |
| "step": 841 |
| }, |
| { |
| "completion_length": 19.296875, |
| "epoch": 3.523012552301255, |
| "grad_norm": 6.485559463500977, |
| "kl": 0.16796875, |
| "learning_rate": 1.202928870292887e-07, |
| "loss": 0.0067, |
| "reward": 0.6677060425281525, |
| "reward_std": 0.0645143985748291, |
| "reward_std_all": 0.06768122501671314, |
| "rewards-max/forensics_iou_reward": 0.9324324131011963, |
| "rewards/forensics_iou_reward": 0.6677060425281525, |
| "step": 842 |
| }, |
| { |
| "completion_length": 19.421875, |
| "epoch": 3.5271966527196654, |
| "grad_norm": 5.696123123168945, |
| "kl": 0.21240234375, |
| "learning_rate": 1.1924686192468618e-07, |
| "loss": 0.0085, |
| "reward": 0.6144144237041473, |
| "reward_std": 0.045997804030776024, |
| "reward_std_all": 0.05627049691975117, |
| "rewards-max/forensics_iou_reward": 0.9843234717845917, |
| "rewards/forensics_iou_reward": 0.6144144237041473, |
| "step": 843 |
| }, |
| { |
| "completion_length": 19.828125, |
| "epoch": 3.5313807531380754, |
| "grad_norm": 7.018336296081543, |
| "kl": 0.19775390625, |
| "learning_rate": 1.1820083682008368e-07, |
| "loss": 0.0079, |
| "reward": 0.5155273824930191, |
| "reward_std": 0.06591426953673363, |
| "reward_std_all": 0.08132782950997353, |
| "rewards-max/forensics_iou_reward": 0.9117594659328461, |
| "rewards/forensics_iou_reward": 0.5155273824930191, |
| "step": 844 |
| }, |
| { |
| "completion_length": 19.734375, |
| "epoch": 3.5355648535564854, |
| "grad_norm": 6.565451622009277, |
| "kl": 0.21484375, |
| "learning_rate": 1.1715481171548117e-07, |
| "loss": 0.0086, |
| "reward": 0.6210409700870514, |
| "reward_std": 0.08988313376903534, |
| "reward_std_all": 0.07980865985155106, |
| "rewards-max/forensics_iou_reward": 0.9169935286045074, |
| "rewards/forensics_iou_reward": 0.6210409700870514, |
| "step": 845 |
| }, |
| { |
| "completion_length": 20.203125, |
| "epoch": 3.5397489539748954, |
| "grad_norm": 7.227148056030273, |
| "kl": 0.21630859375, |
| "learning_rate": 1.1610878661087866e-07, |
| "loss": 0.0086, |
| "reward": 0.564423143863678, |
| "reward_std": 0.05244209244847298, |
| "reward_std_all": 0.07277482002973557, |
| "rewards-max/forensics_iou_reward": 0.8546729683876038, |
| "rewards/forensics_iou_reward": 0.564423143863678, |
| "step": 846 |
| }, |
| { |
| "completion_length": 19.3125, |
| "epoch": 3.5439330543933054, |
| "grad_norm": 4.580743312835693, |
| "kl": 0.18408203125, |
| "learning_rate": 1.1506276150627614e-07, |
| "loss": 0.0074, |
| "reward": 0.621886134147644, |
| "reward_std": 0.036260420456528664, |
| "reward_std_all": 0.04356165789067745, |
| "rewards-max/forensics_iou_reward": 0.9584574997425079, |
| "rewards/forensics_iou_reward": 0.621886134147644, |
| "step": 847 |
| }, |
| { |
| "completion_length": 19.515625, |
| "epoch": 3.5481171548117154, |
| "grad_norm": 7.6898651123046875, |
| "kl": 0.2333984375, |
| "learning_rate": 1.1401673640167364e-07, |
| "loss": 0.0093, |
| "reward": 0.6460304856300354, |
| "reward_std": 0.08518640324473381, |
| "reward_std_all": 0.0868108868598938, |
| "rewards-max/forensics_iou_reward": 0.9650106132030487, |
| "rewards/forensics_iou_reward": 0.6460304856300354, |
| "step": 848 |
| }, |
| { |
| "completion_length": 19.125, |
| "epoch": 3.5523012552301254, |
| "grad_norm": 5.7955451011657715, |
| "kl": 0.216796875, |
| "learning_rate": 1.1297071129707113e-07, |
| "loss": 0.0087, |
| "reward": 0.6700129806995392, |
| "reward_std": 0.05392867513000965, |
| "reward_std_all": 0.06470882520079613, |
| "rewards-max/forensics_iou_reward": 0.9881129264831543, |
| "rewards/forensics_iou_reward": 0.6700129806995392, |
| "step": 849 |
| }, |
| { |
| "completion_length": 19.609375, |
| "epoch": 3.5564853556485354, |
| "grad_norm": 4.244430065155029, |
| "kl": 0.18359375, |
| "learning_rate": 1.1192468619246861e-07, |
| "loss": 0.0073, |
| "reward": 0.6347448825836182, |
| "reward_std": 0.08710183203220367, |
| "reward_std_all": 0.08829249441623688, |
| "rewards-max/forensics_iou_reward": 0.9869525134563446, |
| "rewards/forensics_iou_reward": 0.6347448825836182, |
| "step": 850 |
| }, |
| { |
| "completion_length": 19.34375, |
| "epoch": 3.560669456066946, |
| "grad_norm": 6.103775501251221, |
| "kl": 0.19384765625, |
| "learning_rate": 1.108786610878661e-07, |
| "loss": 0.0078, |
| "reward": 0.5000211298465729, |
| "reward_std": 0.06920674443244934, |
| "reward_std_all": 0.07025819644331932, |
| "rewards-max/forensics_iou_reward": 0.9493116438388824, |
| "rewards/forensics_iou_reward": 0.5000211298465729, |
| "step": 851 |
| }, |
| { |
| "completion_length": 19.390625, |
| "epoch": 3.564853556485356, |
| "grad_norm": 12.602009773254395, |
| "kl": 0.3642578125, |
| "learning_rate": 1.0983263598326359e-07, |
| "loss": 0.0146, |
| "reward": 0.6333672404289246, |
| "reward_std": 0.08720533549785614, |
| "reward_std_all": 0.0852992832660675, |
| "rewards-max/forensics_iou_reward": 0.9461956322193146, |
| "rewards/forensics_iou_reward": 0.6333672404289246, |
| "step": 852 |
| }, |
| { |
| "completion_length": 20.625, |
| "epoch": 3.569037656903766, |
| "grad_norm": 4.943075656890869, |
| "kl": 0.1982421875, |
| "learning_rate": 1.0878661087866109e-07, |
| "loss": 0.0079, |
| "reward": 0.6034579575061798, |
| "reward_std": 0.04621196910738945, |
| "reward_std_all": 0.04425559565424919, |
| "rewards-max/forensics_iou_reward": 0.9778043925762177, |
| "rewards/forensics_iou_reward": 0.6034579575061798, |
| "step": 853 |
| }, |
| { |
| "completion_length": 19.765625, |
| "epoch": 3.573221757322176, |
| "grad_norm": 6.56069278717041, |
| "kl": 0.21533203125, |
| "learning_rate": 1.0774058577405857e-07, |
| "loss": 0.0086, |
| "reward": 0.5611265003681183, |
| "reward_std": 0.0988220889121294, |
| "reward_std_all": 0.0975569486618042, |
| "rewards-max/forensics_iou_reward": 0.9330028593540192, |
| "rewards/forensics_iou_reward": 0.5611265003681183, |
| "step": 854 |
| }, |
| { |
| "completion_length": 19.25, |
| "epoch": 3.577405857740586, |
| "grad_norm": 12.304871559143066, |
| "kl": 0.3515625, |
| "learning_rate": 1.0669456066945606e-07, |
| "loss": 0.0141, |
| "reward": 0.6051933169364929, |
| "reward_std": 0.06462953053414822, |
| "reward_std_all": 0.07783051952719688, |
| "rewards-max/forensics_iou_reward": 0.9340217411518097, |
| "rewards/forensics_iou_reward": 0.6051933169364929, |
| "step": 855 |
| }, |
| { |
| "completion_length": 21.0625, |
| "epoch": 3.581589958158996, |
| "grad_norm": 10.927102088928223, |
| "kl": 0.1875, |
| "learning_rate": 1.0564853556485355e-07, |
| "loss": 0.0075, |
| "reward": 0.4027104675769806, |
| "reward_std": 0.06807921081781387, |
| "reward_std_all": 0.08038963750004768, |
| "rewards-max/forensics_iou_reward": 0.8921922445297241, |
| "rewards/forensics_iou_reward": 0.4027104675769806, |
| "step": 856 |
| }, |
| { |
| "completion_length": 19.53125, |
| "epoch": 3.585774058577406, |
| "grad_norm": 5.2154645919799805, |
| "kl": 0.19970703125, |
| "learning_rate": 1.0460251046025103e-07, |
| "loss": 0.008, |
| "reward": 0.7835457026958466, |
| "reward_std": 0.05376807972788811, |
| "reward_std_all": 0.06143315136432648, |
| "rewards-max/forensics_iou_reward": 0.9808352589607239, |
| "rewards/forensics_iou_reward": 0.7835457026958466, |
| "step": 857 |
| }, |
| { |
| "completion_length": 20.546875, |
| "epoch": 3.589958158995816, |
| "grad_norm": 6.605453968048096, |
| "kl": 0.234375, |
| "learning_rate": 1.0355648535564853e-07, |
| "loss": 0.0094, |
| "reward": 0.579064816236496, |
| "reward_std": 0.06524843536317348, |
| "reward_std_all": 0.07520086504518986, |
| "rewards-max/forensics_iou_reward": 0.8996218144893646, |
| "rewards/forensics_iou_reward": 0.579064816236496, |
| "step": 858 |
| }, |
| { |
| "completion_length": 20.828125, |
| "epoch": 3.594142259414226, |
| "grad_norm": 12.965999603271484, |
| "kl": 0.6572265625, |
| "learning_rate": 1.0251046025104603e-07, |
| "loss": 0.0264, |
| "reward": 0.5335219204425812, |
| "reward_std": 0.06890367716550827, |
| "reward_std_all": 0.06900840066373348, |
| "rewards-max/forensics_iou_reward": 0.8896641135215759, |
| "rewards/forensics_iou_reward": 0.5335219204425812, |
| "step": 859 |
| }, |
| { |
| "completion_length": 21.578125, |
| "epoch": 3.598326359832636, |
| "grad_norm": 6.058223247528076, |
| "kl": 0.3076171875, |
| "learning_rate": 1.0146443514644352e-07, |
| "loss": 0.0123, |
| "reward": 0.5473254770040512, |
| "reward_std": 0.10156495124101639, |
| "reward_std_all": 0.0969674177467823, |
| "rewards-max/forensics_iou_reward": 0.9568967819213867, |
| "rewards/forensics_iou_reward": 0.5473254770040512, |
| "step": 860 |
| }, |
| { |
| "completion_length": 19.765625, |
| "epoch": 3.602510460251046, |
| "grad_norm": 6.109407901763916, |
| "kl": 0.22998046875, |
| "learning_rate": 1.00418410041841e-07, |
| "loss": 0.0092, |
| "reward": 0.567653477191925, |
| "reward_std": 0.05900300294160843, |
| "reward_std_all": 0.08770311623811722, |
| "rewards-max/forensics_iou_reward": 0.9771966338157654, |
| "rewards/forensics_iou_reward": 0.567653477191925, |
| "step": 861 |
| }, |
| { |
| "completion_length": 20.265625, |
| "epoch": 3.606694560669456, |
| "grad_norm": 6.15337610244751, |
| "kl": 0.3427734375, |
| "learning_rate": 9.937238493723849e-08, |
| "loss": 0.0137, |
| "reward": 0.6474016308784485, |
| "reward_std": 0.08485609665513039, |
| "reward_std_all": 0.09446807950735092, |
| "rewards-max/forensics_iou_reward": 0.9717440903186798, |
| "rewards/forensics_iou_reward": 0.6474016308784485, |
| "step": 862 |
| }, |
| { |
| "completion_length": 19.796875, |
| "epoch": 3.6108786610878663, |
| "grad_norm": 7.823073387145996, |
| "kl": 0.25244140625, |
| "learning_rate": 9.832635983263599e-08, |
| "loss": 0.0101, |
| "reward": 0.710582971572876, |
| "reward_std": 0.1033385619521141, |
| "reward_std_all": 0.13115189969539642, |
| "rewards-max/forensics_iou_reward": 0.9509413540363312, |
| "rewards/forensics_iou_reward": 0.710582971572876, |
| "step": 863 |
| }, |
| { |
| "completion_length": 21.21875, |
| "epoch": 3.6150627615062763, |
| "grad_norm": 5.672852993011475, |
| "kl": 0.2109375, |
| "learning_rate": 9.728033472803348e-08, |
| "loss": 0.0084, |
| "reward": 0.6992847323417664, |
| "reward_std": 0.06958222761750221, |
| "reward_std_all": 0.08287582919001579, |
| "rewards-max/forensics_iou_reward": 0.9686293601989746, |
| "rewards/forensics_iou_reward": 0.6992847323417664, |
| "step": 864 |
| }, |
| { |
| "completion_length": 18.921875, |
| "epoch": 3.6192468619246863, |
| "grad_norm": 6.896697998046875, |
| "kl": 0.15380859375, |
| "learning_rate": 9.623430962343096e-08, |
| "loss": 0.0062, |
| "reward": 0.5566319078207016, |
| "reward_std": 0.08208059519529343, |
| "reward_std_all": 0.08027109503746033, |
| "rewards-max/forensics_iou_reward": 0.9737030863761902, |
| "rewards/forensics_iou_reward": 0.5566319078207016, |
| "step": 865 |
| }, |
| { |
| "completion_length": 19.765625, |
| "epoch": 3.6234309623430963, |
| "grad_norm": 6.279343605041504, |
| "kl": 0.2177734375, |
| "learning_rate": 9.518828451882845e-08, |
| "loss": 0.0087, |
| "reward": 0.6908364295959473, |
| "reward_std": 0.07881857454776764, |
| "reward_std_all": 0.07024677656590939, |
| "rewards-max/forensics_iou_reward": 0.9519976377487183, |
| "rewards/forensics_iou_reward": 0.6908364295959473, |
| "step": 866 |
| }, |
| { |
| "completion_length": 20.125, |
| "epoch": 3.6276150627615062, |
| "grad_norm": 4.429011344909668, |
| "kl": 0.25634765625, |
| "learning_rate": 9.414225941422594e-08, |
| "loss": 0.0103, |
| "reward": 0.6016822159290314, |
| "reward_std": 0.07119659706950188, |
| "reward_std_all": 0.09118044748902321, |
| "rewards-max/forensics_iou_reward": 0.9515839219093323, |
| "rewards/forensics_iou_reward": 0.6016822159290314, |
| "step": 867 |
| }, |
| { |
| "completion_length": 20.84375, |
| "epoch": 3.6317991631799162, |
| "grad_norm": 7.274193286895752, |
| "kl": 0.19189453125, |
| "learning_rate": 9.309623430962342e-08, |
| "loss": 0.0077, |
| "reward": 0.4710354208946228, |
| "reward_std": 0.04353770613670349, |
| "reward_std_all": 0.04473784938454628, |
| "rewards-max/forensics_iou_reward": 0.8609022498130798, |
| "rewards/forensics_iou_reward": 0.4710354208946228, |
| "step": 868 |
| }, |
| { |
| "completion_length": 19.734375, |
| "epoch": 3.6359832635983262, |
| "grad_norm": 4.799069404602051, |
| "kl": 0.19384765625, |
| "learning_rate": 9.205020920502092e-08, |
| "loss": 0.0078, |
| "reward": 0.5739415138959885, |
| "reward_std": 0.08350927382707596, |
| "reward_std_all": 0.09198399260640144, |
| "rewards-max/forensics_iou_reward": 0.9829647839069366, |
| "rewards/forensics_iou_reward": 0.5739415138959885, |
| "step": 869 |
| }, |
| { |
| "completion_length": 20.03125, |
| "epoch": 3.6401673640167362, |
| "grad_norm": 6.332769393920898, |
| "kl": 0.318359375, |
| "learning_rate": 9.100418410041841e-08, |
| "loss": 0.0127, |
| "reward": 0.6524845957756042, |
| "reward_std": 0.10143144428730011, |
| "reward_std_all": 0.09619429334998131, |
| "rewards-max/forensics_iou_reward": 0.9746390581130981, |
| "rewards/forensics_iou_reward": 0.6524845957756042, |
| "step": 870 |
| }, |
| { |
| "completion_length": 20.15625, |
| "epoch": 3.6443514644351467, |
| "grad_norm": 11.329800605773926, |
| "kl": 0.20703125, |
| "learning_rate": 8.99581589958159e-08, |
| "loss": 0.0083, |
| "reward": 0.5972310900688171, |
| "reward_std": 0.10238676890730858, |
| "reward_std_all": 0.0945189781486988, |
| "rewards-max/forensics_iou_reward": 0.9727010130882263, |
| "rewards/forensics_iou_reward": 0.5972310900688171, |
| "step": 871 |
| }, |
| { |
| "completion_length": 19.3125, |
| "epoch": 3.6485355648535567, |
| "grad_norm": 6.885110378265381, |
| "kl": 0.29052734375, |
| "learning_rate": 8.891213389121338e-08, |
| "loss": 0.0116, |
| "reward": 0.5921665728092194, |
| "reward_std": 0.04828730598092079, |
| "reward_std_all": 0.056482452899217606, |
| "rewards-max/forensics_iou_reward": 0.8949437737464905, |
| "rewards/forensics_iou_reward": 0.5921665728092194, |
| "step": 872 |
| }, |
| { |
| "completion_length": 19.890625, |
| "epoch": 3.6527196652719667, |
| "grad_norm": 6.114720344543457, |
| "kl": 0.3154296875, |
| "learning_rate": 8.786610878661087e-08, |
| "loss": 0.0126, |
| "reward": 0.5475471317768097, |
| "reward_std": 0.05166175961494446, |
| "reward_std_all": 0.06581001542508602, |
| "rewards-max/forensics_iou_reward": 0.9444534778594971, |
| "rewards/forensics_iou_reward": 0.5475471317768097, |
| "step": 873 |
| }, |
| { |
| "completion_length": 20.25, |
| "epoch": 3.6569037656903767, |
| "grad_norm": 7.025356769561768, |
| "kl": 0.19677734375, |
| "learning_rate": 8.682008368200837e-08, |
| "loss": 0.0079, |
| "reward": 0.6610976159572601, |
| "reward_std": 0.05797549709677696, |
| "reward_std_all": 0.05693148076534271, |
| "rewards-max/forensics_iou_reward": 0.9907407462596893, |
| "rewards/forensics_iou_reward": 0.6610976159572601, |
| "step": 874 |
| }, |
| { |
| "completion_length": 20.09375, |
| "epoch": 3.6610878661087867, |
| "grad_norm": 8.3456392288208, |
| "kl": 0.22998046875, |
| "learning_rate": 8.577405857740585e-08, |
| "loss": 0.0092, |
| "reward": 0.5982494354248047, |
| "reward_std": 0.09129296615719795, |
| "reward_std_all": 0.10824818164110184, |
| "rewards-max/forensics_iou_reward": 0.9370763897895813, |
| "rewards/forensics_iou_reward": 0.5982494354248047, |
| "step": 875 |
| }, |
| { |
| "completion_length": 19.796875, |
| "epoch": 3.6652719665271967, |
| "grad_norm": 7.948225498199463, |
| "kl": 0.26171875, |
| "learning_rate": 8.472803347280334e-08, |
| "loss": 0.0105, |
| "reward": 0.6389199495315552, |
| "reward_std": 0.08725567907094955, |
| "reward_std_all": 0.08453699201345444, |
| "rewards-max/forensics_iou_reward": 0.9473911821842194, |
| "rewards/forensics_iou_reward": 0.6389199495315552, |
| "step": 876 |
| }, |
| { |
| "completion_length": 19.703125, |
| "epoch": 3.6694560669456067, |
| "grad_norm": 11.123475074768066, |
| "kl": 0.2587890625, |
| "learning_rate": 8.368200836820083e-08, |
| "loss": 0.0104, |
| "reward": 0.6004510521888733, |
| "reward_std": 0.12430783361196518, |
| "reward_std_all": 0.12453038617968559, |
| "rewards-max/forensics_iou_reward": 0.9588839411735535, |
| "rewards/forensics_iou_reward": 0.6004510521888733, |
| "step": 877 |
| }, |
| { |
| "completion_length": 20.484375, |
| "epoch": 3.6736401673640167, |
| "grad_norm": 7.31289005279541, |
| "kl": 0.275390625, |
| "learning_rate": 8.263598326359831e-08, |
| "loss": 0.011, |
| "reward": 0.6233519315719604, |
| "reward_std": 0.05743580311536789, |
| "reward_std_all": 0.06288335844874382, |
| "rewards-max/forensics_iou_reward": 0.9567983150482178, |
| "rewards/forensics_iou_reward": 0.6233519315719604, |
| "step": 878 |
| }, |
| { |
| "completion_length": 20.328125, |
| "epoch": 3.6778242677824267, |
| "grad_norm": 4.707196235656738, |
| "kl": 0.18994140625, |
| "learning_rate": 8.158995815899581e-08, |
| "loss": 0.0076, |
| "reward": 0.5084906667470932, |
| "reward_std": 0.03746745362877846, |
| "reward_std_all": 0.07840261422097683, |
| "rewards-max/forensics_iou_reward": 0.8068181872367859, |
| "rewards/forensics_iou_reward": 0.5084906667470932, |
| "step": 879 |
| }, |
| { |
| "completion_length": 19.859375, |
| "epoch": 3.6820083682008367, |
| "grad_norm": 7.006363868713379, |
| "kl": 0.24951171875, |
| "learning_rate": 8.05439330543933e-08, |
| "loss": 0.01, |
| "reward": 0.5710345655679703, |
| "reward_std": 0.10048677027225494, |
| "reward_std_all": 0.09799251332879066, |
| "rewards-max/forensics_iou_reward": 0.9052681922912598, |
| "rewards/forensics_iou_reward": 0.5710345655679703, |
| "step": 880 |
| }, |
| { |
| "completion_length": 19.1875, |
| "epoch": 3.6861924686192467, |
| "grad_norm": 5.907443046569824, |
| "kl": 0.2861328125, |
| "learning_rate": 7.949790794979078e-08, |
| "loss": 0.0114, |
| "reward": 0.5690503418445587, |
| "reward_std": 0.09411999955773354, |
| "reward_std_all": 0.09063517674803734, |
| "rewards-max/forensics_iou_reward": 0.970370352268219, |
| "rewards/forensics_iou_reward": 0.5690503418445587, |
| "step": 881 |
| }, |
| { |
| "completion_length": 19.328125, |
| "epoch": 3.6903765690376567, |
| "grad_norm": 7.9894866943359375, |
| "kl": 0.3125, |
| "learning_rate": 7.845188284518828e-08, |
| "loss": 0.0125, |
| "reward": 0.5827760696411133, |
| "reward_std": 0.062177903950214386, |
| "reward_std_all": 0.08008622750639915, |
| "rewards-max/forensics_iou_reward": 0.9739172458648682, |
| "rewards/forensics_iou_reward": 0.5827760696411133, |
| "step": 882 |
| }, |
| { |
| "completion_length": 20.0625, |
| "epoch": 3.694560669456067, |
| "grad_norm": 5.2307329177856445, |
| "kl": 0.1484375, |
| "learning_rate": 7.740585774058577e-08, |
| "loss": 0.0059, |
| "reward": 0.5873124599456787, |
| "reward_std": 0.07597197219729424, |
| "reward_std_all": 0.07891646027565002, |
| "rewards-max/forensics_iou_reward": 0.9291112422943115, |
| "rewards/forensics_iou_reward": 0.5873124599456787, |
| "step": 883 |
| }, |
| { |
| "completion_length": 21.65625, |
| "epoch": 3.698744769874477, |
| "grad_norm": 5.965217590332031, |
| "kl": 0.23974609375, |
| "learning_rate": 7.635983263598327e-08, |
| "loss": 0.0096, |
| "reward": 0.4191198945045471, |
| "reward_std": 0.08791840262711048, |
| "reward_std_all": 0.09192412346601486, |
| "rewards-max/forensics_iou_reward": 0.843959391117096, |
| "rewards/forensics_iou_reward": 0.4191198945045471, |
| "step": 884 |
| }, |
| { |
| "completion_length": 19.3125, |
| "epoch": 3.702928870292887, |
| "grad_norm": 5.7487592697143555, |
| "kl": 0.220703125, |
| "learning_rate": 7.531380753138076e-08, |
| "loss": 0.0088, |
| "reward": 0.5560978204011917, |
| "reward_std": 0.049271898344159126, |
| "reward_std_all": 0.06957253813743591, |
| "rewards-max/forensics_iou_reward": 0.9043211340904236, |
| "rewards/forensics_iou_reward": 0.5560978204011917, |
| "step": 885 |
| }, |
| { |
| "completion_length": 19.59375, |
| "epoch": 3.707112970711297, |
| "grad_norm": 6.932721138000488, |
| "kl": 0.20556640625, |
| "learning_rate": 7.426778242677824e-08, |
| "loss": 0.0082, |
| "reward": 0.6505179405212402, |
| "reward_std": 0.07352270185947418, |
| "reward_std_all": 0.09711914509534836, |
| "rewards-max/forensics_iou_reward": 0.9456569254398346, |
| "rewards/forensics_iou_reward": 0.6505179405212402, |
| "step": 886 |
| }, |
| { |
| "completion_length": 19.53125, |
| "epoch": 3.711297071129707, |
| "grad_norm": 6.082297325134277, |
| "kl": 0.18603515625, |
| "learning_rate": 7.322175732217573e-08, |
| "loss": 0.0074, |
| "reward": 0.4615498036146164, |
| "reward_std": 0.07139494549483061, |
| "reward_std_all": 0.07947228848934174, |
| "rewards-max/forensics_iou_reward": 0.8992632329463959, |
| "rewards/forensics_iou_reward": 0.4615498036146164, |
| "step": 887 |
| }, |
| { |
| "completion_length": 20.09375, |
| "epoch": 3.715481171548117, |
| "grad_norm": 7.173395156860352, |
| "kl": 0.390625, |
| "learning_rate": 7.217573221757322e-08, |
| "loss": 0.0156, |
| "reward": 0.6124118268489838, |
| "reward_std": 0.14842721074819565, |
| "reward_std_all": 0.13847696036100388, |
| "rewards-max/forensics_iou_reward": 0.9256137609481812, |
| "rewards/forensics_iou_reward": 0.6124118268489838, |
| "step": 888 |
| }, |
| { |
| "completion_length": 21.203125, |
| "epoch": 3.719665271966527, |
| "grad_norm": 4.9449872970581055, |
| "kl": 0.244140625, |
| "learning_rate": 7.112970711297071e-08, |
| "loss": 0.0098, |
| "reward": 0.6536839604377747, |
| "reward_std": 0.05782449617981911, |
| "reward_std_all": 0.06692039966583252, |
| "rewards-max/forensics_iou_reward": 0.9389112591743469, |
| "rewards/forensics_iou_reward": 0.6536839604377747, |
| "step": 889 |
| }, |
| { |
| "completion_length": 19.46875, |
| "epoch": 3.723849372384937, |
| "grad_norm": 6.683010578155518, |
| "kl": 0.19775390625, |
| "learning_rate": 7.00836820083682e-08, |
| "loss": 0.0079, |
| "reward": 0.6946537792682648, |
| "reward_std": 0.051668308675289154, |
| "reward_std_all": 0.04841233417391777, |
| "rewards-max/forensics_iou_reward": 0.978005588054657, |
| "rewards/forensics_iou_reward": 0.6946537792682648, |
| "step": 890 |
| }, |
| { |
| "completion_length": 20.40625, |
| "epoch": 3.7280334728033475, |
| "grad_norm": 8.153006553649902, |
| "kl": 0.48095703125, |
| "learning_rate": 6.903765690376569e-08, |
| "loss": 0.0191, |
| "reward": 0.6587729156017303, |
| "reward_std": 0.09425517916679382, |
| "reward_std_all": 0.09968496114015579, |
| "rewards-max/forensics_iou_reward": 0.9771608114242554, |
| "rewards/forensics_iou_reward": 0.6587729156017303, |
| "step": 891 |
| }, |
| { |
| "completion_length": 19.8125, |
| "epoch": 3.7322175732217575, |
| "grad_norm": 7.088319301605225, |
| "kl": 0.19287109375, |
| "learning_rate": 6.799163179916317e-08, |
| "loss": 0.0077, |
| "reward": 0.5788787603378296, |
| "reward_std": 0.12247762642800808, |
| "reward_std_all": 0.1116358321160078, |
| "rewards-max/forensics_iou_reward": 0.9582516551017761, |
| "rewards/forensics_iou_reward": 0.5788787603378296, |
| "step": 892 |
| }, |
| { |
| "completion_length": 21.265625, |
| "epoch": 3.7364016736401675, |
| "grad_norm": 6.06962251663208, |
| "kl": 0.21875, |
| "learning_rate": 6.694560669456066e-08, |
| "loss": 0.0088, |
| "reward": 0.49342262744903564, |
| "reward_std": 0.05090899020433426, |
| "reward_std_all": 0.06500819884240627, |
| "rewards-max/forensics_iou_reward": 0.6971125304698944, |
| "rewards/forensics_iou_reward": 0.49342262744903564, |
| "step": 893 |
| }, |
| { |
| "completion_length": 19.859375, |
| "epoch": 3.7405857740585775, |
| "grad_norm": 7.396283149719238, |
| "kl": 0.23828125, |
| "learning_rate": 6.589958158995816e-08, |
| "loss": 0.0095, |
| "reward": 0.6140889525413513, |
| "reward_std": 0.11473578214645386, |
| "reward_std_all": 0.12847280502319336, |
| "rewards-max/forensics_iou_reward": 0.9749964773654938, |
| "rewards/forensics_iou_reward": 0.6140889525413513, |
| "step": 894 |
| }, |
| { |
| "completion_length": 18.953125, |
| "epoch": 3.7447698744769875, |
| "grad_norm": 8.541275024414062, |
| "kl": 0.20556640625, |
| "learning_rate": 6.485355648535565e-08, |
| "loss": 0.0082, |
| "reward": 0.6776835322380066, |
| "reward_std": 0.07499459944665432, |
| "reward_std_all": 0.07928700558841228, |
| "rewards-max/forensics_iou_reward": 0.9592314660549164, |
| "rewards/forensics_iou_reward": 0.6776835322380066, |
| "step": 895 |
| }, |
| { |
| "completion_length": 20.203125, |
| "epoch": 3.7489539748953975, |
| "grad_norm": 5.755167007446289, |
| "kl": 0.3515625, |
| "learning_rate": 6.380753138075313e-08, |
| "loss": 0.0141, |
| "reward": 0.6703572273254395, |
| "reward_std": 0.043831586837768555, |
| "reward_std_all": 0.04987550154328346, |
| "rewards-max/forensics_iou_reward": 0.9909776449203491, |
| "rewards/forensics_iou_reward": 0.6703572273254395, |
| "step": 896 |
| }, |
| { |
| "completion_length": 19.421875, |
| "epoch": 3.7531380753138075, |
| "grad_norm": 6.7464070320129395, |
| "kl": 0.19970703125, |
| "learning_rate": 6.276150627615062e-08, |
| "loss": 0.008, |
| "reward": 0.6082351207733154, |
| "reward_std": 0.05140582472085953, |
| "reward_std_all": 0.05595888011157513, |
| "rewards-max/forensics_iou_reward": 0.9711624979972839, |
| "rewards/forensics_iou_reward": 0.6082351207733154, |
| "step": 897 |
| }, |
| { |
| "completion_length": 20.484375, |
| "epoch": 3.7573221757322175, |
| "grad_norm": 5.670766830444336, |
| "kl": 0.25439453125, |
| "learning_rate": 6.171548117154812e-08, |
| "loss": 0.0102, |
| "reward": 0.6505111753940582, |
| "reward_std": 0.06477977707982063, |
| "reward_std_all": 0.06617007590830326, |
| "rewards-max/forensics_iou_reward": 0.9688008725643158, |
| "rewards/forensics_iou_reward": 0.6505111753940582, |
| "step": 898 |
| }, |
| { |
| "completion_length": 19.609375, |
| "epoch": 3.7615062761506275, |
| "grad_norm": 6.501717567443848, |
| "kl": 0.2421875, |
| "learning_rate": 6.06694560669456e-08, |
| "loss": 0.0097, |
| "reward": 0.6044117212295532, |
| "reward_std": 0.06857657060027122, |
| "reward_std_all": 0.06817344389855862, |
| "rewards-max/forensics_iou_reward": 0.9419128596782684, |
| "rewards/forensics_iou_reward": 0.6044117212295532, |
| "step": 899 |
| }, |
| { |
| "completion_length": 19.25, |
| "epoch": 3.7656903765690375, |
| "grad_norm": 6.839367866516113, |
| "kl": 0.1630859375, |
| "learning_rate": 5.962343096234309e-08, |
| "loss": 0.0065, |
| "reward": 0.5652490258216858, |
| "reward_std": 0.053392043337225914, |
| "reward_std_all": 0.056824998930096626, |
| "rewards-max/forensics_iou_reward": 0.8327121436595917, |
| "rewards/forensics_iou_reward": 0.5652490258216858, |
| "step": 900 |
| }, |
| { |
| "completion_length": 19.640625, |
| "epoch": 3.7698744769874475, |
| "grad_norm": 5.036550045013428, |
| "kl": 0.17431640625, |
| "learning_rate": 5.8577405857740584e-08, |
| "loss": 0.007, |
| "reward": 0.6689288318157196, |
| "reward_std": 0.03520103916525841, |
| "reward_std_all": 0.05553664453327656, |
| "rewards-max/forensics_iou_reward": 0.9739476442337036, |
| "rewards/forensics_iou_reward": 0.6689288318157196, |
| "step": 901 |
| }, |
| { |
| "completion_length": 19.75, |
| "epoch": 3.7740585774058575, |
| "grad_norm": 6.15358304977417, |
| "kl": 0.212890625, |
| "learning_rate": 5.753138075313807e-08, |
| "loss": 0.0085, |
| "reward": 0.5467930734157562, |
| "reward_std": 0.07522759400308132, |
| "reward_std_all": 0.07380538806319237, |
| "rewards-max/forensics_iou_reward": 0.8976826965808868, |
| "rewards/forensics_iou_reward": 0.5467930734157562, |
| "step": 902 |
| }, |
| { |
| "completion_length": 19.703125, |
| "epoch": 3.778242677824268, |
| "grad_norm": 8.428044319152832, |
| "kl": 0.2646484375, |
| "learning_rate": 5.6485355648535564e-08, |
| "loss": 0.0106, |
| "reward": 0.5812807977199554, |
| "reward_std": 0.05388137139379978, |
| "reward_std_all": 0.059342194348573685, |
| "rewards-max/forensics_iou_reward": 0.9521642923355103, |
| "rewards/forensics_iou_reward": 0.5812807977199554, |
| "step": 903 |
| }, |
| { |
| "completion_length": 19.84375, |
| "epoch": 3.782426778242678, |
| "grad_norm": 9.170724868774414, |
| "kl": 0.158203125, |
| "learning_rate": 5.543933054393305e-08, |
| "loss": 0.0063, |
| "reward": 0.6977536678314209, |
| "reward_std": 0.06675713136792183, |
| "reward_std_all": 0.06216430850327015, |
| "rewards-max/forensics_iou_reward": 0.9129084944725037, |
| "rewards/forensics_iou_reward": 0.6977536678314209, |
| "step": 904 |
| }, |
| { |
| "completion_length": 19.796875, |
| "epoch": 3.786610878661088, |
| "grad_norm": 7.746022701263428, |
| "kl": 0.19677734375, |
| "learning_rate": 5.439330543933054e-08, |
| "loss": 0.0079, |
| "reward": 0.5257458686828613, |
| "reward_std": 0.10996879637241364, |
| "reward_std_all": 0.10504969209432602, |
| "rewards-max/forensics_iou_reward": 0.945961207151413, |
| "rewards/forensics_iou_reward": 0.5257458686828613, |
| "step": 905 |
| }, |
| { |
| "completion_length": 19.765625, |
| "epoch": 3.790794979079498, |
| "grad_norm": 6.608165264129639, |
| "kl": 0.2177734375, |
| "learning_rate": 5.334728033472803e-08, |
| "loss": 0.0087, |
| "reward": 0.6440927684307098, |
| "reward_std": 0.06265414133667946, |
| "reward_std_all": 0.0682198628783226, |
| "rewards-max/forensics_iou_reward": 0.9583694040775299, |
| "rewards/forensics_iou_reward": 0.6440927684307098, |
| "step": 906 |
| }, |
| { |
| "completion_length": 19.765625, |
| "epoch": 3.794979079497908, |
| "grad_norm": 5.9058308601379395, |
| "kl": 0.298828125, |
| "learning_rate": 5.2301255230125516e-08, |
| "loss": 0.0119, |
| "reward": 0.45921993255615234, |
| "reward_std": 0.08555106446146965, |
| "reward_std_all": 0.07732369005680084, |
| "rewards-max/forensics_iou_reward": 0.9538154900074005, |
| "rewards/forensics_iou_reward": 0.45921993255615234, |
| "step": 907 |
| }, |
| { |
| "completion_length": 20.140625, |
| "epoch": 3.799163179916318, |
| "grad_norm": 6.401925563812256, |
| "kl": 0.20849609375, |
| "learning_rate": 5.1255230125523015e-08, |
| "loss": 0.0084, |
| "reward": 0.6263912916183472, |
| "reward_std": 0.043489690870046616, |
| "reward_std_all": 0.057912785559892654, |
| "rewards-max/forensics_iou_reward": 0.9547283053398132, |
| "rewards/forensics_iou_reward": 0.6263912916183472, |
| "step": 908 |
| }, |
| { |
| "completion_length": 19.9375, |
| "epoch": 3.803347280334728, |
| "grad_norm": 9.388350486755371, |
| "kl": 0.40234375, |
| "learning_rate": 5.02092050209205e-08, |
| "loss": 0.0161, |
| "reward": 0.6753903031349182, |
| "reward_std": 0.1143597662448883, |
| "reward_std_all": 0.10481419041752815, |
| "rewards-max/forensics_iou_reward": 0.9948576092720032, |
| "rewards/forensics_iou_reward": 0.6753903031349182, |
| "step": 909 |
| }, |
| { |
| "completion_length": 19.90625, |
| "epoch": 3.8075313807531384, |
| "grad_norm": 4.768494606018066, |
| "kl": 0.22216796875, |
| "learning_rate": 4.9163179916317995e-08, |
| "loss": 0.0089, |
| "reward": 0.7123938202857971, |
| "reward_std": 0.057293668389320374, |
| "reward_std_all": 0.06704975105822086, |
| "rewards-max/forensics_iou_reward": 0.96685990691185, |
| "rewards/forensics_iou_reward": 0.7123938202857971, |
| "step": 910 |
| }, |
| { |
| "completion_length": 19.84375, |
| "epoch": 3.8117154811715483, |
| "grad_norm": 5.925776481628418, |
| "kl": 0.2333984375, |
| "learning_rate": 4.811715481171548e-08, |
| "loss": 0.0093, |
| "reward": 0.5213495492935181, |
| "reward_std": 0.0646356251090765, |
| "reward_std_all": 0.08138420060276985, |
| "rewards-max/forensics_iou_reward": 0.8708642423152924, |
| "rewards/forensics_iou_reward": 0.5213495492935181, |
| "step": 911 |
| }, |
| { |
| "completion_length": 19.84375, |
| "epoch": 3.8158995815899583, |
| "grad_norm": 8.53896427154541, |
| "kl": 0.23046875, |
| "learning_rate": 4.707112970711297e-08, |
| "loss": 0.0092, |
| "reward": 0.631392627954483, |
| "reward_std": 0.08032131008803844, |
| "reward_std_all": 0.07349587976932526, |
| "rewards-max/forensics_iou_reward": 0.9097904562950134, |
| "rewards/forensics_iou_reward": 0.631392627954483, |
| "step": 912 |
| }, |
| { |
| "completion_length": 20.453125, |
| "epoch": 3.8200836820083683, |
| "grad_norm": 8.391043663024902, |
| "kl": 0.171875, |
| "learning_rate": 4.602510460251046e-08, |
| "loss": 0.0069, |
| "reward": 0.6572259366512299, |
| "reward_std": 0.07529779896140099, |
| "reward_std_all": 0.0772387720644474, |
| "rewards-max/forensics_iou_reward": 0.9632253646850586, |
| "rewards/forensics_iou_reward": 0.6572259366512299, |
| "step": 913 |
| }, |
| { |
| "completion_length": 20.5, |
| "epoch": 3.8242677824267783, |
| "grad_norm": 8.629030227661133, |
| "kl": 0.189453125, |
| "learning_rate": 4.497907949790795e-08, |
| "loss": 0.0076, |
| "reward": 0.5131673514842987, |
| "reward_std": 0.0666593611240387, |
| "reward_std_all": 0.07044555991888046, |
| "rewards-max/forensics_iou_reward": 0.975288450717926, |
| "rewards/forensics_iou_reward": 0.5131673514842987, |
| "step": 914 |
| }, |
| { |
| "completion_length": 21.578125, |
| "epoch": 3.8284518828451883, |
| "grad_norm": 4.55079984664917, |
| "kl": 0.27197265625, |
| "learning_rate": 4.393305439330543e-08, |
| "loss": 0.0109, |
| "reward": 0.5457545220851898, |
| "reward_std": 0.12418095394968987, |
| "reward_std_all": 0.10940432548522949, |
| "rewards-max/forensics_iou_reward": 0.9729763865470886, |
| "rewards/forensics_iou_reward": 0.5457545220851898, |
| "step": 915 |
| }, |
| { |
| "completion_length": 19.78125, |
| "epoch": 3.8326359832635983, |
| "grad_norm": 6.506716728210449, |
| "kl": 0.267578125, |
| "learning_rate": 4.2887029288702926e-08, |
| "loss": 0.0107, |
| "reward": 0.549262285232544, |
| "reward_std": 0.0768287219107151, |
| "reward_std_all": 0.0753881111741066, |
| "rewards-max/forensics_iou_reward": 0.8748505711555481, |
| "rewards/forensics_iou_reward": 0.549262285232544, |
| "step": 916 |
| }, |
| { |
| "completion_length": 19.0625, |
| "epoch": 3.8368200836820083, |
| "grad_norm": 10.692651748657227, |
| "kl": 0.3984375, |
| "learning_rate": 4.184100418410041e-08, |
| "loss": 0.016, |
| "reward": 0.6338827013969421, |
| "reward_std": 0.049754610285162926, |
| "reward_std_all": 0.06150138005614281, |
| "rewards-max/forensics_iou_reward": 0.9085361063480377, |
| "rewards/forensics_iou_reward": 0.6338827013969421, |
| "step": 917 |
| }, |
| { |
| "completion_length": 20.0, |
| "epoch": 3.8410041841004183, |
| "grad_norm": 6.681275367736816, |
| "kl": 0.21484375, |
| "learning_rate": 4.0794979079497906e-08, |
| "loss": 0.0086, |
| "reward": 0.6625626385211945, |
| "reward_std": 0.04502120427787304, |
| "reward_std_all": 0.05242776870727539, |
| "rewards-max/forensics_iou_reward": 0.9891569316387177, |
| "rewards/forensics_iou_reward": 0.6625626385211945, |
| "step": 918 |
| }, |
| { |
| "completion_length": 19.625, |
| "epoch": 3.8451882845188283, |
| "grad_norm": 7.0299482345581055, |
| "kl": 0.2568359375, |
| "learning_rate": 3.974895397489539e-08, |
| "loss": 0.0103, |
| "reward": 0.6665904521942139, |
| "reward_std": 0.06126677989959717, |
| "reward_std_all": 0.05981821566820145, |
| "rewards-max/forensics_iou_reward": 0.9726284742355347, |
| "rewards/forensics_iou_reward": 0.6665904521942139, |
| "step": 919 |
| }, |
| { |
| "completion_length": 20.546875, |
| "epoch": 3.8493723849372383, |
| "grad_norm": 7.071646690368652, |
| "kl": 0.17919921875, |
| "learning_rate": 3.8702928870292885e-08, |
| "loss": 0.0072, |
| "reward": 0.5451256781816483, |
| "reward_std": 0.07304153963923454, |
| "reward_std_all": 0.09387891739606857, |
| "rewards-max/forensics_iou_reward": 0.9293608963489532, |
| "rewards/forensics_iou_reward": 0.5451256781816483, |
| "step": 920 |
| }, |
| { |
| "completion_length": 19.5, |
| "epoch": 3.8535564853556483, |
| "grad_norm": 6.5857672691345215, |
| "kl": 0.20361328125, |
| "learning_rate": 3.765690376569038e-08, |
| "loss": 0.0081, |
| "reward": 0.5826208591461182, |
| "reward_std": 0.04926537908613682, |
| "reward_std_all": 0.06277021393179893, |
| "rewards-max/forensics_iou_reward": 0.9932590425014496, |
| "rewards/forensics_iou_reward": 0.5826208591461182, |
| "step": 921 |
| }, |
| { |
| "completion_length": 20.4375, |
| "epoch": 3.8577405857740583, |
| "grad_norm": 8.101725578308105, |
| "kl": 0.25830078125, |
| "learning_rate": 3.6610878661087864e-08, |
| "loss": 0.0103, |
| "reward": 0.4409870505332947, |
| "reward_std": 0.08444111980497837, |
| "reward_std_all": 0.07580407336354256, |
| "rewards-max/forensics_iou_reward": 0.8819047510623932, |
| "rewards/forensics_iou_reward": 0.4409870505332947, |
| "step": 922 |
| }, |
| { |
| "completion_length": 20.46875, |
| "epoch": 3.8619246861924688, |
| "grad_norm": 4.705468654632568, |
| "kl": 0.19775390625, |
| "learning_rate": 3.556485355648536e-08, |
| "loss": 0.0079, |
| "reward": 0.6545010805130005, |
| "reward_std": 0.08012884110212326, |
| "reward_std_all": 0.0907994881272316, |
| "rewards-max/forensics_iou_reward": 0.9980228841304779, |
| "rewards/forensics_iou_reward": 0.6545010805130005, |
| "step": 923 |
| }, |
| { |
| "completion_length": 19.640625, |
| "epoch": 3.8661087866108788, |
| "grad_norm": 7.469610214233398, |
| "kl": 0.228515625, |
| "learning_rate": 3.4518828451882844e-08, |
| "loss": 0.0091, |
| "reward": 0.6502385437488556, |
| "reward_std": 0.11029687523841858, |
| "reward_std_all": 0.10243702679872513, |
| "rewards-max/forensics_iou_reward": 0.9793510437011719, |
| "rewards/forensics_iou_reward": 0.6502385437488556, |
| "step": 924 |
| }, |
| { |
| "completion_length": 19.796875, |
| "epoch": 3.8702928870292888, |
| "grad_norm": 13.763721466064453, |
| "kl": 0.25537109375, |
| "learning_rate": 3.347280334728033e-08, |
| "loss": 0.0102, |
| "reward": 0.6246716380119324, |
| "reward_std": 0.132647342979908, |
| "reward_std_all": 0.11996451392769814, |
| "rewards-max/forensics_iou_reward": 0.937400758266449, |
| "rewards/forensics_iou_reward": 0.6246716380119324, |
| "step": 925 |
| }, |
| { |
| "completion_length": 20.484375, |
| "epoch": 3.8744769874476988, |
| "grad_norm": 6.838551044464111, |
| "kl": 0.26123046875, |
| "learning_rate": 3.242677824267782e-08, |
| "loss": 0.0104, |
| "reward": 0.7122205197811127, |
| "reward_std": 0.06346860527992249, |
| "reward_std_all": 0.06881479173898697, |
| "rewards-max/forensics_iou_reward": 0.9511702358722687, |
| "rewards/forensics_iou_reward": 0.7122205197811127, |
| "step": 926 |
| }, |
| { |
| "completion_length": 20.0, |
| "epoch": 3.8786610878661087, |
| "grad_norm": 6.7476301193237305, |
| "kl": 0.25341796875, |
| "learning_rate": 3.138075313807531e-08, |
| "loss": 0.0101, |
| "reward": 0.567421555519104, |
| "reward_std": 0.08516626060009003, |
| "reward_std_all": 0.07909991964697838, |
| "rewards-max/forensics_iou_reward": 0.9385251402854919, |
| "rewards/forensics_iou_reward": 0.567421555519104, |
| "step": 927 |
| }, |
| { |
| "completion_length": 19.28125, |
| "epoch": 3.8828451882845187, |
| "grad_norm": 5.330680847167969, |
| "kl": 0.28173828125, |
| "learning_rate": 3.03347280334728e-08, |
| "loss": 0.0113, |
| "reward": 0.5760163962841034, |
| "reward_std": 0.07997453585267067, |
| "reward_std_all": 0.07773560285568237, |
| "rewards-max/forensics_iou_reward": 0.9094879627227783, |
| "rewards/forensics_iou_reward": 0.5760163962841034, |
| "step": 928 |
| }, |
| { |
| "completion_length": 21.515625, |
| "epoch": 3.8870292887029287, |
| "grad_norm": 8.179926872253418, |
| "kl": 0.212890625, |
| "learning_rate": 2.9288702928870292e-08, |
| "loss": 0.0085, |
| "reward": 0.6287315189838409, |
| "reward_std": 0.08616849407553673, |
| "reward_std_all": 0.0841730423271656, |
| "rewards-max/forensics_iou_reward": 0.985221654176712, |
| "rewards/forensics_iou_reward": 0.6287315189838409, |
| "step": 929 |
| }, |
| { |
| "completion_length": 20.125, |
| "epoch": 3.891213389121339, |
| "grad_norm": 6.6296586990356445, |
| "kl": 0.228515625, |
| "learning_rate": 2.8242677824267782e-08, |
| "loss": 0.0092, |
| "reward": 0.622155487537384, |
| "reward_std": 0.08535593748092651, |
| "reward_std_all": 0.07641034200787544, |
| "rewards-max/forensics_iou_reward": 0.9881754815578461, |
| "rewards/forensics_iou_reward": 0.622155487537384, |
| "step": 930 |
| }, |
| { |
| "completion_length": 20.859375, |
| "epoch": 3.895397489539749, |
| "grad_norm": 6.359039306640625, |
| "kl": 0.212890625, |
| "learning_rate": 2.719665271966527e-08, |
| "loss": 0.0085, |
| "reward": 0.4475126415491104, |
| "reward_std": 0.07427852600812912, |
| "reward_std_all": 0.08763879537582397, |
| "rewards-max/forensics_iou_reward": 0.8929985463619232, |
| "rewards/forensics_iou_reward": 0.4475126415491104, |
| "step": 931 |
| }, |
| { |
| "completion_length": 19.171875, |
| "epoch": 3.899581589958159, |
| "grad_norm": 9.47093677520752, |
| "kl": 0.1904296875, |
| "learning_rate": 2.6150627615062758e-08, |
| "loss": 0.0076, |
| "reward": 0.6034533232450485, |
| "reward_std": 0.0440397709608078, |
| "reward_std_all": 0.05571764335036278, |
| "rewards-max/forensics_iou_reward": 0.9567655920982361, |
| "rewards/forensics_iou_reward": 0.6034533232450485, |
| "step": 932 |
| }, |
| { |
| "completion_length": 19.640625, |
| "epoch": 3.903765690376569, |
| "grad_norm": 5.803466796875, |
| "kl": 0.19140625, |
| "learning_rate": 2.510460251046025e-08, |
| "loss": 0.0076, |
| "reward": 0.6087769269943237, |
| "reward_std": 0.08506398648023605, |
| "reward_std_all": 0.07243844121694565, |
| "rewards-max/forensics_iou_reward": 0.937892884016037, |
| "rewards/forensics_iou_reward": 0.6087769269943237, |
| "step": 933 |
| }, |
| { |
| "completion_length": 20.140625, |
| "epoch": 3.907949790794979, |
| "grad_norm": 12.995620727539062, |
| "kl": 0.171875, |
| "learning_rate": 2.405857740585774e-08, |
| "loss": 0.0069, |
| "reward": 0.5822662115097046, |
| "reward_std": 0.08958427608013153, |
| "reward_std_all": 0.08720807358622551, |
| "rewards-max/forensics_iou_reward": 0.9655969738960266, |
| "rewards/forensics_iou_reward": 0.5822662115097046, |
| "step": 934 |
| }, |
| { |
| "completion_length": 19.96875, |
| "epoch": 3.912133891213389, |
| "grad_norm": 6.811155319213867, |
| "kl": 0.26416015625, |
| "learning_rate": 2.301255230125523e-08, |
| "loss": 0.0106, |
| "reward": 0.6628967225551605, |
| "reward_std": 0.053724393248558044, |
| "reward_std_all": 0.057547738775610924, |
| "rewards-max/forensics_iou_reward": 0.9102648794651031, |
| "rewards/forensics_iou_reward": 0.6628967225551605, |
| "step": 935 |
| }, |
| { |
| "completion_length": 19.84375, |
| "epoch": 3.916317991631799, |
| "grad_norm": 9.141552925109863, |
| "kl": 0.32275390625, |
| "learning_rate": 2.1966527196652717e-08, |
| "loss": 0.0129, |
| "reward": 0.6373286843299866, |
| "reward_std": 0.07491157203912735, |
| "reward_std_all": 0.07120720855891705, |
| "rewards-max/forensics_iou_reward": 0.9883822202682495, |
| "rewards/forensics_iou_reward": 0.6373286843299866, |
| "step": 936 |
| }, |
| { |
| "completion_length": 22.203125, |
| "epoch": 3.920502092050209, |
| "grad_norm": 5.841102600097656, |
| "kl": 0.19091796875, |
| "learning_rate": 2.0920502092050206e-08, |
| "loss": 0.0076, |
| "reward": 0.5705366134643555, |
| "reward_std": 0.10749761387705803, |
| "reward_std_all": 0.0967276394367218, |
| "rewards-max/forensics_iou_reward": 0.9762770533561707, |
| "rewards/forensics_iou_reward": 0.5705366134643555, |
| "step": 937 |
| }, |
| { |
| "completion_length": 21.0625, |
| "epoch": 3.924686192468619, |
| "grad_norm": 9.04256820678711, |
| "kl": 0.20703125, |
| "learning_rate": 1.9874476987447696e-08, |
| "loss": 0.0083, |
| "reward": 0.5629846900701523, |
| "reward_std": 0.10904347151517868, |
| "reward_std_all": 0.104925237596035, |
| "rewards-max/forensics_iou_reward": 0.9171201884746552, |
| "rewards/forensics_iou_reward": 0.5629846900701523, |
| "step": 938 |
| }, |
| { |
| "completion_length": 19.84375, |
| "epoch": 3.928870292887029, |
| "grad_norm": 6.77930212020874, |
| "kl": 0.1806640625, |
| "learning_rate": 1.882845188284519e-08, |
| "loss": 0.0072, |
| "reward": 0.5012898445129395, |
| "reward_std": 0.04762773960828781, |
| "reward_std_all": 0.05298398807644844, |
| "rewards-max/forensics_iou_reward": 0.9362242519855499, |
| "rewards/forensics_iou_reward": 0.5012898445129395, |
| "step": 939 |
| }, |
| { |
| "completion_length": 20.203125, |
| "epoch": 3.933054393305439, |
| "grad_norm": 6.475246906280518, |
| "kl": 0.3505859375, |
| "learning_rate": 1.778242677824268e-08, |
| "loss": 0.014, |
| "reward": 0.5372985899448395, |
| "reward_std": 0.08175596408545971, |
| "reward_std_all": 0.08148378692567348, |
| "rewards-max/forensics_iou_reward": 0.9084711968898773, |
| "rewards/forensics_iou_reward": 0.5372985899448395, |
| "step": 940 |
| }, |
| { |
| "completion_length": 21.265625, |
| "epoch": 3.937238493723849, |
| "grad_norm": 6.317575454711914, |
| "kl": 0.22119140625, |
| "learning_rate": 1.6736401673640165e-08, |
| "loss": 0.0089, |
| "reward": 0.6404686570167542, |
| "reward_std": 0.10436028614640236, |
| "reward_std_all": 0.10430000722408295, |
| "rewards-max/forensics_iou_reward": 0.9627142250537872, |
| "rewards/forensics_iou_reward": 0.6404686570167542, |
| "step": 941 |
| }, |
| { |
| "completion_length": 21.359375, |
| "epoch": 3.941422594142259, |
| "grad_norm": 7.682011604309082, |
| "kl": 0.24365234375, |
| "learning_rate": 1.5690376569037655e-08, |
| "loss": 0.0097, |
| "reward": 0.5516895949840546, |
| "reward_std": 0.08274391293525696, |
| "reward_std_all": 0.08605389669537544, |
| "rewards-max/forensics_iou_reward": 0.9450686275959015, |
| "rewards/forensics_iou_reward": 0.5516895949840546, |
| "step": 942 |
| }, |
| { |
| "completion_length": 20.078125, |
| "epoch": 3.9456066945606696, |
| "grad_norm": 6.481320858001709, |
| "kl": 0.2216796875, |
| "learning_rate": 1.4644351464435146e-08, |
| "loss": 0.0089, |
| "reward": 0.5350929945707321, |
| "reward_std": 0.04935399815440178, |
| "reward_std_all": 0.06646456569433212, |
| "rewards-max/forensics_iou_reward": 0.9800728261470795, |
| "rewards/forensics_iou_reward": 0.5350929945707321, |
| "step": 943 |
| }, |
| { |
| "completion_length": 21.109375, |
| "epoch": 3.9497907949790796, |
| "grad_norm": 13.831422805786133, |
| "kl": 0.26611328125, |
| "learning_rate": 1.3598326359832636e-08, |
| "loss": 0.0106, |
| "reward": 0.5788275003433228, |
| "reward_std": 0.08550543338060379, |
| "reward_std_all": 0.09766050055623055, |
| "rewards-max/forensics_iou_reward": 0.9617001414299011, |
| "rewards/forensics_iou_reward": 0.5788275003433228, |
| "step": 944 |
| }, |
| { |
| "completion_length": 20.359375, |
| "epoch": 3.9539748953974896, |
| "grad_norm": 5.313716411590576, |
| "kl": 0.1982421875, |
| "learning_rate": 1.2552301255230125e-08, |
| "loss": 0.0079, |
| "reward": 0.44177675247192383, |
| "reward_std": 0.06517988629639149, |
| "reward_std_all": 0.06525103282183409, |
| "rewards-max/forensics_iou_reward": 0.9018962979316711, |
| "rewards/forensics_iou_reward": 0.44177675247192383, |
| "step": 945 |
| }, |
| { |
| "completion_length": 19.59375, |
| "epoch": 3.9581589958158996, |
| "grad_norm": 8.787726402282715, |
| "kl": 0.21728515625, |
| "learning_rate": 1.1506276150627615e-08, |
| "loss": 0.0087, |
| "reward": 0.5039725452661514, |
| "reward_std": 0.05131026357412338, |
| "reward_std_all": 0.06917222775518894, |
| "rewards-max/forensics_iou_reward": 0.9380869567394257, |
| "rewards/forensics_iou_reward": 0.5039725452661514, |
| "step": 946 |
| }, |
| { |
| "completion_length": 19.875, |
| "epoch": 3.9623430962343096, |
| "grad_norm": 5.468041896820068, |
| "kl": 0.22998046875, |
| "learning_rate": 1.0460251046025103e-08, |
| "loss": 0.0092, |
| "reward": 0.5297128856182098, |
| "reward_std": 0.08246839791536331, |
| "reward_std_all": 0.0859297662973404, |
| "rewards-max/forensics_iou_reward": 0.7993016242980957, |
| "rewards/forensics_iou_reward": 0.5297128856182098, |
| "step": 947 |
| }, |
| { |
| "completion_length": 19.96875, |
| "epoch": 3.9665271966527196, |
| "grad_norm": 6.094367980957031, |
| "kl": 0.20654296875, |
| "learning_rate": 9.414225941422594e-09, |
| "loss": 0.0082, |
| "reward": 0.5609661936759949, |
| "reward_std": 0.11640171334147453, |
| "reward_std_all": 0.11182210221886635, |
| "rewards-max/forensics_iou_reward": 0.8814770877361298, |
| "rewards/forensics_iou_reward": 0.5609661936759949, |
| "step": 948 |
| }, |
| { |
| "completion_length": 19.765625, |
| "epoch": 3.9707112970711296, |
| "grad_norm": 5.223315238952637, |
| "kl": 0.3837890625, |
| "learning_rate": 8.368200836820083e-09, |
| "loss": 0.0153, |
| "reward": 0.6239343583583832, |
| "reward_std": 0.11170723661780357, |
| "reward_std_all": 0.13546176254749298, |
| "rewards-max/forensics_iou_reward": 0.9098612666130066, |
| "rewards/forensics_iou_reward": 0.6239343583583832, |
| "step": 949 |
| }, |
| { |
| "completion_length": 19.890625, |
| "epoch": 3.97489539748954, |
| "grad_norm": 9.110376358032227, |
| "kl": 0.23486328125, |
| "learning_rate": 7.322175732217573e-09, |
| "loss": 0.0094, |
| "reward": 0.4744117259979248, |
| "reward_std": 0.07315591350197792, |
| "reward_std_all": 0.07490979135036469, |
| "rewards-max/forensics_iou_reward": 0.875, |
| "rewards/forensics_iou_reward": 0.4744117259979248, |
| "step": 950 |
| }, |
| { |
| "completion_length": 19.4375, |
| "epoch": 3.97907949790795, |
| "grad_norm": 6.811954498291016, |
| "kl": 0.2265625, |
| "learning_rate": 6.276150627615063e-09, |
| "loss": 0.009, |
| "reward": 0.6259590089321136, |
| "reward_std": 0.06351862661540508, |
| "reward_std_all": 0.060562750324606895, |
| "rewards-max/forensics_iou_reward": 0.9207738041877747, |
| "rewards/forensics_iou_reward": 0.6259590089321136, |
| "step": 951 |
| }, |
| { |
| "completion_length": 20.0625, |
| "epoch": 3.98326359832636, |
| "grad_norm": 4.30385684967041, |
| "kl": 0.20947265625, |
| "learning_rate": 5.2301255230125516e-09, |
| "loss": 0.0084, |
| "reward": 0.5343451797962189, |
| "reward_std": 0.07074539363384247, |
| "reward_std_all": 0.0918283574283123, |
| "rewards-max/forensics_iou_reward": 0.9854446053504944, |
| "rewards/forensics_iou_reward": 0.5343451797962189, |
| "step": 952 |
| }, |
| { |
| "completion_length": 19.484375, |
| "epoch": 3.98744769874477, |
| "grad_norm": 6.667928695678711, |
| "kl": 0.296875, |
| "learning_rate": 4.184100418410041e-09, |
| "loss": 0.0119, |
| "reward": 0.5700589120388031, |
| "reward_std": 0.08708499744534492, |
| "reward_std_all": 0.07966680824756622, |
| "rewards-max/forensics_iou_reward": 0.9366612136363983, |
| "rewards/forensics_iou_reward": 0.5700589120388031, |
| "step": 953 |
| }, |
| { |
| "completion_length": 20.59375, |
| "epoch": 3.99163179916318, |
| "grad_norm": 6.974944591522217, |
| "kl": 0.19189453125, |
| "learning_rate": 3.1380753138075314e-09, |
| "loss": 0.0077, |
| "reward": 0.6919052004814148, |
| "reward_std": 0.09324624761939049, |
| "reward_std_all": 0.08810235187411308, |
| "rewards-max/forensics_iou_reward": 0.9887072741985321, |
| "rewards/forensics_iou_reward": 0.6919052004814148, |
| "step": 954 |
| }, |
| { |
| "completion_length": 20.03125, |
| "epoch": 3.99581589958159, |
| "grad_norm": 5.052855491638184, |
| "kl": 0.19873046875, |
| "learning_rate": 2.0920502092050206e-09, |
| "loss": 0.008, |
| "reward": 0.6752189099788666, |
| "reward_std": 0.07558301649987698, |
| "reward_std_all": 0.07682192325592041, |
| "rewards-max/forensics_iou_reward": 0.9828662574291229, |
| "rewards/forensics_iou_reward": 0.6752189099788666, |
| "step": 955 |
| }, |
| { |
| "completion_length": 18.928571701049805, |
| "epoch": 4.0, |
| "grad_norm": 5.962639331817627, |
| "kl": 0.20703125, |
| "learning_rate": 1.0460251046025103e-09, |
| "loss": 0.0103, |
| "reward": 0.629859447479248, |
| "reward_std": 0.0633792420849204, |
| "reward_std_all": 0.04159746132791042, |
| "rewards-max/forensics_iou_reward": 0.7161838412284851, |
| "rewards/forensics_iou_reward": 0.629859447479248, |
| "step": 956 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 956, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 4, |
| "save_steps": 956, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
