diff --git "a/af/trainer_state.json" "b/af/trainer_state.json" new file mode 100644--- /dev/null +++ "b/af/trainer_state.json" @@ -0,0 +1,23434 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.989247311827957, + "eval_steps": 500, + "global_step": 1300, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "completion_length": 25.46875, + "epoch": 0.0030721966205837174, + "grad_norm": 15.017864227294922, + "kl": 0.0, + "learning_rate": 1e-06, + "loss": 0.107, + "reward": 0.5180116593837738, + "reward_std": 0.3052596151828766, + "reward_std_all": 0.29121407866477966, + "rewards-max/forensics_sinkhorn_reward": 0.8892748057842255, + "rewards-max/forensics_strict_edge_reward": 0.011221585165912984, + "rewards-max/forensics_tiou_reward": 0.7268121540546417, + "rewards/forensics_sinkhorn_reward": 0.42016465961933136, + "rewards/forensics_strict_edge_reward": 0.000849602663265614, + "rewards/forensics_tiou_reward": 0.17821162939071655, + "step": 1 + }, + { + "completion_length": 24.28125, + "epoch": 0.006144393241167435, + "grad_norm": 4.406585216522217, + "kl": 0.00018072128295898438, + "learning_rate": 9.992307692307693e-07, + "loss": 0.1508, + "reward": 0.5576996505260468, + "reward_std": 0.18033110350370407, + "reward_std_all": 0.20349255204200745, + "rewards-max/forensics_sinkhorn_reward": 0.9156321585178375, + "rewards-max/forensics_strict_edge_reward": 0.032440694980323315, + "rewards-max/forensics_tiou_reward": 0.8120884895324707, + "rewards/forensics_sinkhorn_reward": 0.4417954981327057, + "rewards/forensics_strict_edge_reward": 0.001776575983967632, + "rewards/forensics_tiou_reward": 0.2040727362036705, + "step": 2 + }, + { + "completion_length": 24.296875, + "epoch": 0.009216589861751152, + "grad_norm": 6.433550834655762, + "kl": 0.001674652099609375, + "learning_rate": 9.984615384615384e-07, + "loss": 0.0306, + "reward": 0.555063784122467, + "reward_std": 0.2399495244026184, + "reward_std_all": 0.24139754474163055, + "rewards-max/forensics_sinkhorn_reward": 0.8463857471942902, + "rewards-max/forensics_strict_edge_reward": 0.18014888744801283, + "rewards-max/forensics_tiou_reward": 0.6632390916347504, + "rewards/forensics_sinkhorn_reward": 0.42813771963119507, + "rewards/forensics_strict_edge_reward": 0.010839145339559764, + "rewards/forensics_tiou_reward": 0.17328982800245285, + "step": 3 + }, + { + "completion_length": 29.703125, + "epoch": 0.01228878648233487, + "grad_norm": 5.221900463104248, + "kl": 0.002071380615234375, + "learning_rate": 9.976923076923077e-07, + "loss": 0.1275, + "reward": 0.574498325586319, + "reward_std": 0.20493483543395996, + "reward_std_all": 0.21066967397928238, + "rewards-max/forensics_sinkhorn_reward": 0.8603406250476837, + "rewards-max/forensics_strict_edge_reward": 0.006181619595736265, + "rewards-max/forensics_tiou_reward": 0.6930947601795197, + "rewards/forensics_sinkhorn_reward": 0.4606490582227707, + "rewards/forensics_strict_edge_reward": 0.00039696856401860714, + "rewards/forensics_tiou_reward": 0.20066239684820175, + "step": 4 + }, + { + "completion_length": 20.5625, + "epoch": 0.015360983102918587, + "grad_norm": 11.033803939819336, + "kl": 0.0239105224609375, + "learning_rate": 9.969230769230768e-07, + "loss": 0.2085, + "reward": 0.5439513623714447, + "reward_std": 0.2195013090968132, + "reward_std_all": 0.23005983978509903, + "rewards-max/forensics_sinkhorn_reward": 0.9066244661808014, + "rewards-max/forensics_strict_edge_reward": 0.09076120145618916, + "rewards-max/forensics_tiou_reward": 0.8069023191928864, + "rewards/forensics_sinkhorn_reward": 0.44591861963272095, + "rewards/forensics_strict_edge_reward": 0.004822091657842975, + "rewards/forensics_tiou_reward": 0.16517234593629837, + "step": 5 + }, + { + "completion_length": 25.21875, + "epoch": 0.018433179723502304, + "grad_norm": 6.229222774505615, + "kl": 0.0311279296875, + "learning_rate": 9.961538461538461e-07, + "loss": -0.0428, + "reward": 0.5845586657524109, + "reward_std": 0.17881572991609573, + "reward_std_all": 0.18833837658166885, + "rewards-max/forensics_sinkhorn_reward": 0.7797255218029022, + "rewards-max/forensics_strict_edge_reward": 0.20107876020483673, + "rewards-max/forensics_tiou_reward": 0.5945796370506287, + "rewards/forensics_sinkhorn_reward": 0.4104858934879303, + "rewards/forensics_strict_edge_reward": 0.008412680566834752, + "rewards/forensics_tiou_reward": 0.1661985144019127, + "step": 6 + }, + { + "completion_length": 24.375, + "epoch": 0.021505376344086023, + "grad_norm": 4.831221103668213, + "kl": 0.03033447265625, + "learning_rate": 9.953846153846154e-07, + "loss": 0.1293, + "reward": 0.5829033255577087, + "reward_std": 0.16157164424657822, + "reward_std_all": 0.18374278396368027, + "rewards-max/forensics_sinkhorn_reward": 0.8279477059841156, + "rewards-max/forensics_strict_edge_reward": 0.2507351003587246, + "rewards-max/forensics_tiou_reward": 0.6415424346923828, + "rewards/forensics_sinkhorn_reward": 0.4545351564884186, + "rewards/forensics_strict_edge_reward": 0.01046559342648834, + "rewards/forensics_tiou_reward": 0.17269325256347656, + "step": 7 + }, + { + "completion_length": 22.75, + "epoch": 0.02457757296466974, + "grad_norm": 6.224086284637451, + "kl": 0.0322265625, + "learning_rate": 9.946153846153846e-07, + "loss": 0.0499, + "reward": 0.5687319338321686, + "reward_std": 0.17719502747058868, + "reward_std_all": 0.19686701893806458, + "rewards-max/forensics_sinkhorn_reward": 0.8560549914836884, + "rewards-max/forensics_strict_edge_reward": 0.12837854606732435, + "rewards-max/forensics_tiou_reward": 0.6846510469913483, + "rewards/forensics_sinkhorn_reward": 0.4403053820133209, + "rewards/forensics_strict_edge_reward": 0.006619214262173045, + "rewards/forensics_tiou_reward": 0.17232344299554825, + "step": 8 + }, + { + "completion_length": 23.859375, + "epoch": 0.027649769585253458, + "grad_norm": 4.864814281463623, + "kl": 0.0582275390625, + "learning_rate": 9.938461538461539e-07, + "loss": 0.164, + "reward": 0.6267366409301758, + "reward_std": 0.1526922509074211, + "reward_std_all": 0.15724144876003265, + "rewards-max/forensics_sinkhorn_reward": 0.7881395220756531, + "rewards-max/forensics_strict_edge_reward": 0.001239844277733937, + "rewards-max/forensics_tiou_reward": 0.527285248041153, + "rewards/forensics_sinkhorn_reward": 0.4773080050945282, + "rewards/forensics_strict_edge_reward": 8.952419375418685e-05, + "rewards/forensics_tiou_reward": 0.1607183963060379, + "step": 9 + }, + { + "completion_length": 23.5625, + "epoch": 0.030721966205837174, + "grad_norm": 5.202534198760986, + "kl": 0.1240234375, + "learning_rate": 9.93076923076923e-07, + "loss": 0.1894, + "reward": 0.6019327938556671, + "reward_std": 0.1898573786020279, + "reward_std_all": 0.1930967941880226, + "rewards-max/forensics_sinkhorn_reward": 0.9236713647842407, + "rewards-max/forensics_strict_edge_reward": 0.40908672753721476, + "rewards-max/forensics_tiou_reward": 0.8152942657470703, + "rewards/forensics_sinkhorn_reward": 0.489246666431427, + "rewards/forensics_strict_edge_reward": 0.01760488306172192, + "rewards/forensics_tiou_reward": 0.2231603041291237, + "step": 10 + }, + { + "completion_length": 25.375, + "epoch": 0.03379416282642089, + "grad_norm": 4.573007106781006, + "kl": 0.1376953125, + "learning_rate": 9.923076923076923e-07, + "loss": 0.3039, + "reward": 0.5911915600299835, + "reward_std": 0.18369656056165695, + "reward_std_all": 0.1889195740222931, + "rewards-max/forensics_sinkhorn_reward": 0.9465669691562653, + "rewards-max/forensics_strict_edge_reward": 0.19454996660351753, + "rewards-max/forensics_tiou_reward": 0.8690944910049438, + "rewards/forensics_sinkhorn_reward": 0.4587595611810684, + "rewards/forensics_strict_edge_reward": 0.008360049920156598, + "rewards/forensics_tiou_reward": 0.22827893495559692, + "step": 11 + }, + { + "completion_length": 28.53125, + "epoch": 0.03686635944700461, + "grad_norm": 3.8537378311157227, + "kl": 0.135498046875, + "learning_rate": 9.915384615384616e-07, + "loss": 0.1056, + "reward": 0.5832359194755554, + "reward_std": 0.16323647648096085, + "reward_std_all": 0.1834820806980133, + "rewards-max/forensics_sinkhorn_reward": 0.8054409921169281, + "rewards-max/forensics_strict_edge_reward": 0.05968565680086613, + "rewards-max/forensics_tiou_reward": 0.5568203777074814, + "rewards/forensics_sinkhorn_reward": 0.4369667321443558, + "rewards/forensics_strict_edge_reward": 0.002946040127426386, + "rewards/forensics_tiou_reward": 0.20132677257061005, + "step": 12 + }, + { + "completion_length": 23.484375, + "epoch": 0.039938556067588324, + "grad_norm": 4.013275146484375, + "kl": 0.15478515625, + "learning_rate": 9.907692307692307e-07, + "loss": 0.1594, + "reward": 0.6000135838985443, + "reward_std": 0.15791091322898865, + "reward_std_all": 0.17073581367731094, + "rewards-max/forensics_sinkhorn_reward": 0.894521564245224, + "rewards-max/forensics_strict_edge_reward": 0.16486333357170224, + "rewards-max/forensics_tiou_reward": 0.74928879737854, + "rewards/forensics_sinkhorn_reward": 0.4649927169084549, + "rewards/forensics_strict_edge_reward": 0.007751789584290236, + "rewards/forensics_tiou_reward": 0.16230841726064682, + "step": 13 + }, + { + "completion_length": 27.484375, + "epoch": 0.043010752688172046, + "grad_norm": 4.52585506439209, + "kl": 0.1484375, + "learning_rate": 9.9e-07, + "loss": 0.192, + "reward": 0.6182315051555634, + "reward_std": 0.14992449805140495, + "reward_std_all": 0.1647532805800438, + "rewards-max/forensics_sinkhorn_reward": 0.900205135345459, + "rewards-max/forensics_strict_edge_reward": 0.24015378719673208, + "rewards-max/forensics_tiou_reward": 0.7603848874568939, + "rewards/forensics_sinkhorn_reward": 0.4823654890060425, + "rewards/forensics_strict_edge_reward": 0.01763432740717974, + "rewards/forensics_tiou_reward": 0.2435074895620346, + "step": 14 + }, + { + "completion_length": 25.078125, + "epoch": 0.04608294930875576, + "grad_norm": 4.064450263977051, + "kl": 0.18408203125, + "learning_rate": 9.892307692307692e-07, + "loss": 0.2124, + "reward": 0.632150799036026, + "reward_std": 0.13552513718605042, + "reward_std_all": 0.1466834768652916, + "rewards-max/forensics_sinkhorn_reward": 0.9189243614673615, + "rewards-max/forensics_strict_edge_reward": 0.005588223808445036, + "rewards-max/forensics_tiou_reward": 0.8251618444919586, + "rewards/forensics_sinkhorn_reward": 0.49615079164505005, + "rewards/forensics_strict_edge_reward": 0.00027150452297064476, + "rewards/forensics_tiou_reward": 0.205951526761055, + "step": 15 + }, + { + "completion_length": 23.0, + "epoch": 0.04915514592933948, + "grad_norm": 4.368458271026611, + "kl": 0.1552734375, + "learning_rate": 9.884615384615385e-07, + "loss": 0.1397, + "reward": 0.6765664219856262, + "reward_std": 0.1414380744099617, + "reward_std_all": 0.13489285856485367, + "rewards-max/forensics_sinkhorn_reward": 0.8514679074287415, + "rewards-max/forensics_strict_edge_reward": 0.060797755140811205, + "rewards-max/forensics_tiou_reward": 0.7594341337680817, + "rewards/forensics_sinkhorn_reward": 0.5536627173423767, + "rewards/forensics_strict_edge_reward": 0.0026750845718197525, + "rewards/forensics_tiou_reward": 0.2135370969772339, + "step": 16 + }, + { + "completion_length": 24.1875, + "epoch": 0.05222734254992319, + "grad_norm": 4.304068565368652, + "kl": 0.1767578125, + "learning_rate": 9.876923076923076e-07, + "loss": 0.0236, + "reward": 0.6036165952682495, + "reward_std": 0.184492290019989, + "reward_std_all": 0.1857067495584488, + "rewards-max/forensics_sinkhorn_reward": 0.7728465795516968, + "rewards-max/forensics_strict_edge_reward": 0.001324848418569502, + "rewards-max/forensics_tiou_reward": 0.5176638960838318, + "rewards/forensics_sinkhorn_reward": 0.4566531926393509, + "rewards/forensics_strict_edge_reward": 0.00010577176610870875, + "rewards/forensics_tiou_reward": 0.14773979783058167, + "step": 17 + }, + { + "completion_length": 22.484375, + "epoch": 0.055299539170506916, + "grad_norm": 3.614719867706299, + "kl": 0.158203125, + "learning_rate": 9.86923076923077e-07, + "loss": 0.1551, + "reward": 0.6553654372692108, + "reward_std": 0.1630876660346985, + "reward_std_all": 0.1567959226667881, + "rewards-max/forensics_sinkhorn_reward": 0.9024694561958313, + "rewards-max/forensics_strict_edge_reward": 0.034112306660972536, + "rewards-max/forensics_tiou_reward": 0.7450892627239227, + "rewards/forensics_sinkhorn_reward": 0.5492920130491257, + "rewards/forensics_strict_edge_reward": 0.0016751309594837949, + "rewards/forensics_tiou_reward": 0.22265133261680603, + "step": 18 + }, + { + "completion_length": 24.078125, + "epoch": 0.05837173579109063, + "grad_norm": 4.501729488372803, + "kl": 0.154296875, + "learning_rate": 9.86153846153846e-07, + "loss": 0.3348, + "reward": 0.608467698097229, + "reward_std": 0.12505021691322327, + "reward_std_all": 0.15759079158306122, + "rewards-max/forensics_sinkhorn_reward": 0.9497201442718506, + "rewards-max/forensics_strict_edge_reward": 0.2449169121682644, + "rewards-max/forensics_tiou_reward": 0.902175635099411, + "rewards/forensics_sinkhorn_reward": 0.4779987782239914, + "rewards/forensics_strict_edge_reward": 0.015391590422950685, + "rewards/forensics_tiou_reward": 0.22183404117822647, + "step": 19 + }, + { + "completion_length": 26.953125, + "epoch": 0.06144393241167435, + "grad_norm": 4.428674697875977, + "kl": 0.16455078125, + "learning_rate": 9.853846153846153e-07, + "loss": 0.0797, + "reward": 0.632140725851059, + "reward_std": 0.13943550735712051, + "reward_std_all": 0.15233267843723297, + "rewards-max/forensics_sinkhorn_reward": 0.9294396340847015, + "rewards-max/forensics_strict_edge_reward": 0.22317428514361382, + "rewards-max/forensics_tiou_reward": 0.8067216277122498, + "rewards/forensics_sinkhorn_reward": 0.5219174772500992, + "rewards/forensics_strict_edge_reward": 0.00986797921359539, + "rewards/forensics_tiou_reward": 0.2766689658164978, + "step": 20 + }, + { + "completion_length": 21.515625, + "epoch": 0.06451612903225806, + "grad_norm": 5.803476810455322, + "kl": 0.19580078125, + "learning_rate": 9.846153846153847e-07, + "loss": 0.0279, + "reward": 0.6667121946811676, + "reward_std": 0.1413332335650921, + "reward_std_all": 0.13805663585662842, + "rewards-max/forensics_sinkhorn_reward": 0.9582386016845703, + "rewards-max/forensics_strict_edge_reward": 0.19716041162610054, + "rewards-max/forensics_tiou_reward": 0.8959644138813019, + "rewards/forensics_sinkhorn_reward": 0.5764413177967072, + "rewards/forensics_strict_edge_reward": 0.00842533609829843, + "rewards/forensics_tiou_reward": 0.26099659502506256, + "step": 21 + }, + { + "completion_length": 21.53125, + "epoch": 0.06758832565284179, + "grad_norm": 5.035747528076172, + "kl": 0.189453125, + "learning_rate": 9.838461538461538e-07, + "loss": 0.3299, + "reward": 0.6473861932754517, + "reward_std": 0.11725450679659843, + "reward_std_all": 0.1336938515305519, + "rewards-max/forensics_sinkhorn_reward": 0.8794032037258148, + "rewards-max/forensics_strict_edge_reward": 0.2591388002038002, + "rewards-max/forensics_tiou_reward": 0.7185758948326111, + "rewards/forensics_sinkhorn_reward": 0.5196727216243744, + "rewards/forensics_strict_edge_reward": 0.011108127422630787, + "rewards/forensics_tiou_reward": 0.19572000205516815, + "step": 22 + }, + { + "completion_length": 22.625, + "epoch": 0.0706605222734255, + "grad_norm": 3.953895092010498, + "kl": 0.21728515625, + "learning_rate": 9.83076923076923e-07, + "loss": 0.1886, + "reward": 0.7067916989326477, + "reward_std": 0.10218081623315811, + "reward_std_all": 0.09943752363324165, + "rewards-max/forensics_sinkhorn_reward": 0.9515096843242645, + "rewards-max/forensics_strict_edge_reward": 0.008859581929073101, + "rewards-max/forensics_tiou_reward": 0.8916186690330505, + "rewards/forensics_sinkhorn_reward": 0.5869141221046448, + "rewards/forensics_strict_edge_reward": 0.0003941693866451601, + "rewards/forensics_tiou_reward": 0.24589042365550995, + "step": 23 + }, + { + "completion_length": 23.359375, + "epoch": 0.07373271889400922, + "grad_norm": 4.959726333618164, + "kl": 0.2177734375, + "learning_rate": 9.823076923076922e-07, + "loss": 0.1375, + "reward": 0.6897447109222412, + "reward_std": 0.12353962659835815, + "reward_std_all": 0.11931761354207993, + "rewards-max/forensics_sinkhorn_reward": 0.8468116819858551, + "rewards-max/forensics_strict_edge_reward": 0.010405359556898475, + "rewards-max/forensics_tiou_reward": 0.6636875718832016, + "rewards/forensics_sinkhorn_reward": 0.5422674417495728, + "rewards/forensics_strict_edge_reward": 0.0005064609867986292, + "rewards/forensics_tiou_reward": 0.2164381891489029, + "step": 24 + }, + { + "completion_length": 21.828125, + "epoch": 0.07680491551459294, + "grad_norm": 4.350214958190918, + "kl": 0.21337890625, + "learning_rate": 9.815384615384615e-07, + "loss": 0.102, + "reward": 0.6275688409805298, + "reward_std": 0.1428019404411316, + "reward_std_all": 0.15440218895673752, + "rewards-max/forensics_sinkhorn_reward": 0.8997052311897278, + "rewards-max/forensics_strict_edge_reward": 0.08751886337995529, + "rewards-max/forensics_tiou_reward": 0.7850223779678345, + "rewards/forensics_sinkhorn_reward": 0.49290797114372253, + "rewards/forensics_strict_edge_reward": 0.0037620963412337005, + "rewards/forensics_tiou_reward": 0.1822633370757103, + "step": 25 + }, + { + "completion_length": 21.875, + "epoch": 0.07987711213517665, + "grad_norm": 4.740930557250977, + "kl": 0.181640625, + "learning_rate": 9.807692307692306e-07, + "loss": 0.1825, + "reward": 0.6844587922096252, + "reward_std": 0.07503917440772057, + "reward_std_all": 0.09391132742166519, + "rewards-max/forensics_sinkhorn_reward": 0.9563069045543671, + "rewards-max/forensics_strict_edge_reward": 0.034915102645754814, + "rewards-max/forensics_tiou_reward": 0.8909980654716492, + "rewards/forensics_sinkhorn_reward": 0.587503582239151, + "rewards/forensics_strict_edge_reward": 0.00266269293933874, + "rewards/forensics_tiou_reward": 0.23406606912612915, + "step": 26 + }, + { + "completion_length": 22.375, + "epoch": 0.08294930875576037, + "grad_norm": 4.87563943862915, + "kl": 0.2373046875, + "learning_rate": 9.8e-07, + "loss": 0.1549, + "reward": 0.7015930414199829, + "reward_std": 0.0898250862956047, + "reward_std_all": 0.09820303320884705, + "rewards-max/forensics_sinkhorn_reward": 0.8987598717212677, + "rewards-max/forensics_strict_edge_reward": 0.1721096499968553, + "rewards-max/forensics_tiou_reward": 0.7972362637519836, + "rewards/forensics_sinkhorn_reward": 0.5949448347091675, + "rewards/forensics_strict_edge_reward": 0.01092665949545335, + "rewards/forensics_tiou_reward": 0.2721514031291008, + "step": 27 + }, + { + "completion_length": 22.71875, + "epoch": 0.08602150537634409, + "grad_norm": 4.094585418701172, + "kl": 0.21533203125, + "learning_rate": 9.792307692307693e-07, + "loss": 0.1776, + "reward": 0.6963971257209778, + "reward_std": 0.110671017318964, + "reward_std_all": 0.10849200934171677, + "rewards-max/forensics_sinkhorn_reward": 0.869853675365448, + "rewards-max/forensics_strict_edge_reward": 0.06123139193005045, + "rewards-max/forensics_tiou_reward": 0.7176607251167297, + "rewards/forensics_sinkhorn_reward": 0.5665201246738434, + "rewards/forensics_strict_edge_reward": 0.0037466138417983075, + "rewards/forensics_tiou_reward": 0.25579555332660675, + "step": 28 + }, + { + "completion_length": 20.28125, + "epoch": 0.0890937019969278, + "grad_norm": 3.3565733432769775, + "kl": 0.21923828125, + "learning_rate": 9.784615384615384e-07, + "loss": 0.1804, + "reward": 0.7206996083259583, + "reward_std": 0.09284093230962753, + "reward_std_all": 0.09009625762701035, + "rewards-max/forensics_sinkhorn_reward": 0.901806503534317, + "rewards-max/forensics_strict_edge_reward": 0.00027616339139058255, + "rewards-max/forensics_tiou_reward": 0.7685517072677612, + "rewards/forensics_sinkhorn_reward": 0.6337465345859528, + "rewards/forensics_strict_edge_reward": 1.568083189340541e-05, + "rewards/forensics_tiou_reward": 0.25556742399930954, + "step": 29 + }, + { + "completion_length": 26.296875, + "epoch": 0.09216589861751152, + "grad_norm": 4.205411434173584, + "kl": 0.22265625, + "learning_rate": 9.776923076923077e-07, + "loss": 0.137, + "reward": 0.6926176249980927, + "reward_std": 0.10651151835918427, + "reward_std_all": 0.10896393284201622, + "rewards-max/forensics_sinkhorn_reward": 0.893149733543396, + "rewards-max/forensics_strict_edge_reward": 0.010465137199616947, + "rewards-max/forensics_tiou_reward": 0.7412552833557129, + "rewards/forensics_sinkhorn_reward": 0.5474705994129181, + "rewards/forensics_strict_edge_reward": 0.00043609598907323743, + "rewards/forensics_tiou_reward": 0.25376835465431213, + "step": 30 + }, + { + "completion_length": 22.421875, + "epoch": 0.09523809523809523, + "grad_norm": 4.098252296447754, + "kl": 0.26513671875, + "learning_rate": 9.769230769230768e-07, + "loss": 0.1449, + "reward": 0.6537618041038513, + "reward_std": 0.1365450844168663, + "reward_std_all": 0.13945938646793365, + "rewards-max/forensics_sinkhorn_reward": 0.9562013149261475, + "rewards-max/forensics_strict_edge_reward": 0.046110697789117694, + "rewards-max/forensics_tiou_reward": 0.9126094877719879, + "rewards/forensics_sinkhorn_reward": 0.5584604442119598, + "rewards/forensics_strict_edge_reward": 0.002574242407717975, + "rewards/forensics_tiou_reward": 0.23961374163627625, + "step": 31 + }, + { + "completion_length": 23.796875, + "epoch": 0.09831029185867896, + "grad_norm": 3.966362237930298, + "kl": 0.240234375, + "learning_rate": 9.761538461538461e-07, + "loss": 0.1145, + "reward": 0.7059866189956665, + "reward_std": 0.10657758638262749, + "reward_std_all": 0.10349911451339722, + "rewards-max/forensics_sinkhorn_reward": 0.9019323587417603, + "rewards-max/forensics_strict_edge_reward": 0.055678056087344885, + "rewards-max/forensics_tiou_reward": 0.7724812030792236, + "rewards/forensics_sinkhorn_reward": 0.5888490080833435, + "rewards/forensics_strict_edge_reward": 0.0023298441228689626, + "rewards/forensics_tiou_reward": 0.24707330763339996, + "step": 32 + }, + { + "completion_length": 24.328125, + "epoch": 0.10138248847926268, + "grad_norm": 9.751164436340332, + "kl": 0.5810546875, + "learning_rate": 9.753846153846154e-07, + "loss": 0.1245, + "reward": 0.6651782691478729, + "reward_std": 0.15380705147981644, + "reward_std_all": 0.1472957357764244, + "rewards-max/forensics_sinkhorn_reward": 0.843212753534317, + "rewards-max/forensics_strict_edge_reward": 1.925441793559912e-06, + "rewards-max/forensics_tiou_reward": 0.6215561926364899, + "rewards/forensics_sinkhorn_reward": 0.5301893651485443, + "rewards/forensics_strict_edge_reward": 1.0429783436904927e-07, + "rewards/forensics_tiou_reward": 0.18785811960697174, + "step": 33 + }, + { + "completion_length": 24.625, + "epoch": 0.10445468509984639, + "grad_norm": 4.721068382263184, + "kl": 0.2548828125, + "learning_rate": 9.746153846153845e-07, + "loss": 0.1501, + "reward": 0.7296205163002014, + "reward_std": 0.10103139281272888, + "reward_std_all": 0.0928499847650528, + "rewards-max/forensics_sinkhorn_reward": 0.9130180478096008, + "rewards-max/forensics_strict_edge_reward": 0.0496609378606081, + "rewards-max/forensics_tiou_reward": 0.8003481030464172, + "rewards/forensics_sinkhorn_reward": 0.6497354209423065, + "rewards/forensics_strict_edge_reward": 0.002575725200586021, + "rewards/forensics_tiou_reward": 0.3447698801755905, + "step": 34 + }, + { + "completion_length": 25.484375, + "epoch": 0.10752688172043011, + "grad_norm": 4.871171474456787, + "kl": 0.271484375, + "learning_rate": 9.738461538461539e-07, + "loss": 0.2363, + "reward": 0.6924614310264587, + "reward_std": 0.12934990972280502, + "reward_std_all": 0.12262880057096481, + "rewards-max/forensics_sinkhorn_reward": 0.8651344776153564, + "rewards-max/forensics_strict_edge_reward": 0.207058347761631, + "rewards-max/forensics_tiou_reward": 0.7703021168708801, + "rewards/forensics_sinkhorn_reward": 0.5740231871604919, + "rewards/forensics_strict_edge_reward": 0.009137192042544484, + "rewards/forensics_tiou_reward": 0.25311972200870514, + "step": 35 + }, + { + "completion_length": 24.703125, + "epoch": 0.11059907834101383, + "grad_norm": 4.893945217132568, + "kl": 0.220703125, + "learning_rate": 9.73076923076923e-07, + "loss": 0.2399, + "reward": 0.6458395719528198, + "reward_std": 0.14350562915205956, + "reward_std_all": 0.146080132573843, + "rewards-max/forensics_sinkhorn_reward": 0.9618299901485443, + "rewards-max/forensics_strict_edge_reward": 0.051337226293981075, + "rewards-max/forensics_tiou_reward": 0.9148248136043549, + "rewards/forensics_sinkhorn_reward": 0.5320371687412262, + "rewards/forensics_strict_edge_reward": 0.0025789429782889783, + "rewards/forensics_tiou_reward": 0.2595372051000595, + "step": 36 + }, + { + "completion_length": 28.546875, + "epoch": 0.11367127496159754, + "grad_norm": 7.315536022186279, + "kl": 0.33349609375, + "learning_rate": 9.723076923076923e-07, + "loss": 0.0941, + "reward": 0.5944620668888092, + "reward_std": 0.17075667530298233, + "reward_std_all": 0.1797226294875145, + "rewards-max/forensics_sinkhorn_reward": 0.8862639665603638, + "rewards-max/forensics_strict_edge_reward": 0.007195596816018224, + "rewards-max/forensics_tiou_reward": 0.7356691658496857, + "rewards/forensics_sinkhorn_reward": 0.4611501097679138, + "rewards/forensics_strict_edge_reward": 0.0003317449227324687, + "rewards/forensics_tiou_reward": 0.20564496517181396, + "step": 37 + }, + { + "completion_length": 26.515625, + "epoch": 0.11674347158218126, + "grad_norm": 6.7749433517456055, + "kl": 0.224609375, + "learning_rate": 9.715384615384616e-07, + "loss": 0.2718, + "reward": 0.6252135634422302, + "reward_std": 0.10418764129281044, + "reward_std_all": 0.13701419159770012, + "rewards-max/forensics_sinkhorn_reward": 0.9175335764884949, + "rewards-max/forensics_strict_edge_reward": 0.057416749667027034, + "rewards-max/forensics_tiou_reward": 0.7973452806472778, + "rewards/forensics_sinkhorn_reward": 0.4581119120121002, + "rewards/forensics_strict_edge_reward": 0.0024699985224287957, + "rewards/forensics_tiou_reward": 0.24084115773439407, + "step": 38 + }, + { + "completion_length": 23.671875, + "epoch": 0.11981566820276497, + "grad_norm": 46.21137619018555, + "kl": 1.2744140625, + "learning_rate": 9.707692307692307e-07, + "loss": 0.1687, + "reward": 0.6690804064273834, + "reward_std": 0.1606164649128914, + "reward_std_all": 0.15074552595615387, + "rewards-max/forensics_sinkhorn_reward": 0.8939070105552673, + "rewards-max/forensics_strict_edge_reward": 0.0502003189176321, + "rewards-max/forensics_tiou_reward": 0.7734197676181793, + "rewards/forensics_sinkhorn_reward": 0.5736702680587769, + "rewards/forensics_strict_edge_reward": 0.0024668710539117455, + "rewards/forensics_tiou_reward": 0.2794101685285568, + "step": 39 + }, + { + "completion_length": 30.65625, + "epoch": 0.1228878648233487, + "grad_norm": 4.798606872558594, + "kl": 0.23095703125, + "learning_rate": 9.7e-07, + "loss": 0.2249, + "reward": 0.6888239681720734, + "reward_std": 0.11732276529073715, + "reward_std_all": 0.1155242957174778, + "rewards-max/forensics_sinkhorn_reward": 0.9628092050552368, + "rewards-max/forensics_strict_edge_reward": 0.36176968924701214, + "rewards-max/forensics_tiou_reward": 0.9164037704467773, + "rewards/forensics_sinkhorn_reward": 0.5608715713024139, + "rewards/forensics_strict_edge_reward": 0.015562411397695541, + "rewards/forensics_tiou_reward": 0.3138734996318817, + "step": 40 + }, + { + "completion_length": 27.0625, + "epoch": 0.1259600614439324, + "grad_norm": 3.441364288330078, + "kl": 0.22021484375, + "learning_rate": 9.692307692307691e-07, + "loss": 0.1674, + "reward": 0.6893076598644257, + "reward_std": 0.09722088277339935, + "reward_std_all": 0.10284698382019997, + "rewards-max/forensics_sinkhorn_reward": 0.7840943038463593, + "rewards-max/forensics_strict_edge_reward": 0.04733707755804062, + "rewards-max/forensics_tiou_reward": 0.5683804154396057, + "rewards/forensics_sinkhorn_reward": 0.5331307053565979, + "rewards/forensics_strict_edge_reward": 0.0025204174453392625, + "rewards/forensics_tiou_reward": 0.20762912184000015, + "step": 41 + }, + { + "completion_length": 22.28125, + "epoch": 0.12903225806451613, + "grad_norm": 3.6256587505340576, + "kl": 0.20458984375, + "learning_rate": 9.684615384615385e-07, + "loss": 0.1897, + "reward": 0.6984226405620575, + "reward_std": 0.09183249622583389, + "reward_std_all": 0.09872140362858772, + "rewards-max/forensics_sinkhorn_reward": 0.9195894002914429, + "rewards-max/forensics_strict_edge_reward": 0.01554880803450942, + "rewards-max/forensics_tiou_reward": 0.8265445828437805, + "rewards/forensics_sinkhorn_reward": 0.5985351204872131, + "rewards/forensics_strict_edge_reward": 0.0008536297536920756, + "rewards/forensics_tiou_reward": 0.26105591654777527, + "step": 42 + }, + { + "completion_length": 23.921875, + "epoch": 0.13210445468509985, + "grad_norm": 3.9005630016326904, + "kl": 0.236328125, + "learning_rate": 9.676923076923078e-07, + "loss": 0.1461, + "reward": 0.690881609916687, + "reward_std": 0.1071852371096611, + "reward_std_all": 0.10698951780796051, + "rewards-max/forensics_sinkhorn_reward": 0.9011575281620026, + "rewards-max/forensics_strict_edge_reward": 0.030406708335704025, + "rewards-max/forensics_tiou_reward": 0.7303362190723419, + "rewards/forensics_sinkhorn_reward": 0.5674743056297302, + "rewards/forensics_strict_edge_reward": 0.0022400905567110385, + "rewards/forensics_tiou_reward": 0.23169638216495514, + "step": 43 + }, + { + "completion_length": 21.6875, + "epoch": 0.13517665130568357, + "grad_norm": 5.054514408111572, + "kl": 0.251953125, + "learning_rate": 9.669230769230769e-07, + "loss": 0.149, + "reward": 0.6498146057128906, + "reward_std": 0.14491017162799835, + "reward_std_all": 0.1494777724146843, + "rewards-max/forensics_sinkhorn_reward": 0.8948346078395844, + "rewards-max/forensics_strict_edge_reward": 0.0036034940276294947, + "rewards-max/forensics_tiou_reward": 0.729841947555542, + "rewards/forensics_sinkhorn_reward": 0.5558753311634064, + "rewards/forensics_strict_edge_reward": 0.00018954599727294408, + "rewards/forensics_tiou_reward": 0.20807908475399017, + "step": 44 + }, + { + "completion_length": 21.140625, + "epoch": 0.1382488479262673, + "grad_norm": 4.39292049407959, + "kl": 0.236328125, + "learning_rate": 9.66153846153846e-07, + "loss": 0.0997, + "reward": 0.7239247560501099, + "reward_std": 0.09397222101688385, + "reward_std_all": 0.09062012284994125, + "rewards-max/forensics_sinkhorn_reward": 0.9156840443611145, + "rewards-max/forensics_strict_edge_reward": 0.0769482451651129, + "rewards-max/forensics_tiou_reward": 0.7765783071517944, + "rewards/forensics_sinkhorn_reward": 0.6273707449436188, + "rewards/forensics_strict_edge_reward": 0.0043578289009928994, + "rewards/forensics_tiou_reward": 0.29927656054496765, + "step": 45 + }, + { + "completion_length": 23.171875, + "epoch": 0.141321044546851, + "grad_norm": 4.1861467361450195, + "kl": 0.22216796875, + "learning_rate": 9.653846153846153e-07, + "loss": 0.1489, + "reward": 0.6566336452960968, + "reward_std": 0.10703098773956299, + "reward_std_all": 0.12527744472026825, + "rewards-max/forensics_sinkhorn_reward": 0.9572404623031616, + "rewards-max/forensics_strict_edge_reward": 0.05255119130015373, + "rewards-max/forensics_tiou_reward": 0.9040226936340332, + "rewards/forensics_sinkhorn_reward": 0.5541320145130157, + "rewards/forensics_strict_edge_reward": 0.0023231139057315886, + "rewards/forensics_tiou_reward": 0.2234741747379303, + "step": 46 + }, + { + "completion_length": 26.65625, + "epoch": 0.1443932411674347, + "grad_norm": 4.707549095153809, + "kl": 0.20458984375, + "learning_rate": 9.646153846153846e-07, + "loss": 0.1611, + "reward": 0.6305238306522369, + "reward_std": 0.11589157581329346, + "reward_std_all": 0.13760708272457123, + "rewards-max/forensics_sinkhorn_reward": 0.9492619633674622, + "rewards-max/forensics_strict_edge_reward": 0.3341981876292266, + "rewards-max/forensics_tiou_reward": 0.8688937723636627, + "rewards/forensics_sinkhorn_reward": 0.5091026425361633, + "rewards/forensics_strict_edge_reward": 0.015043503499327926, + "rewards/forensics_tiou_reward": 0.24566750973463058, + "step": 47 + }, + { + "completion_length": 22.65625, + "epoch": 0.14746543778801843, + "grad_norm": 3.7328853607177734, + "kl": 0.20458984375, + "learning_rate": 9.638461538461537e-07, + "loss": 0.0421, + "reward": 0.7223207354545593, + "reward_std": 0.10369808971881866, + "reward_std_all": 0.09747456014156342, + "rewards-max/forensics_sinkhorn_reward": 0.9050062894821167, + "rewards-max/forensics_strict_edge_reward": 0.027080100786406547, + "rewards-max/forensics_tiou_reward": 0.7692039012908936, + "rewards/forensics_sinkhorn_reward": 0.6056917309761047, + "rewards/forensics_strict_edge_reward": 0.0017733722415869124, + "rewards/forensics_tiou_reward": 0.32249465584754944, + "step": 48 + }, + { + "completion_length": 22.21875, + "epoch": 0.15053763440860216, + "grad_norm": 4.30291223526001, + "kl": 0.1875, + "learning_rate": 9.63076923076923e-07, + "loss": 0.1412, + "reward": 0.6968431174755096, + "reward_std": 0.09105860069394112, + "reward_std_all": 0.09917454421520233, + "rewards-max/forensics_sinkhorn_reward": 0.8528808653354645, + "rewards-max/forensics_strict_edge_reward": 0.21249387064017355, + "rewards-max/forensics_tiou_reward": 0.7393182218074799, + "rewards/forensics_sinkhorn_reward": 0.5790915489196777, + "rewards/forensics_strict_edge_reward": 0.009221838088706136, + "rewards/forensics_tiou_reward": 0.2383236587047577, + "step": 49 + }, + { + "completion_length": 23.75, + "epoch": 0.15360983102918588, + "grad_norm": 3.7066025733947754, + "kl": 0.18505859375, + "learning_rate": 9.623076923076922e-07, + "loss": 0.1107, + "reward": 0.6307157576084137, + "reward_std": 0.12678582221269608, + "reward_std_all": 0.14595621824264526, + "rewards-max/forensics_sinkhorn_reward": 0.8976386189460754, + "rewards-max/forensics_strict_edge_reward": 0.010299882822437212, + "rewards-max/forensics_tiou_reward": 0.7904693782329559, + "rewards/forensics_sinkhorn_reward": 0.5127425491809845, + "rewards/forensics_strict_edge_reward": 0.00044766769042325905, + "rewards/forensics_tiou_reward": 0.2263328731060028, + "step": 50 + }, + { + "completion_length": 23.53125, + "epoch": 0.15668202764976957, + "grad_norm": 4.422190189361572, + "kl": 0.23095703125, + "learning_rate": 9.615384615384615e-07, + "loss": 0.1632, + "reward": 0.6311408579349518, + "reward_std": 0.16348673403263092, + "reward_std_all": 0.1627218797802925, + "rewards-max/forensics_sinkhorn_reward": 0.9665747880935669, + "rewards-max/forensics_strict_edge_reward": 0.3628446846269071, + "rewards-max/forensics_tiou_reward": 0.9235945641994476, + "rewards/forensics_sinkhorn_reward": 0.5221361219882965, + "rewards/forensics_strict_edge_reward": 0.0163595755584538, + "rewards/forensics_tiou_reward": 0.21028399467468262, + "step": 51 + }, + { + "completion_length": 24.25, + "epoch": 0.1597542242703533, + "grad_norm": 4.625850200653076, + "kl": 0.20751953125, + "learning_rate": 9.607692307692308e-07, + "loss": 0.1408, + "reward": 0.7033702731132507, + "reward_std": 0.09014922380447388, + "reward_std_all": 0.09488804638385773, + "rewards-max/forensics_sinkhorn_reward": 0.9278042316436768, + "rewards-max/forensics_strict_edge_reward": 0.3450232893228531, + "rewards-max/forensics_tiou_reward": 0.814490020275116, + "rewards/forensics_sinkhorn_reward": 0.5736911594867706, + "rewards/forensics_strict_edge_reward": 0.016445573419332504, + "rewards/forensics_tiou_reward": 0.26632116734981537, + "step": 52 + }, + { + "completion_length": 27.21875, + "epoch": 0.16282642089093702, + "grad_norm": 5.017592906951904, + "kl": 0.18359375, + "learning_rate": 9.6e-07, + "loss": 0.1623, + "reward": 0.682520717382431, + "reward_std": 0.13656894862651825, + "reward_std_all": 0.129991352558136, + "rewards-max/forensics_sinkhorn_reward": 0.9650632739067078, + "rewards-max/forensics_strict_edge_reward": 0.2566752042621374, + "rewards-max/forensics_tiou_reward": 0.9321500062942505, + "rewards/forensics_sinkhorn_reward": 0.5664448738098145, + "rewards/forensics_strict_edge_reward": 0.016468226676806808, + "rewards/forensics_tiou_reward": 0.31293870508670807, + "step": 53 + }, + { + "completion_length": 22.203125, + "epoch": 0.16589861751152074, + "grad_norm": 3.896657705307007, + "kl": 0.1875, + "learning_rate": 9.592307692307692e-07, + "loss": 0.131, + "reward": 0.7218937873840332, + "reward_std": 0.11752534657716751, + "reward_std_all": 0.10685992985963821, + "rewards-max/forensics_sinkhorn_reward": 0.8681782782077789, + "rewards-max/forensics_strict_edge_reward": 0.016425664070993662, + "rewards-max/forensics_tiou_reward": 0.7052515149116516, + "rewards/forensics_sinkhorn_reward": 0.6078203618526459, + "rewards/forensics_strict_edge_reward": 0.0008322869543917477, + "rewards/forensics_tiou_reward": 0.2941286712884903, + "step": 54 + }, + { + "completion_length": 25.296875, + "epoch": 0.16897081413210446, + "grad_norm": 3.8157012462615967, + "kl": 0.240234375, + "learning_rate": 9.584615384615383e-07, + "loss": 0.1741, + "reward": 0.638936460018158, + "reward_std": 0.12546135485172272, + "reward_std_all": 0.14166024327278137, + "rewards-max/forensics_sinkhorn_reward": 0.8304384648799896, + "rewards-max/forensics_strict_edge_reward": 0.2049986784986686, + "rewards-max/forensics_tiou_reward": 0.636310875415802, + "rewards/forensics_sinkhorn_reward": 0.5025323331356049, + "rewards/forensics_strict_edge_reward": 0.009392281303007621, + "rewards/forensics_tiou_reward": 0.2108977809548378, + "step": 55 + }, + { + "completion_length": 24.09375, + "epoch": 0.17204301075268819, + "grad_norm": 4.767069339752197, + "kl": 0.2275390625, + "learning_rate": 9.576923076923077e-07, + "loss": 0.1483, + "reward": 0.646376222372055, + "reward_std": 0.13816715776920319, + "reward_std_all": 0.14594414085149765, + "rewards-max/forensics_sinkhorn_reward": 0.9598250091075897, + "rewards-max/forensics_strict_edge_reward": 0.25721245509339496, + "rewards-max/forensics_tiou_reward": 0.9178726375102997, + "rewards/forensics_sinkhorn_reward": 0.5298498272895813, + "rewards/forensics_strict_edge_reward": 0.012558289590742788, + "rewards/forensics_tiou_reward": 0.2588675767183304, + "step": 56 + }, + { + "completion_length": 22.78125, + "epoch": 0.17511520737327188, + "grad_norm": 4.066549777984619, + "kl": 0.2138671875, + "learning_rate": 9.569230769230768e-07, + "loss": 0.1984, + "reward": 0.7052267789840698, + "reward_std": 0.08699789270758629, + "reward_std_all": 0.09216586500406265, + "rewards-max/forensics_sinkhorn_reward": 0.9250836670398712, + "rewards-max/forensics_strict_edge_reward": 0.3139483230188489, + "rewards-max/forensics_tiou_reward": 0.8475916981697083, + "rewards/forensics_sinkhorn_reward": 0.6019813418388367, + "rewards/forensics_strict_edge_reward": 0.013121950541972183, + "rewards/forensics_tiou_reward": 0.29025667905807495, + "step": 57 + }, + { + "completion_length": 22.34375, + "epoch": 0.1781874039938556, + "grad_norm": 3.9059276580810547, + "kl": 0.17529296875, + "learning_rate": 9.56153846153846e-07, + "loss": 0.1339, + "reward": 0.6920393407344818, + "reward_std": 0.09614486992359161, + "reward_std_all": 0.10390296205878258, + "rewards-max/forensics_sinkhorn_reward": 0.8812635540962219, + "rewards-max/forensics_strict_edge_reward": 0.0004413227021586863, + "rewards-max/forensics_tiou_reward": 0.6789155006408691, + "rewards/forensics_sinkhorn_reward": 0.5790511667728424, + "rewards/forensics_strict_edge_reward": 3.70383228398552e-05, + "rewards/forensics_tiou_reward": 0.259396456182003, + "step": 58 + }, + { + "completion_length": 25.109375, + "epoch": 0.18125960061443933, + "grad_norm": 6.517645359039307, + "kl": 0.20263671875, + "learning_rate": 9.553846153846154e-07, + "loss": 0.086, + "reward": 0.5851137638092041, + "reward_std": 0.18654906749725342, + "reward_std_all": 0.19271792471408844, + "rewards-max/forensics_sinkhorn_reward": 0.9343521296977997, + "rewards-max/forensics_strict_edge_reward": 0.01706346543505788, + "rewards-max/forensics_tiou_reward": 0.859970360994339, + "rewards/forensics_sinkhorn_reward": 0.45987568795681, + "rewards/forensics_strict_edge_reward": 0.0008587828488089144, + "rewards/forensics_tiou_reward": 0.24702078104019165, + "step": 59 + }, + { + "completion_length": 27.15625, + "epoch": 0.18433179723502305, + "grad_norm": 5.147050380706787, + "kl": 0.20947265625, + "learning_rate": 9.546153846153845e-07, + "loss": 0.236, + "reward": 0.6183677315711975, + "reward_std": 0.11934096366167068, + "reward_std_all": 0.14652671664953232, + "rewards-max/forensics_sinkhorn_reward": 0.8205731213092804, + "rewards-max/forensics_strict_edge_reward": 0.13720293163189545, + "rewards-max/forensics_tiou_reward": 0.570318341255188, + "rewards/forensics_sinkhorn_reward": 0.4628327637910843, + "rewards/forensics_strict_edge_reward": 0.005788231239846731, + "rewards/forensics_tiou_reward": 0.19485292583703995, + "step": 60 + }, + { + "completion_length": 28.765625, + "epoch": 0.18740399385560677, + "grad_norm": 4.42332124710083, + "kl": 0.21435546875, + "learning_rate": 9.538461538461538e-07, + "loss": 0.0691, + "reward": 0.5644819140434265, + "reward_std": 0.15561850368976593, + "reward_std_all": 0.1879860684275627, + "rewards-max/forensics_sinkhorn_reward": 0.8644483983516693, + "rewards-max/forensics_strict_edge_reward": 0.03130201669409871, + "rewards-max/forensics_tiou_reward": 0.7233558893203735, + "rewards/forensics_sinkhorn_reward": 0.4369947612285614, + "rewards/forensics_strict_edge_reward": 0.0019250416371505708, + "rewards/forensics_tiou_reward": 0.21143464744091034, + "step": 61 + }, + { + "completion_length": 26.40625, + "epoch": 0.19047619047619047, + "grad_norm": 3.830873727798462, + "kl": 0.20263671875, + "learning_rate": 9.53076923076923e-07, + "loss": 0.1673, + "reward": 0.6624014675617218, + "reward_std": 0.1361984834074974, + "reward_std_all": 0.1393170729279518, + "rewards-max/forensics_sinkhorn_reward": 0.9244153797626495, + "rewards-max/forensics_strict_edge_reward": 0.11928796071515535, + "rewards-max/forensics_tiou_reward": 0.8288418650627136, + "rewards/forensics_sinkhorn_reward": 0.5254128277301788, + "rewards/forensics_strict_edge_reward": 0.005013348507418414, + "rewards/forensics_tiou_reward": 0.2782427668571472, + "step": 62 + }, + { + "completion_length": 24.6875, + "epoch": 0.1935483870967742, + "grad_norm": 4.00238561630249, + "kl": 0.17822265625, + "learning_rate": 9.523076923076923e-07, + "loss": 0.1349, + "reward": 0.6643558144569397, + "reward_std": 0.12070654332637787, + "reward_std_all": 0.12809887155890465, + "rewards-max/forensics_sinkhorn_reward": 0.9329976737499237, + "rewards-max/forensics_strict_edge_reward": 0.15252999030053616, + "rewards-max/forensics_tiou_reward": 0.872031956911087, + "rewards/forensics_sinkhorn_reward": 0.5409306585788727, + "rewards/forensics_strict_edge_reward": 0.011568704037927091, + "rewards/forensics_tiou_reward": 0.28472019731998444, + "step": 63 + }, + { + "completion_length": 22.984375, + "epoch": 0.1966205837173579, + "grad_norm": 3.7629308700561523, + "kl": 0.1923828125, + "learning_rate": 9.515384615384615e-07, + "loss": 0.2561, + "reward": 0.7279493808746338, + "reward_std": 0.08048248291015625, + "reward_std_all": 0.08007504791021347, + "rewards-max/forensics_sinkhorn_reward": 0.9248572885990143, + "rewards-max/forensics_strict_edge_reward": 0.031121840700507164, + "rewards-max/forensics_tiou_reward": 0.807826817035675, + "rewards/forensics_sinkhorn_reward": 0.6169094443321228, + "rewards/forensics_strict_edge_reward": 0.0013431737897917628, + "rewards/forensics_tiou_reward": 0.33287231624126434, + "step": 64 + }, + { + "completion_length": 25.28125, + "epoch": 0.19969278033794163, + "grad_norm": 3.622135639190674, + "kl": 0.1611328125, + "learning_rate": 9.507692307692308e-07, + "loss": 0.1846, + "reward": 0.7345190644264221, + "reward_std": 0.09329094365239143, + "reward_std_all": 0.08730331063270569, + "rewards-max/forensics_sinkhorn_reward": 0.9324921369552612, + "rewards-max/forensics_strict_edge_reward": 0.15024219080805779, + "rewards-max/forensics_tiou_reward": 0.8379725217819214, + "rewards/forensics_sinkhorn_reward": 0.6402496695518494, + "rewards/forensics_strict_edge_reward": 0.0144422622397542, + "rewards/forensics_tiou_reward": 0.3540066331624985, + "step": 65 + }, + { + "completion_length": 24.625, + "epoch": 0.20276497695852536, + "grad_norm": 3.1995506286621094, + "kl": 0.1826171875, + "learning_rate": 9.499999999999999e-07, + "loss": 0.1065, + "reward": 0.6665518283843994, + "reward_std": 0.10989505052566528, + "reward_std_all": 0.12193476781249046, + "rewards-max/forensics_sinkhorn_reward": 0.878681093454361, + "rewards-max/forensics_strict_edge_reward": 0.2140314243733883, + "rewards-max/forensics_tiou_reward": 0.7677238583564758, + "rewards/forensics_sinkhorn_reward": 0.5457462221384048, + "rewards/forensics_strict_edge_reward": 0.008924176217988133, + "rewards/forensics_tiou_reward": 0.23620882630348206, + "step": 66 + }, + { + "completion_length": 25.15625, + "epoch": 0.20583717357910905, + "grad_norm": 4.427835941314697, + "kl": 0.21240234375, + "learning_rate": 9.492307692307691e-07, + "loss": 0.1686, + "reward": 0.6350033581256866, + "reward_std": 0.16110683232545853, + "reward_std_all": 0.16180281341075897, + "rewards-max/forensics_sinkhorn_reward": 0.849644809961319, + "rewards-max/forensics_strict_edge_reward": 0.05052844842430204, + "rewards-max/forensics_tiou_reward": 0.6836467087268829, + "rewards/forensics_sinkhorn_reward": 0.5017452239990234, + "rewards/forensics_strict_edge_reward": 0.002389253988440032, + "rewards/forensics_tiou_reward": 0.22804592549800873, + "step": 67 + }, + { + "completion_length": 26.78125, + "epoch": 0.20890937019969277, + "grad_norm": 6.342324256896973, + "kl": 0.2275390625, + "learning_rate": 9.484615384615384e-07, + "loss": 0.3313, + "reward": 0.6393630504608154, + "reward_std": 0.10910747200250626, + "reward_std_all": 0.14022473990917206, + "rewards-max/forensics_sinkhorn_reward": 0.9655687808990479, + "rewards-max/forensics_strict_edge_reward": 0.29384535178542137, + "rewards-max/forensics_tiou_reward": 0.9263907670974731, + "rewards/forensics_sinkhorn_reward": 0.514805018901825, + "rewards/forensics_strict_edge_reward": 0.015237036161124706, + "rewards/forensics_tiou_reward": 0.309795081615448, + "step": 68 + }, + { + "completion_length": 23.609375, + "epoch": 0.2119815668202765, + "grad_norm": 4.322883129119873, + "kl": 0.2119140625, + "learning_rate": 9.476923076923077e-07, + "loss": 0.2671, + "reward": 0.6456633508205414, + "reward_std": 0.15162302553653717, + "reward_std_all": 0.15469364076852798, + "rewards-max/forensics_sinkhorn_reward": 0.9651179015636444, + "rewards-max/forensics_strict_edge_reward": 0.15973610430955887, + "rewards-max/forensics_tiou_reward": 0.9332000613212585, + "rewards/forensics_sinkhorn_reward": 0.533229261636734, + "rewards/forensics_strict_edge_reward": 0.009436949621886015, + "rewards/forensics_tiou_reward": 0.24955782294273376, + "step": 69 + }, + { + "completion_length": 27.0625, + "epoch": 0.21505376344086022, + "grad_norm": 3.8001153469085693, + "kl": 0.17724609375, + "learning_rate": 9.46923076923077e-07, + "loss": 0.2237, + "reward": 0.6269749402999878, + "reward_std": 0.16121762990951538, + "reward_std_all": 0.16440441459417343, + "rewards-max/forensics_sinkhorn_reward": 0.8956453800201416, + "rewards-max/forensics_strict_edge_reward": 0.2976976758800447, + "rewards-max/forensics_tiou_reward": 0.7841113805770874, + "rewards/forensics_sinkhorn_reward": 0.5230332314968109, + "rewards/forensics_strict_edge_reward": 0.012598375382367522, + "rewards/forensics_tiou_reward": 0.2392704263329506, + "step": 70 + }, + { + "completion_length": 23.203125, + "epoch": 0.21812596006144394, + "grad_norm": 4.411831378936768, + "kl": 0.16845703125, + "learning_rate": 9.461538461538461e-07, + "loss": 0.1692, + "reward": 0.6721272468566895, + "reward_std": 0.11389965936541557, + "reward_std_all": 0.12060859054327011, + "rewards-max/forensics_sinkhorn_reward": 0.9477923810482025, + "rewards-max/forensics_strict_edge_reward": 0.028016774886054918, + "rewards-max/forensics_tiou_reward": 0.8860475420951843, + "rewards/forensics_sinkhorn_reward": 0.5640097856521606, + "rewards/forensics_strict_edge_reward": 0.001529935580038, + "rewards/forensics_tiou_reward": 0.262196309864521, + "step": 71 + }, + { + "completion_length": 22.546875, + "epoch": 0.22119815668202766, + "grad_norm": 3.9083778858184814, + "kl": 0.1806640625, + "learning_rate": 9.453846153846153e-07, + "loss": 0.1036, + "reward": 0.6355668902397156, + "reward_std": 0.10109730809926987, + "reward_std_all": 0.1256670020520687, + "rewards-max/forensics_sinkhorn_reward": 0.8744306564331055, + "rewards-max/forensics_strict_edge_reward": 0.26607583928853273, + "rewards-max/forensics_tiou_reward": 0.7440514266490936, + "rewards/forensics_sinkhorn_reward": 0.5264429152011871, + "rewards/forensics_strict_edge_reward": 0.012903651033411734, + "rewards/forensics_tiou_reward": 0.1887867972254753, + "step": 72 + }, + { + "completion_length": 24.09375, + "epoch": 0.22427035330261136, + "grad_norm": 4.125799655914307, + "kl": 0.173828125, + "learning_rate": 9.446153846153846e-07, + "loss": 0.2083, + "reward": 0.6890977621078491, + "reward_std": 0.1318235844373703, + "reward_std_all": 0.12599167227745056, + "rewards-max/forensics_sinkhorn_reward": 0.9233121871948242, + "rewards-max/forensics_strict_edge_reward": 0.012860890092269983, + "rewards-max/forensics_tiou_reward": 0.8131066858768463, + "rewards/forensics_sinkhorn_reward": 0.5808150470256805, + "rewards/forensics_strict_edge_reward": 0.0010097334250076528, + "rewards/forensics_tiou_reward": 0.30945560336112976, + "step": 73 + }, + { + "completion_length": 24.296875, + "epoch": 0.22734254992319508, + "grad_norm": 3.712376356124878, + "kl": 0.2109375, + "learning_rate": 9.438461538461538e-07, + "loss": 0.2391, + "reward": 0.7019245624542236, + "reward_std": 0.09485341608524323, + "reward_std_all": 0.09764466434717178, + "rewards-max/forensics_sinkhorn_reward": 0.9105406403541565, + "rewards-max/forensics_strict_edge_reward": 0.004944490967318416, + "rewards-max/forensics_tiou_reward": 0.8050450384616852, + "rewards/forensics_sinkhorn_reward": 0.5733276307582855, + "rewards/forensics_strict_edge_reward": 0.0002185533958254382, + "rewards/forensics_tiou_reward": 0.2537904232740402, + "step": 74 + }, + { + "completion_length": 20.90625, + "epoch": 0.2304147465437788, + "grad_norm": 4.187369346618652, + "kl": 0.19921875, + "learning_rate": 9.43076923076923e-07, + "loss": 0.1895, + "reward": 0.6719456911087036, + "reward_std": 0.12575048208236694, + "reward_std_all": 0.13022523000836372, + "rewards-max/forensics_sinkhorn_reward": 0.9394395649433136, + "rewards-max/forensics_strict_edge_reward": 0.10222964407876134, + "rewards-max/forensics_tiou_reward": 0.8596875965595245, + "rewards/forensics_sinkhorn_reward": 0.5835058093070984, + "rewards/forensics_strict_edge_reward": 0.0070977353025227785, + "rewards/forensics_tiou_reward": 0.2767096608877182, + "step": 75 + }, + { + "completion_length": 20.296875, + "epoch": 0.23348694316436253, + "grad_norm": 5.361637115478516, + "kl": 0.17138671875, + "learning_rate": 9.423076923076923e-07, + "loss": 0.2297, + "reward": 0.6588501930236816, + "reward_std": 0.15574993193149567, + "reward_std_all": 0.14941616356372833, + "rewards-max/forensics_sinkhorn_reward": 0.9748798608779907, + "rewards-max/forensics_strict_edge_reward": 0.4182229759171605, + "rewards-max/forensics_tiou_reward": 0.9483819305896759, + "rewards/forensics_sinkhorn_reward": 0.5749777257442474, + "rewards/forensics_strict_edge_reward": 0.026566245476715267, + "rewards/forensics_tiou_reward": 0.25277596712112427, + "step": 76 + }, + { + "completion_length": 22.5, + "epoch": 0.23655913978494625, + "grad_norm": 3.8059961795806885, + "kl": 0.20849609375, + "learning_rate": 9.415384615384615e-07, + "loss": 0.0982, + "reward": 0.6287039220333099, + "reward_std": 0.13925857469439507, + "reward_std_all": 0.150296188890934, + "rewards-max/forensics_sinkhorn_reward": 0.9129684865474701, + "rewards-max/forensics_strict_edge_reward": 0.25712111484608613, + "rewards-max/forensics_tiou_reward": 0.8046395480632782, + "rewards/forensics_sinkhorn_reward": 0.5083759725093842, + "rewards/forensics_strict_edge_reward": 0.010724043857408105, + "rewards/forensics_tiou_reward": 0.1809721142053604, + "step": 77 + }, + { + "completion_length": 22.609375, + "epoch": 0.23963133640552994, + "grad_norm": 4.051192760467529, + "kl": 0.18505859375, + "learning_rate": 9.407692307692308e-07, + "loss": 0.1593, + "reward": 0.6857080459594727, + "reward_std": 0.10171542689204216, + "reward_std_all": 0.10795003175735474, + "rewards-max/forensics_sinkhorn_reward": 0.9587143361568451, + "rewards-max/forensics_strict_edge_reward": 0.10717548429965973, + "rewards-max/forensics_tiou_reward": 0.9101665019989014, + "rewards/forensics_sinkhorn_reward": 0.5763312578201294, + "rewards/forensics_strict_edge_reward": 0.00468941661529243, + "rewards/forensics_tiou_reward": 0.28753335773944855, + "step": 78 + }, + { + "completion_length": 21.90625, + "epoch": 0.24270353302611367, + "grad_norm": 3.9337351322174072, + "kl": 0.19580078125, + "learning_rate": 9.399999999999999e-07, + "loss": 0.1605, + "reward": 0.6915411651134491, + "reward_std": 0.12236332893371582, + "reward_std_all": 0.11845763027667999, + "rewards-max/forensics_sinkhorn_reward": 0.8650675415992737, + "rewards-max/forensics_strict_edge_reward": 0.007773047014779877, + "rewards-max/forensics_tiou_reward": 0.6755749583244324, + "rewards/forensics_sinkhorn_reward": 0.585264652967453, + "rewards/forensics_strict_edge_reward": 0.00046818140458526614, + "rewards/forensics_tiou_reward": 0.26720503717660904, + "step": 79 + }, + { + "completion_length": 21.1875, + "epoch": 0.2457757296466974, + "grad_norm": 4.4997992515563965, + "kl": 0.2099609375, + "learning_rate": 9.392307692307691e-07, + "loss": 0.1615, + "reward": 0.6953153908252716, + "reward_std": 0.10426390543580055, + "reward_std_all": 0.10599767416715622, + "rewards-max/forensics_sinkhorn_reward": 0.9791377782821655, + "rewards-max/forensics_strict_edge_reward": 0.4446859806776047, + "rewards-max/forensics_tiou_reward": 0.9746571481227875, + "rewards/forensics_sinkhorn_reward": 0.6205668151378632, + "rewards/forensics_strict_edge_reward": 0.02061697654426098, + "rewards/forensics_tiou_reward": 0.3444516807794571, + "step": 80 + }, + { + "completion_length": 21.921875, + "epoch": 0.2488479262672811, + "grad_norm": 4.331244945526123, + "kl": 0.23974609375, + "learning_rate": 9.384615384615384e-07, + "loss": 0.1524, + "reward": 0.6960994005203247, + "reward_std": 0.0772039070725441, + "reward_std_all": 0.09455164894461632, + "rewards-max/forensics_sinkhorn_reward": 0.9573794007301331, + "rewards-max/forensics_strict_edge_reward": 0.08088952163234353, + "rewards-max/forensics_tiou_reward": 0.9155758023262024, + "rewards/forensics_sinkhorn_reward": 0.6170915961265564, + "rewards/forensics_strict_edge_reward": 0.0035790751862805337, + "rewards/forensics_tiou_reward": 0.3374028503894806, + "step": 81 + }, + { + "completion_length": 23.6875, + "epoch": 0.2519201228878648, + "grad_norm": 4.328030586242676, + "kl": 0.228515625, + "learning_rate": 9.376923076923076e-07, + "loss": 0.1324, + "reward": 0.701356828212738, + "reward_std": 0.09558944031596184, + "reward_std_all": 0.09685412049293518, + "rewards-max/forensics_sinkhorn_reward": 0.9719968438148499, + "rewards-max/forensics_strict_edge_reward": 0.07400898262858391, + "rewards-max/forensics_tiou_reward": 0.9518124163150787, + "rewards/forensics_sinkhorn_reward": 0.6087078750133514, + "rewards/forensics_strict_edge_reward": 0.00457785848993808, + "rewards/forensics_tiou_reward": 0.34356580674648285, + "step": 82 + }, + { + "completion_length": 23.390625, + "epoch": 0.25499231950844853, + "grad_norm": 3.5037682056427, + "kl": 0.2001953125, + "learning_rate": 9.36923076923077e-07, + "loss": -0.005, + "reward": 0.6757905781269073, + "reward_std": 0.1240309476852417, + "reward_std_all": 0.12413552030920982, + "rewards-max/forensics_sinkhorn_reward": 0.941081702709198, + "rewards-max/forensics_strict_edge_reward": 0.20424521761015058, + "rewards-max/forensics_tiou_reward": 0.8895507752895355, + "rewards/forensics_sinkhorn_reward": 0.5656755566596985, + "rewards/forensics_strict_edge_reward": 0.016454264521598816, + "rewards/forensics_tiou_reward": 0.29610635340213776, + "step": 83 + }, + { + "completion_length": 20.484375, + "epoch": 0.25806451612903225, + "grad_norm": 3.4992806911468506, + "kl": 0.21533203125, + "learning_rate": 9.361538461538461e-07, + "loss": 0.1622, + "reward": 0.6472349166870117, + "reward_std": 0.13562829792499542, + "reward_std_all": 0.14832288026809692, + "rewards-max/forensics_sinkhorn_reward": 0.9625630378723145, + "rewards-max/forensics_strict_edge_reward": 0.21023141866680817, + "rewards-max/forensics_tiou_reward": 0.9293815195560455, + "rewards/forensics_sinkhorn_reward": 0.5734779238700867, + "rewards/forensics_strict_edge_reward": 0.015493951004827977, + "rewards/forensics_tiou_reward": 0.2746552377939224, + "step": 84 + }, + { + "completion_length": 22.875, + "epoch": 0.261136712749616, + "grad_norm": 4.916399955749512, + "kl": 0.232421875, + "learning_rate": 9.353846153846153e-07, + "loss": 0.1979, + "reward": 0.6760595440864563, + "reward_std": 0.1069946363568306, + "reward_std_all": 0.11384347081184387, + "rewards-max/forensics_sinkhorn_reward": 0.8725222647190094, + "rewards-max/forensics_strict_edge_reward": 0.053732478991150856, + "rewards-max/forensics_tiou_reward": 0.7284293174743652, + "rewards/forensics_sinkhorn_reward": 0.5648144781589508, + "rewards/forensics_strict_edge_reward": 0.002775570610538125, + "rewards/forensics_tiou_reward": 0.24881857633590698, + "step": 85 + }, + { + "completion_length": 23.75, + "epoch": 0.2642089093701997, + "grad_norm": 3.5425710678100586, + "kl": 0.2255859375, + "learning_rate": 9.346153846153846e-07, + "loss": 0.2573, + "reward": 0.7276632487773895, + "reward_std": 0.10008051991462708, + "reward_std_all": 0.0938415452837944, + "rewards-max/forensics_sinkhorn_reward": 0.8527841866016388, + "rewards-max/forensics_strict_edge_reward": 0.16422211844474077, + "rewards-max/forensics_tiou_reward": 0.7506654560565948, + "rewards/forensics_sinkhorn_reward": 0.6196058690547943, + "rewards/forensics_strict_edge_reward": 0.006954091775696725, + "rewards/forensics_tiou_reward": 0.3206038475036621, + "step": 86 + }, + { + "completion_length": 24.8125, + "epoch": 0.2672811059907834, + "grad_norm": 3.8099334239959717, + "kl": 0.21484375, + "learning_rate": 9.338461538461538e-07, + "loss": 0.0946, + "reward": 0.6605036556720734, + "reward_std": 0.13560855388641357, + "reward_std_all": 0.13625069707632065, + "rewards-max/forensics_sinkhorn_reward": 0.9190075993537903, + "rewards-max/forensics_strict_edge_reward": 0.0030495249084196985, + "rewards-max/forensics_tiou_reward": 0.7845393717288971, + "rewards/forensics_sinkhorn_reward": 0.5473629832267761, + "rewards/forensics_strict_edge_reward": 0.00028253570417291485, + "rewards/forensics_tiou_reward": 0.3080778121948242, + "step": 87 + }, + { + "completion_length": 22.34375, + "epoch": 0.27035330261136714, + "grad_norm": 4.6378021240234375, + "kl": 0.2265625, + "learning_rate": 9.33076923076923e-07, + "loss": 0.2377, + "reward": 0.6438799798488617, + "reward_std": 0.12486575543880463, + "reward_std_all": 0.13972703367471695, + "rewards-max/forensics_sinkhorn_reward": 0.9698070585727692, + "rewards-max/forensics_strict_edge_reward": 0.1447548232972622, + "rewards-max/forensics_tiou_reward": 0.9458337724208832, + "rewards/forensics_sinkhorn_reward": 0.5507292747497559, + "rewards/forensics_strict_edge_reward": 0.011621426790952682, + "rewards/forensics_tiou_reward": 0.30661405622959137, + "step": 88 + }, + { + "completion_length": 24.484375, + "epoch": 0.27342549923195086, + "grad_norm": 4.496413230895996, + "kl": 0.22900390625, + "learning_rate": 9.323076923076922e-07, + "loss": -0.0205, + "reward": 0.6933127343654633, + "reward_std": 0.1370667703449726, + "reward_std_all": 0.1293877400457859, + "rewards-max/forensics_sinkhorn_reward": 0.9622092247009277, + "rewards-max/forensics_strict_edge_reward": 0.15034117922186852, + "rewards-max/forensics_tiou_reward": 0.9271878004074097, + "rewards/forensics_sinkhorn_reward": 0.5898105502128601, + "rewards/forensics_strict_edge_reward": 0.008264946984127164, + "rewards/forensics_tiou_reward": 0.2883637845516205, + "step": 89 + }, + { + "completion_length": 24.234375, + "epoch": 0.2764976958525346, + "grad_norm": 5.358239650726318, + "kl": 0.333984375, + "learning_rate": 9.315384615384615e-07, + "loss": 0.1339, + "reward": 0.706805408000946, + "reward_std": 0.0966373085975647, + "reward_std_all": 0.09736977145075798, + "rewards-max/forensics_sinkhorn_reward": 0.8995195925235748, + "rewards-max/forensics_strict_edge_reward": 0.5865238755941391, + "rewards-max/forensics_tiou_reward": 0.8217556774616241, + "rewards/forensics_sinkhorn_reward": 0.5903966128826141, + "rewards/forensics_strict_edge_reward": 0.036557674407958984, + "rewards/forensics_tiou_reward": 0.2897799015045166, + "step": 90 + }, + { + "completion_length": 21.21875, + "epoch": 0.27956989247311825, + "grad_norm": 5.560170650482178, + "kl": 0.38330078125, + "learning_rate": 9.307692307692308e-07, + "loss": 0.0782, + "reward": 0.6970610916614532, + "reward_std": 0.10604973882436752, + "reward_std_all": 0.10630039498209953, + "rewards-max/forensics_sinkhorn_reward": 0.9712274074554443, + "rewards-max/forensics_strict_edge_reward": 0.4235072433948517, + "rewards-max/forensics_tiou_reward": 0.9486882984638214, + "rewards/forensics_sinkhorn_reward": 0.6043425500392914, + "rewards/forensics_strict_edge_reward": 0.018872656859457493, + "rewards/forensics_tiou_reward": 0.3318966031074524, + "step": 91 + }, + { + "completion_length": 23.59375, + "epoch": 0.282642089093702, + "grad_norm": 5.441155433654785, + "kl": 0.267578125, + "learning_rate": 9.3e-07, + "loss": 0.0586, + "reward": 0.6570391058921814, + "reward_std": 0.10116149857640266, + "reward_std_all": 0.11586211621761322, + "rewards-max/forensics_sinkhorn_reward": 0.8785561919212341, + "rewards-max/forensics_strict_edge_reward": 0.04850808991068334, + "rewards-max/forensics_tiou_reward": 0.6982896029949188, + "rewards/forensics_sinkhorn_reward": 0.5439478605985641, + "rewards/forensics_strict_edge_reward": 0.002915806944884025, + "rewards/forensics_tiou_reward": 0.2632350027561188, + "step": 92 + }, + { + "completion_length": 23.671875, + "epoch": 0.2857142857142857, + "grad_norm": 5.0850982666015625, + "kl": 0.23876953125, + "learning_rate": 9.292307692307692e-07, + "loss": -0.0495, + "reward": 0.6548646688461304, + "reward_std": 0.1352827399969101, + "reward_std_all": 0.14039505273103714, + "rewards-max/forensics_sinkhorn_reward": 0.9103423357009888, + "rewards-max/forensics_strict_edge_reward": 0.01138365245424211, + "rewards-max/forensics_tiou_reward": 0.7777450382709503, + "rewards/forensics_sinkhorn_reward": 0.5173227936029434, + "rewards/forensics_strict_edge_reward": 0.0005133267695782706, + "rewards/forensics_tiou_reward": 0.2578863129019737, + "step": 93 + }, + { + "completion_length": 22.40625, + "epoch": 0.2887864823348694, + "grad_norm": 3.8024299144744873, + "kl": 0.2294921875, + "learning_rate": 9.284615384615384e-07, + "loss": 0.1856, + "reward": 0.6957218647003174, + "reward_std": 0.08649839833378792, + "reward_std_all": 0.09683015197515488, + "rewards-max/forensics_sinkhorn_reward": 0.8764113783836365, + "rewards-max/forensics_strict_edge_reward": 0.032248427625745535, + "rewards-max/forensics_tiou_reward": 0.7285012900829315, + "rewards/forensics_sinkhorn_reward": 0.5593695640563965, + "rewards/forensics_strict_edge_reward": 0.001649704048759304, + "rewards/forensics_tiou_reward": 0.21728968620300293, + "step": 94 + }, + { + "completion_length": 25.78125, + "epoch": 0.29185867895545314, + "grad_norm": 4.724862575531006, + "kl": 0.2333984375, + "learning_rate": 9.276923076923076e-07, + "loss": 0.1921, + "reward": 0.6736975312232971, + "reward_std": 0.11513564735651016, + "reward_std_all": 0.12287253141403198, + "rewards-max/forensics_sinkhorn_reward": 0.9487711787223816, + "rewards-max/forensics_strict_edge_reward": 0.07196865044534206, + "rewards-max/forensics_tiou_reward": 0.8871055841445923, + "rewards/forensics_sinkhorn_reward": 0.5613449811935425, + "rewards/forensics_strict_edge_reward": 0.003906527766957879, + "rewards/forensics_tiou_reward": 0.3213351368904114, + "step": 95 + }, + { + "completion_length": 24.765625, + "epoch": 0.29493087557603687, + "grad_norm": 4.212848663330078, + "kl": 0.2060546875, + "learning_rate": 9.26923076923077e-07, + "loss": 0.2825, + "reward": 0.6505046486854553, + "reward_std": 0.09777124226093292, + "reward_std_all": 0.126369871199131, + "rewards-max/forensics_sinkhorn_reward": 0.9240042269229889, + "rewards-max/forensics_strict_edge_reward": 0.13348280638456345, + "rewards-max/forensics_tiou_reward": 0.8351553976535797, + "rewards/forensics_sinkhorn_reward": 0.5552400946617126, + "rewards/forensics_strict_edge_reward": 0.00636933371424675, + "rewards/forensics_tiou_reward": 0.2601988613605499, + "step": 96 + }, + { + "completion_length": 22.984375, + "epoch": 0.2980030721966206, + "grad_norm": 4.29916524887085, + "kl": 0.22314453125, + "learning_rate": 9.261538461538461e-07, + "loss": 0.0602, + "reward": 0.7244614660739899, + "reward_std": 0.0521519985049963, + "reward_std_all": 0.06706277467310429, + "rewards-max/forensics_sinkhorn_reward": 0.9406714737415314, + "rewards-max/forensics_strict_edge_reward": 0.15579338232055306, + "rewards-max/forensics_tiou_reward": 0.8681555390357971, + "rewards/forensics_sinkhorn_reward": 0.6328344941139221, + "rewards/forensics_strict_edge_reward": 0.006677003373624757, + "rewards/forensics_tiou_reward": 0.3672473728656769, + "step": 97 + }, + { + "completion_length": 23.234375, + "epoch": 0.3010752688172043, + "grad_norm": 4.290058135986328, + "kl": 0.22119140625, + "learning_rate": 9.253846153846153e-07, + "loss": 0.166, + "reward": 0.6957856714725494, + "reward_std": 0.04879263788461685, + "reward_std_all": 0.07731668651103973, + "rewards-max/forensics_sinkhorn_reward": 0.9389293193817139, + "rewards-max/forensics_strict_edge_reward": 0.17567812325432897, + "rewards-max/forensics_tiou_reward": 0.8657706379890442, + "rewards/forensics_sinkhorn_reward": 0.5744253695011139, + "rewards/forensics_strict_edge_reward": 0.007391329854726791, + "rewards/forensics_tiou_reward": 0.28932029008865356, + "step": 98 + }, + { + "completion_length": 23.109375, + "epoch": 0.30414746543778803, + "grad_norm": 3.890082359313965, + "kl": 0.220703125, + "learning_rate": 9.246153846153846e-07, + "loss": 0.101, + "reward": 0.6924510300159454, + "reward_std": 0.08551393076777458, + "reward_std_all": 0.09815030172467232, + "rewards-max/forensics_sinkhorn_reward": 0.9581768214702606, + "rewards-max/forensics_strict_edge_reward": 0.1318028528571631, + "rewards-max/forensics_tiou_reward": 0.9160454571247101, + "rewards/forensics_sinkhorn_reward": 0.5873491168022156, + "rewards/forensics_strict_edge_reward": 0.00575523924794652, + "rewards/forensics_tiou_reward": 0.27605554461479187, + "step": 99 + }, + { + "completion_length": 23.21875, + "epoch": 0.30721966205837176, + "grad_norm": 3.589049816131592, + "kl": 0.19873046875, + "learning_rate": 9.238461538461538e-07, + "loss": 0.1663, + "reward": 0.6844547986984253, + "reward_std": 0.08850681409239769, + "reward_std_all": 0.10347802937030792, + "rewards-max/forensics_sinkhorn_reward": 0.9212190806865692, + "rewards-max/forensics_strict_edge_reward": 0.3135915455459326, + "rewards-max/forensics_tiou_reward": 0.8457333147525787, + "rewards/forensics_sinkhorn_reward": 0.5580151379108429, + "rewards/forensics_strict_edge_reward": 0.013377045636616458, + "rewards/forensics_tiou_reward": 0.2672649547457695, + "step": 100 + }, + { + "completion_length": 22.765625, + "epoch": 0.3102918586789555, + "grad_norm": 3.9354519844055176, + "kl": 0.22802734375, + "learning_rate": 9.230769230769231e-07, + "loss": 0.1565, + "reward": 0.6872659921646118, + "reward_std": 0.11261536180973053, + "reward_std_all": 0.11257554963231087, + "rewards-max/forensics_sinkhorn_reward": 0.9030170738697052, + "rewards-max/forensics_strict_edge_reward": 0.03473378345370293, + "rewards-max/forensics_tiou_reward": 0.7757658958435059, + "rewards/forensics_sinkhorn_reward": 0.5586119294166565, + "rewards/forensics_strict_edge_reward": 0.001564702542964369, + "rewards/forensics_tiou_reward": 0.24468161165714264, + "step": 101 + }, + { + "completion_length": 21.40625, + "epoch": 0.31336405529953915, + "grad_norm": 4.923160076141357, + "kl": 0.21337890625, + "learning_rate": 9.223076923076922e-07, + "loss": 0.1542, + "reward": 0.6657630205154419, + "reward_std": 0.12049636244773865, + "reward_std_all": 0.1293082870543003, + "rewards-max/forensics_sinkhorn_reward": 0.9313574135303497, + "rewards-max/forensics_strict_edge_reward": 0.03338917932705954, + "rewards-max/forensics_tiou_reward": 0.8127835392951965, + "rewards/forensics_sinkhorn_reward": 0.5931272506713867, + "rewards/forensics_strict_edge_reward": 0.0017060141399269924, + "rewards/forensics_tiou_reward": 0.3461000993847847, + "step": 102 + }, + { + "completion_length": 23.296875, + "epoch": 0.31643625192012287, + "grad_norm": 4.148444175720215, + "kl": 0.22705078125, + "learning_rate": 9.215384615384614e-07, + "loss": 0.0762, + "reward": 0.6947645843029022, + "reward_std": 0.10226647928357124, + "reward_std_all": 0.10436034202575684, + "rewards-max/forensics_sinkhorn_reward": 0.8548120558261871, + "rewards-max/forensics_strict_edge_reward": 0.033331906190142035, + "rewards-max/forensics_tiou_reward": 0.690924882888794, + "rewards/forensics_sinkhorn_reward": 0.5710434317588806, + "rewards/forensics_strict_edge_reward": 0.001513260678621009, + "rewards/forensics_tiou_reward": 0.2318892478942871, + "step": 103 + }, + { + "completion_length": 25.203125, + "epoch": 0.3195084485407066, + "grad_norm": 4.12461519241333, + "kl": 0.20361328125, + "learning_rate": 9.207692307692308e-07, + "loss": -0.0172, + "reward": 0.6694366931915283, + "reward_std": 0.10524770617485046, + "reward_std_all": 0.11805282160639763, + "rewards-max/forensics_sinkhorn_reward": 0.9044953882694244, + "rewards-max/forensics_strict_edge_reward": 0.1601156342549075, + "rewards-max/forensics_tiou_reward": 0.719698965549469, + "rewards/forensics_sinkhorn_reward": 0.5621131956577301, + "rewards/forensics_strict_edge_reward": 0.01051399650032181, + "rewards/forensics_tiou_reward": 0.2672673836350441, + "step": 104 + }, + { + "completion_length": 22.78125, + "epoch": 0.3225806451612903, + "grad_norm": 4.298645496368408, + "kl": 0.23876953125, + "learning_rate": 9.2e-07, + "loss": 0.0331, + "reward": 0.7117058038711548, + "reward_std": 0.0655699335038662, + "reward_std_all": 0.07772942259907722, + "rewards-max/forensics_sinkhorn_reward": 0.9235458970069885, + "rewards-max/forensics_strict_edge_reward": 0.00015740497877914095, + "rewards-max/forensics_tiou_reward": 0.7862889468669891, + "rewards/forensics_sinkhorn_reward": 0.5850164890289307, + "rewards/forensics_strict_edge_reward": 6.875397602357225e-06, + "rewards/forensics_tiou_reward": 0.23823951929807663, + "step": 105 + }, + { + "completion_length": 25.265625, + "epoch": 0.32565284178187404, + "grad_norm": 4.026068210601807, + "kl": 0.23193359375, + "learning_rate": 9.192307692307692e-07, + "loss": 0.0731, + "reward": 0.667449027299881, + "reward_std": 0.1200784407556057, + "reward_std_all": 0.12348349019885063, + "rewards-max/forensics_sinkhorn_reward": 0.9029175341129303, + "rewards-max/forensics_strict_edge_reward": 0.17711446493922267, + "rewards-max/forensics_tiou_reward": 0.7897762358188629, + "rewards/forensics_sinkhorn_reward": 0.5350763201713562, + "rewards/forensics_strict_edge_reward": 0.008124451092953677, + "rewards/forensics_tiou_reward": 0.2532248944044113, + "step": 106 + }, + { + "completion_length": 24.640625, + "epoch": 0.32872503840245776, + "grad_norm": 4.430719375610352, + "kl": 0.22021484375, + "learning_rate": 9.184615384615384e-07, + "loss": 0.0675, + "reward": 0.6817147433757782, + "reward_std": 0.12615060806274414, + "reward_std_all": 0.12298290804028511, + "rewards-max/forensics_sinkhorn_reward": 0.9654110968112946, + "rewards-max/forensics_strict_edge_reward": 0.13498795218765736, + "rewards-max/forensics_tiou_reward": 0.9333766996860504, + "rewards/forensics_sinkhorn_reward": 0.5496302843093872, + "rewards/forensics_strict_edge_reward": 0.009609624161385, + "rewards/forensics_tiou_reward": 0.37069061398506165, + "step": 107 + }, + { + "completion_length": 21.390625, + "epoch": 0.3317972350230415, + "grad_norm": 4.30990743637085, + "kl": 0.22216796875, + "learning_rate": 9.176923076923076e-07, + "loss": 0.1537, + "reward": 0.6927423179149628, + "reward_std": 0.09959318116307259, + "reward_std_all": 0.10166923701763153, + "rewards-max/forensics_sinkhorn_reward": 0.9202134907245636, + "rewards-max/forensics_strict_edge_reward": 0.09888562001287937, + "rewards-max/forensics_tiou_reward": 0.8277974724769592, + "rewards/forensics_sinkhorn_reward": 0.5873937904834747, + "rewards/forensics_strict_edge_reward": 0.0061213315930217505, + "rewards/forensics_tiou_reward": 0.30292922258377075, + "step": 108 + }, + { + "completion_length": 25.09375, + "epoch": 0.3348694316436252, + "grad_norm": 6.311233043670654, + "kl": 0.2060546875, + "learning_rate": 9.169230769230769e-07, + "loss": 0.2655, + "reward": 0.6595729291439056, + "reward_std": 0.14442402869462967, + "reward_std_all": 0.14441829919815063, + "rewards-max/forensics_sinkhorn_reward": 0.9060747027397156, + "rewards-max/forensics_strict_edge_reward": 0.048058446729555726, + "rewards-max/forensics_tiou_reward": 0.8055365681648254, + "rewards/forensics_sinkhorn_reward": 0.5297877788543701, + "rewards/forensics_strict_edge_reward": 0.002192532701883465, + "rewards/forensics_tiou_reward": 0.23667298257350922, + "step": 109 + }, + { + "completion_length": 24.0, + "epoch": 0.3379416282642089, + "grad_norm": 3.720932722091675, + "kl": 0.1982421875, + "learning_rate": 9.161538461538462e-07, + "loss": 0.2406, + "reward": 0.6776750087738037, + "reward_std": 0.08617264032363892, + "reward_std_all": 0.1029491052031517, + "rewards-max/forensics_sinkhorn_reward": 0.9326959848403931, + "rewards-max/forensics_strict_edge_reward": 0.054147507064044476, + "rewards-max/forensics_tiou_reward": 0.8395191133022308, + "rewards/forensics_sinkhorn_reward": 0.5576974451541901, + "rewards/forensics_strict_edge_reward": 0.004949394842697075, + "rewards/forensics_tiou_reward": 0.2663838416337967, + "step": 110 + }, + { + "completion_length": 23.203125, + "epoch": 0.34101382488479265, + "grad_norm": 4.838845252990723, + "kl": 0.205078125, + "learning_rate": 9.153846153846153e-07, + "loss": 0.2081, + "reward": 0.6363730728626251, + "reward_std": 0.1630859076976776, + "reward_std_all": 0.1617242842912674, + "rewards-max/forensics_sinkhorn_reward": 0.9561496675014496, + "rewards-max/forensics_strict_edge_reward": 0.20801136828958988, + "rewards-max/forensics_tiou_reward": 0.9159163236618042, + "rewards/forensics_sinkhorn_reward": 0.5394165813922882, + "rewards/forensics_strict_edge_reward": 0.01510257029440254, + "rewards/forensics_tiou_reward": 0.3037831336259842, + "step": 111 + }, + { + "completion_length": 24.0625, + "epoch": 0.34408602150537637, + "grad_norm": 4.398743152618408, + "kl": 0.2060546875, + "learning_rate": 9.146153846153846e-07, + "loss": 0.0645, + "reward": 0.6825672090053558, + "reward_std": 0.08426640927791595, + "reward_std_all": 0.10293347388505936, + "rewards-max/forensics_sinkhorn_reward": 0.9459599256515503, + "rewards-max/forensics_strict_edge_reward": 0.04354281071573496, + "rewards-max/forensics_tiou_reward": 0.8803559243679047, + "rewards/forensics_sinkhorn_reward": 0.5887058973312378, + "rewards/forensics_strict_edge_reward": 0.0031064029317349195, + "rewards/forensics_tiou_reward": 0.3194667100906372, + "step": 112 + }, + { + "completion_length": 22.21875, + "epoch": 0.34715821812596004, + "grad_norm": 3.8565878868103027, + "kl": 0.2216796875, + "learning_rate": 9.138461538461538e-07, + "loss": 0.0522, + "reward": 0.7415899336338043, + "reward_std": 0.0682296734303236, + "reward_std_all": 0.06688150577247143, + "rewards-max/forensics_sinkhorn_reward": 0.9574632048606873, + "rewards-max/forensics_strict_edge_reward": 0.23501590639352798, + "rewards-max/forensics_tiou_reward": 0.9101371467113495, + "rewards/forensics_sinkhorn_reward": 0.6325962841510773, + "rewards/forensics_strict_edge_reward": 0.011119115632027388, + "rewards/forensics_tiou_reward": 0.3948024958372116, + "step": 113 + }, + { + "completion_length": 25.4375, + "epoch": 0.35023041474654376, + "grad_norm": 4.073228359222412, + "kl": 0.2451171875, + "learning_rate": 9.130769230769231e-07, + "loss": 0.1935, + "reward": 0.6715356111526489, + "reward_std": 0.14489839226007462, + "reward_std_all": 0.13861507922410965, + "rewards-max/forensics_sinkhorn_reward": 0.9010035693645477, + "rewards-max/forensics_strict_edge_reward": 0.13581120502203703, + "rewards-max/forensics_tiou_reward": 0.766438752412796, + "rewards/forensics_sinkhorn_reward": 0.5201206654310226, + "rewards/forensics_strict_edge_reward": 0.0063774207956157625, + "rewards/forensics_tiou_reward": 0.25938789546489716, + "step": 114 + }, + { + "completion_length": 25.109375, + "epoch": 0.3533026113671275, + "grad_norm": 4.346466541290283, + "kl": 0.220703125, + "learning_rate": 9.123076923076922e-07, + "loss": 0.1856, + "reward": 0.6933984756469727, + "reward_std": 0.10482052341103554, + "reward_std_all": 0.10755699872970581, + "rewards-max/forensics_sinkhorn_reward": 0.8481972515583038, + "rewards-max/forensics_strict_edge_reward": 0.38210204988718033, + "rewards-max/forensics_tiou_reward": 0.7116467952728271, + "rewards/forensics_sinkhorn_reward": 0.5857203304767609, + "rewards/forensics_strict_edge_reward": 0.0174578744918108, + "rewards/forensics_tiou_reward": 0.2690346986055374, + "step": 115 + }, + { + "completion_length": 24.0, + "epoch": 0.3563748079877112, + "grad_norm": 2.7970802783966064, + "kl": 0.2294921875, + "learning_rate": 9.115384615384614e-07, + "loss": 0.1001, + "reward": 0.6888096332550049, + "reward_std": 0.08613978326320648, + "reward_std_all": 0.09808462113142014, + "rewards-max/forensics_sinkhorn_reward": 0.8831377327442169, + "rewards-max/forensics_strict_edge_reward": 0.06125769135724113, + "rewards-max/forensics_tiou_reward": 0.7491612732410431, + "rewards/forensics_sinkhorn_reward": 0.5615533590316772, + "rewards/forensics_strict_edge_reward": 0.003510911319949628, + "rewards/forensics_tiou_reward": 0.2830739766359329, + "step": 116 + }, + { + "completion_length": 22.40625, + "epoch": 0.35944700460829493, + "grad_norm": 4.220727920532227, + "kl": 0.2841796875, + "learning_rate": 9.107692307692308e-07, + "loss": 0.0513, + "reward": 0.637539803981781, + "reward_std": 0.13100459426641464, + "reward_std_all": 0.14497264474630356, + "rewards-max/forensics_sinkhorn_reward": 0.9010443985462189, + "rewards-max/forensics_strict_edge_reward": 0.08486683992668986, + "rewards-max/forensics_tiou_reward": 0.7755345702171326, + "rewards/forensics_sinkhorn_reward": 0.5120584666728973, + "rewards/forensics_strict_edge_reward": 0.0040563614456914365, + "rewards/forensics_tiou_reward": 0.2040066123008728, + "step": 117 + }, + { + "completion_length": 25.28125, + "epoch": 0.36251920122887865, + "grad_norm": 3.690009117126465, + "kl": 0.265625, + "learning_rate": 9.1e-07, + "loss": 0.0301, + "reward": 0.6850385069847107, + "reward_std": 0.0797569565474987, + "reward_std_all": 0.09914300218224525, + "rewards-max/forensics_sinkhorn_reward": 0.9291922748088837, + "rewards-max/forensics_strict_edge_reward": 0.05623254366219044, + "rewards-max/forensics_tiou_reward": 0.8533933162689209, + "rewards/forensics_sinkhorn_reward": 0.5739975273609161, + "rewards/forensics_strict_edge_reward": 0.002608317183330655, + "rewards/forensics_tiou_reward": 0.2976900041103363, + "step": 118 + }, + { + "completion_length": 24.1875, + "epoch": 0.3655913978494624, + "grad_norm": 4.585857391357422, + "kl": 0.208984375, + "learning_rate": 9.092307692307692e-07, + "loss": 0.2374, + "reward": 0.6731116473674774, + "reward_std": 0.13847170770168304, + "reward_std_all": 0.13340026512742043, + "rewards-max/forensics_sinkhorn_reward": 0.9567011892795563, + "rewards-max/forensics_strict_edge_reward": 0.32668518647551537, + "rewards-max/forensics_tiou_reward": 0.9206094145774841, + "rewards/forensics_sinkhorn_reward": 0.5774285793304443, + "rewards/forensics_strict_edge_reward": 0.01596722286194563, + "rewards/forensics_tiou_reward": 0.31757375597953796, + "step": 119 + }, + { + "completion_length": 23.265625, + "epoch": 0.3686635944700461, + "grad_norm": 3.7241950035095215, + "kl": 0.21728515625, + "learning_rate": 9.084615384615384e-07, + "loss": 0.261, + "reward": 0.6047624945640564, + "reward_std": 0.18897893279790878, + "reward_std_all": 0.18850626796483994, + "rewards-max/forensics_sinkhorn_reward": 0.9317514300346375, + "rewards-max/forensics_strict_edge_reward": 0.18009157478809357, + "rewards-max/forensics_tiou_reward": 0.8655781447887421, + "rewards/forensics_sinkhorn_reward": 0.5139330923557281, + "rewards/forensics_strict_edge_reward": 0.009497123304754496, + "rewards/forensics_tiou_reward": 0.2568623721599579, + "step": 120 + }, + { + "completion_length": 23.3125, + "epoch": 0.3717357910906298, + "grad_norm": 4.315061092376709, + "kl": 0.248046875, + "learning_rate": 9.076923076923076e-07, + "loss": 0.1684, + "reward": 0.6617870330810547, + "reward_std": 0.10856487974524498, + "reward_std_all": 0.12323053926229477, + "rewards-max/forensics_sinkhorn_reward": 0.8662592172622681, + "rewards-max/forensics_strict_edge_reward": 0.04269141051918268, + "rewards-max/forensics_tiou_reward": 0.7012982666492462, + "rewards/forensics_sinkhorn_reward": 0.5385458171367645, + "rewards/forensics_strict_edge_reward": 0.0020776151213794947, + "rewards/forensics_tiou_reward": 0.24903445690870285, + "step": 121 + }, + { + "completion_length": 23.4375, + "epoch": 0.37480798771121354, + "grad_norm": 5.281167507171631, + "kl": 0.20068359375, + "learning_rate": 9.069230769230769e-07, + "loss": 0.1593, + "reward": 0.7140557169914246, + "reward_std": 0.09010198339819908, + "reward_std_all": 0.08945224061608315, + "rewards-max/forensics_sinkhorn_reward": 0.8820695877075195, + "rewards-max/forensics_strict_edge_reward": 0.028187068179249763, + "rewards-max/forensics_tiou_reward": 0.758122444152832, + "rewards/forensics_sinkhorn_reward": 0.5989495813846588, + "rewards/forensics_strict_edge_reward": 0.0024063187593128532, + "rewards/forensics_tiou_reward": 0.26727380603551865, + "step": 122 + }, + { + "completion_length": 25.609375, + "epoch": 0.3778801843317972, + "grad_norm": 3.854313850402832, + "kl": 0.2568359375, + "learning_rate": 9.061538461538461e-07, + "loss": 0.2807, + "reward": 0.6821293532848358, + "reward_std": 0.1121002770960331, + "reward_std_all": 0.11519825458526611, + "rewards-max/forensics_sinkhorn_reward": 0.8913854360580444, + "rewards-max/forensics_strict_edge_reward": 0.00881742947967723, + "rewards-max/forensics_tiou_reward": 0.7405750155448914, + "rewards/forensics_sinkhorn_reward": 0.5903197526931763, + "rewards/forensics_strict_edge_reward": 0.0009448721029912122, + "rewards/forensics_tiou_reward": 0.3173769861459732, + "step": 123 + }, + { + "completion_length": 22.765625, + "epoch": 0.38095238095238093, + "grad_norm": 4.516040802001953, + "kl": 0.21630859375, + "learning_rate": 9.053846153846153e-07, + "loss": 0.082, + "reward": 0.7123021483421326, + "reward_std": 0.08651966974139214, + "reward_std_all": 0.08858177810907364, + "rewards-max/forensics_sinkhorn_reward": 0.9399447441101074, + "rewards-max/forensics_strict_edge_reward": 0.1505280442070216, + "rewards-max/forensics_tiou_reward": 0.8554825484752655, + "rewards/forensics_sinkhorn_reward": 0.5940777659416199, + "rewards/forensics_strict_edge_reward": 0.007886966479418334, + "rewards/forensics_tiou_reward": 0.34725768864154816, + "step": 124 + }, + { + "completion_length": 26.625, + "epoch": 0.38402457757296465, + "grad_norm": 3.8571105003356934, + "kl": 0.32568359375, + "learning_rate": 9.046153846153846e-07, + "loss": 0.0948, + "reward": 0.6969379782676697, + "reward_std": 0.10468807443976402, + "reward_std_all": 0.10701176151633263, + "rewards-max/forensics_sinkhorn_reward": 0.9258082509040833, + "rewards-max/forensics_strict_edge_reward": 0.3837217539548874, + "rewards-max/forensics_tiou_reward": 0.8352214395999908, + "rewards/forensics_sinkhorn_reward": 0.5608004331588745, + "rewards/forensics_strict_edge_reward": 0.01697367150336504, + "rewards/forensics_tiou_reward": 0.33248570561408997, + "step": 125 + }, + { + "completion_length": 22.875, + "epoch": 0.3870967741935484, + "grad_norm": 4.316264629364014, + "kl": 0.2421875, + "learning_rate": 9.038461538461538e-07, + "loss": 0.076, + "reward": 0.650079071521759, + "reward_std": 0.11816218495368958, + "reward_std_all": 0.13349854201078415, + "rewards-max/forensics_sinkhorn_reward": 0.9284176230430603, + "rewards-max/forensics_strict_edge_reward": 0.2694640066474676, + "rewards-max/forensics_tiou_reward": 0.8472996056079865, + "rewards/forensics_sinkhorn_reward": 0.5359291136264801, + "rewards/forensics_strict_edge_reward": 0.019585855770856142, + "rewards/forensics_tiou_reward": 0.22632277756929398, + "step": 126 + }, + { + "completion_length": 22.40625, + "epoch": 0.3901689708141321, + "grad_norm": 3.202813148498535, + "kl": 0.21875, + "learning_rate": 9.030769230769231e-07, + "loss": 0.1398, + "reward": 0.6926111578941345, + "reward_std": 0.1253463588654995, + "reward_std_all": 0.1194092258810997, + "rewards-max/forensics_sinkhorn_reward": 0.9485138356685638, + "rewards-max/forensics_strict_edge_reward": 0.21517616137862206, + "rewards-max/forensics_tiou_reward": 0.8932360708713531, + "rewards/forensics_sinkhorn_reward": 0.5947855114936829, + "rewards/forensics_strict_edge_reward": 0.010347850387915969, + "rewards/forensics_tiou_reward": 0.29858388006687164, + "step": 127 + }, + { + "completion_length": 22.90625, + "epoch": 0.3932411674347158, + "grad_norm": 4.459503173828125, + "kl": 0.2548828125, + "learning_rate": 9.023076923076922e-07, + "loss": 0.1542, + "reward": 0.7183249294757843, + "reward_std": 0.09675052762031555, + "reward_std_all": 0.09335853904485703, + "rewards-max/forensics_sinkhorn_reward": 0.9397114217281342, + "rewards-max/forensics_strict_edge_reward": 0.014296650420874357, + "rewards-max/forensics_tiou_reward": 0.8695799708366394, + "rewards/forensics_sinkhorn_reward": 0.629786878824234, + "rewards/forensics_strict_edge_reward": 0.0007682663563173264, + "rewards/forensics_tiou_reward": 0.31069953739643097, + "step": 128 + }, + { + "completion_length": 22.234375, + "epoch": 0.39631336405529954, + "grad_norm": 3.7544853687286377, + "kl": 0.2958984375, + "learning_rate": 9.015384615384614e-07, + "loss": 0.095, + "reward": 0.6924610435962677, + "reward_std": 0.09210295230150223, + "reward_std_all": 0.1019834503531456, + "rewards-max/forensics_sinkhorn_reward": 0.8641164600849152, + "rewards-max/forensics_strict_edge_reward": 0.03520354279316962, + "rewards-max/forensics_tiou_reward": 0.7145825624465942, + "rewards/forensics_sinkhorn_reward": 0.5922704935073853, + "rewards/forensics_strict_edge_reward": 0.0024178895037039183, + "rewards/forensics_tiou_reward": 0.2834678143262863, + "step": 129 + }, + { + "completion_length": 22.9375, + "epoch": 0.39938556067588327, + "grad_norm": 4.27451753616333, + "kl": 0.251953125, + "learning_rate": 9.007692307692307e-07, + "loss": 0.1224, + "reward": 0.652874767780304, + "reward_std": 0.13551395386457443, + "reward_std_all": 0.13752444088459015, + "rewards-max/forensics_sinkhorn_reward": 0.8487919270992279, + "rewards-max/forensics_strict_edge_reward": 0.13041074387729168, + "rewards-max/forensics_tiou_reward": 0.7252244055271149, + "rewards/forensics_sinkhorn_reward": 0.5211644172668457, + "rewards/forensics_strict_edge_reward": 0.007890368578955531, + "rewards/forensics_tiou_reward": 0.25808917731046677, + "step": 130 + }, + { + "completion_length": 24.53125, + "epoch": 0.402457757296467, + "grad_norm": 4.083869934082031, + "kl": 0.23291015625, + "learning_rate": 9e-07, + "loss": 0.2568, + "reward": 0.6886385977268219, + "reward_std": 0.1148500069975853, + "reward_std_all": 0.11281560361385345, + "rewards-max/forensics_sinkhorn_reward": 0.94572713971138, + "rewards-max/forensics_strict_edge_reward": 0.28211354464292526, + "rewards-max/forensics_tiou_reward": 0.8777989447116852, + "rewards/forensics_sinkhorn_reward": 0.5738532841205597, + "rewards/forensics_strict_edge_reward": 0.01500180410221219, + "rewards/forensics_tiou_reward": 0.3015487492084503, + "step": 131 + }, + { + "completion_length": 22.640625, + "epoch": 0.4055299539170507, + "grad_norm": 4.5037922859191895, + "kl": 0.25048828125, + "learning_rate": 8.992307692307693e-07, + "loss": 0.2045, + "reward": 0.6340287923812866, + "reward_std": 0.16724693775177002, + "reward_std_all": 0.16394642740488052, + "rewards-max/forensics_sinkhorn_reward": 0.8745878040790558, + "rewards-max/forensics_strict_edge_reward": 0.006886035902425647, + "rewards-max/forensics_tiou_reward": 0.7082546949386597, + "rewards/forensics_sinkhorn_reward": 0.5353283733129501, + "rewards/forensics_strict_edge_reward": 0.0004993397888028994, + "rewards/forensics_tiou_reward": 0.22552456706762314, + "step": 132 + }, + { + "completion_length": 23.671875, + "epoch": 0.40860215053763443, + "grad_norm": 5.011726379394531, + "kl": 0.22509765625, + "learning_rate": 8.984615384615384e-07, + "loss": 0.2317, + "reward": 0.6773346662521362, + "reward_std": 0.13451136648654938, + "reward_std_all": 0.13386524468660355, + "rewards-max/forensics_sinkhorn_reward": 0.9145402312278748, + "rewards-max/forensics_strict_edge_reward": 0.2873297519981861, + "rewards-max/forensics_tiou_reward": 0.7927023768424988, + "rewards/forensics_sinkhorn_reward": 0.5710183382034302, + "rewards/forensics_strict_edge_reward": 0.012377217411994934, + "rewards/forensics_tiou_reward": 0.32164807617664337, + "step": 133 + }, + { + "completion_length": 23.515625, + "epoch": 0.4116743471582181, + "grad_norm": 4.280043601989746, + "kl": 0.2177734375, + "learning_rate": 8.976923076923076e-07, + "loss": 0.1303, + "reward": 0.7159011662006378, + "reward_std": 0.09728031978011131, + "reward_std_all": 0.09400174766778946, + "rewards-max/forensics_sinkhorn_reward": 0.9500273168087006, + "rewards-max/forensics_strict_edge_reward": 0.06041394383646548, + "rewards-max/forensics_tiou_reward": 0.9017269313335419, + "rewards/forensics_sinkhorn_reward": 0.5907571017742157, + "rewards/forensics_strict_edge_reward": 0.0028968896222067997, + "rewards/forensics_tiou_reward": 0.29888391494750977, + "step": 134 + }, + { + "completion_length": 21.125, + "epoch": 0.4147465437788018, + "grad_norm": 5.453299522399902, + "kl": 0.26123046875, + "learning_rate": 8.969230769230769e-07, + "loss": 0.1222, + "reward": 0.701539546251297, + "reward_std": 0.08112169802188873, + "reward_std_all": 0.09152692928910255, + "rewards-max/forensics_sinkhorn_reward": 0.9259195327758789, + "rewards-max/forensics_strict_edge_reward": 0.03841971047222614, + "rewards-max/forensics_tiou_reward": 0.8545956313610077, + "rewards/forensics_sinkhorn_reward": 0.6039856672286987, + "rewards/forensics_strict_edge_reward": 0.0027199106698390096, + "rewards/forensics_tiou_reward": 0.3434341251850128, + "step": 135 + }, + { + "completion_length": 25.25, + "epoch": 0.41781874039938555, + "grad_norm": 4.1620707511901855, + "kl": 0.24169921875, + "learning_rate": 8.961538461538461e-07, + "loss": 0.1429, + "reward": 0.7006269097328186, + "reward_std": 0.11105448007583618, + "reward_std_all": 0.11129093915224075, + "rewards-max/forensics_sinkhorn_reward": 0.9276590943336487, + "rewards-max/forensics_strict_edge_reward": 0.13495026365853846, + "rewards-max/forensics_tiou_reward": 0.8340420126914978, + "rewards/forensics_sinkhorn_reward": 0.6017210185527802, + "rewards/forensics_strict_edge_reward": 0.0064337069052271545, + "rewards/forensics_tiou_reward": 0.36927713453769684, + "step": 136 + }, + { + "completion_length": 23.71875, + "epoch": 0.42089093701996927, + "grad_norm": 5.365396976470947, + "kl": 0.2607421875, + "learning_rate": 8.953846153846152e-07, + "loss": 0.0734, + "reward": 0.6951124668121338, + "reward_std": 0.12408466637134552, + "reward_std_all": 0.11867202445864677, + "rewards-max/forensics_sinkhorn_reward": 0.9747278690338135, + "rewards-max/forensics_strict_edge_reward": 0.09088217839598656, + "rewards-max/forensics_tiou_reward": 0.956510990858078, + "rewards/forensics_sinkhorn_reward": 0.6027211248874664, + "rewards/forensics_strict_edge_reward": 0.00396210141479969, + "rewards/forensics_tiou_reward": 0.3380693942308426, + "step": 137 + }, + { + "completion_length": 22.859375, + "epoch": 0.423963133640553, + "grad_norm": 3.618079423904419, + "kl": 0.25, + "learning_rate": 8.946153846153846e-07, + "loss": 0.2135, + "reward": 0.6422411203384399, + "reward_std": 0.11638273671269417, + "reward_std_all": 0.13570674508810043, + "rewards-max/forensics_sinkhorn_reward": 0.9522103071212769, + "rewards-max/forensics_strict_edge_reward": 0.037840028351638466, + "rewards-max/forensics_tiou_reward": 0.9053370654582977, + "rewards/forensics_sinkhorn_reward": 0.5288629084825516, + "rewards/forensics_strict_edge_reward": 0.002650756694492884, + "rewards/forensics_tiou_reward": 0.25404299050569534, + "step": 138 + }, + { + "completion_length": 24.078125, + "epoch": 0.4270353302611367, + "grad_norm": 3.6575751304626465, + "kl": 0.2685546875, + "learning_rate": 8.938461538461538e-07, + "loss": 0.1356, + "reward": 0.7068271338939667, + "reward_std": 0.08130757510662079, + "reward_std_all": 0.09050598554313183, + "rewards-max/forensics_sinkhorn_reward": 0.9258757531642914, + "rewards-max/forensics_strict_edge_reward": 0.05875555274542421, + "rewards-max/forensics_tiou_reward": 0.8129273951053619, + "rewards/forensics_sinkhorn_reward": 0.6127643883228302, + "rewards/forensics_strict_edge_reward": 0.0027045828028349206, + "rewards/forensics_tiou_reward": 0.3742029666900635, + "step": 139 + }, + { + "completion_length": 24.0, + "epoch": 0.43010752688172044, + "grad_norm": 4.5209879875183105, + "kl": 0.3447265625, + "learning_rate": 8.930769230769231e-07, + "loss": 0.2319, + "reward": 0.7012770771980286, + "reward_std": 0.08972135558724403, + "reward_std_all": 0.09781694039702415, + "rewards-max/forensics_sinkhorn_reward": 0.9519439041614532, + "rewards-max/forensics_strict_edge_reward": 0.17608270980417728, + "rewards-max/forensics_tiou_reward": 0.884095311164856, + "rewards/forensics_sinkhorn_reward": 0.6020748913288116, + "rewards/forensics_strict_edge_reward": 0.008171076769940555, + "rewards/forensics_tiou_reward": 0.3118024468421936, + "step": 140 + }, + { + "completion_length": 23.4375, + "epoch": 0.43317972350230416, + "grad_norm": 4.351680278778076, + "kl": 0.2353515625, + "learning_rate": 8.923076923076923e-07, + "loss": 0.1895, + "reward": 0.6991091966629028, + "reward_std": 0.09135016798973083, + "reward_std_all": 0.09552010893821716, + "rewards-max/forensics_sinkhorn_reward": 0.9815509617328644, + "rewards-max/forensics_strict_edge_reward": 0.4636124148964882, + "rewards-max/forensics_tiou_reward": 0.972128838300705, + "rewards/forensics_sinkhorn_reward": 0.6041278839111328, + "rewards/forensics_strict_edge_reward": 0.019991565495729446, + "rewards/forensics_tiou_reward": 0.33904577791690826, + "step": 141 + }, + { + "completion_length": 22.546875, + "epoch": 0.4362519201228879, + "grad_norm": 4.747137546539307, + "kl": 0.2607421875, + "learning_rate": 8.915384615384614e-07, + "loss": 0.065, + "reward": 0.748094230890274, + "reward_std": 0.046626804396510124, + "reward_std_all": 0.05134431831538677, + "rewards-max/forensics_sinkhorn_reward": 0.9724660217761993, + "rewards-max/forensics_strict_edge_reward": 0.293256051838398, + "rewards-max/forensics_tiou_reward": 0.9490809738636017, + "rewards/forensics_sinkhorn_reward": 0.6564283072948456, + "rewards/forensics_strict_edge_reward": 0.013563599903136492, + "rewards/forensics_tiou_reward": 0.3301175683736801, + "step": 142 + }, + { + "completion_length": 23.4375, + "epoch": 0.4393241167434716, + "grad_norm": 3.942147731781006, + "kl": 0.21630859375, + "learning_rate": 8.907692307692307e-07, + "loss": 0.1082, + "reward": 0.7212665677070618, + "reward_std": 0.0732341855764389, + "reward_std_all": 0.07924721390008926, + "rewards-max/forensics_sinkhorn_reward": 0.9217577576637268, + "rewards-max/forensics_strict_edge_reward": 0.35709598287940025, + "rewards-max/forensics_tiou_reward": 0.8485811352729797, + "rewards/forensics_sinkhorn_reward": 0.6167725026607513, + "rewards/forensics_strict_edge_reward": 0.020257187716197222, + "rewards/forensics_tiou_reward": 0.3246060609817505, + "step": 143 + }, + { + "completion_length": 23.859375, + "epoch": 0.4423963133640553, + "grad_norm": 3.1664581298828125, + "kl": 0.24755859375, + "learning_rate": 8.9e-07, + "loss": 0.1811, + "reward": 0.6848470568656921, + "reward_std": 0.09318595007061958, + "reward_std_all": 0.10795457288622856, + "rewards-max/forensics_sinkhorn_reward": 0.8616154789924622, + "rewards-max/forensics_strict_edge_reward": 0.05089401826262474, + "rewards-max/forensics_tiou_reward": 0.6745442152023315, + "rewards/forensics_sinkhorn_reward": 0.5565349459648132, + "rewards/forensics_strict_edge_reward": 0.002297774626640603, + "rewards/forensics_tiou_reward": 0.2737082988023758, + "step": 144 + }, + { + "completion_length": 24.15625, + "epoch": 0.445468509984639, + "grad_norm": 5.463983535766602, + "kl": 0.251953125, + "learning_rate": 8.892307692307693e-07, + "loss": 0.194, + "reward": 0.7296844124794006, + "reward_std": 0.07569712772965431, + "reward_std_all": 0.07562362588942051, + "rewards-max/forensics_sinkhorn_reward": 0.9725532233715057, + "rewards-max/forensics_strict_edge_reward": 0.16966115683317184, + "rewards-max/forensics_tiou_reward": 0.9489418566226959, + "rewards/forensics_sinkhorn_reward": 0.6540018916130066, + "rewards/forensics_strict_edge_reward": 0.01026965118944645, + "rewards/forensics_tiou_reward": 0.3985828757286072, + "step": 145 + }, + { + "completion_length": 22.921875, + "epoch": 0.4485407066052227, + "grad_norm": 6.082848072052002, + "kl": 0.2294921875, + "learning_rate": 8.884615384615384e-07, + "loss": 0.1938, + "reward": 0.6763088703155518, + "reward_std": 0.08521247655153275, + "reward_std_all": 0.09993977472186089, + "rewards-max/forensics_sinkhorn_reward": 0.9389910995960236, + "rewards-max/forensics_strict_edge_reward": 0.1133190244436264, + "rewards-max/forensics_tiou_reward": 0.8565824925899506, + "rewards/forensics_sinkhorn_reward": 0.554753452539444, + "rewards/forensics_strict_edge_reward": 0.0056412366684526205, + "rewards/forensics_tiou_reward": 0.26040689647197723, + "step": 146 + }, + { + "completion_length": 22.328125, + "epoch": 0.45161290322580644, + "grad_norm": 5.547555446624756, + "kl": 0.22802734375, + "learning_rate": 8.876923076923076e-07, + "loss": 0.1314, + "reward": 0.7281382381916046, + "reward_std": 0.08586161956191063, + "reward_std_all": 0.08260054141283035, + "rewards-max/forensics_sinkhorn_reward": 0.9322580397129059, + "rewards-max/forensics_strict_edge_reward": 0.3334420909159235, + "rewards-max/forensics_tiou_reward": 0.8450202941894531, + "rewards/forensics_sinkhorn_reward": 0.6557379961013794, + "rewards/forensics_strict_edge_reward": 0.014980803833168466, + "rewards/forensics_tiou_reward": 0.4224454015493393, + "step": 147 + }, + { + "completion_length": 23.328125, + "epoch": 0.45468509984639016, + "grad_norm": 5.878541469573975, + "kl": 0.24267578125, + "learning_rate": 8.869230769230769e-07, + "loss": 0.23, + "reward": 0.7029889523983002, + "reward_std": 0.08867561630904675, + "reward_std_all": 0.09356383606791496, + "rewards-max/forensics_sinkhorn_reward": 0.9368222653865814, + "rewards-max/forensics_strict_edge_reward": 0.18768735975027084, + "rewards-max/forensics_tiou_reward": 0.8617589473724365, + "rewards/forensics_sinkhorn_reward": 0.6098122894763947, + "rewards/forensics_strict_edge_reward": 0.011572680377867073, + "rewards/forensics_tiou_reward": 0.35329461097717285, + "step": 148 + }, + { + "completion_length": 23.125, + "epoch": 0.4577572964669739, + "grad_norm": 4.796446800231934, + "kl": 0.267578125, + "learning_rate": 8.861538461538461e-07, + "loss": 0.0757, + "reward": 0.7459108829498291, + "reward_std": 0.05388874560594559, + "reward_std_all": 0.056251270696520805, + "rewards-max/forensics_sinkhorn_reward": 0.97633495926857, + "rewards-max/forensics_strict_edge_reward": 0.5787998139858246, + "rewards-max/forensics_tiou_reward": 0.9713801443576813, + "rewards/forensics_sinkhorn_reward": 0.6384888291358948, + "rewards/forensics_strict_edge_reward": 0.02647012285888195, + "rewards/forensics_tiou_reward": 0.37177012860774994, + "step": 149 + }, + { + "completion_length": 24.5625, + "epoch": 0.4608294930875576, + "grad_norm": 4.62745475769043, + "kl": 0.275390625, + "learning_rate": 8.853846153846154e-07, + "loss": 0.0795, + "reward": 0.7194972634315491, + "reward_std": 0.09402461722493172, + "reward_std_all": 0.09084320813417435, + "rewards-max/forensics_sinkhorn_reward": 0.9459083676338196, + "rewards-max/forensics_strict_edge_reward": 0.19106926769018173, + "rewards-max/forensics_tiou_reward": 0.8885945379734039, + "rewards/forensics_sinkhorn_reward": 0.6011901497840881, + "rewards/forensics_strict_edge_reward": 0.010894888080656528, + "rewards/forensics_tiou_reward": 0.3864038735628128, + "step": 150 + }, + { + "completion_length": 24.671875, + "epoch": 0.46390168970814133, + "grad_norm": 5.024142265319824, + "kl": 0.2724609375, + "learning_rate": 8.846153846153846e-07, + "loss": 0.1939, + "reward": 0.6833930313587189, + "reward_std": 0.09441274404525757, + "reward_std_all": 0.10993984714150429, + "rewards-max/forensics_sinkhorn_reward": 0.9551512002944946, + "rewards-max/forensics_strict_edge_reward": 0.27751971781253815, + "rewards-max/forensics_tiou_reward": 0.8990756273269653, + "rewards/forensics_sinkhorn_reward": 0.58431276679039, + "rewards/forensics_strict_edge_reward": 0.02145094983279705, + "rewards/forensics_tiou_reward": 0.3659052848815918, + "step": 151 + }, + { + "completion_length": 24.328125, + "epoch": 0.46697388632872505, + "grad_norm": 5.524312973022461, + "kl": 0.306640625, + "learning_rate": 8.838461538461538e-07, + "loss": 0.2345, + "reward": 0.6929260790348053, + "reward_std": 0.10787983983755112, + "reward_std_all": 0.11100343614816666, + "rewards-max/forensics_sinkhorn_reward": 0.9779742956161499, + "rewards-max/forensics_strict_edge_reward": 0.5314679890871048, + "rewards-max/forensics_tiou_reward": 0.9726690351963043, + "rewards/forensics_sinkhorn_reward": 0.5898255109786987, + "rewards/forensics_strict_edge_reward": 0.032960399985313416, + "rewards/forensics_tiou_reward": 0.3798806965351105, + "step": 152 + }, + { + "completion_length": 24.109375, + "epoch": 0.4700460829493088, + "grad_norm": 5.210085391998291, + "kl": 0.3291015625, + "learning_rate": 8.830769230769231e-07, + "loss": 0.0508, + "reward": 0.7373276948928833, + "reward_std": 0.0917121097445488, + "reward_std_all": 0.08530154451727867, + "rewards-max/forensics_sinkhorn_reward": 0.9597291946411133, + "rewards-max/forensics_strict_edge_reward": 0.1688495147973299, + "rewards-max/forensics_tiou_reward": 0.9201908111572266, + "rewards/forensics_sinkhorn_reward": 0.6553941071033478, + "rewards/forensics_strict_edge_reward": 0.021906610505538993, + "rewards/forensics_tiou_reward": 0.47814053297042847, + "step": 153 + }, + { + "completion_length": 23.875, + "epoch": 0.4731182795698925, + "grad_norm": 5.167613983154297, + "kl": 0.3095703125, + "learning_rate": 8.823076923076923e-07, + "loss": 0.0522, + "reward": 0.7036270797252655, + "reward_std": 0.09016740694642067, + "reward_std_all": 0.09840375557541847, + "rewards-max/forensics_sinkhorn_reward": 0.9407858848571777, + "rewards-max/forensics_strict_edge_reward": 0.45600031479261816, + "rewards-max/forensics_tiou_reward": 0.8861222267150879, + "rewards/forensics_sinkhorn_reward": 0.6126600503921509, + "rewards/forensics_strict_edge_reward": 0.02241386688547209, + "rewards/forensics_tiou_reward": 0.33710893988609314, + "step": 154 + }, + { + "completion_length": 22.90625, + "epoch": 0.47619047619047616, + "grad_norm": 4.6537370681762695, + "kl": 0.2958984375, + "learning_rate": 8.815384615384614e-07, + "loss": 0.0813, + "reward": 0.7659644484519958, + "reward_std": 0.06373994797468185, + "reward_std_all": 0.05776103585958481, + "rewards-max/forensics_sinkhorn_reward": 0.9748291373252869, + "rewards-max/forensics_strict_edge_reward": 0.4231441020965576, + "rewards-max/forensics_tiou_reward": 0.9604324400424957, + "rewards/forensics_sinkhorn_reward": 0.6985376179218292, + "rewards/forensics_strict_edge_reward": 0.033067259937524796, + "rewards/forensics_tiou_reward": 0.43954357504844666, + "step": 155 + }, + { + "completion_length": 22.90625, + "epoch": 0.4792626728110599, + "grad_norm": 4.877557754516602, + "kl": 0.27587890625, + "learning_rate": 8.807692307692307e-07, + "loss": 0.1425, + "reward": 0.7060821652412415, + "reward_std": 0.07937849871814251, + "reward_std_all": 0.0886163990944624, + "rewards-max/forensics_sinkhorn_reward": 0.9337949156761169, + "rewards-max/forensics_strict_edge_reward": 0.4212241992354393, + "rewards-max/forensics_tiou_reward": 0.8211544752120972, + "rewards/forensics_sinkhorn_reward": 0.5678708255290985, + "rewards/forensics_strict_edge_reward": 0.030737133231014013, + "rewards/forensics_tiou_reward": 0.3414607048034668, + "step": 156 + }, + { + "completion_length": 22.859375, + "epoch": 0.4823348694316436, + "grad_norm": 5.740560531616211, + "kl": 0.24072265625, + "learning_rate": 8.799999999999999e-07, + "loss": 0.1181, + "reward": 0.7314294874668121, + "reward_std": 0.06861027888953686, + "reward_std_all": 0.07215794920921326, + "rewards-max/forensics_sinkhorn_reward": 0.90804523229599, + "rewards-max/forensics_strict_edge_reward": 0.018368746154010296, + "rewards-max/forensics_tiou_reward": 0.7879106104373932, + "rewards/forensics_sinkhorn_reward": 0.6420372426509857, + "rewards/forensics_strict_edge_reward": 0.0008881782268872485, + "rewards/forensics_tiou_reward": 0.3619990646839142, + "step": 157 + }, + { + "completion_length": 23.828125, + "epoch": 0.48540706605222733, + "grad_norm": 4.756355285644531, + "kl": 0.3046875, + "learning_rate": 8.792307692307693e-07, + "loss": 0.017, + "reward": 0.7327983975410461, + "reward_std": 0.06962185353040695, + "reward_std_all": 0.07310526818037033, + "rewards-max/forensics_sinkhorn_reward": 0.9197427332401276, + "rewards-max/forensics_strict_edge_reward": 0.08056749776005745, + "rewards-max/forensics_tiou_reward": 0.831045925617218, + "rewards/forensics_sinkhorn_reward": 0.633264809846878, + "rewards/forensics_strict_edge_reward": 0.004047969006933272, + "rewards/forensics_tiou_reward": 0.3575607240200043, + "step": 158 + }, + { + "completion_length": 24.578125, + "epoch": 0.48847926267281105, + "grad_norm": 4.860424041748047, + "kl": 0.318359375, + "learning_rate": 8.784615384615385e-07, + "loss": 0.052, + "reward": 0.7029368579387665, + "reward_std": 0.09675122424960136, + "reward_std_all": 0.09866562485694885, + "rewards-max/forensics_sinkhorn_reward": 0.8791836202144623, + "rewards-max/forensics_strict_edge_reward": 0.003879512834828347, + "rewards-max/forensics_tiou_reward": 0.7339401543140411, + "rewards/forensics_sinkhorn_reward": 0.5755033493041992, + "rewards/forensics_strict_edge_reward": 0.00029290682869032025, + "rewards/forensics_tiou_reward": 0.33716467022895813, + "step": 159 + }, + { + "completion_length": 23.578125, + "epoch": 0.4915514592933948, + "grad_norm": 6.818739414215088, + "kl": 0.298828125, + "learning_rate": 8.776923076923076e-07, + "loss": 0.1024, + "reward": 0.7247419357299805, + "reward_std": 0.08934526890516281, + "reward_std_all": 0.08711329475045204, + "rewards-max/forensics_sinkhorn_reward": 0.9552367031574249, + "rewards-max/forensics_strict_edge_reward": 0.292797539383173, + "rewards-max/forensics_tiou_reward": 0.9026874899864197, + "rewards/forensics_sinkhorn_reward": 0.6304663419723511, + "rewards/forensics_strict_edge_reward": 0.01422774326056242, + "rewards/forensics_tiou_reward": 0.37258289754390717, + "step": 160 + }, + { + "completion_length": 22.390625, + "epoch": 0.4946236559139785, + "grad_norm": 5.301138877868652, + "kl": 0.24072265625, + "learning_rate": 8.769230769230769e-07, + "loss": 0.1484, + "reward": 0.6984923481941223, + "reward_std": 0.08923887461423874, + "reward_std_all": 0.09794702380895615, + "rewards-max/forensics_sinkhorn_reward": 0.9661749303340912, + "rewards-max/forensics_strict_edge_reward": 0.20731769874691963, + "rewards-max/forensics_tiou_reward": 0.9388338625431061, + "rewards/forensics_sinkhorn_reward": 0.6060479283332825, + "rewards/forensics_strict_edge_reward": 0.010489012114703655, + "rewards/forensics_tiou_reward": 0.3585057705640793, + "step": 161 + }, + { + "completion_length": 22.65625, + "epoch": 0.4976958525345622, + "grad_norm": 4.997905254364014, + "kl": 0.27734375, + "learning_rate": 8.761538461538461e-07, + "loss": 0.0577, + "reward": 0.7569016814231873, + "reward_std": 0.05141668580472469, + "reward_std_all": 0.051660552620887756, + "rewards-max/forensics_sinkhorn_reward": 0.9740080237388611, + "rewards-max/forensics_strict_edge_reward": 0.36628034710884094, + "rewards-max/forensics_tiou_reward": 0.9654203355312347, + "rewards/forensics_sinkhorn_reward": 0.6774893403053284, + "rewards/forensics_strict_edge_reward": 0.02970217482652515, + "rewards/forensics_tiou_reward": 0.4134531468153, + "step": 162 + }, + { + "completion_length": 22.625, + "epoch": 0.500768049155146, + "grad_norm": 6.329458713531494, + "kl": 0.2900390625, + "learning_rate": 8.753846153846154e-07, + "loss": 0.1078, + "reward": 0.6971228718757629, + "reward_std": 0.08526931703090668, + "reward_std_all": 0.09514347463846207, + "rewards-max/forensics_sinkhorn_reward": 0.8815804719924927, + "rewards-max/forensics_strict_edge_reward": 0.009534470271319151, + "rewards-max/forensics_tiou_reward": 0.7389010488986969, + "rewards/forensics_sinkhorn_reward": 0.5835424959659576, + "rewards/forensics_strict_edge_reward": 0.000561914435820654, + "rewards/forensics_tiou_reward": 0.2724136859178543, + "step": 163 + }, + { + "completion_length": 23.796875, + "epoch": 0.5038402457757296, + "grad_norm": 7.687970161437988, + "kl": 0.4111328125, + "learning_rate": 8.746153846153845e-07, + "loss": 0.1348, + "reward": 0.7153229117393494, + "reward_std": 0.07628047093749046, + "reward_std_all": 0.08026968315243721, + "rewards-max/forensics_sinkhorn_reward": 0.9366816580295563, + "rewards-max/forensics_strict_edge_reward": 0.01384083554148674, + "rewards-max/forensics_tiou_reward": 0.8534445464611053, + "rewards/forensics_sinkhorn_reward": 0.5860257744789124, + "rewards/forensics_strict_edge_reward": 0.0008857654756866395, + "rewards/forensics_tiou_reward": 0.29041050374507904, + "step": 164 + }, + { + "completion_length": 24.21875, + "epoch": 0.5069124423963134, + "grad_norm": 4.240787506103516, + "kl": 0.28125, + "learning_rate": 8.738461538461538e-07, + "loss": 0.1534, + "reward": 0.687178909778595, + "reward_std": 0.09143042378127575, + "reward_std_all": 0.10405869781970978, + "rewards-max/forensics_sinkhorn_reward": 0.9846219420433044, + "rewards-max/forensics_strict_edge_reward": 0.2540656663477421, + "rewards-max/forensics_tiou_reward": 0.9852466881275177, + "rewards/forensics_sinkhorn_reward": 0.5920986831188202, + "rewards/forensics_strict_edge_reward": 0.02367315674200654, + "rewards/forensics_tiou_reward": 0.380330890417099, + "step": 165 + }, + { + "completion_length": 23.203125, + "epoch": 0.5099846390168971, + "grad_norm": 4.79766321182251, + "kl": 0.27685546875, + "learning_rate": 8.730769230769231e-07, + "loss": 0.1744, + "reward": 0.7059165835380554, + "reward_std": 0.09453429281711578, + "reward_std_all": 0.09763649851083755, + "rewards-max/forensics_sinkhorn_reward": 0.9648345112800598, + "rewards-max/forensics_strict_edge_reward": 0.15669209882616997, + "rewards-max/forensics_tiou_reward": 0.9403949975967407, + "rewards/forensics_sinkhorn_reward": 0.6266611814498901, + "rewards/forensics_strict_edge_reward": 0.0124759532045573, + "rewards/forensics_tiou_reward": 0.3913957327604294, + "step": 166 + }, + { + "completion_length": 22.140625, + "epoch": 0.5130568356374808, + "grad_norm": 5.0165324211120605, + "kl": 0.462890625, + "learning_rate": 8.723076923076923e-07, + "loss": 0.1963, + "reward": 0.7149491608142853, + "reward_std": 0.08285246416926384, + "reward_std_all": 0.08766943961381912, + "rewards-max/forensics_sinkhorn_reward": 0.9336681067943573, + "rewards-max/forensics_strict_edge_reward": 0.15616539865732193, + "rewards-max/forensics_tiou_reward": 0.8424167931079865, + "rewards/forensics_sinkhorn_reward": 0.6323482990264893, + "rewards/forensics_strict_edge_reward": 0.016019734321162105, + "rewards/forensics_tiou_reward": 0.3658367991447449, + "step": 167 + }, + { + "completion_length": 22.96875, + "epoch": 0.5161290322580645, + "grad_norm": 4.35988712310791, + "kl": 0.2939453125, + "learning_rate": 8.715384615384614e-07, + "loss": 0.0362, + "reward": 0.7080722451210022, + "reward_std": 0.06830081716179848, + "reward_std_all": 0.07993410527706146, + "rewards-max/forensics_sinkhorn_reward": 0.9730753898620605, + "rewards-max/forensics_strict_edge_reward": 0.08082165569067001, + "rewards-max/forensics_tiou_reward": 0.948140025138855, + "rewards/forensics_sinkhorn_reward": 0.6189902722835541, + "rewards/forensics_strict_edge_reward": 0.00621787435375154, + "rewards/forensics_tiou_reward": 0.38648365437984467, + "step": 168 + }, + { + "completion_length": 23.1875, + "epoch": 0.5192012288786483, + "grad_norm": 4.587300777435303, + "kl": 0.26171875, + "learning_rate": 8.707692307692307e-07, + "loss": 0.0966, + "reward": 0.7196620106697083, + "reward_std": 0.09115762263536453, + "reward_std_all": 0.0901147648692131, + "rewards-max/forensics_sinkhorn_reward": 0.9148958325386047, + "rewards-max/forensics_strict_edge_reward": 0.2091898899525404, + "rewards-max/forensics_tiou_reward": 0.8202216923236847, + "rewards/forensics_sinkhorn_reward": 0.5932208299636841, + "rewards/forensics_strict_edge_reward": 0.013519332685973495, + "rewards/forensics_tiou_reward": 0.3280673176050186, + "step": 169 + }, + { + "completion_length": 25.125, + "epoch": 0.522273425499232, + "grad_norm": 4.39460563659668, + "kl": 0.267578125, + "learning_rate": 8.699999999999999e-07, + "loss": 0.1082, + "reward": 0.7092729806900024, + "reward_std": 0.07853101193904877, + "reward_std_all": 0.0887550637125969, + "rewards-max/forensics_sinkhorn_reward": 0.9564026892185211, + "rewards-max/forensics_strict_edge_reward": 0.1542465090751648, + "rewards-max/forensics_tiou_reward": 0.9119175672531128, + "rewards/forensics_sinkhorn_reward": 0.6033995449542999, + "rewards/forensics_strict_edge_reward": 0.00863920385017991, + "rewards/forensics_tiou_reward": 0.3623429834842682, + "step": 170 + }, + { + "completion_length": 24.890625, + "epoch": 0.5253456221198156, + "grad_norm": 4.278872013092041, + "kl": 0.25146484375, + "learning_rate": 8.692307692307692e-07, + "loss": 0.038, + "reward": 0.6384803056716919, + "reward_std": 0.09285912662744522, + "reward_std_all": 0.12085049599409103, + "rewards-max/forensics_sinkhorn_reward": 0.8703413903713226, + "rewards-max/forensics_strict_edge_reward": 0.05728777372951299, + "rewards-max/forensics_tiou_reward": 0.6935077011585236, + "rewards/forensics_sinkhorn_reward": 0.5276561379432678, + "rewards/forensics_strict_edge_reward": 0.0038299563294685868, + "rewards/forensics_tiou_reward": 0.23734784126281738, + "step": 171 + }, + { + "completion_length": 25.234375, + "epoch": 0.5284178187403994, + "grad_norm": 4.8349409103393555, + "kl": 0.26025390625, + "learning_rate": 8.684615384615385e-07, + "loss": 0.2277, + "reward": 0.7005079984664917, + "reward_std": 0.08073207177221775, + "reward_std_all": 0.095767080783844, + "rewards-max/forensics_sinkhorn_reward": 0.9511317610740662, + "rewards-max/forensics_strict_edge_reward": 0.2300645187497139, + "rewards-max/forensics_tiou_reward": 0.8996531963348389, + "rewards/forensics_sinkhorn_reward": 0.6027364730834961, + "rewards/forensics_strict_edge_reward": 0.016157409641891718, + "rewards/forensics_tiou_reward": 0.37454867362976074, + "step": 172 + }, + { + "completion_length": 22.6875, + "epoch": 0.5314900153609831, + "grad_norm": 4.342402458190918, + "kl": 0.2900390625, + "learning_rate": 8.676923076923076e-07, + "loss": 0.1632, + "reward": 0.7140294313430786, + "reward_std": 0.08427875488996506, + "reward_std_all": 0.08999517932534218, + "rewards-max/forensics_sinkhorn_reward": 0.9825942516326904, + "rewards-max/forensics_strict_edge_reward": 0.636265903711319, + "rewards-max/forensics_tiou_reward": 0.9822497069835663, + "rewards/forensics_sinkhorn_reward": 0.6383833289146423, + "rewards/forensics_strict_edge_reward": 0.029298923444002867, + "rewards/forensics_tiou_reward": 0.3954204171895981, + "step": 173 + }, + { + "completion_length": 24.296875, + "epoch": 0.5345622119815668, + "grad_norm": 4.709314346313477, + "kl": 0.2685546875, + "learning_rate": 8.669230769230769e-07, + "loss": 0.1152, + "reward": 0.7348947525024414, + "reward_std": 0.08922901749610901, + "reward_std_all": 0.0835517905652523, + "rewards-max/forensics_sinkhorn_reward": 0.9592138528823853, + "rewards-max/forensics_strict_edge_reward": 0.29804089665412903, + "rewards-max/forensics_tiou_reward": 0.9162280559539795, + "rewards/forensics_sinkhorn_reward": 0.6412870287895203, + "rewards/forensics_strict_edge_reward": 0.019955235067754984, + "rewards/forensics_tiou_reward": 0.378370076417923, + "step": 174 + }, + { + "completion_length": 26.390625, + "epoch": 0.5376344086021505, + "grad_norm": 3.943525791168213, + "kl": 0.279296875, + "learning_rate": 8.661538461538461e-07, + "loss": 0.1996, + "reward": 0.7029397785663605, + "reward_std": 0.08133542910218239, + "reward_std_all": 0.08807859942317009, + "rewards-max/forensics_sinkhorn_reward": 0.8826854228973389, + "rewards-max/forensics_strict_edge_reward": 0.03055455646244809, + "rewards-max/forensics_tiou_reward": 0.7356361746788025, + "rewards/forensics_sinkhorn_reward": 0.5515104234218597, + "rewards/forensics_strict_edge_reward": 0.001606866640941007, + "rewards/forensics_tiou_reward": 0.30770444869995117, + "step": 175 + }, + { + "completion_length": 24.296875, + "epoch": 0.5407066052227343, + "grad_norm": 4.051927089691162, + "kl": 0.283203125, + "learning_rate": 8.653846153846154e-07, + "loss": 0.081, + "reward": 0.6987928450107574, + "reward_std": 0.11811195686459541, + "reward_std_all": 0.11472094058990479, + "rewards-max/forensics_sinkhorn_reward": 0.9400665462017059, + "rewards-max/forensics_strict_edge_reward": 0.23571141064167023, + "rewards-max/forensics_tiou_reward": 0.8800113797187805, + "rewards/forensics_sinkhorn_reward": 0.6031342148780823, + "rewards/forensics_strict_edge_reward": 0.01245335629209876, + "rewards/forensics_tiou_reward": 0.31622888147830963, + "step": 176 + }, + { + "completion_length": 23.234375, + "epoch": 0.543778801843318, + "grad_norm": 4.779592514038086, + "kl": 0.23828125, + "learning_rate": 8.646153846153845e-07, + "loss": 0.2004, + "reward": 0.694795697927475, + "reward_std": 0.1318778470158577, + "reward_std_all": 0.12499909102916718, + "rewards-max/forensics_sinkhorn_reward": 0.9618554711341858, + "rewards-max/forensics_strict_edge_reward": 0.12836418766528368, + "rewards-max/forensics_tiou_reward": 0.9299925863742828, + "rewards/forensics_sinkhorn_reward": 0.6145021021366119, + "rewards/forensics_strict_edge_reward": 0.009608907013898715, + "rewards/forensics_tiou_reward": 0.3998347669839859, + "step": 177 + }, + { + "completion_length": 23.3125, + "epoch": 0.5468509984639017, + "grad_norm": 4.061398506164551, + "kl": 0.326171875, + "learning_rate": 8.638461538461537e-07, + "loss": 0.1541, + "reward": 0.6948331296443939, + "reward_std": 0.08065468445420265, + "reward_std_all": 0.09095072373747826, + "rewards-max/forensics_sinkhorn_reward": 0.9167454838752747, + "rewards-max/forensics_strict_edge_reward": 0.16976426332257688, + "rewards-max/forensics_tiou_reward": 0.8233742117881775, + "rewards/forensics_sinkhorn_reward": 0.5801450312137604, + "rewards/forensics_strict_edge_reward": 0.007731599471298978, + "rewards/forensics_tiou_reward": 0.2988611310720444, + "step": 178 + }, + { + "completion_length": 25.328125, + "epoch": 0.5499231950844854, + "grad_norm": 4.5148539543151855, + "kl": 0.2705078125, + "learning_rate": 8.630769230769231e-07, + "loss": 0.1663, + "reward": 0.7373583912849426, + "reward_std": 0.07148118317127228, + "reward_std_all": 0.07285136356949806, + "rewards-max/forensics_sinkhorn_reward": 0.8916943073272705, + "rewards-max/forensics_strict_edge_reward": 0.16804073750972748, + "rewards-max/forensics_tiou_reward": 0.7726555168628693, + "rewards/forensics_sinkhorn_reward": 0.6033420860767365, + "rewards/forensics_strict_edge_reward": 0.010529003571718931, + "rewards/forensics_tiou_reward": 0.3529042452573776, + "step": 179 + }, + { + "completion_length": 23.484375, + "epoch": 0.5529953917050692, + "grad_norm": 5.211463451385498, + "kl": 0.2978515625, + "learning_rate": 8.623076923076923e-07, + "loss": 0.0848, + "reward": 0.6967598795890808, + "reward_std": 0.08927623927593231, + "reward_std_all": 0.097600556910038, + "rewards-max/forensics_sinkhorn_reward": 0.8913310170173645, + "rewards-max/forensics_strict_edge_reward": 0.20077420212328434, + "rewards-max/forensics_tiou_reward": 0.7813799977302551, + "rewards/forensics_sinkhorn_reward": 0.5901829302310944, + "rewards/forensics_strict_edge_reward": 0.013753959210589528, + "rewards/forensics_tiou_reward": 0.31651489436626434, + "step": 180 + }, + { + "completion_length": 24.421875, + "epoch": 0.5560675883256528, + "grad_norm": 4.746703147888184, + "kl": 0.2783203125, + "learning_rate": 8.615384615384616e-07, + "loss": 0.1183, + "reward": 0.7314884066581726, + "reward_std": 0.06424232758581638, + "reward_std_all": 0.06891374662518501, + "rewards-max/forensics_sinkhorn_reward": 0.9076421558856964, + "rewards-max/forensics_strict_edge_reward": 0.027607446259935386, + "rewards-max/forensics_tiou_reward": 0.7781879603862762, + "rewards/forensics_sinkhorn_reward": 0.6634561121463776, + "rewards/forensics_strict_edge_reward": 0.0016652763479214627, + "rewards/forensics_tiou_reward": 0.4034568518400192, + "step": 181 + }, + { + "completion_length": 24.546875, + "epoch": 0.5591397849462365, + "grad_norm": 3.8786840438842773, + "kl": 0.287109375, + "learning_rate": 8.607692307692307e-07, + "loss": 0.0833, + "reward": 0.7564681470394135, + "reward_std": 0.05415179207921028, + "reward_std_all": 0.052966609597206116, + "rewards-max/forensics_sinkhorn_reward": 0.9860397279262543, + "rewards-max/forensics_strict_edge_reward": 0.47682616859674454, + "rewards-max/forensics_tiou_reward": 0.9908013045787811, + "rewards/forensics_sinkhorn_reward": 0.6761336326599121, + "rewards/forensics_strict_edge_reward": 0.024913211353123188, + "rewards/forensics_tiou_reward": 0.4289712756872177, + "step": 182 + }, + { + "completion_length": 22.453125, + "epoch": 0.5622119815668203, + "grad_norm": 4.479518890380859, + "kl": 0.27734375, + "learning_rate": 8.599999999999999e-07, + "loss": -0.1005, + "reward": 0.739555150270462, + "reward_std": 0.0433834670111537, + "reward_std_all": 0.05280047468841076, + "rewards-max/forensics_sinkhorn_reward": 0.954725593328476, + "rewards-max/forensics_strict_edge_reward": 0.18269184976816177, + "rewards-max/forensics_tiou_reward": 0.9126606583595276, + "rewards/forensics_sinkhorn_reward": 0.6565777659416199, + "rewards/forensics_strict_edge_reward": 0.008344953879714012, + "rewards/forensics_tiou_reward": 0.34265217185020447, + "step": 183 + }, + { + "completion_length": 24.140625, + "epoch": 0.565284178187404, + "grad_norm": 4.3795952796936035, + "kl": 0.318359375, + "learning_rate": 8.592307692307692e-07, + "loss": 0.1841, + "reward": 0.7202824056148529, + "reward_std": 0.04310087487101555, + "reward_std_all": 0.063652029260993, + "rewards-max/forensics_sinkhorn_reward": 0.91774582862854, + "rewards-max/forensics_strict_edge_reward": 0.18009740114212036, + "rewards-max/forensics_tiou_reward": 0.8391572833061218, + "rewards/forensics_sinkhorn_reward": 0.6341715157032013, + "rewards/forensics_strict_edge_reward": 0.009425753261893988, + "rewards/forensics_tiou_reward": 0.3246881365776062, + "step": 184 + }, + { + "completion_length": 22.4375, + "epoch": 0.5683563748079877, + "grad_norm": 5.0383477210998535, + "kl": 0.3837890625, + "learning_rate": 8.584615384615385e-07, + "loss": 0.1036, + "reward": 0.718267410993576, + "reward_std": 0.08874190039932728, + "reward_std_all": 0.0874653123319149, + "rewards-max/forensics_sinkhorn_reward": 0.9464313387870789, + "rewards-max/forensics_strict_edge_reward": 0.25080026499927044, + "rewards-max/forensics_tiou_reward": 0.8727917075157166, + "rewards/forensics_sinkhorn_reward": 0.5991983413696289, + "rewards/forensics_strict_edge_reward": 0.01853908272460103, + "rewards/forensics_tiou_reward": 0.3144614100456238, + "step": 185 + }, + { + "completion_length": 23.171875, + "epoch": 0.5714285714285714, + "grad_norm": 4.799901008605957, + "kl": 0.29345703125, + "learning_rate": 8.576923076923076e-07, + "loss": 0.0741, + "reward": 0.7332278192043304, + "reward_std": 0.0647634994238615, + "reward_std_all": 0.06994923576712608, + "rewards-max/forensics_sinkhorn_reward": 0.8803726732730865, + "rewards-max/forensics_strict_edge_reward": 0.24026965606026351, + "rewards-max/forensics_tiou_reward": 0.7244375348091125, + "rewards/forensics_sinkhorn_reward": 0.6269162893295288, + "rewards/forensics_strict_edge_reward": 0.012712356328847818, + "rewards/forensics_tiou_reward": 0.3276898264884949, + "step": 186 + }, + { + "completion_length": 25.484375, + "epoch": 0.5745007680491552, + "grad_norm": 5.480781555175781, + "kl": 0.30859375, + "learning_rate": 8.569230769230769e-07, + "loss": 0.1363, + "reward": 0.7131676077842712, + "reward_std": 0.09796738997101784, + "reward_std_all": 0.09660201519727707, + "rewards-max/forensics_sinkhorn_reward": 0.8436143696308136, + "rewards-max/forensics_strict_edge_reward": 0.15896787494421005, + "rewards-max/forensics_tiou_reward": 0.7126357555389404, + "rewards/forensics_sinkhorn_reward": 0.5809040367603302, + "rewards/forensics_strict_edge_reward": 0.011847929417854175, + "rewards/forensics_tiou_reward": 0.3212510496377945, + "step": 187 + }, + { + "completion_length": 23.65625, + "epoch": 0.5775729646697388, + "grad_norm": 4.658575534820557, + "kl": 0.28759765625, + "learning_rate": 8.561538461538461e-07, + "loss": 0.0323, + "reward": 0.719889223575592, + "reward_std": 0.07441386207938194, + "reward_std_all": 0.08002793416380882, + "rewards-max/forensics_sinkhorn_reward": 0.8493509888648987, + "rewards-max/forensics_strict_edge_reward": 0.03849071941658622, + "rewards-max/forensics_tiou_reward": 0.6682513952255249, + "rewards/forensics_sinkhorn_reward": 0.6147553622722626, + "rewards/forensics_strict_edge_reward": 0.0018939625929306203, + "rewards/forensics_tiou_reward": 0.3531155288219452, + "step": 188 + }, + { + "completion_length": 22.625, + "epoch": 0.5806451612903226, + "grad_norm": 4.704739570617676, + "kl": 0.283203125, + "learning_rate": 8.553846153846154e-07, + "loss": 0.0286, + "reward": 0.7222743332386017, + "reward_std": 0.09748847037553787, + "reward_std_all": 0.09240596741437912, + "rewards-max/forensics_sinkhorn_reward": 0.9768180549144745, + "rewards-max/forensics_strict_edge_reward": 0.2866882011294365, + "rewards-max/forensics_tiou_reward": 0.9650482535362244, + "rewards/forensics_sinkhorn_reward": 0.6514858901500702, + "rewards/forensics_strict_edge_reward": 0.018457645550370216, + "rewards/forensics_tiou_reward": 0.41272783279418945, + "step": 189 + }, + { + "completion_length": 25.71875, + "epoch": 0.5837173579109063, + "grad_norm": 3.661839246749878, + "kl": 0.310546875, + "learning_rate": 8.546153846153846e-07, + "loss": 0.0829, + "reward": 0.7291776835918427, + "reward_std": 0.08929050713777542, + "reward_std_all": 0.08495304733514786, + "rewards-max/forensics_sinkhorn_reward": 0.9235555231571198, + "rewards-max/forensics_strict_edge_reward": 0.08024366199970245, + "rewards-max/forensics_tiou_reward": 0.8198020458221436, + "rewards/forensics_sinkhorn_reward": 0.6135996878147125, + "rewards/forensics_strict_edge_reward": 0.004300287691876292, + "rewards/forensics_tiou_reward": 0.3421984314918518, + "step": 190 + }, + { + "completion_length": 23.09375, + "epoch": 0.5867895545314901, + "grad_norm": 4.811416149139404, + "kl": 0.26953125, + "learning_rate": 8.538461538461537e-07, + "loss": 0.1874, + "reward": 0.7120346426963806, + "reward_std": 0.07175162062048912, + "reward_std_all": 0.08251991868019104, + "rewards-max/forensics_sinkhorn_reward": 0.9219699800014496, + "rewards-max/forensics_strict_edge_reward": 0.1930617094039917, + "rewards-max/forensics_tiou_reward": 0.8402746915817261, + "rewards/forensics_sinkhorn_reward": 0.6097568273544312, + "rewards/forensics_strict_edge_reward": 0.008671073708683252, + "rewards/forensics_tiou_reward": 0.332598477602005, + "step": 191 + }, + { + "completion_length": 24.4375, + "epoch": 0.5898617511520737, + "grad_norm": 4.692314147949219, + "kl": 0.2841796875, + "learning_rate": 8.530769230769231e-07, + "loss": 0.0732, + "reward": 0.7306415736675262, + "reward_std": 0.07254943996667862, + "reward_std_all": 0.07866290770471096, + "rewards-max/forensics_sinkhorn_reward": 0.9537197351455688, + "rewards-max/forensics_strict_edge_reward": 0.1928788386285305, + "rewards-max/forensics_tiou_reward": 0.9003921151161194, + "rewards/forensics_sinkhorn_reward": 0.6856890022754669, + "rewards/forensics_strict_edge_reward": 0.017195293679833412, + "rewards/forensics_tiou_reward": 0.47999343276023865, + "step": 192 + }, + { + "completion_length": 24.265625, + "epoch": 0.5929339477726574, + "grad_norm": 5.462874412536621, + "kl": 0.314453125, + "learning_rate": 8.523076923076923e-07, + "loss": 0.1163, + "reward": 0.7191320061683655, + "reward_std": 0.08904673531651497, + "reward_std_all": 0.08716024085879326, + "rewards-max/forensics_sinkhorn_reward": 0.966437965631485, + "rewards-max/forensics_strict_edge_reward": 0.44934946298599243, + "rewards-max/forensics_tiou_reward": 0.9464797079563141, + "rewards/forensics_sinkhorn_reward": 0.6098307073116302, + "rewards/forensics_strict_edge_reward": 0.04012378863990307, + "rewards/forensics_tiou_reward": 0.364554762840271, + "step": 193 + }, + { + "completion_length": 23.34375, + "epoch": 0.5960061443932412, + "grad_norm": 4.681181907653809, + "kl": 0.255859375, + "learning_rate": 8.515384615384616e-07, + "loss": 0.084, + "reward": 0.702605277299881, + "reward_std": 0.08387291058897972, + "reward_std_all": 0.09662043862044811, + "rewards-max/forensics_sinkhorn_reward": 0.9299135208129883, + "rewards-max/forensics_strict_edge_reward": 0.30157603323459625, + "rewards-max/forensics_tiou_reward": 0.8452514410018921, + "rewards/forensics_sinkhorn_reward": 0.616420716047287, + "rewards/forensics_strict_edge_reward": 0.016373366466723382, + "rewards/forensics_tiou_reward": 0.3861594498157501, + "step": 194 + }, + { + "completion_length": 25.46875, + "epoch": 0.5990783410138248, + "grad_norm": 4.658320426940918, + "kl": 0.333984375, + "learning_rate": 8.507692307692307e-07, + "loss": 0.1405, + "reward": 0.6565106213092804, + "reward_std": 0.12342692911624908, + "reward_std_all": 0.13812396675348282, + "rewards-max/forensics_sinkhorn_reward": 0.9414268434047699, + "rewards-max/forensics_strict_edge_reward": 0.13911163434386253, + "rewards-max/forensics_tiou_reward": 0.8779670894145966, + "rewards/forensics_sinkhorn_reward": 0.5654163062572479, + "rewards/forensics_strict_edge_reward": 0.009558814344927669, + "rewards/forensics_tiou_reward": 0.37493276596069336, + "step": 195 + }, + { + "completion_length": 25.15625, + "epoch": 0.6021505376344086, + "grad_norm": 4.034542083740234, + "kl": 0.287109375, + "learning_rate": 8.499999999999999e-07, + "loss": 0.1514, + "reward": 0.652818113565445, + "reward_std": 0.10544068180024624, + "reward_std_all": 0.12803980708122253, + "rewards-max/forensics_sinkhorn_reward": 0.9449011981487274, + "rewards-max/forensics_strict_edge_reward": 0.1491641066968441, + "rewards-max/forensics_tiou_reward": 0.8870493471622467, + "rewards/forensics_sinkhorn_reward": 0.5569591820240021, + "rewards/forensics_strict_edge_reward": 0.01348023593891412, + "rewards/forensics_tiou_reward": 0.3069441020488739, + "step": 196 + }, + { + "completion_length": 27.0, + "epoch": 0.6052227342549923, + "grad_norm": 4.1774187088012695, + "kl": 0.26171875, + "learning_rate": 8.492307692307692e-07, + "loss": 0.1878, + "reward": 0.6620131134986877, + "reward_std": 0.09720595180988312, + "reward_std_all": 0.11560238897800446, + "rewards-max/forensics_sinkhorn_reward": 0.9585044682025909, + "rewards-max/forensics_strict_edge_reward": 0.19777581095695496, + "rewards-max/forensics_tiou_reward": 0.9244745075702667, + "rewards/forensics_sinkhorn_reward": 0.5412047207355499, + "rewards/forensics_strict_edge_reward": 0.015251215547323227, + "rewards/forensics_tiou_reward": 0.3038925975561142, + "step": 197 + }, + { + "completion_length": 26.9375, + "epoch": 0.6082949308755761, + "grad_norm": 5.638034820556641, + "kl": 0.2958984375, + "learning_rate": 8.484615384615384e-07, + "loss": 0.2025, + "reward": 0.6975597143173218, + "reward_std": 0.09462213143706322, + "reward_std_all": 0.10122406482696533, + "rewards-max/forensics_sinkhorn_reward": 0.9443848133087158, + "rewards-max/forensics_strict_edge_reward": 0.04891967959702015, + "rewards-max/forensics_tiou_reward": 0.8736696243286133, + "rewards/forensics_sinkhorn_reward": 0.5734523236751556, + "rewards/forensics_strict_edge_reward": 0.0022745825117453933, + "rewards/forensics_tiou_reward": 0.3065049946308136, + "step": 198 + }, + { + "completion_length": 22.6875, + "epoch": 0.6113671274961597, + "grad_norm": 4.222497940063477, + "kl": 0.2666015625, + "learning_rate": 8.476923076923077e-07, + "loss": 0.1639, + "reward": 0.7274366021156311, + "reward_std": 0.07506725192070007, + "reward_std_all": 0.07849356904625893, + "rewards-max/forensics_sinkhorn_reward": 0.8973833918571472, + "rewards-max/forensics_strict_edge_reward": 0.10564448079094291, + "rewards-max/forensics_tiou_reward": 0.7733281254768372, + "rewards/forensics_sinkhorn_reward": 0.6433585584163666, + "rewards/forensics_strict_edge_reward": 0.00692066833289573, + "rewards/forensics_tiou_reward": 0.350215420126915, + "step": 199 + }, + { + "completion_length": 24.71875, + "epoch": 0.6144393241167435, + "grad_norm": 5.526483535766602, + "kl": 0.2744140625, + "learning_rate": 8.469230769230769e-07, + "loss": 0.0444, + "reward": 0.6976716816425323, + "reward_std": 0.09576700069010258, + "reward_std_all": 0.09889519214630127, + "rewards-max/forensics_sinkhorn_reward": 0.9141288995742798, + "rewards-max/forensics_strict_edge_reward": 0.023652314208447933, + "rewards-max/forensics_tiou_reward": 0.8171792328357697, + "rewards/forensics_sinkhorn_reward": 0.5569734573364258, + "rewards/forensics_strict_edge_reward": 0.0013691651838598773, + "rewards/forensics_tiou_reward": 0.354269415140152, + "step": 200 + }, + { + "completion_length": 23.359375, + "epoch": 0.6175115207373272, + "grad_norm": 4.912312030792236, + "kl": 0.267578125, + "learning_rate": 8.461538461538461e-07, + "loss": 0.2846, + "reward": 0.6621492505073547, + "reward_std": 0.14270032942295074, + "reward_std_all": 0.14033664762973785, + "rewards-max/forensics_sinkhorn_reward": 0.9830332100391388, + "rewards-max/forensics_strict_edge_reward": 0.19966012984514236, + "rewards-max/forensics_tiou_reward": 0.9853990375995636, + "rewards/forensics_sinkhorn_reward": 0.5761830508708954, + "rewards/forensics_strict_edge_reward": 0.014580311719328165, + "rewards/forensics_tiou_reward": 0.37743403017520905, + "step": 201 + }, + { + "completion_length": 25.3125, + "epoch": 0.620583717357911, + "grad_norm": 3.8421294689178467, + "kl": 0.29931640625, + "learning_rate": 8.453846153846154e-07, + "loss": 0.1615, + "reward": 0.6668963134288788, + "reward_std": 0.13564813137054443, + "reward_std_all": 0.13603613525629044, + "rewards-max/forensics_sinkhorn_reward": 0.9349228143692017, + "rewards-max/forensics_strict_edge_reward": 0.004329758230596781, + "rewards-max/forensics_tiou_reward": 0.8382982313632965, + "rewards/forensics_sinkhorn_reward": 0.5632874965667725, + "rewards/forensics_strict_edge_reward": 0.0001903837219288107, + "rewards/forensics_tiou_reward": 0.2999918460845947, + "step": 202 + }, + { + "completion_length": 24.03125, + "epoch": 0.6236559139784946, + "grad_norm": 4.297637939453125, + "kl": 0.2822265625, + "learning_rate": 8.446153846153846e-07, + "loss": 0.1677, + "reward": 0.7404661178588867, + "reward_std": 0.07013120874762535, + "reward_std_all": 0.0683651901781559, + "rewards-max/forensics_sinkhorn_reward": 0.9202860295772552, + "rewards-max/forensics_strict_edge_reward": 0.3268406540155411, + "rewards-max/forensics_tiou_reward": 0.836931973695755, + "rewards/forensics_sinkhorn_reward": 0.626623272895813, + "rewards/forensics_strict_edge_reward": 0.020151381380856037, + "rewards/forensics_tiou_reward": 0.31919705867767334, + "step": 203 + }, + { + "completion_length": 24.4375, + "epoch": 0.6267281105990783, + "grad_norm": 5.890398025512695, + "kl": 0.326171875, + "learning_rate": 8.438461538461537e-07, + "loss": 0.0915, + "reward": 0.7308272123336792, + "reward_std": 0.05651008524000645, + "reward_std_all": 0.06365048140287399, + "rewards-max/forensics_sinkhorn_reward": 0.9595033526420593, + "rewards-max/forensics_strict_edge_reward": 0.1105187889188528, + "rewards-max/forensics_tiou_reward": 0.9177838265895844, + "rewards/forensics_sinkhorn_reward": 0.5793072581291199, + "rewards/forensics_strict_edge_reward": 0.007096098619513214, + "rewards/forensics_tiou_reward": 0.3709576725959778, + "step": 204 + }, + { + "completion_length": 25.828125, + "epoch": 0.6298003072196621, + "grad_norm": 4.594563007354736, + "kl": 0.22119140625, + "learning_rate": 8.43076923076923e-07, + "loss": -0.0743, + "reward": 0.7233611941337585, + "reward_std": 0.06915610283613205, + "reward_std_all": 0.07961881905794144, + "rewards-max/forensics_sinkhorn_reward": 0.9181832075119019, + "rewards-max/forensics_strict_edge_reward": 0.3300565853714943, + "rewards-max/forensics_tiou_reward": 0.8389251232147217, + "rewards/forensics_sinkhorn_reward": 0.6260569989681244, + "rewards/forensics_strict_edge_reward": 0.020955322543159127, + "rewards/forensics_tiou_reward": 0.3947932571172714, + "step": 205 + }, + { + "completion_length": 24.203125, + "epoch": 0.6328725038402457, + "grad_norm": 4.163965225219727, + "kl": 0.2421875, + "learning_rate": 8.423076923076923e-07, + "loss": 0.1722, + "reward": 0.6945083737373352, + "reward_std": 0.12503063678741455, + "reward_std_all": 0.12389468401670456, + "rewards-max/forensics_sinkhorn_reward": 0.9599899649620056, + "rewards-max/forensics_strict_edge_reward": 0.21238384395837784, + "rewards-max/forensics_tiou_reward": 0.9202646315097809, + "rewards/forensics_sinkhorn_reward": 0.5898474752902985, + "rewards/forensics_strict_edge_reward": 0.009913695743307471, + "rewards/forensics_tiou_reward": 0.3750559836626053, + "step": 206 + }, + { + "completion_length": 24.578125, + "epoch": 0.6359447004608295, + "grad_norm": 4.170827865600586, + "kl": 0.318359375, + "learning_rate": 8.415384615384616e-07, + "loss": 0.0417, + "reward": 0.7571667730808258, + "reward_std": 0.05445228889584541, + "reward_std_all": 0.05326808616518974, + "rewards-max/forensics_sinkhorn_reward": 0.9275070428848267, + "rewards-max/forensics_strict_edge_reward": 0.12515248451381922, + "rewards-max/forensics_tiou_reward": 0.7947824597358704, + "rewards/forensics_sinkhorn_reward": 0.6675137281417847, + "rewards/forensics_strict_edge_reward": 0.010084148729220033, + "rewards/forensics_tiou_reward": 0.39768998324871063, + "step": 207 + }, + { + "completion_length": 22.84375, + "epoch": 0.6390168970814132, + "grad_norm": 4.46564245223999, + "kl": 0.2255859375, + "learning_rate": 8.407692307692308e-07, + "loss": 0.1908, + "reward": 0.7112807929515839, + "reward_std": 0.11189296841621399, + "reward_std_all": 0.10546229034662247, + "rewards-max/forensics_sinkhorn_reward": 0.9461995363235474, + "rewards-max/forensics_strict_edge_reward": 0.26311759650707245, + "rewards-max/forensics_tiou_reward": 0.8855649530887604, + "rewards/forensics_sinkhorn_reward": 0.6048116087913513, + "rewards/forensics_strict_edge_reward": 0.02035059593617916, + "rewards/forensics_tiou_reward": 0.37126851081848145, + "step": 208 + }, + { + "completion_length": 24.671875, + "epoch": 0.642089093701997, + "grad_norm": 5.153226852416992, + "kl": 0.349609375, + "learning_rate": 8.399999999999999e-07, + "loss": -0.0362, + "reward": 0.6756743788719177, + "reward_std": 0.10151879489421844, + "reward_std_all": 0.1147964559495449, + "rewards-max/forensics_sinkhorn_reward": 0.9462151825428009, + "rewards-max/forensics_strict_edge_reward": 0.26143594086170197, + "rewards-max/forensics_tiou_reward": 0.90052330493927, + "rewards/forensics_sinkhorn_reward": 0.5764406323432922, + "rewards/forensics_strict_edge_reward": 0.019122425466775894, + "rewards/forensics_tiou_reward": 0.32568277418613434, + "step": 209 + }, + { + "completion_length": 24.890625, + "epoch": 0.6451612903225806, + "grad_norm": 5.347108364105225, + "kl": 0.25146484375, + "learning_rate": 8.392307692307692e-07, + "loss": 0.0471, + "reward": 0.676558256149292, + "reward_std": 0.09277436882257462, + "reward_std_all": 0.107269037514925, + "rewards-max/forensics_sinkhorn_reward": 0.8969626724720001, + "rewards-max/forensics_strict_edge_reward": 0.17957914376165718, + "rewards-max/forensics_tiou_reward": 0.7705712914466858, + "rewards/forensics_sinkhorn_reward": 0.5669704377651215, + "rewards/forensics_strict_edge_reward": 0.008593461403506808, + "rewards/forensics_tiou_reward": 0.2934158444404602, + "step": 210 + }, + { + "completion_length": 21.890625, + "epoch": 0.6482334869431644, + "grad_norm": 4.79758358001709, + "kl": 0.2978515625, + "learning_rate": 8.384615384615384e-07, + "loss": 0.0925, + "reward": 0.7324140667915344, + "reward_std": 0.05741269513964653, + "reward_std_all": 0.06454817205667496, + "rewards-max/forensics_sinkhorn_reward": 0.9279763102531433, + "rewards-max/forensics_strict_edge_reward": 0.24143732897937298, + "rewards-max/forensics_tiou_reward": 0.8513725996017456, + "rewards/forensics_sinkhorn_reward": 0.667876124382019, + "rewards/forensics_strict_edge_reward": 0.014650390221504495, + "rewards/forensics_tiou_reward": 0.38004928827285767, + "step": 211 + }, + { + "completion_length": 22.703125, + "epoch": 0.6513056835637481, + "grad_norm": 4.645277500152588, + "kl": 0.2509765625, + "learning_rate": 8.376923076923076e-07, + "loss": 0.2679, + "reward": 0.6718730330467224, + "reward_std": 0.10768386721611023, + "reward_std_all": 0.1185753270983696, + "rewards-max/forensics_sinkhorn_reward": 0.9630520641803741, + "rewards-max/forensics_strict_edge_reward": 0.3449285253882408, + "rewards-max/forensics_tiou_reward": 0.9273639619350433, + "rewards/forensics_sinkhorn_reward": 0.5992787480354309, + "rewards/forensics_strict_edge_reward": 0.021490140818059444, + "rewards/forensics_tiou_reward": 0.33658483624458313, + "step": 212 + }, + { + "completion_length": 24.953125, + "epoch": 0.6543778801843319, + "grad_norm": 5.609621047973633, + "kl": 0.3017578125, + "learning_rate": 8.369230769230769e-07, + "loss": 0.1205, + "reward": 0.6846341192722321, + "reward_std": 0.0810730792582035, + "reward_std_all": 0.09943366423249245, + "rewards-max/forensics_sinkhorn_reward": 0.9590608775615692, + "rewards-max/forensics_strict_edge_reward": 0.23915943503379822, + "rewards-max/forensics_tiou_reward": 0.9220583438873291, + "rewards/forensics_sinkhorn_reward": 0.5470294654369354, + "rewards/forensics_strict_edge_reward": 0.013748967554420233, + "rewards/forensics_tiou_reward": 0.3436366468667984, + "step": 213 + }, + { + "completion_length": 24.1875, + "epoch": 0.6574500768049155, + "grad_norm": 5.192232131958008, + "kl": 0.294921875, + "learning_rate": 8.361538461538461e-07, + "loss": -0.0177, + "reward": 0.7082805931568146, + "reward_std": 0.07112857326865196, + "reward_std_all": 0.08271702751517296, + "rewards-max/forensics_sinkhorn_reward": 0.9679759442806244, + "rewards-max/forensics_strict_edge_reward": 0.39789116382598877, + "rewards-max/forensics_tiou_reward": 0.9443395733833313, + "rewards/forensics_sinkhorn_reward": 0.6033357083797455, + "rewards/forensics_strict_edge_reward": 0.028404884040355682, + "rewards/forensics_tiou_reward": 0.33957549929618835, + "step": 214 + }, + { + "completion_length": 24.359375, + "epoch": 0.6605222734254992, + "grad_norm": 3.8261630535125732, + "kl": 0.263671875, + "learning_rate": 8.353846153846154e-07, + "loss": 0.1753, + "reward": 0.6988897919654846, + "reward_std": 0.05783780664205551, + "reward_std_all": 0.08036784827709198, + "rewards-max/forensics_sinkhorn_reward": 0.8996050357818604, + "rewards-max/forensics_strict_edge_reward": 0.12816077284514904, + "rewards-max/forensics_tiou_reward": 0.7919896245002747, + "rewards/forensics_sinkhorn_reward": 0.5964063107967377, + "rewards/forensics_strict_edge_reward": 0.009381013922393322, + "rewards/forensics_tiou_reward": 0.34849393367767334, + "step": 215 + }, + { + "completion_length": 22.765625, + "epoch": 0.663594470046083, + "grad_norm": 5.056163311004639, + "kl": 0.2646484375, + "learning_rate": 8.346153846153846e-07, + "loss": 0.0881, + "reward": 0.7111643254756927, + "reward_std": 0.08816630393266678, + "reward_std_all": 0.0895097404718399, + "rewards-max/forensics_sinkhorn_reward": 0.9518909752368927, + "rewards-max/forensics_strict_edge_reward": 0.14467177167534828, + "rewards-max/forensics_tiou_reward": 0.9030710756778717, + "rewards/forensics_sinkhorn_reward": 0.6108566522598267, + "rewards/forensics_strict_edge_reward": 0.00974245322868228, + "rewards/forensics_tiou_reward": 0.35260792076587677, + "step": 216 + }, + { + "completion_length": 24.0625, + "epoch": 0.6666666666666666, + "grad_norm": 4.490585803985596, + "kl": 0.271484375, + "learning_rate": 8.338461538461537e-07, + "loss": 0.0657, + "reward": 0.778574675321579, + "reward_std": 0.03097326960414648, + "reward_std_all": 0.030970269814133644, + "rewards-max/forensics_sinkhorn_reward": 0.9657407104969025, + "rewards-max/forensics_strict_edge_reward": 0.2356414794921875, + "rewards-max/forensics_tiou_reward": 0.9379616379737854, + "rewards/forensics_sinkhorn_reward": 0.6699217855930328, + "rewards/forensics_strict_edge_reward": 0.011655317153781652, + "rewards/forensics_tiou_reward": 0.42483246326446533, + "step": 217 + }, + { + "completion_length": 26.171875, + "epoch": 0.6697388632872504, + "grad_norm": 4.6335601806640625, + "kl": 0.345703125, + "learning_rate": 8.33076923076923e-07, + "loss": 0.0387, + "reward": 0.6979228854179382, + "reward_std": 0.05157735198736191, + "reward_std_all": 0.08105840533971786, + "rewards-max/forensics_sinkhorn_reward": 0.9685066938400269, + "rewards-max/forensics_strict_edge_reward": 0.18722280114889145, + "rewards-max/forensics_tiou_reward": 0.9494372606277466, + "rewards/forensics_sinkhorn_reward": 0.5776991844177246, + "rewards/forensics_strict_edge_reward": 0.008155738527420908, + "rewards/forensics_tiou_reward": 0.3957546055316925, + "step": 218 + }, + { + "completion_length": 22.765625, + "epoch": 0.6728110599078341, + "grad_norm": 5.615726947784424, + "kl": 0.2353515625, + "learning_rate": 8.323076923076922e-07, + "loss": 0.0642, + "reward": 0.7503718435764313, + "reward_std": 0.04160647466778755, + "reward_std_all": 0.049786658957600594, + "rewards-max/forensics_sinkhorn_reward": 0.9543173611164093, + "rewards-max/forensics_strict_edge_reward": 0.28100673854351044, + "rewards-max/forensics_tiou_reward": 0.916229635477066, + "rewards/forensics_sinkhorn_reward": 0.6681643724441528, + "rewards/forensics_strict_edge_reward": 0.01662857737392187, + "rewards/forensics_tiou_reward": 0.37427589297294617, + "step": 219 + }, + { + "completion_length": 26.84375, + "epoch": 0.6758832565284179, + "grad_norm": 4.582182884216309, + "kl": 0.3125, + "learning_rate": 8.315384615384616e-07, + "loss": 0.1263, + "reward": 0.6239263713359833, + "reward_std": 0.15130925923585892, + "reward_std_all": 0.16330285370349884, + "rewards-max/forensics_sinkhorn_reward": 0.9430363178253174, + "rewards-max/forensics_strict_edge_reward": 0.18044526944868267, + "rewards-max/forensics_tiou_reward": 0.8769235610961914, + "rewards/forensics_sinkhorn_reward": 0.4962846338748932, + "rewards/forensics_strict_edge_reward": 0.01140399705764139, + "rewards/forensics_tiou_reward": 0.26040908694267273, + "step": 220 + }, + { + "completion_length": 23.109375, + "epoch": 0.6789554531490015, + "grad_norm": 5.471200942993164, + "kl": 0.2958984375, + "learning_rate": 8.307692307692308e-07, + "loss": -0.0905, + "reward": 0.6999920904636383, + "reward_std": 0.0800362080335617, + "reward_std_all": 0.09363102167844772, + "rewards-max/forensics_sinkhorn_reward": 0.9581837356090546, + "rewards-max/forensics_strict_edge_reward": 0.4284459017217159, + "rewards-max/forensics_tiou_reward": 0.9197541773319244, + "rewards/forensics_sinkhorn_reward": 0.5910536646842957, + "rewards/forensics_strict_edge_reward": 0.034624316496774554, + "rewards/forensics_tiou_reward": 0.3688165098428726, + "step": 221 + }, + { + "completion_length": 23.515625, + "epoch": 0.6820276497695853, + "grad_norm": 4.920078754425049, + "kl": 0.28515625, + "learning_rate": 8.299999999999999e-07, + "loss": 0.0116, + "reward": 0.7174539566040039, + "reward_std": 0.061887577176094055, + "reward_std_all": 0.07571524754166603, + "rewards-max/forensics_sinkhorn_reward": 0.9754831194877625, + "rewards-max/forensics_strict_edge_reward": 0.32514405623078346, + "rewards-max/forensics_tiou_reward": 0.960500568151474, + "rewards/forensics_sinkhorn_reward": 0.5750015079975128, + "rewards/forensics_strict_edge_reward": 0.021295584505423903, + "rewards/forensics_tiou_reward": 0.37012213468551636, + "step": 222 + }, + { + "completion_length": 23.6875, + "epoch": 0.685099846390169, + "grad_norm": 6.180229187011719, + "kl": 0.2900390625, + "learning_rate": 8.292307692307692e-07, + "loss": 0.1125, + "reward": 0.7514900863170624, + "reward_std": 0.06542219780385494, + "reward_std_all": 0.062283700332045555, + "rewards-max/forensics_sinkhorn_reward": 0.9428972005844116, + "rewards-max/forensics_strict_edge_reward": 0.22760766744613647, + "rewards-max/forensics_tiou_reward": 0.8849954605102539, + "rewards/forensics_sinkhorn_reward": 0.6330709159374237, + "rewards/forensics_strict_edge_reward": 0.011610731482505798, + "rewards/forensics_tiou_reward": 0.42869894206523895, + "step": 223 + }, + { + "completion_length": 23.59375, + "epoch": 0.6881720430107527, + "grad_norm": 4.6472601890563965, + "kl": 0.3486328125, + "learning_rate": 8.284615384615384e-07, + "loss": 0.0836, + "reward": 0.6818655729293823, + "reward_std": 0.07786452025175095, + "reward_std_all": 0.09937890991568565, + "rewards-max/forensics_sinkhorn_reward": 0.9500558972358704, + "rewards-max/forensics_strict_edge_reward": 0.18722065724432468, + "rewards-max/forensics_tiou_reward": 0.9098608493804932, + "rewards/forensics_sinkhorn_reward": 0.5727400481700897, + "rewards/forensics_strict_edge_reward": 0.009900421951897442, + "rewards/forensics_tiou_reward": 0.33162978291511536, + "step": 224 + }, + { + "completion_length": 23.6875, + "epoch": 0.6912442396313364, + "grad_norm": 12.661725044250488, + "kl": 0.513671875, + "learning_rate": 8.276923076923077e-07, + "loss": 0.1775, + "reward": 0.7215786874294281, + "reward_std": 0.10135510563850403, + "reward_std_all": 0.09604734554886818, + "rewards-max/forensics_sinkhorn_reward": 0.9606700241565704, + "rewards-max/forensics_strict_edge_reward": 0.23242570832371712, + "rewards-max/forensics_tiou_reward": 0.9230126142501831, + "rewards/forensics_sinkhorn_reward": 0.616377204656601, + "rewards/forensics_strict_edge_reward": 0.011455519823357463, + "rewards/forensics_tiou_reward": 0.3570120334625244, + "step": 225 + }, + { + "completion_length": 24.5, + "epoch": 0.6943164362519201, + "grad_norm": 4.682663440704346, + "kl": 0.328125, + "learning_rate": 8.269230769230768e-07, + "loss": 0.146, + "reward": 0.6669906079769135, + "reward_std": 0.07932916283607483, + "reward_std_all": 0.10572408512234688, + "rewards-max/forensics_sinkhorn_reward": 0.9598767757415771, + "rewards-max/forensics_strict_edge_reward": 0.15840823762118816, + "rewards-max/forensics_tiou_reward": 0.9178245365619659, + "rewards/forensics_sinkhorn_reward": 0.5596180856227875, + "rewards/forensics_strict_edge_reward": 0.006661540945060551, + "rewards/forensics_tiou_reward": 0.31433430314064026, + "step": 226 + }, + { + "completion_length": 21.0625, + "epoch": 0.6973886328725039, + "grad_norm": 5.534582138061523, + "kl": 0.294921875, + "learning_rate": 8.261538461538461e-07, + "loss": 0.0151, + "reward": 0.7593927681446075, + "reward_std": 0.0283294094260782, + "reward_std_all": 0.03852370195090771, + "rewards-max/forensics_sinkhorn_reward": 0.968143492937088, + "rewards-max/forensics_strict_edge_reward": 0.5172730684280396, + "rewards-max/forensics_tiou_reward": 0.9476513564586639, + "rewards/forensics_sinkhorn_reward": 0.6642113029956818, + "rewards/forensics_strict_edge_reward": 0.03822988737374544, + "rewards/forensics_tiou_reward": 0.3943367153406143, + "step": 227 + }, + { + "completion_length": 21.765625, + "epoch": 0.7004608294930875, + "grad_norm": 5.549066543579102, + "kl": 0.26123046875, + "learning_rate": 8.253846153846154e-07, + "loss": 0.035, + "reward": 0.7131383121013641, + "reward_std": 0.0603334978222847, + "reward_std_all": 0.07572950050234795, + "rewards-max/forensics_sinkhorn_reward": 0.9131183624267578, + "rewards-max/forensics_strict_edge_reward": 0.10072415508329868, + "rewards-max/forensics_tiou_reward": 0.815105676651001, + "rewards/forensics_sinkhorn_reward": 0.6295586228370667, + "rewards/forensics_strict_edge_reward": 0.0074425352504476905, + "rewards/forensics_tiou_reward": 0.36337870359420776, + "step": 228 + }, + { + "completion_length": 22.28125, + "epoch": 0.7035330261136713, + "grad_norm": 6.048102378845215, + "kl": 0.2744140625, + "learning_rate": 8.246153846153846e-07, + "loss": -0.0559, + "reward": 0.7462117373943329, + "reward_std": 0.0724974125623703, + "reward_std_all": 0.06886955350637436, + "rewards-max/forensics_sinkhorn_reward": 0.9562582671642303, + "rewards-max/forensics_strict_edge_reward": 0.4211990535259247, + "rewards-max/forensics_tiou_reward": 0.9034560024738312, + "rewards/forensics_sinkhorn_reward": 0.6717406213283539, + "rewards/forensics_strict_edge_reward": 0.036542175337672234, + "rewards/forensics_tiou_reward": 0.39790216088294983, + "step": 229 + }, + { + "completion_length": 22.078125, + "epoch": 0.706605222734255, + "grad_norm": 4.65255880355835, + "kl": 0.25537109375, + "learning_rate": 8.238461538461538e-07, + "loss": 0.0277, + "reward": 0.732766181230545, + "reward_std": 0.07642906159162521, + "reward_std_all": 0.07860720530152321, + "rewards-max/forensics_sinkhorn_reward": 0.980228990316391, + "rewards-max/forensics_strict_edge_reward": 0.21599318459630013, + "rewards-max/forensics_tiou_reward": 0.9694370329380035, + "rewards/forensics_sinkhorn_reward": 0.6579843163490295, + "rewards/forensics_strict_edge_reward": 0.022009216248989105, + "rewards/forensics_tiou_reward": 0.36330100893974304, + "step": 230 + }, + { + "completion_length": 22.296875, + "epoch": 0.7096774193548387, + "grad_norm": 5.068573474884033, + "kl": 0.32421875, + "learning_rate": 8.23076923076923e-07, + "loss": 0.1828, + "reward": 0.6915961503982544, + "reward_std": 0.08689066395163536, + "reward_std_all": 0.0977242961525917, + "rewards-max/forensics_sinkhorn_reward": 0.9318402707576752, + "rewards-max/forensics_strict_edge_reward": 0.10246447287499905, + "rewards-max/forensics_tiou_reward": 0.8618768155574799, + "rewards/forensics_sinkhorn_reward": 0.5940840840339661, + "rewards/forensics_strict_edge_reward": 0.006993778282776475, + "rewards/forensics_tiou_reward": 0.3316196799278259, + "step": 231 + }, + { + "completion_length": 22.296875, + "epoch": 0.7127496159754224, + "grad_norm": 4.197347640991211, + "kl": 0.33984375, + "learning_rate": 8.223076923076922e-07, + "loss": 0.1077, + "reward": 0.7391844689846039, + "reward_std": 0.05358040891587734, + "reward_std_all": 0.06263267807662487, + "rewards-max/forensics_sinkhorn_reward": 0.9616542458534241, + "rewards-max/forensics_strict_edge_reward": 0.3243762031197548, + "rewards-max/forensics_tiou_reward": 0.9276926815509796, + "rewards/forensics_sinkhorn_reward": 0.6435149013996124, + "rewards/forensics_strict_edge_reward": 0.014335863990709186, + "rewards/forensics_tiou_reward": 0.3985525816679001, + "step": 232 + }, + { + "completion_length": 22.359375, + "epoch": 0.7158218125960062, + "grad_norm": 4.857463359832764, + "kl": 0.25048828125, + "learning_rate": 8.215384615384616e-07, + "loss": 0.0548, + "reward": 0.7257083058357239, + "reward_std": 0.08433020487427711, + "reward_std_all": 0.08338311314582825, + "rewards-max/forensics_sinkhorn_reward": 0.9270947873592377, + "rewards-max/forensics_strict_edge_reward": 0.21190487511921674, + "rewards-max/forensics_tiou_reward": 0.8547790050506592, + "rewards/forensics_sinkhorn_reward": 0.619289219379425, + "rewards/forensics_strict_edge_reward": 0.011219646374229342, + "rewards/forensics_tiou_reward": 0.35668139159679413, + "step": 233 + }, + { + "completion_length": 24.03125, + "epoch": 0.7188940092165899, + "grad_norm": 4.445806503295898, + "kl": 0.3330078125, + "learning_rate": 8.207692307692308e-07, + "loss": 0.1589, + "reward": 0.6759050488471985, + "reward_std": 0.1321227103471756, + "reward_std_all": 0.13267767429351807, + "rewards-max/forensics_sinkhorn_reward": 0.929634690284729, + "rewards-max/forensics_strict_edge_reward": 0.23621871322393417, + "rewards-max/forensics_tiou_reward": 0.8637755513191223, + "rewards/forensics_sinkhorn_reward": 0.5878159701824188, + "rewards/forensics_strict_edge_reward": 0.015553776640444994, + "rewards/forensics_tiou_reward": 0.35790547728538513, + "step": 234 + }, + { + "completion_length": 25.84375, + "epoch": 0.7219662058371735, + "grad_norm": 3.8648154735565186, + "kl": 0.248046875, + "learning_rate": 8.199999999999999e-07, + "loss": 0.0574, + "reward": 0.6911400258541107, + "reward_std": 0.11202827841043472, + "reward_std_all": 0.11271603032946587, + "rewards-max/forensics_sinkhorn_reward": 0.8931375741958618, + "rewards-max/forensics_strict_edge_reward": 0.07045251866838953, + "rewards-max/forensics_tiou_reward": 0.7779825925827026, + "rewards/forensics_sinkhorn_reward": 0.5861916542053223, + "rewards/forensics_strict_edge_reward": 0.0029398804856555216, + "rewards/forensics_tiou_reward": 0.31671208143234253, + "step": 235 + }, + { + "completion_length": 22.375, + "epoch": 0.7250384024577573, + "grad_norm": 6.704954147338867, + "kl": 0.2802734375, + "learning_rate": 8.192307692307692e-07, + "loss": 0.1165, + "reward": 0.6990451812744141, + "reward_std": 0.08690277114510536, + "reward_std_all": 0.0958322174847126, + "rewards-max/forensics_sinkhorn_reward": 0.9311128556728363, + "rewards-max/forensics_strict_edge_reward": 0.12253231182694435, + "rewards-max/forensics_tiou_reward": 0.8607686460018158, + "rewards/forensics_sinkhorn_reward": 0.5620723068714142, + "rewards/forensics_strict_edge_reward": 0.007610304397530854, + "rewards/forensics_tiou_reward": 0.28768257796764374, + "step": 236 + }, + { + "completion_length": 24.15625, + "epoch": 0.728110599078341, + "grad_norm": 4.642776966094971, + "kl": 0.298828125, + "learning_rate": 8.184615384615384e-07, + "loss": 0.0964, + "reward": 0.7387734651565552, + "reward_std": 0.06865400448441505, + "reward_std_all": 0.06749605759978294, + "rewards-max/forensics_sinkhorn_reward": 0.9245585799217224, + "rewards-max/forensics_strict_edge_reward": 0.061164241284132004, + "rewards-max/forensics_tiou_reward": 0.8505880534648895, + "rewards/forensics_sinkhorn_reward": 0.6342878043651581, + "rewards/forensics_strict_edge_reward": 0.0029471800662577152, + "rewards/forensics_tiou_reward": 0.3491790145635605, + "step": 237 + }, + { + "completion_length": 22.0, + "epoch": 0.7311827956989247, + "grad_norm": 4.08975076675415, + "kl": 0.28515625, + "learning_rate": 8.176923076923077e-07, + "loss": 0.0867, + "reward": 0.7412900924682617, + "reward_std": 0.04931442253291607, + "reward_std_all": 0.05714048817753792, + "rewards-max/forensics_sinkhorn_reward": 0.9192050397396088, + "rewards-max/forensics_strict_edge_reward": 0.05068525788374245, + "rewards-max/forensics_tiou_reward": 0.8444569706916809, + "rewards/forensics_sinkhorn_reward": 0.6452302932739258, + "rewards/forensics_strict_edge_reward": 0.0021763444165117107, + "rewards/forensics_tiou_reward": 0.34626784920692444, + "step": 238 + }, + { + "completion_length": 25.921875, + "epoch": 0.7342549923195084, + "grad_norm": 3.7304515838623047, + "kl": 0.2724609375, + "learning_rate": 8.169230769230769e-07, + "loss": 0.1428, + "reward": 0.7328917980194092, + "reward_std": 0.0781587902456522, + "reward_std_all": 0.07572526298463345, + "rewards-max/forensics_sinkhorn_reward": 0.9630175530910492, + "rewards-max/forensics_strict_edge_reward": 0.3800387680530548, + "rewards-max/forensics_tiou_reward": 0.9268750548362732, + "rewards/forensics_sinkhorn_reward": 0.6384807825088501, + "rewards/forensics_strict_edge_reward": 0.0326362419873476, + "rewards/forensics_tiou_reward": 0.43791691958904266, + "step": 239 + }, + { + "completion_length": 24.921875, + "epoch": 0.7373271889400922, + "grad_norm": 3.771592617034912, + "kl": 0.26025390625, + "learning_rate": 8.16153846153846e-07, + "loss": 0.106, + "reward": 0.7054490447044373, + "reward_std": 0.08369350805878639, + "reward_std_all": 0.09152785316109657, + "rewards-max/forensics_sinkhorn_reward": 0.9177144467830658, + "rewards-max/forensics_strict_edge_reward": 0.016351817408576608, + "rewards-max/forensics_tiou_reward": 0.828071117401123, + "rewards/forensics_sinkhorn_reward": 0.587011307477951, + "rewards/forensics_strict_edge_reward": 0.0012493317481130362, + "rewards/forensics_tiou_reward": 0.36582112312316895, + "step": 240 + }, + { + "completion_length": 26.65625, + "epoch": 0.7403993855606759, + "grad_norm": 4.78831148147583, + "kl": 0.3203125, + "learning_rate": 8.153846153846154e-07, + "loss": 0.2087, + "reward": 0.7408247590065002, + "reward_std": 0.06548817455768585, + "reward_std_all": 0.06598393991589546, + "rewards-max/forensics_sinkhorn_reward": 0.9377689957618713, + "rewards-max/forensics_strict_edge_reward": 0.07006018981337547, + "rewards-max/forensics_tiou_reward": 0.8793863654136658, + "rewards/forensics_sinkhorn_reward": 0.6609873473644257, + "rewards/forensics_strict_edge_reward": 0.004578528692945838, + "rewards/forensics_tiou_reward": 0.40381766855716705, + "step": 241 + }, + { + "completion_length": 23.0625, + "epoch": 0.7434715821812596, + "grad_norm": 5.041311264038086, + "kl": 0.24609375, + "learning_rate": 8.146153846153846e-07, + "loss": 0.1852, + "reward": 0.727935254573822, + "reward_std": 0.07508930563926697, + "reward_std_all": 0.0767226368188858, + "rewards-max/forensics_sinkhorn_reward": 0.9462548196315765, + "rewards-max/forensics_strict_edge_reward": 0.28277574479579926, + "rewards-max/forensics_tiou_reward": 0.8968824744224548, + "rewards/forensics_sinkhorn_reward": 0.6251331269741058, + "rewards/forensics_strict_edge_reward": 0.027950317598879337, + "rewards/forensics_tiou_reward": 0.35435751080513, + "step": 242 + }, + { + "completion_length": 23.578125, + "epoch": 0.7465437788018433, + "grad_norm": 5.848924160003662, + "kl": 0.216796875, + "learning_rate": 8.138461538461538e-07, + "loss": -0.006, + "reward": 0.7374731302261353, + "reward_std": 0.04754258878529072, + "reward_std_all": 0.058399416506290436, + "rewards-max/forensics_sinkhorn_reward": 0.9633606374263763, + "rewards-max/forensics_strict_edge_reward": 0.2558319717645645, + "rewards-max/forensics_tiou_reward": 0.9375512301921844, + "rewards/forensics_sinkhorn_reward": 0.6685427725315094, + "rewards/forensics_strict_edge_reward": 0.01767348125576973, + "rewards/forensics_tiou_reward": 0.48664185404777527, + "step": 243 + }, + { + "completion_length": 26.796875, + "epoch": 0.7496159754224271, + "grad_norm": 4.878171443939209, + "kl": 0.24267578125, + "learning_rate": 8.13076923076923e-07, + "loss": 0.1596, + "reward": 0.7491817772388458, + "reward_std": 0.0650259517133236, + "reward_std_all": 0.062379222363233566, + "rewards-max/forensics_sinkhorn_reward": 0.963793009519577, + "rewards-max/forensics_strict_edge_reward": 0.4259333088994026, + "rewards-max/forensics_tiou_reward": 0.9352646172046661, + "rewards/forensics_sinkhorn_reward": 0.6298830211162567, + "rewards/forensics_strict_edge_reward": 0.020321783144026995, + "rewards/forensics_tiou_reward": 0.3882705569267273, + "step": 244 + }, + { + "completion_length": 22.8125, + "epoch": 0.7526881720430108, + "grad_norm": 3.9103622436523438, + "kl": 0.2890625, + "learning_rate": 8.123076923076922e-07, + "loss": -0.083, + "reward": 0.7093183696269989, + "reward_std": 0.08782194182276726, + "reward_std_all": 0.09166519343852997, + "rewards-max/forensics_sinkhorn_reward": 0.9654275476932526, + "rewards-max/forensics_strict_edge_reward": 0.23440391570329666, + "rewards-max/forensics_tiou_reward": 0.9386910200119019, + "rewards/forensics_sinkhorn_reward": 0.6281572580337524, + "rewards/forensics_strict_edge_reward": 0.0167582374997437, + "rewards/forensics_tiou_reward": 0.355268269777298, + "step": 245 + }, + { + "completion_length": 22.53125, + "epoch": 0.7557603686635944, + "grad_norm": 5.782008171081543, + "kl": 0.2568359375, + "learning_rate": 8.115384615384615e-07, + "loss": -0.0067, + "reward": 0.7440560758113861, + "reward_std": 0.07869035750627518, + "reward_std_all": 0.07383634895086288, + "rewards-max/forensics_sinkhorn_reward": 0.913645476102829, + "rewards-max/forensics_strict_edge_reward": 0.22221739031374454, + "rewards-max/forensics_tiou_reward": 0.8267126679420471, + "rewards/forensics_sinkhorn_reward": 0.6680632531642914, + "rewards/forensics_strict_edge_reward": 0.01876695896498859, + "rewards/forensics_tiou_reward": 0.39049482345581055, + "step": 246 + }, + { + "completion_length": 23.4375, + "epoch": 0.7588325652841782, + "grad_norm": 4.526020050048828, + "kl": 0.3359375, + "learning_rate": 8.107692307692308e-07, + "loss": 0.102, + "reward": 0.7071051597595215, + "reward_std": 0.06589528918266296, + "reward_std_all": 0.0801386907696724, + "rewards-max/forensics_sinkhorn_reward": 0.8443098068237305, + "rewards-max/forensics_strict_edge_reward": 0.07764224521815777, + "rewards-max/forensics_tiou_reward": 0.6187262237071991, + "rewards/forensics_sinkhorn_reward": 0.6028242409229279, + "rewards/forensics_strict_edge_reward": 0.004032158991321921, + "rewards/forensics_tiou_reward": 0.34156084060668945, + "step": 247 + }, + { + "completion_length": 25.03125, + "epoch": 0.7619047619047619, + "grad_norm": 4.261857509613037, + "kl": 0.3125, + "learning_rate": 8.1e-07, + "loss": 0.1251, + "reward": 0.7476575970649719, + "reward_std": 0.04904384259134531, + "reward_std_all": 0.056137796491384506, + "rewards-max/forensics_sinkhorn_reward": 0.9488013088703156, + "rewards-max/forensics_strict_edge_reward": 0.4284205287694931, + "rewards-max/forensics_tiou_reward": 0.899651825428009, + "rewards/forensics_sinkhorn_reward": 0.6337864995002747, + "rewards/forensics_strict_edge_reward": 0.022946853190660477, + "rewards/forensics_tiou_reward": 0.3936910331249237, + "step": 248 + }, + { + "completion_length": 25.953125, + "epoch": 0.7649769585253456, + "grad_norm": 3.8970911502838135, + "kl": 0.29296875, + "learning_rate": 8.092307692307692e-07, + "loss": 0.0351, + "reward": 0.7501348555088043, + "reward_std": 0.05261159688234329, + "reward_std_all": 0.055149881169199944, + "rewards-max/forensics_sinkhorn_reward": 0.9452458024024963, + "rewards-max/forensics_strict_edge_reward": 0.1865822672843933, + "rewards-max/forensics_tiou_reward": 0.8990406394004822, + "rewards/forensics_sinkhorn_reward": 0.6469317972660065, + "rewards/forensics_strict_edge_reward": 0.013807457871735096, + "rewards/forensics_tiou_reward": 0.44210201501846313, + "step": 249 + }, + { + "completion_length": 24.515625, + "epoch": 0.7680491551459293, + "grad_norm": 3.8398568630218506, + "kl": 0.3701171875, + "learning_rate": 8.084615384615384e-07, + "loss": 0.1144, + "reward": 0.7231443822383881, + "reward_std": 0.046780864242464304, + "reward_std_all": 0.06344280205667019, + "rewards-max/forensics_sinkhorn_reward": 0.893463134765625, + "rewards-max/forensics_strict_edge_reward": 0.25104339129757136, + "rewards-max/forensics_tiou_reward": 0.7912636995315552, + "rewards/forensics_sinkhorn_reward": 0.6403678059577942, + "rewards/forensics_strict_edge_reward": 0.01529813559318427, + "rewards/forensics_tiou_reward": 0.35052844882011414, + "step": 250 + }, + { + "completion_length": 26.625, + "epoch": 0.7711213517665131, + "grad_norm": 4.721198081970215, + "kl": 0.314453125, + "learning_rate": 8.076923076923077e-07, + "loss": 0.0132, + "reward": 0.6623117923736572, + "reward_std": 0.09147607162594795, + "reward_std_all": 0.1116991937160492, + "rewards-max/forensics_sinkhorn_reward": 0.9186314940452576, + "rewards-max/forensics_strict_edge_reward": 0.24197828979231417, + "rewards-max/forensics_tiou_reward": 0.8256825804710388, + "rewards/forensics_sinkhorn_reward": 0.5419380068778992, + "rewards/forensics_strict_edge_reward": 0.010943988978397101, + "rewards/forensics_tiou_reward": 0.32481707632541656, + "step": 251 + }, + { + "completion_length": 22.609375, + "epoch": 0.7741935483870968, + "grad_norm": 4.991756439208984, + "kl": 0.31640625, + "learning_rate": 8.069230769230769e-07, + "loss": 0.1004, + "reward": 0.7106851935386658, + "reward_std": 0.0935148149728775, + "reward_std_all": 0.09603073447942734, + "rewards-max/forensics_sinkhorn_reward": 0.9437682628631592, + "rewards-max/forensics_strict_edge_reward": 0.15158757381141186, + "rewards-max/forensics_tiou_reward": 0.8770780265331268, + "rewards/forensics_sinkhorn_reward": 0.6053285002708435, + "rewards/forensics_strict_edge_reward": 0.014673061668872833, + "rewards/forensics_tiou_reward": 0.3398743271827698, + "step": 252 + }, + { + "completion_length": 25.3125, + "epoch": 0.7772657450076805, + "grad_norm": 4.59052848815918, + "kl": 0.326171875, + "learning_rate": 8.06153846153846e-07, + "loss": 0.0738, + "reward": 0.7424440383911133, + "reward_std": 0.05845397710800171, + "reward_std_all": 0.05982530489563942, + "rewards-max/forensics_sinkhorn_reward": 0.9632135331630707, + "rewards-max/forensics_strict_edge_reward": 0.042227813974022865, + "rewards-max/forensics_tiou_reward": 0.9337040185928345, + "rewards/forensics_sinkhorn_reward": 0.6155940592288971, + "rewards/forensics_strict_edge_reward": 0.003075718996115029, + "rewards/forensics_tiou_reward": 0.3926156908273697, + "step": 253 + }, + { + "completion_length": 24.703125, + "epoch": 0.7803379416282642, + "grad_norm": 4.315326690673828, + "kl": 0.3818359375, + "learning_rate": 8.053846153846154e-07, + "loss": 0.0131, + "reward": 0.6960194110870361, + "reward_std": 0.07845650985836983, + "reward_std_all": 0.09581027552485466, + "rewards-max/forensics_sinkhorn_reward": 0.9670716524124146, + "rewards-max/forensics_strict_edge_reward": 0.47232839092612267, + "rewards-max/forensics_tiou_reward": 0.9379361569881439, + "rewards/forensics_sinkhorn_reward": 0.5998211205005646, + "rewards/forensics_strict_edge_reward": 0.023342353524640203, + "rewards/forensics_tiou_reward": 0.31814301013946533, + "step": 254 + }, + { + "completion_length": 23.71875, + "epoch": 0.783410138248848, + "grad_norm": 4.6057963371276855, + "kl": 0.3369140625, + "learning_rate": 8.046153846153846e-07, + "loss": 0.2182, + "reward": 0.7175907492637634, + "reward_std": 0.07896478846669197, + "reward_std_all": 0.08270680904388428, + "rewards-max/forensics_sinkhorn_reward": 0.932514488697052, + "rewards-max/forensics_strict_edge_reward": 0.10350316390395164, + "rewards-max/forensics_tiou_reward": 0.8751953840255737, + "rewards/forensics_sinkhorn_reward": 0.6330399811267853, + "rewards/forensics_strict_edge_reward": 0.004943060223013163, + "rewards/forensics_tiou_reward": 0.3528422713279724, + "step": 255 + }, + { + "completion_length": 28.828125, + "epoch": 0.7864823348694316, + "grad_norm": 4.665998458862305, + "kl": 0.25927734375, + "learning_rate": 8.038461538461538e-07, + "loss": 0.2234, + "reward": 0.7106801569461823, + "reward_std": 0.10485461726784706, + "reward_std_all": 0.10152747482061386, + "rewards-max/forensics_sinkhorn_reward": 0.9444971084594727, + "rewards-max/forensics_strict_edge_reward": 0.45999419689178467, + "rewards-max/forensics_tiou_reward": 0.891936331987381, + "rewards/forensics_sinkhorn_reward": 0.5966432690620422, + "rewards/forensics_strict_edge_reward": 0.0338777843862772, + "rewards/forensics_tiou_reward": 0.3931726664304733, + "step": 256 + }, + { + "completion_length": 25.890625, + "epoch": 0.7895545314900153, + "grad_norm": 3.9793288707733154, + "kl": 0.25732421875, + "learning_rate": 8.03076923076923e-07, + "loss": 0.0659, + "reward": 0.6593087017536163, + "reward_std": 0.08945821225643158, + "reward_std_all": 0.11411511525511742, + "rewards-max/forensics_sinkhorn_reward": 0.9691407978534698, + "rewards-max/forensics_strict_edge_reward": 0.3400171995162964, + "rewards-max/forensics_tiou_reward": 0.946025937795639, + "rewards/forensics_sinkhorn_reward": 0.5197015255689621, + "rewards/forensics_strict_edge_reward": 0.019780904054641724, + "rewards/forensics_tiou_reward": 0.39218343794345856, + "step": 257 + }, + { + "completion_length": 24.203125, + "epoch": 0.7926267281105991, + "grad_norm": 4.518051624298096, + "kl": 0.23095703125, + "learning_rate": 8.023076923076922e-07, + "loss": 0.1794, + "reward": 0.7115864157676697, + "reward_std": 0.09381898120045662, + "reward_std_all": 0.09357175603508949, + "rewards-max/forensics_sinkhorn_reward": 0.9061840176582336, + "rewards-max/forensics_strict_edge_reward": 0.06455939821898937, + "rewards-max/forensics_tiou_reward": 0.8030552864074707, + "rewards/forensics_sinkhorn_reward": 0.5771409571170807, + "rewards/forensics_strict_edge_reward": 0.007490046788007021, + "rewards/forensics_tiou_reward": 0.3517443984746933, + "step": 258 + }, + { + "completion_length": 26.125, + "epoch": 0.7956989247311828, + "grad_norm": 3.8115756511688232, + "kl": 0.34375, + "learning_rate": 8.015384615384615e-07, + "loss": 0.2418, + "reward": 0.6946418285369873, + "reward_std": 0.08816361427307129, + "reward_std_all": 0.09894756227731705, + "rewards-max/forensics_sinkhorn_reward": 0.9687096178531647, + "rewards-max/forensics_strict_edge_reward": 0.5371420830488205, + "rewards-max/forensics_tiou_reward": 0.9432619214057922, + "rewards/forensics_sinkhorn_reward": 0.6054859161376953, + "rewards/forensics_strict_edge_reward": 0.042473329231143, + "rewards/forensics_tiou_reward": 0.3770364075899124, + "step": 259 + }, + { + "completion_length": 24.28125, + "epoch": 0.7987711213517665, + "grad_norm": 4.141703128814697, + "kl": 0.3994140625, + "learning_rate": 8.007692307692308e-07, + "loss": 0.1124, + "reward": 0.7268043458461761, + "reward_std": 0.08434447646141052, + "reward_std_all": 0.08653546869754791, + "rewards-max/forensics_sinkhorn_reward": 0.96342733502388, + "rewards-max/forensics_strict_edge_reward": 0.290697917342186, + "rewards-max/forensics_tiou_reward": 0.9273158311843872, + "rewards/forensics_sinkhorn_reward": 0.6240766644477844, + "rewards/forensics_strict_edge_reward": 0.02855434361845255, + "rewards/forensics_tiou_reward": 0.38509438931941986, + "step": 260 + }, + { + "completion_length": 25.796875, + "epoch": 0.8018433179723502, + "grad_norm": 4.127774238586426, + "kl": 0.2939453125, + "learning_rate": 8e-07, + "loss": 0.107, + "reward": 0.7437496483325958, + "reward_std": 0.052546096965670586, + "reward_std_all": 0.056853191927075386, + "rewards-max/forensics_sinkhorn_reward": 0.9572671949863434, + "rewards-max/forensics_strict_edge_reward": 0.34197375178337097, + "rewards-max/forensics_tiou_reward": 0.9188871085643768, + "rewards/forensics_sinkhorn_reward": 0.6447758376598358, + "rewards/forensics_strict_edge_reward": 0.02205142006278038, + "rewards/forensics_tiou_reward": 0.45330557227134705, + "step": 261 + }, + { + "completion_length": 22.953125, + "epoch": 0.804915514592934, + "grad_norm": 5.720284461975098, + "kl": 0.306640625, + "learning_rate": 7.992307692307692e-07, + "loss": 0.1226, + "reward": 0.7425169348716736, + "reward_std": 0.05342675931751728, + "reward_std_all": 0.057438235729932785, + "rewards-max/forensics_sinkhorn_reward": 0.9405239820480347, + "rewards-max/forensics_strict_edge_reward": 0.15211109817028046, + "rewards-max/forensics_tiou_reward": 0.8774083852767944, + "rewards/forensics_sinkhorn_reward": 0.6318778991699219, + "rewards/forensics_strict_edge_reward": 0.01029948377981782, + "rewards/forensics_tiou_reward": 0.3736691474914551, + "step": 262 + }, + { + "completion_length": 26.578125, + "epoch": 0.8079877112135176, + "grad_norm": 4.332508563995361, + "kl": 0.2431640625, + "learning_rate": 7.984615384615384e-07, + "loss": 0.0307, + "reward": 0.7386213839054108, + "reward_std": 0.05176846124231815, + "reward_std_all": 0.062343928031623363, + "rewards-max/forensics_sinkhorn_reward": 0.9086247980594635, + "rewards-max/forensics_strict_edge_reward": 0.19158204458653927, + "rewards-max/forensics_tiou_reward": 0.7998854517936707, + "rewards/forensics_sinkhorn_reward": 0.6626463532447815, + "rewards/forensics_strict_edge_reward": 0.011266602668911219, + "rewards/forensics_tiou_reward": 0.4105944186449051, + "step": 263 + }, + { + "completion_length": 27.15625, + "epoch": 0.8110599078341014, + "grad_norm": 3.6532812118530273, + "kl": 0.23486328125, + "learning_rate": 7.976923076923077e-07, + "loss": 0.1223, + "reward": 0.7153191268444061, + "reward_std": 0.09431452676653862, + "reward_std_all": 0.09724964946508408, + "rewards-max/forensics_sinkhorn_reward": 0.973476380109787, + "rewards-max/forensics_strict_edge_reward": 0.5422053933143616, + "rewards-max/forensics_tiou_reward": 0.9616832137107849, + "rewards/forensics_sinkhorn_reward": 0.6040070354938507, + "rewards/forensics_strict_edge_reward": 0.030986913479864597, + "rewards/forensics_tiou_reward": 0.44369882345199585, + "step": 264 + }, + { + "completion_length": 25.59375, + "epoch": 0.8141321044546851, + "grad_norm": 4.57415246963501, + "kl": 0.3291015625, + "learning_rate": 7.969230769230769e-07, + "loss": 0.2204, + "reward": 0.6845948994159698, + "reward_std": 0.09572556987404823, + "reward_std_all": 0.10515008866786957, + "rewards-max/forensics_sinkhorn_reward": 0.8458346724510193, + "rewards-max/forensics_strict_edge_reward": 0.0880789184011519, + "rewards-max/forensics_tiou_reward": 0.6531646549701691, + "rewards/forensics_sinkhorn_reward": 0.525790274143219, + "rewards/forensics_strict_edge_reward": 0.006392695184331387, + "rewards/forensics_tiou_reward": 0.29261599481105804, + "step": 265 + }, + { + "completion_length": 24.609375, + "epoch": 0.8172043010752689, + "grad_norm": 5.472289562225342, + "kl": 0.263671875, + "learning_rate": 7.96153846153846e-07, + "loss": 0.0803, + "reward": 0.6920785307884216, + "reward_std": 0.09302779287099838, + "reward_std_all": 0.0991813987493515, + "rewards-max/forensics_sinkhorn_reward": 0.9468184411525726, + "rewards-max/forensics_strict_edge_reward": 0.5373985022306442, + "rewards-max/forensics_tiou_reward": 0.8705691397190094, + "rewards/forensics_sinkhorn_reward": 0.6045105159282684, + "rewards/forensics_strict_edge_reward": 0.038881988264620304, + "rewards/forensics_tiou_reward": 0.35355839133262634, + "step": 266 + }, + { + "completion_length": 30.53125, + "epoch": 0.8202764976958525, + "grad_norm": 4.123321533203125, + "kl": 0.2626953125, + "learning_rate": 7.953846153846154e-07, + "loss": 0.1253, + "reward": 0.6966725587844849, + "reward_std": 0.07694688206538558, + "reward_std_all": 0.09053950384259224, + "rewards-max/forensics_sinkhorn_reward": 0.9273853898048401, + "rewards-max/forensics_strict_edge_reward": 0.11588936671614647, + "rewards-max/forensics_tiou_reward": 0.8539571762084961, + "rewards/forensics_sinkhorn_reward": 0.5328914374113083, + "rewards/forensics_strict_edge_reward": 0.006678856676444411, + "rewards/forensics_tiou_reward": 0.3164275735616684, + "step": 267 + }, + { + "completion_length": 26.09375, + "epoch": 0.8233486943164362, + "grad_norm": 4.714967727661133, + "kl": 0.296875, + "learning_rate": 7.946153846153846e-07, + "loss": 0.138, + "reward": 0.6729004085063934, + "reward_std": 0.09862134605646133, + "reward_std_all": 0.11456365510821342, + "rewards-max/forensics_sinkhorn_reward": 0.8896144032478333, + "rewards-max/forensics_strict_edge_reward": 0.15007326751947403, + "rewards-max/forensics_tiou_reward": 0.7453144490718842, + "rewards/forensics_sinkhorn_reward": 0.5418865978717804, + "rewards/forensics_strict_edge_reward": 0.00792012084275484, + "rewards/forensics_tiou_reward": 0.32062844932079315, + "step": 268 + }, + { + "completion_length": 24.328125, + "epoch": 0.82642089093702, + "grad_norm": 4.736511707305908, + "kl": 0.2890625, + "learning_rate": 7.938461538461538e-07, + "loss": 0.1304, + "reward": 0.6941148042678833, + "reward_std": 0.049449341371655464, + "reward_std_all": 0.07796476781368256, + "rewards-max/forensics_sinkhorn_reward": 0.978146880865097, + "rewards-max/forensics_strict_edge_reward": 0.43684302270412445, + "rewards-max/forensics_tiou_reward": 0.9663721919059753, + "rewards/forensics_sinkhorn_reward": 0.591905415058136, + "rewards/forensics_strict_edge_reward": 0.03663163632154465, + "rewards/forensics_tiou_reward": 0.3928965777158737, + "step": 269 + }, + { + "completion_length": 25.5625, + "epoch": 0.8294930875576036, + "grad_norm": 4.5523858070373535, + "kl": 0.2294921875, + "learning_rate": 7.930769230769231e-07, + "loss": 0.1788, + "reward": 0.7297359704971313, + "reward_std": 0.0925518274307251, + "reward_std_all": 0.08712068572640419, + "rewards-max/forensics_sinkhorn_reward": 0.9740355014801025, + "rewards-max/forensics_strict_edge_reward": 0.24571846798062325, + "rewards-max/forensics_tiou_reward": 0.9626171588897705, + "rewards/forensics_sinkhorn_reward": 0.5947600603103638, + "rewards/forensics_strict_edge_reward": 0.014442565385252237, + "rewards/forensics_tiou_reward": 0.4457249194383621, + "step": 270 + }, + { + "completion_length": 25.34375, + "epoch": 0.8325652841781874, + "grad_norm": 4.442654609680176, + "kl": 0.24755859375, + "learning_rate": 7.923076923076922e-07, + "loss": 0.125, + "reward": 0.668671190738678, + "reward_std": 0.08998775854706764, + "reward_std_all": 0.10858012363314629, + "rewards-max/forensics_sinkhorn_reward": 0.9728262424468994, + "rewards-max/forensics_strict_edge_reward": 0.2780006229877472, + "rewards-max/forensics_tiou_reward": 0.9499583840370178, + "rewards/forensics_sinkhorn_reward": 0.5740452408790588, + "rewards/forensics_strict_edge_reward": 0.01314231101423502, + "rewards/forensics_tiou_reward": 0.31365641951560974, + "step": 271 + }, + { + "completion_length": 24.296875, + "epoch": 0.8356374807987711, + "grad_norm": 4.575419902801514, + "kl": 0.2802734375, + "learning_rate": 7.915384615384615e-07, + "loss": 0.0909, + "reward": 0.7647362351417542, + "reward_std": 0.06384154409170151, + "reward_std_all": 0.05822557955980301, + "rewards-max/forensics_sinkhorn_reward": 0.9679271876811981, + "rewards-max/forensics_strict_edge_reward": 0.42223699390888214, + "rewards-max/forensics_tiou_reward": 0.9474490880966187, + "rewards/forensics_sinkhorn_reward": 0.7037330567836761, + "rewards/forensics_strict_edge_reward": 0.05074458569288254, + "rewards/forensics_tiou_reward": 0.47354599833488464, + "step": 272 + }, + { + "completion_length": 24.25, + "epoch": 0.8387096774193549, + "grad_norm": 6.3699188232421875, + "kl": 0.4150390625, + "learning_rate": 7.907692307692307e-07, + "loss": 0.0629, + "reward": 0.7130439877510071, + "reward_std": 0.0794653594493866, + "reward_std_all": 0.09174156188964844, + "rewards-max/forensics_sinkhorn_reward": 0.9663920402526855, + "rewards-max/forensics_strict_edge_reward": 0.10023855790495872, + "rewards-max/forensics_tiou_reward": 0.9405170679092407, + "rewards/forensics_sinkhorn_reward": 0.6277886629104614, + "rewards/forensics_strict_edge_reward": 0.00918862596154213, + "rewards/forensics_tiou_reward": 0.4251601845026016, + "step": 273 + }, + { + "completion_length": 27.390625, + "epoch": 0.8417818740399385, + "grad_norm": 4.445045471191406, + "kl": 0.314453125, + "learning_rate": 7.9e-07, + "loss": 0.1524, + "reward": 0.6541199684143066, + "reward_std": 0.09398846328258514, + "reward_std_all": 0.1183457113802433, + "rewards-max/forensics_sinkhorn_reward": 0.907780259847641, + "rewards-max/forensics_strict_edge_reward": 0.014825146936345845, + "rewards-max/forensics_tiou_reward": 0.8080417215824127, + "rewards/forensics_sinkhorn_reward": 0.5332810878753662, + "rewards/forensics_strict_edge_reward": 0.0009895624789351132, + "rewards/forensics_tiou_reward": 0.31547582149505615, + "step": 274 + }, + { + "completion_length": 24.484375, + "epoch": 0.8448540706605223, + "grad_norm": 4.495822429656982, + "kl": 0.240234375, + "learning_rate": 7.892307692307692e-07, + "loss": 0.1677, + "reward": 0.7056180834770203, + "reward_std": 0.09221664071083069, + "reward_std_all": 0.09717745333909988, + "rewards-max/forensics_sinkhorn_reward": 0.9551101624965668, + "rewards-max/forensics_strict_edge_reward": 0.2783775255084038, + "rewards-max/forensics_tiou_reward": 0.9208045601844788, + "rewards/forensics_sinkhorn_reward": 0.6268576979637146, + "rewards/forensics_strict_edge_reward": 0.017612780909985304, + "rewards/forensics_tiou_reward": 0.3781178295612335, + "step": 275 + }, + { + "completion_length": 22.890625, + "epoch": 0.847926267281106, + "grad_norm": 4.9715752601623535, + "kl": 0.2919921875, + "learning_rate": 7.884615384615384e-07, + "loss": 0.0787, + "reward": 0.7736489176750183, + "reward_std": 0.04658856615424156, + "reward_std_all": 0.043540678918361664, + "rewards-max/forensics_sinkhorn_reward": 0.9247339367866516, + "rewards-max/forensics_strict_edge_reward": 0.11354262754321098, + "rewards-max/forensics_tiou_reward": 0.8449335396289825, + "rewards/forensics_sinkhorn_reward": 0.6905417442321777, + "rewards/forensics_strict_edge_reward": 0.011137581313960254, + "rewards/forensics_tiou_reward": 0.43126633763313293, + "step": 276 + }, + { + "completion_length": 24.03125, + "epoch": 0.8509984639016898, + "grad_norm": 5.900105953216553, + "kl": 0.2724609375, + "learning_rate": 7.876923076923077e-07, + "loss": 0.1048, + "reward": 0.7407378852367401, + "reward_std": 0.07161376252770424, + "reward_std_all": 0.06947515904903412, + "rewards-max/forensics_sinkhorn_reward": 0.9192933142185211, + "rewards-max/forensics_strict_edge_reward": 0.15559563599526882, + "rewards-max/forensics_tiou_reward": 0.8294923305511475, + "rewards/forensics_sinkhorn_reward": 0.6584147214889526, + "rewards/forensics_strict_edge_reward": 0.015500941779464483, + "rewards/forensics_tiou_reward": 0.3938266634941101, + "step": 277 + }, + { + "completion_length": 23.546875, + "epoch": 0.8540706605222734, + "grad_norm": 5.473773002624512, + "kl": 0.2451171875, + "learning_rate": 7.869230769230769e-07, + "loss": -0.0513, + "reward": 0.7543129622936249, + "reward_std": 0.04583900794386864, + "reward_std_all": 0.0487405750900507, + "rewards-max/forensics_sinkhorn_reward": 0.9518743753433228, + "rewards-max/forensics_strict_edge_reward": 0.26349517051130533, + "rewards-max/forensics_tiou_reward": 0.9008292257785797, + "rewards/forensics_sinkhorn_reward": 0.6303423941135406, + "rewards/forensics_strict_edge_reward": 0.012384936853777617, + "rewards/forensics_tiou_reward": 0.3377538025379181, + "step": 278 + }, + { + "completion_length": 22.03125, + "epoch": 0.8571428571428571, + "grad_norm": 4.113809585571289, + "kl": 0.32421875, + "learning_rate": 7.861538461538461e-07, + "loss": 0.0679, + "reward": 0.7134242653846741, + "reward_std": 0.0495495293289423, + "reward_std_all": 0.07174791675060987, + "rewards-max/forensics_sinkhorn_reward": 0.9631771147251129, + "rewards-max/forensics_strict_edge_reward": 0.2967786379158497, + "rewards-max/forensics_tiou_reward": 0.9373732805252075, + "rewards/forensics_sinkhorn_reward": 0.6235267817974091, + "rewards/forensics_strict_edge_reward": 0.01930539822205901, + "rewards/forensics_tiou_reward": 0.39375361800193787, + "step": 279 + }, + { + "completion_length": 25.359375, + "epoch": 0.8602150537634409, + "grad_norm": 5.708009719848633, + "kl": 0.2783203125, + "learning_rate": 7.853846153846153e-07, + "loss": 0.1194, + "reward": 0.7572133541107178, + "reward_std": 0.055386537685990334, + "reward_std_all": 0.0537457000464201, + "rewards-max/forensics_sinkhorn_reward": 0.9469651579856873, + "rewards-max/forensics_strict_edge_reward": 0.4635082483291626, + "rewards-max/forensics_tiou_reward": 0.8823886513710022, + "rewards/forensics_sinkhorn_reward": 0.6755785048007965, + "rewards/forensics_strict_edge_reward": 0.03215466998517513, + "rewards/forensics_tiou_reward": 0.4276740849018097, + "step": 280 + }, + { + "completion_length": 22.546875, + "epoch": 0.8632872503840245, + "grad_norm": 4.733345985412598, + "kl": 0.4443359375, + "learning_rate": 7.846153846153846e-07, + "loss": 0.0936, + "reward": 0.746419221162796, + "reward_std": 0.06652728281915188, + "reward_std_all": 0.068289864808321, + "rewards-max/forensics_sinkhorn_reward": 0.9631118178367615, + "rewards-max/forensics_strict_edge_reward": 0.4231441020965576, + "rewards-max/forensics_tiou_reward": 0.9361554980278015, + "rewards/forensics_sinkhorn_reward": 0.6758661866188049, + "rewards/forensics_strict_edge_reward": 0.02534772828221321, + "rewards/forensics_tiou_reward": 0.4313697814941406, + "step": 281 + }, + { + "completion_length": 23.84375, + "epoch": 0.8663594470046083, + "grad_norm": 5.208113193511963, + "kl": 0.4248046875, + "learning_rate": 7.838461538461539e-07, + "loss": 0.0476, + "reward": 0.7140003740787506, + "reward_std": 0.044077424332499504, + "reward_std_all": 0.06802232563495636, + "rewards-max/forensics_sinkhorn_reward": 0.8830539584159851, + "rewards-max/forensics_strict_edge_reward": 0.011644638689176645, + "rewards-max/forensics_tiou_reward": 0.7661143839359283, + "rewards/forensics_sinkhorn_reward": 0.5948031842708588, + "rewards/forensics_strict_edge_reward": 0.0016733843467591214, + "rewards/forensics_tiou_reward": 0.34004613757133484, + "step": 282 + }, + { + "completion_length": 24.203125, + "epoch": 0.869431643625192, + "grad_norm": 4.341249465942383, + "kl": 0.390625, + "learning_rate": 7.830769230769231e-07, + "loss": 0.0897, + "reward": 0.6806188523769379, + "reward_std": 0.09868648275732994, + "reward_std_all": 0.11285363510251045, + "rewards-max/forensics_sinkhorn_reward": 0.9684899151325226, + "rewards-max/forensics_strict_edge_reward": 0.3082154020667076, + "rewards-max/forensics_tiou_reward": 0.9477654695510864, + "rewards/forensics_sinkhorn_reward": 0.6104531288146973, + "rewards/forensics_strict_edge_reward": 0.021809647791087627, + "rewards/forensics_tiou_reward": 0.4205946624279022, + "step": 283 + }, + { + "completion_length": 22.828125, + "epoch": 0.8725038402457758, + "grad_norm": 4.542240619659424, + "kl": 0.29296875, + "learning_rate": 7.823076923076922e-07, + "loss": 0.0382, + "reward": 0.7625644505023956, + "reward_std": 0.055217018350958824, + "reward_std_all": 0.0525503046810627, + "rewards-max/forensics_sinkhorn_reward": 0.9378936886787415, + "rewards-max/forensics_strict_edge_reward": 0.24143806472420692, + "rewards-max/forensics_tiou_reward": 0.862334132194519, + "rewards/forensics_sinkhorn_reward": 0.7009402811527252, + "rewards/forensics_strict_edge_reward": 0.018557623028755188, + "rewards/forensics_tiou_reward": 0.43346475064754486, + "step": 284 + }, + { + "completion_length": 26.265625, + "epoch": 0.8755760368663594, + "grad_norm": 4.648009300231934, + "kl": 0.447265625, + "learning_rate": 7.815384615384615e-07, + "loss": 0.1065, + "reward": 0.7101974487304688, + "reward_std": 0.053088683634996414, + "reward_std_all": 0.07271947711706161, + "rewards-max/forensics_sinkhorn_reward": 0.8787192404270172, + "rewards-max/forensics_strict_edge_reward": 0.27216063253581524, + "rewards-max/forensics_tiou_reward": 0.7170387804508209, + "rewards/forensics_sinkhorn_reward": 0.5726018846035004, + "rewards/forensics_strict_edge_reward": 0.018229559529572725, + "rewards/forensics_tiou_reward": 0.315731406211853, + "step": 285 + }, + { + "completion_length": 24.96875, + "epoch": 0.8786482334869432, + "grad_norm": 6.58450984954834, + "kl": 0.31640625, + "learning_rate": 7.807692307692307e-07, + "loss": 0.0794, + "reward": 0.7139856219291687, + "reward_std": 0.0668281763792038, + "reward_std_all": 0.07946572825312614, + "rewards-max/forensics_sinkhorn_reward": 0.9461396038532257, + "rewards-max/forensics_strict_edge_reward": 0.2242102473974228, + "rewards-max/forensics_tiou_reward": 0.8993589282035828, + "rewards/forensics_sinkhorn_reward": 0.5816857218742371, + "rewards/forensics_strict_edge_reward": 0.014076043851673603, + "rewards/forensics_tiou_reward": 0.3369050920009613, + "step": 286 + }, + { + "completion_length": 22.953125, + "epoch": 0.8817204301075269, + "grad_norm": 5.222975254058838, + "kl": 0.39453125, + "learning_rate": 7.799999999999999e-07, + "loss": 0.0875, + "reward": 0.6936221122741699, + "reward_std": 0.06618125550448895, + "reward_std_all": 0.08748974185436964, + "rewards-max/forensics_sinkhorn_reward": 0.8927105665206909, + "rewards-max/forensics_strict_edge_reward": 0.2959351986646652, + "rewards-max/forensics_tiou_reward": 0.7913608253002167, + "rewards/forensics_sinkhorn_reward": 0.5938299149274826, + "rewards/forensics_strict_edge_reward": 0.0251399134285748, + "rewards/forensics_tiou_reward": 0.35446858406066895, + "step": 287 + }, + { + "completion_length": 25.671875, + "epoch": 0.8847926267281107, + "grad_norm": 5.3687310218811035, + "kl": 0.3232421875, + "learning_rate": 7.792307692307693e-07, + "loss": 0.0396, + "reward": 0.6802255213260651, + "reward_std": 0.0746820792555809, + "reward_std_all": 0.09616716578602791, + "rewards-max/forensics_sinkhorn_reward": 0.9052620828151703, + "rewards-max/forensics_strict_edge_reward": 0.10705345217138529, + "rewards-max/forensics_tiou_reward": 0.8115925192832947, + "rewards/forensics_sinkhorn_reward": 0.57325080037117, + "rewards/forensics_strict_edge_reward": 0.005281430785544217, + "rewards/forensics_tiou_reward": 0.30486370623111725, + "step": 288 + }, + { + "completion_length": 23.578125, + "epoch": 0.8878648233486943, + "grad_norm": 5.02519416809082, + "kl": 0.3642578125, + "learning_rate": 7.784615384615384e-07, + "loss": 0.2161, + "reward": 0.7066982388496399, + "reward_std": 0.06028667278587818, + "reward_std_all": 0.07740962132811546, + "rewards-max/forensics_sinkhorn_reward": 0.9646220505237579, + "rewards-max/forensics_strict_edge_reward": 0.08097248245030642, + "rewards-max/forensics_tiou_reward": 0.9355520606040955, + "rewards/forensics_sinkhorn_reward": 0.6132147312164307, + "rewards/forensics_strict_edge_reward": 0.00505592324770987, + "rewards/forensics_tiou_reward": 0.33694982528686523, + "step": 289 + }, + { + "completion_length": 28.875, + "epoch": 0.890937019969278, + "grad_norm": 4.741220474243164, + "kl": 0.375, + "learning_rate": 7.776923076923077e-07, + "loss": 0.1798, + "reward": 0.7008544206619263, + "reward_std": 0.08837309293448925, + "reward_std_all": 0.09317173436284065, + "rewards-max/forensics_sinkhorn_reward": 0.9655011892318726, + "rewards-max/forensics_strict_edge_reward": 0.2466004453599453, + "rewards-max/forensics_tiou_reward": 0.9407805800437927, + "rewards/forensics_sinkhorn_reward": 0.5938329994678497, + "rewards/forensics_strict_edge_reward": 0.016469838563352823, + "rewards/forensics_tiou_reward": 0.37957102060317993, + "step": 290 + }, + { + "completion_length": 25.90625, + "epoch": 0.8940092165898618, + "grad_norm": 4.419071674346924, + "kl": 0.322265625, + "learning_rate": 7.769230769230769e-07, + "loss": 0.1201, + "reward": 0.7274551093578339, + "reward_std": 0.07102775201201439, + "reward_std_all": 0.07709991559386253, + "rewards-max/forensics_sinkhorn_reward": 0.9648423790931702, + "rewards-max/forensics_strict_edge_reward": 0.31975166499614716, + "rewards-max/forensics_tiou_reward": 0.9327276051044464, + "rewards/forensics_sinkhorn_reward": 0.6543098390102386, + "rewards/forensics_strict_edge_reward": 0.0281339674256742, + "rewards/forensics_tiou_reward": 0.46311041712760925, + "step": 291 + }, + { + "completion_length": 23.34375, + "epoch": 0.8970814132104454, + "grad_norm": 5.074737071990967, + "kl": 0.27734375, + "learning_rate": 7.761538461538461e-07, + "loss": 0.0752, + "reward": 0.7126930058002472, + "reward_std": 0.09671108052134514, + "reward_std_all": 0.09430134668946266, + "rewards-max/forensics_sinkhorn_reward": 0.9560413062572479, + "rewards-max/forensics_strict_edge_reward": 0.18028822541236877, + "rewards-max/forensics_tiou_reward": 0.9123115837574005, + "rewards/forensics_sinkhorn_reward": 0.5965386629104614, + "rewards/forensics_strict_edge_reward": 0.01728414325043559, + "rewards/forensics_tiou_reward": 0.3651319444179535, + "step": 292 + }, + { + "completion_length": 27.15625, + "epoch": 0.9001536098310292, + "grad_norm": 4.676672458648682, + "kl": 0.27734375, + "learning_rate": 7.753846153846153e-07, + "loss": 0.1472, + "reward": 0.7366510033607483, + "reward_std": 0.08088576793670654, + "reward_std_all": 0.07730383798480034, + "rewards-max/forensics_sinkhorn_reward": 0.9683663845062256, + "rewards-max/forensics_strict_edge_reward": 0.039355408400297165, + "rewards-max/forensics_tiou_reward": 0.9422372877597809, + "rewards/forensics_sinkhorn_reward": 0.6166719198226929, + "rewards/forensics_strict_edge_reward": 0.0033536480041220784, + "rewards/forensics_tiou_reward": 0.41560471057891846, + "step": 293 + }, + { + "completion_length": 30.109375, + "epoch": 0.9032258064516129, + "grad_norm": 5.176265239715576, + "kl": 0.32421875, + "learning_rate": 7.746153846153845e-07, + "loss": 0.0505, + "reward": 0.6624761819839478, + "reward_std": 0.09937167540192604, + "reward_std_all": 0.11218783259391785, + "rewards-max/forensics_sinkhorn_reward": 0.9371714890003204, + "rewards-max/forensics_strict_edge_reward": 0.2757803685963154, + "rewards-max/forensics_tiou_reward": 0.8645956218242645, + "rewards/forensics_sinkhorn_reward": 0.5276483297348022, + "rewards/forensics_strict_edge_reward": 0.012450448120944202, + "rewards/forensics_tiou_reward": 0.3578186184167862, + "step": 294 + }, + { + "completion_length": 25.34375, + "epoch": 0.9062980030721967, + "grad_norm": 4.575304985046387, + "kl": 0.291015625, + "learning_rate": 7.738461538461539e-07, + "loss": 0.0928, + "reward": 0.7387601733207703, + "reward_std": 0.06150093860924244, + "reward_std_all": 0.06566666811704636, + "rewards-max/forensics_sinkhorn_reward": 0.9653928279876709, + "rewards-max/forensics_strict_edge_reward": 0.3808731082826853, + "rewards-max/forensics_tiou_reward": 0.9171061515808105, + "rewards/forensics_sinkhorn_reward": 0.6286410093307495, + "rewards/forensics_strict_edge_reward": 0.02290543491835706, + "rewards/forensics_tiou_reward": 0.40016302466392517, + "step": 295 + }, + { + "completion_length": 25.84375, + "epoch": 0.9093701996927803, + "grad_norm": 4.864716053009033, + "kl": 0.2744140625, + "learning_rate": 7.730769230769231e-07, + "loss": 0.2032, + "reward": 0.6962189972400665, + "reward_std": 0.10988016054034233, + "reward_std_all": 0.10990787297487259, + "rewards-max/forensics_sinkhorn_reward": 0.9133947789669037, + "rewards-max/forensics_strict_edge_reward": 0.20421066135168076, + "rewards-max/forensics_tiou_reward": 0.8168717622756958, + "rewards/forensics_sinkhorn_reward": 0.5853303670883179, + "rewards/forensics_strict_edge_reward": 0.01256085210479796, + "rewards/forensics_tiou_reward": 0.3206677883863449, + "step": 296 + }, + { + "completion_length": 27.0625, + "epoch": 0.9124423963133641, + "grad_norm": 6.470847129821777, + "kl": 0.349609375, + "learning_rate": 7.723076923076923e-07, + "loss": 0.2032, + "reward": 0.6920434534549713, + "reward_std": 0.09951072186231613, + "reward_std_all": 0.10605645179748535, + "rewards-max/forensics_sinkhorn_reward": 0.8577607572078705, + "rewards-max/forensics_strict_edge_reward": 0.1442453321069479, + "rewards-max/forensics_tiou_reward": 0.7557521164417267, + "rewards/forensics_sinkhorn_reward": 0.5160533785820007, + "rewards/forensics_strict_edge_reward": 0.010414977208711207, + "rewards/forensics_tiou_reward": 0.33125317096710205, + "step": 297 + }, + { + "completion_length": 26.453125, + "epoch": 0.9155145929339478, + "grad_norm": 4.880552768707275, + "kl": 0.36328125, + "learning_rate": 7.715384615384615e-07, + "loss": 0.0867, + "reward": 0.7367932796478271, + "reward_std": 0.06621407717466354, + "reward_std_all": 0.0663176141679287, + "rewards-max/forensics_sinkhorn_reward": 0.9663507044315338, + "rewards-max/forensics_strict_edge_reward": 0.2675460949540138, + "rewards-max/forensics_tiou_reward": 0.9408946335315704, + "rewards/forensics_sinkhorn_reward": 0.6275045275688171, + "rewards/forensics_strict_edge_reward": 0.014373746700584888, + "rewards/forensics_tiou_reward": 0.3981892764568329, + "step": 298 + }, + { + "completion_length": 26.328125, + "epoch": 0.9185867895545314, + "grad_norm": 5.761527061462402, + "kl": 0.341796875, + "learning_rate": 7.707692307692307e-07, + "loss": 0.0828, + "reward": 0.6984827518463135, + "reward_std": 0.062392085790634155, + "reward_std_all": 0.08300604298710823, + "rewards-max/forensics_sinkhorn_reward": 0.9741306602954865, + "rewards-max/forensics_strict_edge_reward": 0.10804803110659122, + "rewards-max/forensics_tiou_reward": 0.9538995325565338, + "rewards/forensics_sinkhorn_reward": 0.6025853455066681, + "rewards/forensics_strict_edge_reward": 0.006986595690250397, + "rewards/forensics_tiou_reward": 0.4512675702571869, + "step": 299 + }, + { + "completion_length": 24.46875, + "epoch": 0.9216589861751152, + "grad_norm": 5.32295560836792, + "kl": 0.3642578125, + "learning_rate": 7.699999999999999e-07, + "loss": 0.1425, + "reward": 0.7269871830940247, + "reward_std": 0.08539652824401855, + "reward_std_all": 0.08385958895087242, + "rewards-max/forensics_sinkhorn_reward": 0.9656636118888855, + "rewards-max/forensics_strict_edge_reward": 0.46338212490081787, + "rewards-max/forensics_tiou_reward": 0.9417959749698639, + "rewards/forensics_sinkhorn_reward": 0.6359501779079437, + "rewards/forensics_strict_edge_reward": 0.02970899548381567, + "rewards/forensics_tiou_reward": 0.4012828916311264, + "step": 300 + }, + { + "completion_length": 22.6875, + "epoch": 0.9247311827956989, + "grad_norm": 4.8707051277160645, + "kl": 0.3212890625, + "learning_rate": 7.692307692307693e-07, + "loss": 0.0998, + "reward": 0.7511292994022369, + "reward_std": 0.05759917199611664, + "reward_std_all": 0.05675631761550903, + "rewards-max/forensics_sinkhorn_reward": 0.949347198009491, + "rewards-max/forensics_strict_edge_reward": 0.25752775371074677, + "rewards-max/forensics_tiou_reward": 0.896720290184021, + "rewards/forensics_sinkhorn_reward": 0.6762354373931885, + "rewards/forensics_strict_edge_reward": 0.014949758537113667, + "rewards/forensics_tiou_reward": 0.4537089914083481, + "step": 301 + }, + { + "completion_length": 21.71875, + "epoch": 0.9278033794162827, + "grad_norm": 5.352730751037598, + "kl": 0.283203125, + "learning_rate": 7.684615384615384e-07, + "loss": 0.1095, + "reward": 0.7209315001964569, + "reward_std": 0.09202454797923565, + "reward_std_all": 0.09006195515394211, + "rewards-max/forensics_sinkhorn_reward": 0.9648624658584595, + "rewards-max/forensics_strict_edge_reward": 0.3801230601966381, + "rewards-max/forensics_tiou_reward": 0.9351328313350677, + "rewards/forensics_sinkhorn_reward": 0.6766051352024078, + "rewards/forensics_strict_edge_reward": 0.04098324850201607, + "rewards/forensics_tiou_reward": 0.45464181900024414, + "step": 302 + }, + { + "completion_length": 24.03125, + "epoch": 0.9308755760368663, + "grad_norm": 3.7228689193725586, + "kl": 0.32421875, + "learning_rate": 7.676923076923077e-07, + "loss": 0.1228, + "reward": 0.7246046364307404, + "reward_std": 0.05923820938915014, + "reward_std_all": 0.06768497452139854, + "rewards-max/forensics_sinkhorn_reward": 0.9235231876373291, + "rewards-max/forensics_strict_edge_reward": 0.24057345755863935, + "rewards-max/forensics_tiou_reward": 0.802823007106781, + "rewards/forensics_sinkhorn_reward": 0.5996642410755157, + "rewards/forensics_strict_edge_reward": 0.01724496198585257, + "rewards/forensics_tiou_reward": 0.36693549156188965, + "step": 303 + }, + { + "completion_length": 22.46875, + "epoch": 0.9339477726574501, + "grad_norm": 5.2105302810668945, + "kl": 0.3525390625, + "learning_rate": 7.669230769230769e-07, + "loss": 0.0389, + "reward": 0.7224811017513275, + "reward_std": 0.06564026512205601, + "reward_std_all": 0.07426079548895359, + "rewards-max/forensics_sinkhorn_reward": 0.9821589589118958, + "rewards-max/forensics_strict_edge_reward": 0.5996727645397186, + "rewards-max/forensics_tiou_reward": 0.9802457988262177, + "rewards/forensics_sinkhorn_reward": 0.664673924446106, + "rewards/forensics_strict_edge_reward": 0.03379734791815281, + "rewards/forensics_tiou_reward": 0.4387475699186325, + "step": 304 + }, + { + "completion_length": 25.203125, + "epoch": 0.9370199692780338, + "grad_norm": 5.146493911743164, + "kl": 0.3251953125, + "learning_rate": 7.661538461538461e-07, + "loss": 0.1403, + "reward": 0.7003929615020752, + "reward_std": 0.08599345199763775, + "reward_std_all": 0.09145430848002434, + "rewards-max/forensics_sinkhorn_reward": 0.9636951088905334, + "rewards-max/forensics_strict_edge_reward": 0.34775497764348984, + "rewards-max/forensics_tiou_reward": 0.9099462330341339, + "rewards/forensics_sinkhorn_reward": 0.6244082748889923, + "rewards/forensics_strict_edge_reward": 0.02390663861297071, + "rewards/forensics_tiou_reward": 0.38980430364608765, + "step": 305 + }, + { + "completion_length": 24.546875, + "epoch": 0.9400921658986175, + "grad_norm": 5.043769836425781, + "kl": 0.3037109375, + "learning_rate": 7.653846153846153e-07, + "loss": 0.1517, + "reward": 0.6934415698051453, + "reward_std": 0.09526025131344795, + "reward_std_all": 0.09894277900457382, + "rewards-max/forensics_sinkhorn_reward": 0.9592207074165344, + "rewards-max/forensics_strict_edge_reward": 0.12415425665676594, + "rewards-max/forensics_tiou_reward": 0.9189711809158325, + "rewards/forensics_sinkhorn_reward": 0.5865125358104706, + "rewards/forensics_strict_edge_reward": 0.006049115443602204, + "rewards/forensics_tiou_reward": 0.3866477757692337, + "step": 306 + }, + { + "completion_length": 24.78125, + "epoch": 0.9431643625192012, + "grad_norm": 5.066221237182617, + "kl": 0.3037109375, + "learning_rate": 7.646153846153845e-07, + "loss": 0.1531, + "reward": 0.7685718238353729, + "reward_std": 0.05410110950469971, + "reward_std_all": 0.0502421036362648, + "rewards-max/forensics_sinkhorn_reward": 0.9572390913963318, + "rewards-max/forensics_strict_edge_reward": 0.43321889638900757, + "rewards-max/forensics_tiou_reward": 0.9098191857337952, + "rewards/forensics_sinkhorn_reward": 0.6763054728507996, + "rewards/forensics_strict_edge_reward": 0.02969723753631115, + "rewards/forensics_tiou_reward": 0.43936440348625183, + "step": 307 + }, + { + "completion_length": 26.46875, + "epoch": 0.946236559139785, + "grad_norm": 4.861406326293945, + "kl": 0.369140625, + "learning_rate": 7.638461538461539e-07, + "loss": 0.1537, + "reward": 0.6728861629962921, + "reward_std": 0.07244201749563217, + "reward_std_all": 0.09836094826459885, + "rewards-max/forensics_sinkhorn_reward": 0.950255811214447, + "rewards-max/forensics_strict_edge_reward": 0.23113226145505905, + "rewards-max/forensics_tiou_reward": 0.8918741941452026, + "rewards/forensics_sinkhorn_reward": 0.5712105333805084, + "rewards/forensics_strict_edge_reward": 0.011677945032715797, + "rewards/forensics_tiou_reward": 0.3493334949016571, + "step": 308 + }, + { + "completion_length": 25.90625, + "epoch": 0.9493087557603687, + "grad_norm": 5.339264869689941, + "kl": 0.3134765625, + "learning_rate": 7.630769230769231e-07, + "loss": 0.1422, + "reward": 0.7045566737651825, + "reward_std": 0.07387732341885567, + "reward_std_all": 0.08789612352848053, + "rewards-max/forensics_sinkhorn_reward": 0.8898962140083313, + "rewards-max/forensics_strict_edge_reward": 0.2754755327478051, + "rewards-max/forensics_tiou_reward": 0.802675873041153, + "rewards/forensics_sinkhorn_reward": 0.6015571653842926, + "rewards/forensics_strict_edge_reward": 0.013527949922718108, + "rewards/forensics_tiou_reward": 0.38355641067028046, + "step": 309 + }, + { + "completion_length": 24.65625, + "epoch": 0.9523809523809523, + "grad_norm": 4.529643535614014, + "kl": 0.3046875, + "learning_rate": 7.623076923076923e-07, + "loss": 0.0908, + "reward": 0.773051917552948, + "reward_std": 0.0329817533493042, + "reward_std_all": 0.03369777463376522, + "rewards-max/forensics_sinkhorn_reward": 0.9440655708312988, + "rewards-max/forensics_strict_edge_reward": 0.07652523089200258, + "rewards-max/forensics_tiou_reward": 0.8813478946685791, + "rewards/forensics_sinkhorn_reward": 0.6768611669540405, + "rewards/forensics_strict_edge_reward": 0.007688293058890849, + "rewards/forensics_tiou_reward": 0.4521648734807968, + "step": 310 + }, + { + "completion_length": 23.125, + "epoch": 0.9554531490015361, + "grad_norm": 4.393594741821289, + "kl": 0.27294921875, + "learning_rate": 7.615384615384615e-07, + "loss": 0.0814, + "reward": 0.6669122576713562, + "reward_std": 0.11300359666347504, + "reward_std_all": 0.12015179544687271, + "rewards-max/forensics_sinkhorn_reward": 0.9314441680908203, + "rewards-max/forensics_strict_edge_reward": 0.3200232982635498, + "rewards-max/forensics_tiou_reward": 0.8667590618133545, + "rewards/forensics_sinkhorn_reward": 0.5754857659339905, + "rewards/forensics_strict_edge_reward": 0.021529422141611576, + "rewards/forensics_tiou_reward": 0.3182229548692703, + "step": 311 + }, + { + "completion_length": 24.296875, + "epoch": 0.9585253456221198, + "grad_norm": 5.664923667907715, + "kl": 0.2978515625, + "learning_rate": 7.607692307692307e-07, + "loss": 0.1221, + "reward": 0.7510881423950195, + "reward_std": 0.058933280408382416, + "reward_std_all": 0.05811823159456253, + "rewards-max/forensics_sinkhorn_reward": 0.9009714424610138, + "rewards-max/forensics_strict_edge_reward": 0.21949207037687302, + "rewards-max/forensics_tiou_reward": 0.8033139407634735, + "rewards/forensics_sinkhorn_reward": 0.6511667668819427, + "rewards/forensics_strict_edge_reward": 0.01278314832597971, + "rewards/forensics_tiou_reward": 0.4010283052921295, + "step": 312 + }, + { + "completion_length": 22.78125, + "epoch": 0.9615975422427036, + "grad_norm": 4.6637749671936035, + "kl": 0.451171875, + "learning_rate": 7.599999999999999e-07, + "loss": 0.065, + "reward": 0.6741020381450653, + "reward_std": 0.10536402463912964, + "reward_std_all": 0.11588501185178757, + "rewards-max/forensics_sinkhorn_reward": 0.9474435150623322, + "rewards-max/forensics_strict_edge_reward": 0.3397783748805523, + "rewards-max/forensics_tiou_reward": 0.8911609947681427, + "rewards/forensics_sinkhorn_reward": 0.6070590615272522, + "rewards/forensics_strict_edge_reward": 0.03718672273680568, + "rewards/forensics_tiou_reward": 0.36937354505062103, + "step": 313 + }, + { + "completion_length": 21.546875, + "epoch": 0.9646697388632872, + "grad_norm": 5.5299601554870605, + "kl": 0.294921875, + "learning_rate": 7.592307692307692e-07, + "loss": 0.1334, + "reward": 0.7525681257247925, + "reward_std": 0.03700352553278208, + "reward_std_all": 0.046439029276371, + "rewards-max/forensics_sinkhorn_reward": 0.9649516940116882, + "rewards-max/forensics_strict_edge_reward": 0.24218390509486198, + "rewards-max/forensics_tiou_reward": 0.9351536929607391, + "rewards/forensics_sinkhorn_reward": 0.6704866290092468, + "rewards/forensics_strict_edge_reward": 0.012836641166359186, + "rewards/forensics_tiou_reward": 0.39709655940532684, + "step": 314 + }, + { + "completion_length": 23.359375, + "epoch": 0.967741935483871, + "grad_norm": 4.054599285125732, + "kl": 0.3818359375, + "learning_rate": 7.584615384615384e-07, + "loss": 0.1287, + "reward": 0.6817216277122498, + "reward_std": 0.10694526880979538, + "reward_std_all": 0.11474675312638283, + "rewards-max/forensics_sinkhorn_reward": 0.9357359111309052, + "rewards-max/forensics_strict_edge_reward": 0.05869557219557464, + "rewards-max/forensics_tiou_reward": 0.864985316991806, + "rewards/forensics_sinkhorn_reward": 0.6013718843460083, + "rewards/forensics_strict_edge_reward": 0.004610923082509544, + "rewards/forensics_tiou_reward": 0.35950691998004913, + "step": 315 + }, + { + "completion_length": 25.59375, + "epoch": 0.9708141321044547, + "grad_norm": 5.375238418579102, + "kl": 0.322265625, + "learning_rate": 7.576923076923077e-07, + "loss": 0.0792, + "reward": 0.6997182965278625, + "reward_std": 0.0854736939072609, + "reward_std_all": 0.09249943122267723, + "rewards-max/forensics_sinkhorn_reward": 0.9523607492446899, + "rewards-max/forensics_strict_edge_reward": 0.2455889694392681, + "rewards-max/forensics_tiou_reward": 0.9048992991447449, + "rewards/forensics_sinkhorn_reward": 0.5549071431159973, + "rewards/forensics_strict_edge_reward": 0.012343284673988819, + "rewards/forensics_tiou_reward": 0.394333153963089, + "step": 316 + }, + { + "completion_length": 21.65625, + "epoch": 0.9738863287250384, + "grad_norm": 5.053328514099121, + "kl": 0.3115234375, + "learning_rate": 7.569230769230769e-07, + "loss": -0.0282, + "reward": 0.7809976935386658, + "reward_std": 0.029330095276236534, + "reward_std_all": 0.029442020691931248, + "rewards-max/forensics_sinkhorn_reward": 0.9720262289047241, + "rewards-max/forensics_strict_edge_reward": 0.2168741226196289, + "rewards-max/forensics_tiou_reward": 0.9562920331954956, + "rewards/forensics_sinkhorn_reward": 0.7027194201946259, + "rewards/forensics_strict_edge_reward": 0.01355644827708602, + "rewards/forensics_tiou_reward": 0.44426998496055603, + "step": 317 + }, + { + "completion_length": 24.25, + "epoch": 0.9769585253456221, + "grad_norm": 6.010988235473633, + "kl": 0.3125, + "learning_rate": 7.561538461538461e-07, + "loss": -0.007, + "reward": 0.7300270795822144, + "reward_std": 0.04776727594435215, + "reward_std_all": 0.06615010276436806, + "rewards-max/forensics_sinkhorn_reward": 0.9361280202865601, + "rewards-max/forensics_strict_edge_reward": 0.11217646300792694, + "rewards-max/forensics_tiou_reward": 0.8650317490100861, + "rewards/forensics_sinkhorn_reward": 0.65818852186203, + "rewards/forensics_strict_edge_reward": 0.00878047151491046, + "rewards/forensics_tiou_reward": 0.4126182645559311, + "step": 318 + }, + { + "completion_length": 23.484375, + "epoch": 0.9800307219662059, + "grad_norm": 4.487271308898926, + "kl": 0.31640625, + "learning_rate": 7.553846153846154e-07, + "loss": 0.0196, + "reward": 0.7150301337242126, + "reward_std": 0.09007314220070839, + "reward_std_all": 0.08949360623955727, + "rewards-max/forensics_sinkhorn_reward": 0.9525832235813141, + "rewards-max/forensics_strict_edge_reward": 0.2997382767498493, + "rewards-max/forensics_tiou_reward": 0.8831390142440796, + "rewards/forensics_sinkhorn_reward": 0.5875089466571808, + "rewards/forensics_strict_edge_reward": 0.013578426791355014, + "rewards/forensics_tiou_reward": 0.3610745668411255, + "step": 319 + }, + { + "completion_length": 23.09375, + "epoch": 0.9831029185867896, + "grad_norm": 5.209065914154053, + "kl": 0.3154296875, + "learning_rate": 7.546153846153845e-07, + "loss": 0.1204, + "reward": 0.749356746673584, + "reward_std": 0.04611128196120262, + "reward_std_all": 0.05443473160266876, + "rewards-max/forensics_sinkhorn_reward": 0.9700175523757935, + "rewards-max/forensics_strict_edge_reward": 0.3845585584640503, + "rewards-max/forensics_tiou_reward": 0.9482194781303406, + "rewards/forensics_sinkhorn_reward": 0.7214811444282532, + "rewards/forensics_strict_edge_reward": 0.04080923553556204, + "rewards/forensics_tiou_reward": 0.5150417536497116, + "step": 320 + }, + { + "completion_length": 24.375, + "epoch": 0.9861751152073732, + "grad_norm": 4.143730640411377, + "kl": 0.3203125, + "learning_rate": 7.538461538461538e-07, + "loss": 0.1166, + "reward": 0.7199468016624451, + "reward_std": 0.06837109103798866, + "reward_std_all": 0.0768524594604969, + "rewards-max/forensics_sinkhorn_reward": 0.9795160889625549, + "rewards-max/forensics_strict_edge_reward": 0.6620846390724182, + "rewards-max/forensics_tiou_reward": 0.9765048921108246, + "rewards/forensics_sinkhorn_reward": 0.5999864637851715, + "rewards/forensics_strict_edge_reward": 0.03276384063065052, + "rewards/forensics_tiou_reward": 0.40455175936222076, + "step": 321 + }, + { + "completion_length": 26.015625, + "epoch": 0.989247311827957, + "grad_norm": 5.414015769958496, + "kl": 0.361328125, + "learning_rate": 7.530769230769231e-07, + "loss": 0.1367, + "reward": 0.7139945328235626, + "reward_std": 0.07174751535058022, + "reward_std_all": 0.0781993418931961, + "rewards-max/forensics_sinkhorn_reward": 0.9654916822910309, + "rewards-max/forensics_strict_edge_reward": 0.48420320451259613, + "rewards-max/forensics_tiou_reward": 0.9403140842914581, + "rewards/forensics_sinkhorn_reward": 0.6132687926292419, + "rewards/forensics_strict_edge_reward": 0.03634472191333771, + "rewards/forensics_tiou_reward": 0.3843799978494644, + "step": 322 + }, + { + "completion_length": 24.515625, + "epoch": 0.9923195084485407, + "grad_norm": 4.899774074554443, + "kl": 0.3251953125, + "learning_rate": 7.523076923076923e-07, + "loss": 0.0989, + "reward": 0.7027028203010559, + "reward_std": 0.09656979888677597, + "reward_std_all": 0.09876926988363266, + "rewards-max/forensics_sinkhorn_reward": 0.941217452287674, + "rewards-max/forensics_strict_edge_reward": 0.22357474267482758, + "rewards-max/forensics_tiou_reward": 0.8804083168506622, + "rewards/forensics_sinkhorn_reward": 0.60868901014328, + "rewards/forensics_strict_edge_reward": 0.019822613801807165, + "rewards/forensics_tiou_reward": 0.37126821279525757, + "step": 323 + }, + { + "completion_length": 24.65625, + "epoch": 0.9953917050691244, + "grad_norm": 5.173336029052734, + "kl": 0.34765625, + "learning_rate": 7.515384615384615e-07, + "loss": 0.1522, + "reward": 0.7732657492160797, + "reward_std": 0.046203747391700745, + "reward_std_all": 0.04298561066389084, + "rewards-max/forensics_sinkhorn_reward": 0.9713297784328461, + "rewards-max/forensics_strict_edge_reward": 0.21456831507384777, + "rewards-max/forensics_tiou_reward": 0.9540721476078033, + "rewards/forensics_sinkhorn_reward": 0.6768431067466736, + "rewards/forensics_strict_edge_reward": 0.019588002003729343, + "rewards/forensics_tiou_reward": 0.5212017297744751, + "step": 324 + }, + { + "completion_length": 26.5, + "epoch": 0.9984639016897081, + "grad_norm": 5.377089977264404, + "kl": 0.3173828125, + "learning_rate": 7.507692307692307e-07, + "loss": 0.1892, + "reward": 0.7544234097003937, + "reward_std": 0.05196477100253105, + "reward_std_all": 0.05191406235098839, + "rewards-max/forensics_sinkhorn_reward": 0.9460877478122711, + "rewards-max/forensics_strict_edge_reward": 0.30727942287921906, + "rewards-max/forensics_tiou_reward": 0.896590530872345, + "rewards/forensics_sinkhorn_reward": 0.6752963364124298, + "rewards/forensics_strict_edge_reward": 0.02253210637718439, + "rewards/forensics_tiou_reward": 0.4259446710348129, + "step": 325 + }, + { + "completion_length": 19.0, + "epoch": 1.0, + "grad_norm": 3.7793543338775635, + "kl": 0.25390625, + "learning_rate": 7.5e-07, + "loss": -0.049, + "reward": 0.7629415392875671, + "reward_std": 0.023196620866656303, + "reward_std_all": 0.03680272400379181, + "rewards-max/forensics_sinkhorn_reward": 0.6226826906204224, + "rewards-max/forensics_strict_edge_reward": 0.013494607992470264, + "rewards-max/forensics_tiou_reward": 0.6360785365104675, + "rewards/forensics_sinkhorn_reward": 0.47761034965515137, + "rewards/forensics_strict_edge_reward": 0.0038718681316822767, + "rewards/forensics_tiou_reward": 0.3307580053806305, + "step": 326 + }, + { + "completion_length": 29.828125, + "epoch": 1.0030721966205838, + "grad_norm": 4.0019989013671875, + "kl": 0.3515625, + "learning_rate": 7.492307692307692e-07, + "loss": 0.1039, + "reward": 0.7075544595718384, + "reward_std": 0.048847222700715065, + "reward_std_all": 0.07012537121772766, + "rewards-max/forensics_sinkhorn_reward": 0.9471245408058167, + "rewards-max/forensics_strict_edge_reward": 0.24680054932832718, + "rewards-max/forensics_tiou_reward": 0.8915128111839294, + "rewards/forensics_sinkhorn_reward": 0.5866128206253052, + "rewards/forensics_strict_edge_reward": 0.023038113489747047, + "rewards/forensics_tiou_reward": 0.4108453840017319, + "step": 327 + }, + { + "completion_length": 24.875, + "epoch": 1.0061443932411673, + "grad_norm": 5.28901481628418, + "kl": 0.341796875, + "learning_rate": 7.484615384615384e-07, + "loss": 0.106, + "reward": 0.7175799310207367, + "reward_std": 0.08775641024112701, + "reward_std_all": 0.0886281244456768, + "rewards-max/forensics_sinkhorn_reward": 0.9821582734584808, + "rewards-max/forensics_strict_edge_reward": 0.6076286435127258, + "rewards-max/forensics_tiou_reward": 0.9840353429317474, + "rewards/forensics_sinkhorn_reward": 0.6396852731704712, + "rewards/forensics_strict_edge_reward": 0.04493985138833523, + "rewards/forensics_tiou_reward": 0.43842628598213196, + "step": 328 + }, + { + "completion_length": 29.953125, + "epoch": 1.0092165898617511, + "grad_norm": 5.482292175292969, + "kl": 0.314453125, + "learning_rate": 7.476923076923077e-07, + "loss": 0.1647, + "reward": 0.7185519933700562, + "reward_std": 0.07944702729582787, + "reward_std_all": 0.08249113708734512, + "rewards-max/forensics_sinkhorn_reward": 0.9776462912559509, + "rewards-max/forensics_strict_edge_reward": 0.34675130993127823, + "rewards-max/forensics_tiou_reward": 0.9637279808521271, + "rewards/forensics_sinkhorn_reward": 0.651394248008728, + "rewards/forensics_strict_edge_reward": 0.03136633709073067, + "rewards/forensics_tiou_reward": 0.4728996157646179, + "step": 329 + }, + { + "completion_length": 27.015625, + "epoch": 1.012288786482335, + "grad_norm": 4.000410556793213, + "kl": 0.3134765625, + "learning_rate": 7.469230769230769e-07, + "loss": 0.059, + "reward": 0.7388818860054016, + "reward_std": 0.07890772819519043, + "reward_std_all": 0.07505331933498383, + "rewards-max/forensics_sinkhorn_reward": 0.9795801043510437, + "rewards-max/forensics_strict_edge_reward": 0.6340298652648926, + "rewards-max/forensics_tiou_reward": 0.9741640388965607, + "rewards/forensics_sinkhorn_reward": 0.5928322672843933, + "rewards/forensics_strict_edge_reward": 0.033123794943094254, + "rewards/forensics_tiou_reward": 0.455412894487381, + "step": 330 + }, + { + "completion_length": 28.28125, + "epoch": 1.0153609831029187, + "grad_norm": 4.439173698425293, + "kl": 0.353515625, + "learning_rate": 7.461538461538461e-07, + "loss": 0.136, + "reward": 0.7144956290721893, + "reward_std": 0.06884869933128357, + "reward_std_all": 0.08088387548923492, + "rewards-max/forensics_sinkhorn_reward": 0.9543624520301819, + "rewards-max/forensics_strict_edge_reward": 0.24513883888721466, + "rewards-max/forensics_tiou_reward": 0.8998555839061737, + "rewards/forensics_sinkhorn_reward": 0.6447970271110535, + "rewards/forensics_strict_edge_reward": 0.02038629539310932, + "rewards/forensics_tiou_reward": 0.47167766094207764, + "step": 331 + }, + { + "completion_length": 29.0625, + "epoch": 1.0184331797235022, + "grad_norm": 4.693502426147461, + "kl": 0.3154296875, + "learning_rate": 7.453846153846154e-07, + "loss": 0.171, + "reward": 0.6865242719650269, + "reward_std": 0.0839446634054184, + "reward_std_all": 0.0978422649204731, + "rewards-max/forensics_sinkhorn_reward": 0.9691823422908783, + "rewards-max/forensics_strict_edge_reward": 0.1789868026971817, + "rewards-max/forensics_tiou_reward": 0.944126546382904, + "rewards/forensics_sinkhorn_reward": 0.5580706894397736, + "rewards/forensics_strict_edge_reward": 0.00838437769562006, + "rewards/forensics_tiou_reward": 0.377357617020607, + "step": 332 + }, + { + "completion_length": 23.25, + "epoch": 1.021505376344086, + "grad_norm": 4.815438270568848, + "kl": 0.3232421875, + "learning_rate": 7.446153846153845e-07, + "loss": 0.0157, + "reward": 0.7750045359134674, + "reward_std": 0.034660330042243004, + "reward_std_all": 0.035521147307008505, + "rewards-max/forensics_sinkhorn_reward": 0.9409091174602509, + "rewards-max/forensics_strict_edge_reward": 0.028995479457080364, + "rewards-max/forensics_tiou_reward": 0.8620310425758362, + "rewards/forensics_sinkhorn_reward": 0.6636941730976105, + "rewards/forensics_strict_edge_reward": 0.003393936436623335, + "rewards/forensics_tiou_reward": 0.4481171816587448, + "step": 333 + }, + { + "completion_length": 25.984375, + "epoch": 1.0245775729646698, + "grad_norm": 4.833273410797119, + "kl": 0.3330078125, + "learning_rate": 7.438461538461538e-07, + "loss": 0.1393, + "reward": 0.7082056105136871, + "reward_std": 0.07370427250862122, + "reward_std_all": 0.08605899848043919, + "rewards-max/forensics_sinkhorn_reward": 0.9651411473751068, + "rewards-max/forensics_strict_edge_reward": 0.36554911732673645, + "rewards-max/forensics_tiou_reward": 0.9378240406513214, + "rewards/forensics_sinkhorn_reward": 0.6331778168678284, + "rewards/forensics_strict_edge_reward": 0.04250917211174965, + "rewards/forensics_tiou_reward": 0.4198363721370697, + "step": 334 + }, + { + "completion_length": 26.25, + "epoch": 1.0276497695852536, + "grad_norm": 6.203298568725586, + "kl": 0.400390625, + "learning_rate": 7.43076923076923e-07, + "loss": 0.0156, + "reward": 0.6787010133266449, + "reward_std": 0.059032166842371225, + "reward_std_all": 0.08920978009700775, + "rewards-max/forensics_sinkhorn_reward": 0.9443674385547638, + "rewards-max/forensics_strict_edge_reward": 0.17549064382910728, + "rewards-max/forensics_tiou_reward": 0.8785075843334198, + "rewards/forensics_sinkhorn_reward": 0.5824621021747589, + "rewards/forensics_strict_edge_reward": 0.0076753339963033795, + "rewards/forensics_tiou_reward": 0.39375950396060944, + "step": 335 + }, + { + "completion_length": 26.828125, + "epoch": 1.0307219662058371, + "grad_norm": 5.918509006500244, + "kl": 0.3935546875, + "learning_rate": 7.423076923076923e-07, + "loss": 0.1194, + "reward": 0.7528422772884369, + "reward_std": 0.04746235907077789, + "reward_std_all": 0.051986876875162125, + "rewards-max/forensics_sinkhorn_reward": 0.9711869955062866, + "rewards-max/forensics_strict_edge_reward": 0.24109935015439987, + "rewards-max/forensics_tiou_reward": 0.9515853524208069, + "rewards/forensics_sinkhorn_reward": 0.6236337423324585, + "rewards/forensics_strict_edge_reward": 0.018091090954840183, + "rewards/forensics_tiou_reward": 0.4577018469572067, + "step": 336 + }, + { + "completion_length": 21.953125, + "epoch": 1.033794162826421, + "grad_norm": 5.646511554718018, + "kl": 0.47265625, + "learning_rate": 7.415384615384616e-07, + "loss": 0.0373, + "reward": 0.7673309743404388, + "reward_std": 0.036532243713736534, + "reward_std_all": 0.03866518288850784, + "rewards-max/forensics_sinkhorn_reward": 0.8851254880428314, + "rewards-max/forensics_strict_edge_reward": 0.05350061599165201, + "rewards-max/forensics_tiou_reward": 0.7176854610443115, + "rewards/forensics_sinkhorn_reward": 0.6771005690097809, + "rewards/forensics_strict_edge_reward": 0.0035218041157349944, + "rewards/forensics_tiou_reward": 0.3697197586297989, + "step": 337 + }, + { + "completion_length": 24.40625, + "epoch": 1.0368663594470047, + "grad_norm": 5.7692131996154785, + "kl": 0.3251953125, + "learning_rate": 7.407692307692307e-07, + "loss": 0.0347, + "reward": 0.7718167006969452, + "reward_std": 0.04271078296005726, + "reward_std_all": 0.041441697627305984, + "rewards-max/forensics_sinkhorn_reward": 0.9580941796302795, + "rewards-max/forensics_strict_edge_reward": 0.22323003225028515, + "rewards-max/forensics_tiou_reward": 0.8977168798446655, + "rewards/forensics_sinkhorn_reward": 0.6609190404415131, + "rewards/forensics_strict_edge_reward": 0.012805871665477753, + "rewards/forensics_tiou_reward": 0.3810769319534302, + "step": 338 + }, + { + "completion_length": 25.328125, + "epoch": 1.0399385560675882, + "grad_norm": 4.7784318923950195, + "kl": 0.31640625, + "learning_rate": 7.4e-07, + "loss": 0.1182, + "reward": 0.7367228269577026, + "reward_std": 0.06238082982599735, + "reward_std_all": 0.0668216161429882, + "rewards-max/forensics_sinkhorn_reward": 0.9735032618045807, + "rewards-max/forensics_strict_edge_reward": 0.36220263689756393, + "rewards-max/forensics_tiou_reward": 0.956063836812973, + "rewards/forensics_sinkhorn_reward": 0.6229265332221985, + "rewards/forensics_strict_edge_reward": 0.026850621215999126, + "rewards/forensics_tiou_reward": 0.4425107091665268, + "step": 339 + }, + { + "completion_length": 25.671875, + "epoch": 1.043010752688172, + "grad_norm": 4.482677936553955, + "kl": 0.24951171875, + "learning_rate": 7.392307692307692e-07, + "loss": 0.1244, + "reward": 0.7618813216686249, + "reward_std": 0.047925807535648346, + "reward_std_all": 0.0477024931460619, + "rewards-max/forensics_sinkhorn_reward": 0.9319343268871307, + "rewards-max/forensics_strict_edge_reward": 0.22602936252951622, + "rewards-max/forensics_tiou_reward": 0.8660261631011963, + "rewards/forensics_sinkhorn_reward": 0.6732661426067352, + "rewards/forensics_strict_edge_reward": 0.015148819889873266, + "rewards/forensics_tiou_reward": 0.43928657472133636, + "step": 340 + }, + { + "completion_length": 22.359375, + "epoch": 1.0460829493087558, + "grad_norm": 5.131192207336426, + "kl": 0.294921875, + "learning_rate": 7.384615384615384e-07, + "loss": 0.1111, + "reward": 0.7520062327384949, + "reward_std": 0.025750347413122654, + "reward_std_all": 0.039821796119213104, + "rewards-max/forensics_sinkhorn_reward": 0.9451664686203003, + "rewards-max/forensics_strict_edge_reward": 0.19309754483401775, + "rewards-max/forensics_tiou_reward": 0.8890703618526459, + "rewards/forensics_sinkhorn_reward": 0.6573351323604584, + "rewards/forensics_strict_edge_reward": 0.013728287420235574, + "rewards/forensics_tiou_reward": 0.4289247542619705, + "step": 341 + }, + { + "completion_length": 25.515625, + "epoch": 1.0491551459293396, + "grad_norm": 4.759541034698486, + "kl": 0.3134765625, + "learning_rate": 7.376923076923076e-07, + "loss": 0.1383, + "reward": 0.7035471796989441, + "reward_std": 0.0808527059853077, + "reward_std_all": 0.09190060198307037, + "rewards-max/forensics_sinkhorn_reward": 0.9506345689296722, + "rewards-max/forensics_strict_edge_reward": 0.4581599831581116, + "rewards-max/forensics_tiou_reward": 0.8963432312011719, + "rewards/forensics_sinkhorn_reward": 0.5952533483505249, + "rewards/forensics_strict_edge_reward": 0.03700068173930049, + "rewards/forensics_tiou_reward": 0.42522579431533813, + "step": 342 + }, + { + "completion_length": 24.84375, + "epoch": 1.0522273425499231, + "grad_norm": 5.4345927238464355, + "kl": 0.2763671875, + "learning_rate": 7.369230769230769e-07, + "loss": 0.1835, + "reward": 0.6800759136676788, + "reward_std": 0.10477724298834801, + "reward_std_all": 0.11094288527965546, + "rewards-max/forensics_sinkhorn_reward": 0.9820021688938141, + "rewards-max/forensics_strict_edge_reward": 0.25994260609149933, + "rewards-max/forensics_tiou_reward": 0.9792389571666718, + "rewards/forensics_sinkhorn_reward": 0.564499706029892, + "rewards/forensics_strict_edge_reward": 0.016122108325362206, + "rewards/forensics_tiou_reward": 0.38218703866004944, + "step": 343 + }, + { + "completion_length": 26.46875, + "epoch": 1.055299539170507, + "grad_norm": 4.73542594909668, + "kl": 0.25146484375, + "learning_rate": 7.361538461538461e-07, + "loss": 0.1077, + "reward": 0.725156307220459, + "reward_std": 0.07487682159990072, + "reward_std_all": 0.07634837180376053, + "rewards-max/forensics_sinkhorn_reward": 0.9752906262874603, + "rewards-max/forensics_strict_edge_reward": 0.561550498008728, + "rewards-max/forensics_tiou_reward": 0.9641872048377991, + "rewards/forensics_sinkhorn_reward": 0.5995948910713196, + "rewards/forensics_strict_edge_reward": 0.036112419329583645, + "rewards/forensics_tiou_reward": 0.4544921964406967, + "step": 344 + }, + { + "completion_length": 23.859375, + "epoch": 1.0583717357910907, + "grad_norm": 5.810218811035156, + "kl": 0.2919921875, + "learning_rate": 7.353846153846154e-07, + "loss": 0.0616, + "reward": 0.7510368824005127, + "reward_std": 0.02878657542169094, + "reward_std_all": 0.04087079502642155, + "rewards-max/forensics_sinkhorn_reward": 0.9765608310699463, + "rewards-max/forensics_strict_edge_reward": 0.477030485868454, + "rewards-max/forensics_tiou_reward": 0.9667108058929443, + "rewards/forensics_sinkhorn_reward": 0.6763992309570312, + "rewards/forensics_strict_edge_reward": 0.034286096692085266, + "rewards/forensics_tiou_reward": 0.48928235471248627, + "step": 345 + }, + { + "completion_length": 26.390625, + "epoch": 1.0614439324116745, + "grad_norm": 4.59224271774292, + "kl": 0.3017578125, + "learning_rate": 7.346153846153846e-07, + "loss": 0.1533, + "reward": 0.7500328123569489, + "reward_std": 0.05039358697831631, + "reward_std_all": 0.05457146465778351, + "rewards-max/forensics_sinkhorn_reward": 0.955653578042984, + "rewards-max/forensics_strict_edge_reward": 0.3079311400651932, + "rewards-max/forensics_tiou_reward": 0.9137133061885834, + "rewards/forensics_sinkhorn_reward": 0.6335264444351196, + "rewards/forensics_strict_edge_reward": 0.029879581183195114, + "rewards/forensics_tiou_reward": 0.4567214995622635, + "step": 346 + }, + { + "completion_length": 24.6875, + "epoch": 1.064516129032258, + "grad_norm": 4.505839824676514, + "kl": 0.28662109375, + "learning_rate": 7.338461538461538e-07, + "loss": 0.0941, + "reward": 0.7628391683101654, + "reward_std": 0.042934706434607506, + "reward_std_all": 0.04374571330845356, + "rewards-max/forensics_sinkhorn_reward": 0.9745825529098511, + "rewards-max/forensics_strict_edge_reward": 0.402368500828743, + "rewards-max/forensics_tiou_reward": 0.9579017758369446, + "rewards/forensics_sinkhorn_reward": 0.6691254675388336, + "rewards/forensics_strict_edge_reward": 0.022019704803824425, + "rewards/forensics_tiou_reward": 0.43055470287799835, + "step": 347 + }, + { + "completion_length": 24.140625, + "epoch": 1.0675883256528418, + "grad_norm": 5.400676250457764, + "kl": 0.3154296875, + "learning_rate": 7.33076923076923e-07, + "loss": 0.0857, + "reward": 0.7430866658687592, + "reward_std": 0.043824924156069756, + "reward_std_all": 0.054085325449705124, + "rewards-max/forensics_sinkhorn_reward": 0.9694579243659973, + "rewards-max/forensics_strict_edge_reward": 0.5258456766605377, + "rewards-max/forensics_tiou_reward": 0.9453162550926208, + "rewards/forensics_sinkhorn_reward": 0.689431369304657, + "rewards/forensics_strict_edge_reward": 0.02842133864760399, + "rewards/forensics_tiou_reward": 0.42517146468162537, + "step": 348 + }, + { + "completion_length": 23.75, + "epoch": 1.0706605222734256, + "grad_norm": 5.247318267822266, + "kl": 0.3564453125, + "learning_rate": 7.323076923076923e-07, + "loss": 0.0279, + "reward": 0.7485235333442688, + "reward_std": 0.05044532660394907, + "reward_std_all": 0.055078293196856976, + "rewards-max/forensics_sinkhorn_reward": 0.9677152037620544, + "rewards-max/forensics_strict_edge_reward": 0.2738724797964096, + "rewards-max/forensics_tiou_reward": 0.9499778747558594, + "rewards/forensics_sinkhorn_reward": 0.7046493291854858, + "rewards/forensics_strict_edge_reward": 0.028104862198233604, + "rewards/forensics_tiou_reward": 0.5216944962739944, + "step": 349 + }, + { + "completion_length": 25.140625, + "epoch": 1.0737327188940091, + "grad_norm": 5.637718200683594, + "kl": 0.298828125, + "learning_rate": 7.315384615384616e-07, + "loss": 0.0386, + "reward": 0.7632519006729126, + "reward_std": 0.040037893457338214, + "reward_std_all": 0.041819028090685606, + "rewards-max/forensics_sinkhorn_reward": 0.9833661019802094, + "rewards-max/forensics_strict_edge_reward": 0.8959447741508484, + "rewards-max/forensics_tiou_reward": 0.9872135519981384, + "rewards/forensics_sinkhorn_reward": 0.7371451556682587, + "rewards/forensics_strict_edge_reward": 0.049379006028175354, + "rewards/forensics_tiou_reward": 0.5165356397628784, + "step": 350 + }, + { + "completion_length": 27.09375, + "epoch": 1.076804915514593, + "grad_norm": 5.6277666091918945, + "kl": 0.291015625, + "learning_rate": 7.307692307692307e-07, + "loss": 0.1275, + "reward": 0.7012459933757782, + "reward_std": 0.06986146047711372, + "reward_std_all": 0.0851006731390953, + "rewards-max/forensics_sinkhorn_reward": 0.97532719373703, + "rewards-max/forensics_strict_edge_reward": 0.44371652603149414, + "rewards-max/forensics_tiou_reward": 0.9538633525371552, + "rewards/forensics_sinkhorn_reward": 0.5873594284057617, + "rewards/forensics_strict_edge_reward": 0.041800722014158964, + "rewards/forensics_tiou_reward": 0.40360817313194275, + "step": 351 + }, + { + "completion_length": 24.328125, + "epoch": 1.0798771121351767, + "grad_norm": 5.259487152099609, + "kl": 0.333984375, + "learning_rate": 7.3e-07, + "loss": 0.0004, + "reward": 0.6910519599914551, + "reward_std": 0.1039629839360714, + "reward_std_all": 0.11096694320440292, + "rewards-max/forensics_sinkhorn_reward": 0.9689927101135254, + "rewards-max/forensics_strict_edge_reward": 0.3834627643227577, + "rewards-max/forensics_tiou_reward": 0.9380765557289124, + "rewards/forensics_sinkhorn_reward": 0.6032471060752869, + "rewards/forensics_strict_edge_reward": 0.041495457757264376, + "rewards/forensics_tiou_reward": 0.39016999304294586, + "step": 352 + }, + { + "completion_length": 23.09375, + "epoch": 1.0829493087557605, + "grad_norm": 6.055822849273682, + "kl": 0.4267578125, + "learning_rate": 7.292307692307692e-07, + "loss": 0.1683, + "reward": 0.7566451728343964, + "reward_std": 0.04159712977707386, + "reward_std_all": 0.0486851017922163, + "rewards-max/forensics_sinkhorn_reward": 0.9249313771724701, + "rewards-max/forensics_strict_edge_reward": 0.08205431327223778, + "rewards-max/forensics_tiou_reward": 0.8505389988422394, + "rewards/forensics_sinkhorn_reward": 0.6640781760215759, + "rewards/forensics_strict_edge_reward": 0.007551812916062772, + "rewards/forensics_tiou_reward": 0.38465408980846405, + "step": 353 + }, + { + "completion_length": 24.953125, + "epoch": 1.086021505376344, + "grad_norm": 5.247381687164307, + "kl": 0.2724609375, + "learning_rate": 7.284615384615384e-07, + "loss": 0.114, + "reward": 0.6956038773059845, + "reward_std": 0.03617946617305279, + "reward_std_all": 0.0709722600877285, + "rewards-max/forensics_sinkhorn_reward": 0.9493478238582611, + "rewards-max/forensics_strict_edge_reward": 0.1805974068120122, + "rewards-max/forensics_tiou_reward": 0.8945891261100769, + "rewards/forensics_sinkhorn_reward": 0.5557155907154083, + "rewards/forensics_strict_edge_reward": 0.010189819033257663, + "rewards/forensics_tiou_reward": 0.3379211127758026, + "step": 354 + }, + { + "completion_length": 24.359375, + "epoch": 1.0890937019969278, + "grad_norm": 4.135838031768799, + "kl": 0.34375, + "learning_rate": 7.276923076923076e-07, + "loss": -0.0382, + "reward": 0.7206481397151947, + "reward_std": 0.07631796598434448, + "reward_std_all": 0.0794636681675911, + "rewards-max/forensics_sinkhorn_reward": 0.933102160692215, + "rewards-max/forensics_strict_edge_reward": 0.46168795228004456, + "rewards-max/forensics_tiou_reward": 0.859121561050415, + "rewards/forensics_sinkhorn_reward": 0.6782407462596893, + "rewards/forensics_strict_edge_reward": 0.029701403342187405, + "rewards/forensics_tiou_reward": 0.4355769604444504, + "step": 355 + }, + { + "completion_length": 23.90625, + "epoch": 1.0921658986175116, + "grad_norm": 7.658079147338867, + "kl": 0.50390625, + "learning_rate": 7.269230769230769e-07, + "loss": 0.0858, + "reward": 0.7175653874874115, + "reward_std": 0.08422408998012543, + "reward_std_all": 0.08450832217931747, + "rewards-max/forensics_sinkhorn_reward": 0.9220317304134369, + "rewards-max/forensics_strict_edge_reward": 0.12378453835844994, + "rewards-max/forensics_tiou_reward": 0.8350796699523926, + "rewards/forensics_sinkhorn_reward": 0.6371900141239166, + "rewards/forensics_strict_edge_reward": 0.01163874170742929, + "rewards/forensics_tiou_reward": 0.38099297881126404, + "step": 356 + }, + { + "completion_length": 25.71875, + "epoch": 1.0952380952380953, + "grad_norm": 6.934156894683838, + "kl": 0.390625, + "learning_rate": 7.261538461538461e-07, + "loss": -0.0374, + "reward": 0.7824216485023499, + "reward_std": 0.027158435434103012, + "reward_std_all": 0.02738768234848976, + "rewards-max/forensics_sinkhorn_reward": 0.9583394527435303, + "rewards-max/forensics_strict_edge_reward": 0.09009118005633354, + "rewards-max/forensics_tiou_reward": 0.9239681959152222, + "rewards/forensics_sinkhorn_reward": 0.7577270269393921, + "rewards/forensics_strict_edge_reward": 0.008239396382123232, + "rewards/forensics_tiou_reward": 0.566335141658783, + "step": 357 + }, + { + "completion_length": 26.03125, + "epoch": 1.098310291858679, + "grad_norm": 5.123674392700195, + "kl": 0.31494140625, + "learning_rate": 7.253846153846154e-07, + "loss": 0.027, + "reward": 0.69444739818573, + "reward_std": 0.11147411912679672, + "reward_std_all": 0.10936400294303894, + "rewards-max/forensics_sinkhorn_reward": 0.9647327363491058, + "rewards-max/forensics_strict_edge_reward": 0.4533289223909378, + "rewards-max/forensics_tiou_reward": 0.9455278813838959, + "rewards/forensics_sinkhorn_reward": 0.598064661026001, + "rewards/forensics_strict_edge_reward": 0.032799310982227325, + "rewards/forensics_tiou_reward": 0.40280452370643616, + "step": 358 + }, + { + "completion_length": 26.0625, + "epoch": 1.1013824884792627, + "grad_norm": 4.985512733459473, + "kl": 0.28759765625, + "learning_rate": 7.246153846153846e-07, + "loss": 0.1905, + "reward": 0.7587298154830933, + "reward_std": 0.048470244626514614, + "reward_std_all": 0.04896996356546879, + "rewards-max/forensics_sinkhorn_reward": 0.9558030664920807, + "rewards-max/forensics_strict_edge_reward": 0.33558932319283485, + "rewards-max/forensics_tiou_reward": 0.9179340302944183, + "rewards/forensics_sinkhorn_reward": 0.622359424829483, + "rewards/forensics_strict_edge_reward": 0.021408693864941597, + "rewards/forensics_tiou_reward": 0.4084511697292328, + "step": 359 + }, + { + "completion_length": 24.5625, + "epoch": 1.1044546850998465, + "grad_norm": 4.5964813232421875, + "kl": 0.3623046875, + "learning_rate": 7.238461538461538e-07, + "loss": 0.147, + "reward": 0.7166472673416138, + "reward_std": 0.10470868833363056, + "reward_std_all": 0.09906181693077087, + "rewards-max/forensics_sinkhorn_reward": 0.9769200384616852, + "rewards-max/forensics_strict_edge_reward": 0.37414632737636566, + "rewards-max/forensics_tiou_reward": 0.9600577652454376, + "rewards/forensics_sinkhorn_reward": 0.6167051196098328, + "rewards/forensics_strict_edge_reward": 0.018373331055045128, + "rewards/forensics_tiou_reward": 0.4617867171764374, + "step": 360 + }, + { + "completion_length": 24.0625, + "epoch": 1.10752688172043, + "grad_norm": 6.0490641593933105, + "kl": 0.3505859375, + "learning_rate": 7.23076923076923e-07, + "loss": 0.0729, + "reward": 0.759967714548111, + "reward_std": 0.051214030012488365, + "reward_std_all": 0.05036102421581745, + "rewards-max/forensics_sinkhorn_reward": 0.9845107793807983, + "rewards-max/forensics_strict_edge_reward": 0.8351600170135498, + "rewards-max/forensics_tiou_reward": 0.9882367551326752, + "rewards/forensics_sinkhorn_reward": 0.6797473430633545, + "rewards/forensics_strict_edge_reward": 0.06026815064251423, + "rewards/forensics_tiou_reward": 0.5209641307592392, + "step": 361 + }, + { + "completion_length": 26.140625, + "epoch": 1.1105990783410138, + "grad_norm": 5.360951900482178, + "kl": 0.3515625, + "learning_rate": 7.223076923076922e-07, + "loss": 0.204, + "reward": 0.7138960361480713, + "reward_std": 0.11642644181847572, + "reward_std_all": 0.1082780584692955, + "rewards-max/forensics_sinkhorn_reward": 0.9655773639678955, + "rewards-max/forensics_strict_edge_reward": 0.41016243398189545, + "rewards-max/forensics_tiou_reward": 0.9405366778373718, + "rewards/forensics_sinkhorn_reward": 0.6123556792736053, + "rewards/forensics_strict_edge_reward": 0.04820937663316727, + "rewards/forensics_tiou_reward": 0.4638567864894867, + "step": 362 + }, + { + "completion_length": 27.46875, + "epoch": 1.1136712749615976, + "grad_norm": 4.389161109924316, + "kl": 0.2822265625, + "learning_rate": 7.215384615384616e-07, + "loss": 0.0483, + "reward": 0.7668002545833588, + "reward_std": 0.03525091800838709, + "reward_std_all": 0.037401288747787476, + "rewards-max/forensics_sinkhorn_reward": 0.9720605611801147, + "rewards-max/forensics_strict_edge_reward": 0.2785215973854065, + "rewards-max/forensics_tiou_reward": 0.9567857980728149, + "rewards/forensics_sinkhorn_reward": 0.6795475482940674, + "rewards/forensics_strict_edge_reward": 0.023465372622013092, + "rewards/forensics_tiou_reward": 0.48277056217193604, + "step": 363 + }, + { + "completion_length": 26.90625, + "epoch": 1.1167434715821813, + "grad_norm": 5.115853309631348, + "kl": 0.359375, + "learning_rate": 7.207692307692307e-07, + "loss": 0.0822, + "reward": 0.7163601517677307, + "reward_std": 0.07835433818399906, + "reward_std_all": 0.08863957598805428, + "rewards-max/forensics_sinkhorn_reward": 0.9608883857727051, + "rewards-max/forensics_strict_edge_reward": 0.5761621594429016, + "rewards-max/forensics_tiou_reward": 0.9279641211032867, + "rewards/forensics_sinkhorn_reward": 0.6670863032341003, + "rewards/forensics_strict_edge_reward": 0.0424058772623539, + "rewards/forensics_tiou_reward": 0.4452367275953293, + "step": 364 + }, + { + "completion_length": 22.109375, + "epoch": 1.119815668202765, + "grad_norm": 4.757989883422852, + "kl": 0.513671875, + "learning_rate": 7.2e-07, + "loss": 0.1049, + "reward": 0.772159069776535, + "reward_std": 0.032136221416294575, + "reward_std_all": 0.0361167686060071, + "rewards-max/forensics_sinkhorn_reward": 0.9349735081195831, + "rewards-max/forensics_strict_edge_reward": 0.15878573805093765, + "rewards-max/forensics_tiou_reward": 0.8471101820468903, + "rewards/forensics_sinkhorn_reward": 0.7347148358821869, + "rewards/forensics_strict_edge_reward": 0.008344759698957205, + "rewards/forensics_tiou_reward": 0.4657517373561859, + "step": 365 + }, + { + "completion_length": 22.34375, + "epoch": 1.1228878648233487, + "grad_norm": 6.7500152587890625, + "kl": 0.2939453125, + "learning_rate": 7.192307692307692e-07, + "loss": 0.0173, + "reward": 0.7519676387310028, + "reward_std": 0.03834763285703957, + "reward_std_all": 0.046811847016215324, + "rewards-max/forensics_sinkhorn_reward": 0.9554490745067596, + "rewards-max/forensics_strict_edge_reward": 0.28200098127126694, + "rewards-max/forensics_tiou_reward": 0.9147976636886597, + "rewards/forensics_sinkhorn_reward": 0.7225804924964905, + "rewards/forensics_strict_edge_reward": 0.01772477477788925, + "rewards/forensics_tiou_reward": 0.48285403847694397, + "step": 366 + }, + { + "completion_length": 25.796875, + "epoch": 1.1259600614439325, + "grad_norm": 4.47206449508667, + "kl": 0.328125, + "learning_rate": 7.184615384615384e-07, + "loss": 0.1206, + "reward": 0.7069931626319885, + "reward_std": 0.07009567692875862, + "reward_std_all": 0.08543651923537254, + "rewards-max/forensics_sinkhorn_reward": 0.9531813561916351, + "rewards-max/forensics_strict_edge_reward": 0.3780496418476105, + "rewards-max/forensics_tiou_reward": 0.9133832454681396, + "rewards/forensics_sinkhorn_reward": 0.6267960071563721, + "rewards/forensics_strict_edge_reward": 0.020016880240291357, + "rewards/forensics_tiou_reward": 0.41191457211971283, + "step": 367 + }, + { + "completion_length": 27.265625, + "epoch": 1.129032258064516, + "grad_norm": 5.71698522567749, + "kl": 0.3642578125, + "learning_rate": 7.176923076923077e-07, + "loss": 0.0838, + "reward": 0.7366614043712616, + "reward_std": 0.09178143739700317, + "reward_std_all": 0.08481721580028534, + "rewards-max/forensics_sinkhorn_reward": 0.9638145565986633, + "rewards-max/forensics_strict_edge_reward": 0.41806212067604065, + "rewards-max/forensics_tiou_reward": 0.9379869103431702, + "rewards/forensics_sinkhorn_reward": 0.6424045562744141, + "rewards/forensics_strict_edge_reward": 0.022085627540946007, + "rewards/forensics_tiou_reward": 0.4168125092983246, + "step": 368 + }, + { + "completion_length": 27.375, + "epoch": 1.1321044546850998, + "grad_norm": 4.1147780418396, + "kl": 0.2919921875, + "learning_rate": 7.169230769230768e-07, + "loss": 0.0577, + "reward": 0.7352429628372192, + "reward_std": 0.046666860580444336, + "reward_std_all": 0.06131262145936489, + "rewards-max/forensics_sinkhorn_reward": 0.9737140536308289, + "rewards-max/forensics_strict_edge_reward": 0.2022939845919609, + "rewards-max/forensics_tiou_reward": 0.952908843755722, + "rewards/forensics_sinkhorn_reward": 0.6114528179168701, + "rewards/forensics_strict_edge_reward": 0.01663836487568915, + "rewards/forensics_tiou_reward": 0.4743090271949768, + "step": 369 + }, + { + "completion_length": 25.703125, + "epoch": 1.1351766513056836, + "grad_norm": 5.483392715454102, + "kl": 0.33984375, + "learning_rate": 7.161538461538461e-07, + "loss": 0.2936, + "reward": 0.7125152349472046, + "reward_std": 0.07319260388612747, + "reward_std_all": 0.0816918145865202, + "rewards-max/forensics_sinkhorn_reward": 0.9679828882217407, + "rewards-max/forensics_strict_edge_reward": 0.21340578515082598, + "rewards-max/forensics_tiou_reward": 0.9491170942783356, + "rewards/forensics_sinkhorn_reward": 0.5654825270175934, + "rewards/forensics_strict_edge_reward": 0.026250038063153625, + "rewards/forensics_tiou_reward": 0.4252696931362152, + "step": 370 + }, + { + "completion_length": 22.3125, + "epoch": 1.1382488479262673, + "grad_norm": 5.443384170532227, + "kl": 0.3369140625, + "learning_rate": 7.153846153846154e-07, + "loss": 0.1775, + "reward": 0.7158059179782867, + "reward_std": 0.046892717480659485, + "reward_std_all": 0.06751546636223793, + "rewards-max/forensics_sinkhorn_reward": 0.9835941791534424, + "rewards-max/forensics_strict_edge_reward": 0.6043699085712433, + "rewards-max/forensics_tiou_reward": 0.9842854142189026, + "rewards/forensics_sinkhorn_reward": 0.6243298649787903, + "rewards/forensics_strict_edge_reward": 0.04612768720835447, + "rewards/forensics_tiou_reward": 0.37897029519081116, + "step": 371 + }, + { + "completion_length": 26.34375, + "epoch": 1.141321044546851, + "grad_norm": 4.864809513092041, + "kl": 0.3203125, + "learning_rate": 7.146153846153846e-07, + "loss": 0.0954, + "reward": 0.7395501136779785, + "reward_std": 0.05180380865931511, + "reward_std_all": 0.05961660481989384, + "rewards-max/forensics_sinkhorn_reward": 0.9754343926906586, + "rewards-max/forensics_strict_edge_reward": 0.6149742305278778, + "rewards-max/forensics_tiou_reward": 0.9668407142162323, + "rewards/forensics_sinkhorn_reward": 0.6597909927368164, + "rewards/forensics_strict_edge_reward": 0.046017758548259735, + "rewards/forensics_tiou_reward": 0.4821809381246567, + "step": 372 + }, + { + "completion_length": 24.6875, + "epoch": 1.1443932411674347, + "grad_norm": 5.348287105560303, + "kl": 0.3173828125, + "learning_rate": 7.138461538461538e-07, + "loss": 0.0689, + "reward": 0.7566236555576324, + "reward_std": 0.06723160669207573, + "reward_std_all": 0.06244294159114361, + "rewards-max/forensics_sinkhorn_reward": 0.9668228924274445, + "rewards-max/forensics_strict_edge_reward": 0.2980264276266098, + "rewards-max/forensics_tiou_reward": 0.9464608728885651, + "rewards/forensics_sinkhorn_reward": 0.6707931756973267, + "rewards/forensics_strict_edge_reward": 0.03466887306421995, + "rewards/forensics_tiou_reward": 0.5235595107078552, + "step": 373 + }, + { + "completion_length": 24.453125, + "epoch": 1.1474654377880185, + "grad_norm": 4.7249226570129395, + "kl": 0.35546875, + "learning_rate": 7.13076923076923e-07, + "loss": 0.0546, + "reward": 0.7429163455963135, + "reward_std": 0.03987284656614065, + "reward_std_all": 0.05073003098368645, + "rewards-max/forensics_sinkhorn_reward": 0.9695785343647003, + "rewards-max/forensics_strict_edge_reward": 0.8010868430137634, + "rewards-max/forensics_tiou_reward": 0.9562957286834717, + "rewards/forensics_sinkhorn_reward": 0.6223343908786774, + "rewards/forensics_strict_edge_reward": 0.05037710443139076, + "rewards/forensics_tiou_reward": 0.41377922892570496, + "step": 374 + }, + { + "completion_length": 24.59375, + "epoch": 1.1505376344086022, + "grad_norm": 5.379456520080566, + "kl": 0.4404296875, + "learning_rate": 7.123076923076922e-07, + "loss": 0.1568, + "reward": 0.718727171421051, + "reward_std": 0.0660115722566843, + "reward_std_all": 0.0774443419650197, + "rewards-max/forensics_sinkhorn_reward": 0.9344833791255951, + "rewards-max/forensics_strict_edge_reward": 0.1769215944223106, + "rewards-max/forensics_tiou_reward": 0.8582904934883118, + "rewards/forensics_sinkhorn_reward": 0.6290086805820465, + "rewards/forensics_strict_edge_reward": 0.023987318039871752, + "rewards/forensics_tiou_reward": 0.41265858709812164, + "step": 375 + }, + { + "completion_length": 26.140625, + "epoch": 1.1536098310291858, + "grad_norm": 5.063214302062988, + "kl": 0.3212890625, + "learning_rate": 7.115384615384616e-07, + "loss": 0.0856, + "reward": 0.674953430891037, + "reward_std": 0.04963114857673645, + "reward_std_all": 0.0921156108379364, + "rewards-max/forensics_sinkhorn_reward": 0.9654513001441956, + "rewards-max/forensics_strict_edge_reward": 0.3845585584640503, + "rewards-max/forensics_tiou_reward": 0.941197395324707, + "rewards/forensics_sinkhorn_reward": 0.5880894362926483, + "rewards/forensics_strict_edge_reward": 0.044471679255366325, + "rewards/forensics_tiou_reward": 0.41803722083568573, + "step": 376 + }, + { + "completion_length": 23.765625, + "epoch": 1.1566820276497696, + "grad_norm": 6.81565523147583, + "kl": 0.447265625, + "learning_rate": 7.107692307692308e-07, + "loss": 0.1452, + "reward": 0.7527598142623901, + "reward_std": 0.04838861897587776, + "reward_std_all": 0.05260143242776394, + "rewards-max/forensics_sinkhorn_reward": 0.9240032434463501, + "rewards-max/forensics_strict_edge_reward": 0.31934893876314163, + "rewards-max/forensics_tiou_reward": 0.8222153186798096, + "rewards/forensics_sinkhorn_reward": 0.6750464737415314, + "rewards/forensics_strict_edge_reward": 0.030937922652810812, + "rewards/forensics_tiou_reward": 0.41113921999931335, + "step": 377 + }, + { + "completion_length": 24.546875, + "epoch": 1.1597542242703534, + "grad_norm": 3.976186513900757, + "kl": 0.4267578125, + "learning_rate": 7.1e-07, + "loss": 0.1159, + "reward": 0.7457360327243805, + "reward_std": 0.05102535896003246, + "reward_std_all": 0.05793955735862255, + "rewards-max/forensics_sinkhorn_reward": 0.9853318929672241, + "rewards-max/forensics_strict_edge_reward": 0.6720768958330154, + "rewards-max/forensics_tiou_reward": 0.9897423088550568, + "rewards/forensics_sinkhorn_reward": 0.6725297570228577, + "rewards/forensics_strict_edge_reward": 0.048887886106967926, + "rewards/forensics_tiou_reward": 0.48173151910305023, + "step": 378 + }, + { + "completion_length": 22.484375, + "epoch": 1.1628264208909371, + "grad_norm": 4.719587326049805, + "kl": 0.4931640625, + "learning_rate": 7.092307692307692e-07, + "loss": 0.1097, + "reward": 0.7207637131214142, + "reward_std": 0.06434758007526398, + "reward_std_all": 0.07780451141297817, + "rewards-max/forensics_sinkhorn_reward": 0.9324541389942169, + "rewards-max/forensics_strict_edge_reward": 0.5373985171318054, + "rewards-max/forensics_tiou_reward": 0.8636946976184845, + "rewards/forensics_sinkhorn_reward": 0.6459113657474518, + "rewards/forensics_strict_edge_reward": 0.03133158013224602, + "rewards/forensics_tiou_reward": 0.4107026755809784, + "step": 379 + }, + { + "completion_length": 21.828125, + "epoch": 1.1658986175115207, + "grad_norm": 5.129236221313477, + "kl": 0.388671875, + "learning_rate": 7.084615384615384e-07, + "loss": 0.0224, + "reward": 0.710564136505127, + "reward_std": 0.09177155420184135, + "reward_std_all": 0.09219847992062569, + "rewards-max/forensics_sinkhorn_reward": 0.9609622061252594, + "rewards-max/forensics_strict_edge_reward": 0.13043400272727013, + "rewards-max/forensics_tiou_reward": 0.9330959618091583, + "rewards/forensics_sinkhorn_reward": 0.6755671203136444, + "rewards/forensics_strict_edge_reward": 0.005892978748306632, + "rewards/forensics_tiou_reward": 0.4576268792152405, + "step": 380 + }, + { + "completion_length": 24.515625, + "epoch": 1.1689708141321045, + "grad_norm": 5.421178340911865, + "kl": 0.283203125, + "learning_rate": 7.076923076923077e-07, + "loss": 0.1331, + "reward": 0.7530010640621185, + "reward_std": 0.03784307837486267, + "reward_std_all": 0.04687019623816013, + "rewards-max/forensics_sinkhorn_reward": 0.9468851983547211, + "rewards-max/forensics_strict_edge_reward": 0.23880668729543686, + "rewards-max/forensics_tiou_reward": 0.894121378660202, + "rewards/forensics_sinkhorn_reward": 0.6674286723136902, + "rewards/forensics_strict_edge_reward": 0.01607374637387693, + "rewards/forensics_tiou_reward": 0.43969279527664185, + "step": 381 + }, + { + "completion_length": 22.421875, + "epoch": 1.1720430107526882, + "grad_norm": 4.985270977020264, + "kl": 0.322265625, + "learning_rate": 7.069230769230768e-07, + "loss": 0.0897, + "reward": 0.7319947481155396, + "reward_std": 0.06093113496899605, + "reward_std_all": 0.06485772877931595, + "rewards-max/forensics_sinkhorn_reward": 0.9216109812259674, + "rewards-max/forensics_strict_edge_reward": 0.22657608112785965, + "rewards-max/forensics_tiou_reward": 0.8533684611320496, + "rewards/forensics_sinkhorn_reward": 0.6504931449890137, + "rewards/forensics_strict_edge_reward": 0.015613495750585571, + "rewards/forensics_tiou_reward": 0.40397100150585175, + "step": 382 + }, + { + "completion_length": 24.34375, + "epoch": 1.1751152073732718, + "grad_norm": 4.991052627563477, + "kl": 0.3271484375, + "learning_rate": 7.061538461538462e-07, + "loss": -0.0445, + "reward": 0.7656194567680359, + "reward_std": 0.050472479313611984, + "reward_std_all": 0.049011390656232834, + "rewards-max/forensics_sinkhorn_reward": 0.9742712080478668, + "rewards-max/forensics_strict_edge_reward": 0.51341712474823, + "rewards-max/forensics_tiou_reward": 0.9603660404682159, + "rewards/forensics_sinkhorn_reward": 0.7162416577339172, + "rewards/forensics_strict_edge_reward": 0.052710771560668945, + "rewards/forensics_tiou_reward": 0.49783481657505035, + "step": 383 + }, + { + "completion_length": 27.84375, + "epoch": 1.1781874039938556, + "grad_norm": 4.600424766540527, + "kl": 0.298828125, + "learning_rate": 7.053846153846154e-07, + "loss": 0.0798, + "reward": 0.7364495098590851, + "reward_std": 0.0842756386846304, + "reward_std_all": 0.07957530580461025, + "rewards-max/forensics_sinkhorn_reward": 0.976000040769577, + "rewards-max/forensics_strict_edge_reward": 0.484669953584671, + "rewards-max/forensics_tiou_reward": 0.9629189074039459, + "rewards/forensics_sinkhorn_reward": 0.6802840828895569, + "rewards/forensics_strict_edge_reward": 0.03252525441348553, + "rewards/forensics_tiou_reward": 0.4855533540248871, + "step": 384 + }, + { + "completion_length": 21.65625, + "epoch": 1.1812596006144394, + "grad_norm": 5.47316312789917, + "kl": 0.3212890625, + "learning_rate": 7.046153846153846e-07, + "loss": 0.0783, + "reward": 0.7384134232997894, + "reward_std": 0.06308000162243843, + "reward_std_all": 0.06860711798071861, + "rewards-max/forensics_sinkhorn_reward": 0.9677366614341736, + "rewards-max/forensics_strict_edge_reward": 0.4495786130428314, + "rewards-max/forensics_tiou_reward": 0.9440277516841888, + "rewards/forensics_sinkhorn_reward": 0.6810123324394226, + "rewards/forensics_strict_edge_reward": 0.045176587998867035, + "rewards/forensics_tiou_reward": 0.4563407003879547, + "step": 385 + }, + { + "completion_length": 21.59375, + "epoch": 1.1843317972350231, + "grad_norm": 6.102799892425537, + "kl": 0.27490234375, + "learning_rate": 7.038461538461539e-07, + "loss": -0.0206, + "reward": 0.7356136739253998, + "reward_std": 0.028012202586978674, + "reward_std_all": 0.047119155526161194, + "rewards-max/forensics_sinkhorn_reward": 0.9450488984584808, + "rewards-max/forensics_strict_edge_reward": 0.17275525629520416, + "rewards-max/forensics_tiou_reward": 0.896712601184845, + "rewards/forensics_sinkhorn_reward": 0.6779840886592865, + "rewards/forensics_strict_edge_reward": 0.01789298700168729, + "rewards/forensics_tiou_reward": 0.43732836842536926, + "step": 386 + }, + { + "completion_length": 23.5, + "epoch": 1.1874039938556067, + "grad_norm": 5.711805820465088, + "kl": 0.31494140625, + "learning_rate": 7.03076923076923e-07, + "loss": 0.0348, + "reward": 0.7693705558776855, + "reward_std": 0.04875185713171959, + "reward_std_all": 0.04625207372009754, + "rewards-max/forensics_sinkhorn_reward": 0.9680677056312561, + "rewards-max/forensics_strict_edge_reward": 0.2925925850868225, + "rewards-max/forensics_tiou_reward": 0.9503014385700226, + "rewards/forensics_sinkhorn_reward": 0.7241709530353546, + "rewards/forensics_strict_edge_reward": 0.013654426671564579, + "rewards/forensics_tiou_reward": 0.48750725388526917, + "step": 387 + }, + { + "completion_length": 22.15625, + "epoch": 1.1904761904761905, + "grad_norm": 4.420766353607178, + "kl": 0.3388671875, + "learning_rate": 7.023076923076922e-07, + "loss": 0.0153, + "reward": 0.7828540802001953, + "reward_std": 0.03513702191412449, + "reward_std_all": 0.03309620823711157, + "rewards-max/forensics_sinkhorn_reward": 0.9622016847133636, + "rewards-max/forensics_strict_edge_reward": 0.3932463005185127, + "rewards-max/forensics_tiou_reward": 0.9181552827358246, + "rewards/forensics_sinkhorn_reward": 0.7273237109184265, + "rewards/forensics_strict_edge_reward": 0.031281971372663975, + "rewards/forensics_tiou_reward": 0.4817214906215668, + "step": 388 + }, + { + "completion_length": 25.5625, + "epoch": 1.1935483870967742, + "grad_norm": 5.101705551147461, + "kl": 0.3994140625, + "learning_rate": 7.015384615384615e-07, + "loss": 0.0323, + "reward": 0.736391693353653, + "reward_std": 0.06807653233408928, + "reward_std_all": 0.07032405957579613, + "rewards-max/forensics_sinkhorn_reward": 0.95745649933815, + "rewards-max/forensics_strict_edge_reward": 0.2719302624464035, + "rewards-max/forensics_tiou_reward": 0.9263354539871216, + "rewards/forensics_sinkhorn_reward": 0.6497777998447418, + "rewards/forensics_strict_edge_reward": 0.028636879287660122, + "rewards/forensics_tiou_reward": 0.3821035325527191, + "step": 389 + }, + { + "completion_length": 25.453125, + "epoch": 1.1966205837173578, + "grad_norm": 4.860661506652832, + "kl": 0.33984375, + "learning_rate": 7.007692307692308e-07, + "loss": 0.0394, + "reward": 0.7412030696868896, + "reward_std": 0.06565475650131702, + "reward_std_all": 0.06559702567756176, + "rewards-max/forensics_sinkhorn_reward": 0.9680406153202057, + "rewards-max/forensics_strict_edge_reward": 0.5704504549503326, + "rewards-max/forensics_tiou_reward": 0.948830783367157, + "rewards/forensics_sinkhorn_reward": 0.6573289632797241, + "rewards/forensics_strict_edge_reward": 0.03071805601939559, + "rewards/forensics_tiou_reward": 0.442682221531868, + "step": 390 + }, + { + "completion_length": 26.65625, + "epoch": 1.1996927803379416, + "grad_norm": 5.3087053298950195, + "kl": 0.2939453125, + "learning_rate": 7e-07, + "loss": 0.1942, + "reward": 0.7408894896507263, + "reward_std": 0.04258364252746105, + "reward_std_all": 0.055403491482138634, + "rewards-max/forensics_sinkhorn_reward": 0.939710408449173, + "rewards-max/forensics_strict_edge_reward": 0.15452560782432556, + "rewards-max/forensics_tiou_reward": 0.8747652173042297, + "rewards/forensics_sinkhorn_reward": 0.6359651684761047, + "rewards/forensics_strict_edge_reward": 0.01697385124862194, + "rewards/forensics_tiou_reward": 0.45813336968421936, + "step": 391 + }, + { + "completion_length": 26.546875, + "epoch": 1.2027649769585254, + "grad_norm": 5.029061794281006, + "kl": 0.3271484375, + "learning_rate": 6.992307692307692e-07, + "loss": -0.0297, + "reward": 0.7575129568576813, + "reward_std": 0.03457420039921999, + "reward_std_all": 0.04075098596513271, + "rewards-max/forensics_sinkhorn_reward": 0.9691081941127777, + "rewards-max/forensics_strict_edge_reward": 0.34593143314123154, + "rewards-max/forensics_tiou_reward": 0.9461668133735657, + "rewards/forensics_sinkhorn_reward": 0.6853431463241577, + "rewards/forensics_strict_edge_reward": 0.021514483727514744, + "rewards/forensics_tiou_reward": 0.4849679172039032, + "step": 392 + }, + { + "completion_length": 27.1875, + "epoch": 1.2058371735791091, + "grad_norm": 6.53676700592041, + "kl": 0.3974609375, + "learning_rate": 6.984615384615384e-07, + "loss": 0.0527, + "reward": 0.7203340530395508, + "reward_std": 0.06609672494232655, + "reward_std_all": 0.07400456257164478, + "rewards-max/forensics_sinkhorn_reward": 0.9680027961730957, + "rewards-max/forensics_strict_edge_reward": 0.2906859815120697, + "rewards-max/forensics_tiou_reward": 0.9476833939552307, + "rewards/forensics_sinkhorn_reward": 0.6446403861045837, + "rewards/forensics_strict_edge_reward": 0.01342929620295763, + "rewards/forensics_tiou_reward": 0.38395266234874725, + "step": 393 + }, + { + "completion_length": 21.4375, + "epoch": 1.2089093701996927, + "grad_norm": 4.753777027130127, + "kl": 0.3955078125, + "learning_rate": 6.976923076923077e-07, + "loss": 0.0764, + "reward": 0.7525061964988708, + "reward_std": 0.06310504674911499, + "reward_std_all": 0.06057414226233959, + "rewards-max/forensics_sinkhorn_reward": 0.9591153264045715, + "rewards-max/forensics_strict_edge_reward": 0.3975932002067566, + "rewards-max/forensics_tiou_reward": 0.9296604990959167, + "rewards/forensics_sinkhorn_reward": 0.6797865331172943, + "rewards/forensics_strict_edge_reward": 0.0481446860358119, + "rewards/forensics_tiou_reward": 0.49319152534008026, + "step": 394 + }, + { + "completion_length": 22.796875, + "epoch": 1.2119815668202765, + "grad_norm": 4.856431484222412, + "kl": 0.2666015625, + "learning_rate": 6.969230769230769e-07, + "loss": -0.0337, + "reward": 0.7696183621883392, + "reward_std": 0.04717064090073109, + "reward_std_all": 0.04475591517984867, + "rewards-max/forensics_sinkhorn_reward": 0.9786931872367859, + "rewards-max/forensics_strict_edge_reward": 0.51341712474823, + "rewards-max/forensics_tiou_reward": 0.9730717241764069, + "rewards/forensics_sinkhorn_reward": 0.7102619409561157, + "rewards/forensics_strict_edge_reward": 0.029268890619277954, + "rewards/forensics_tiou_reward": 0.46854376792907715, + "step": 395 + }, + { + "completion_length": 25.15625, + "epoch": 1.2150537634408602, + "grad_norm": 4.861600399017334, + "kl": 0.4140625, + "learning_rate": 6.961538461538461e-07, + "loss": 0.1039, + "reward": 0.7255577743053436, + "reward_std": 0.05262421816587448, + "reward_std_all": 0.0675952136516571, + "rewards-max/forensics_sinkhorn_reward": 0.954625278711319, + "rewards-max/forensics_strict_edge_reward": 0.36587679386138916, + "rewards-max/forensics_tiou_reward": 0.9183148145675659, + "rewards/forensics_sinkhorn_reward": 0.6483844518661499, + "rewards/forensics_strict_edge_reward": 0.029982333071529865, + "rewards/forensics_tiou_reward": 0.4246528297662735, + "step": 396 + }, + { + "completion_length": 26.15625, + "epoch": 1.218125960061444, + "grad_norm": 5.735488414764404, + "kl": 0.3935546875, + "learning_rate": 6.953846153846154e-07, + "loss": 0.0622, + "reward": 0.7211890518665314, + "reward_std": 0.05553986504673958, + "reward_std_all": 0.06888957694172859, + "rewards-max/forensics_sinkhorn_reward": 0.9657706916332245, + "rewards-max/forensics_strict_edge_reward": 0.2946928220335394, + "rewards-max/forensics_tiou_reward": 0.9356180131435394, + "rewards/forensics_sinkhorn_reward": 0.5882454812526703, + "rewards/forensics_strict_edge_reward": 0.02754313212062698, + "rewards/forensics_tiou_reward": 0.43297773599624634, + "step": 397 + }, + { + "completion_length": 22.734375, + "epoch": 1.2211981566820276, + "grad_norm": 4.738451957702637, + "kl": 0.30322265625, + "learning_rate": 6.946153846153846e-07, + "loss": 0.1023, + "reward": 0.7551063895225525, + "reward_std": 0.03617706149816513, + "reward_std_all": 0.04447837918996811, + "rewards-max/forensics_sinkhorn_reward": 0.9690066576004028, + "rewards-max/forensics_strict_edge_reward": 0.30101547949016094, + "rewards-max/forensics_tiou_reward": 0.9389353096485138, + "rewards/forensics_sinkhorn_reward": 0.7014051079750061, + "rewards/forensics_strict_edge_reward": 0.03715067217126489, + "rewards/forensics_tiou_reward": 0.5439224541187286, + "step": 398 + }, + { + "completion_length": 22.8125, + "epoch": 1.2242703533026114, + "grad_norm": 10.845032691955566, + "kl": 0.6982421875, + "learning_rate": 6.938461538461539e-07, + "loss": 0.0569, + "reward": 0.7308311462402344, + "reward_std": 0.07942860201001167, + "reward_std_all": 0.07862435840070248, + "rewards-max/forensics_sinkhorn_reward": 0.9754419326782227, + "rewards-max/forensics_strict_edge_reward": 0.7388437688350677, + "rewards-max/forensics_tiou_reward": 0.96628737449646, + "rewards/forensics_sinkhorn_reward": 0.6854835748672485, + "rewards/forensics_strict_edge_reward": 0.05663909390568733, + "rewards/forensics_tiou_reward": 0.47653114795684814, + "step": 399 + }, + { + "completion_length": 28.265625, + "epoch": 1.2273425499231951, + "grad_norm": 4.582030773162842, + "kl": 0.482421875, + "learning_rate": 6.93076923076923e-07, + "loss": 0.0673, + "reward": 0.7777727842330933, + "reward_std": 0.03752551507204771, + "reward_std_all": 0.035960478708148, + "rewards-max/forensics_sinkhorn_reward": 0.9809081852436066, + "rewards-max/forensics_strict_edge_reward": 0.3887082189321518, + "rewards-max/forensics_tiou_reward": 0.9751941859722137, + "rewards/forensics_sinkhorn_reward": 0.7030717730522156, + "rewards/forensics_strict_edge_reward": 0.04059976525604725, + "rewards/forensics_tiou_reward": 0.5748749375343323, + "step": 400 + }, + { + "completion_length": 25.453125, + "epoch": 1.230414746543779, + "grad_norm": 4.572854042053223, + "kl": 0.34765625, + "learning_rate": 6.923076923076922e-07, + "loss": -0.0748, + "reward": 0.769964724779129, + "reward_std": 0.04900695011019707, + "reward_std_all": 0.04620702378451824, + "rewards-max/forensics_sinkhorn_reward": 0.9529815316200256, + "rewards-max/forensics_strict_edge_reward": 0.2554169371724129, + "rewards-max/forensics_tiou_reward": 0.9032035171985626, + "rewards/forensics_sinkhorn_reward": 0.6518545150756836, + "rewards/forensics_strict_edge_reward": 0.021904422901570797, + "rewards/forensics_tiou_reward": 0.5132041722536087, + "step": 401 + }, + { + "completion_length": 25.453125, + "epoch": 1.2334869431643625, + "grad_norm": 5.026747226715088, + "kl": 0.3330078125, + "learning_rate": 6.915384615384615e-07, + "loss": 0.0772, + "reward": 0.7073542177677155, + "reward_std": 0.06140012992545962, + "reward_std_all": 0.07485535810701549, + "rewards-max/forensics_sinkhorn_reward": 0.9374347031116486, + "rewards-max/forensics_strict_edge_reward": 0.2571959355263971, + "rewards-max/forensics_tiou_reward": 0.881479024887085, + "rewards/forensics_sinkhorn_reward": 0.6098348498344421, + "rewards/forensics_strict_edge_reward": 0.016188406463697902, + "rewards/forensics_tiou_reward": 0.3508513271808624, + "step": 402 + }, + { + "completion_length": 24.796875, + "epoch": 1.2365591397849462, + "grad_norm": 4.68678617477417, + "kl": 0.4091796875, + "learning_rate": 6.907692307692307e-07, + "loss": 0.2115, + "reward": 0.7366432547569275, + "reward_std": 0.09719684347510338, + "reward_std_all": 0.08870003744959831, + "rewards-max/forensics_sinkhorn_reward": 0.984127551317215, + "rewards-max/forensics_strict_edge_reward": 0.5502500236034393, + "rewards-max/forensics_tiou_reward": 0.9871459305286407, + "rewards/forensics_sinkhorn_reward": 0.6725478768348694, + "rewards/forensics_strict_edge_reward": 0.04312048479914665, + "rewards/forensics_tiou_reward": 0.48421208560466766, + "step": 403 + }, + { + "completion_length": 25.640625, + "epoch": 1.23963133640553, + "grad_norm": 4.26107120513916, + "kl": 0.28466796875, + "learning_rate": 6.9e-07, + "loss": 0.1156, + "reward": 0.7404597997665405, + "reward_std": 0.05961910169571638, + "reward_std_all": 0.06618117168545723, + "rewards-max/forensics_sinkhorn_reward": 0.962138295173645, + "rewards-max/forensics_strict_edge_reward": 0.3874639570713043, + "rewards-max/forensics_tiou_reward": 0.9325024783611298, + "rewards/forensics_sinkhorn_reward": 0.708591878414154, + "rewards/forensics_strict_edge_reward": 0.044803861528635025, + "rewards/forensics_tiou_reward": 0.4907665252685547, + "step": 404 + }, + { + "completion_length": 25.546875, + "epoch": 1.2427035330261136, + "grad_norm": 6.23342752456665, + "kl": 0.337890625, + "learning_rate": 6.892307692307692e-07, + "loss": 0.0571, + "reward": 0.7145988047122955, + "reward_std": 0.07490085065364838, + "reward_std_all": 0.08240209147334099, + "rewards-max/forensics_sinkhorn_reward": 0.9741836190223694, + "rewards-max/forensics_strict_edge_reward": 0.6071805953979492, + "rewards-max/forensics_tiou_reward": 0.9637019634246826, + "rewards/forensics_sinkhorn_reward": 0.6667070686817169, + "rewards/forensics_strict_edge_reward": 0.028265921398997307, + "rewards/forensics_tiou_reward": 0.4870159327983856, + "step": 405 + }, + { + "completion_length": 23.859375, + "epoch": 1.2457757296466974, + "grad_norm": 4.592513084411621, + "kl": 0.4462890625, + "learning_rate": 6.884615384615384e-07, + "loss": 0.1471, + "reward": 0.7506864368915558, + "reward_std": 0.04460800066590309, + "reward_std_all": 0.05131254903972149, + "rewards-max/forensics_sinkhorn_reward": 0.9722704291343689, + "rewards-max/forensics_strict_edge_reward": 0.2743289740756154, + "rewards-max/forensics_tiou_reward": 0.9483151137828827, + "rewards/forensics_sinkhorn_reward": 0.6831723749637604, + "rewards/forensics_strict_edge_reward": 0.03236967313569039, + "rewards/forensics_tiou_reward": 0.4677664339542389, + "step": 406 + }, + { + "completion_length": 24.21875, + "epoch": 1.2488479262672811, + "grad_norm": 5.0124077796936035, + "kl": 0.2646484375, + "learning_rate": 6.876923076923077e-07, + "loss": 0.0872, + "reward": 0.7017378807067871, + "reward_std": 0.08155735023319721, + "reward_std_all": 0.09664447605609894, + "rewards-max/forensics_sinkhorn_reward": 0.9687228798866272, + "rewards-max/forensics_strict_edge_reward": 0.4259333088994026, + "rewards-max/forensics_tiou_reward": 0.9422433972358704, + "rewards/forensics_sinkhorn_reward": 0.6475817561149597, + "rewards/forensics_strict_edge_reward": 0.026782572269439697, + "rewards/forensics_tiou_reward": 0.41468942165374756, + "step": 407 + }, + { + "completion_length": 25.03125, + "epoch": 1.251920122887865, + "grad_norm": 5.060030937194824, + "kl": 0.3037109375, + "learning_rate": 6.869230769230769e-07, + "loss": 0.2188, + "reward": 0.744338870048523, + "reward_std": 0.06734199449419975, + "reward_std_all": 0.06705884635448456, + "rewards-max/forensics_sinkhorn_reward": 0.9500089883804321, + "rewards-max/forensics_strict_edge_reward": 0.11381419934332371, + "rewards-max/forensics_tiou_reward": 0.8989403247833252, + "rewards/forensics_sinkhorn_reward": 0.6183555424213409, + "rewards/forensics_strict_edge_reward": 0.006204943754710257, + "rewards/forensics_tiou_reward": 0.40987007319927216, + "step": 408 + }, + { + "completion_length": 25.8125, + "epoch": 1.2549923195084485, + "grad_norm": 5.68923807144165, + "kl": 0.3515625, + "learning_rate": 6.861538461538461e-07, + "loss": 0.1575, + "reward": 0.7262351512908936, + "reward_std": 0.07708801329135895, + "reward_std_all": 0.07930607348680496, + "rewards-max/forensics_sinkhorn_reward": 0.9703485369682312, + "rewards-max/forensics_strict_edge_reward": 0.5081118941307068, + "rewards-max/forensics_tiou_reward": 0.9511630237102509, + "rewards/forensics_sinkhorn_reward": 0.6175929307937622, + "rewards/forensics_strict_edge_reward": 0.030871102586388588, + "rewards/forensics_tiou_reward": 0.4305320531129837, + "step": 409 + }, + { + "completion_length": 25.59375, + "epoch": 1.2580645161290323, + "grad_norm": 4.514561176300049, + "kl": 0.271484375, + "learning_rate": 6.853846153846153e-07, + "loss": -0.0016, + "reward": 0.7636316120624542, + "reward_std": 0.039119794964790344, + "reward_std_all": 0.04138892702758312, + "rewards-max/forensics_sinkhorn_reward": 0.9726528525352478, + "rewards-max/forensics_strict_edge_reward": 0.4152904450893402, + "rewards-max/forensics_tiou_reward": 0.9544188380241394, + "rewards/forensics_sinkhorn_reward": 0.703607439994812, + "rewards/forensics_strict_edge_reward": 0.02754010073840618, + "rewards/forensics_tiou_reward": 0.4907153248786926, + "step": 410 + }, + { + "completion_length": 25.796875, + "epoch": 1.261136712749616, + "grad_norm": 5.232570171356201, + "kl": 0.37109375, + "learning_rate": 6.846153846153846e-07, + "loss": 0.1684, + "reward": 0.7407365441322327, + "reward_std": 0.07581090554594994, + "reward_std_all": 0.07249849289655685, + "rewards-max/forensics_sinkhorn_reward": 0.973134309053421, + "rewards-max/forensics_strict_edge_reward": 0.3287132903933525, + "rewards-max/forensics_tiou_reward": 0.9627195000648499, + "rewards/forensics_sinkhorn_reward": 0.6891093254089355, + "rewards/forensics_strict_edge_reward": 0.02211088640615344, + "rewards/forensics_tiou_reward": 0.4747646450996399, + "step": 411 + }, + { + "completion_length": 28.390625, + "epoch": 1.2642089093701996, + "grad_norm": 5.475403308868408, + "kl": 0.32421875, + "learning_rate": 6.838461538461539e-07, + "loss": 0.2181, + "reward": 0.7228448987007141, + "reward_std": 0.09335097670555115, + "reward_std_all": 0.09128368645906448, + "rewards-max/forensics_sinkhorn_reward": 0.9656164348125458, + "rewards-max/forensics_strict_edge_reward": 0.2924101874232292, + "rewards-max/forensics_tiou_reward": 0.9339672327041626, + "rewards/forensics_sinkhorn_reward": 0.6302942633628845, + "rewards/forensics_strict_edge_reward": 0.01904175989329815, + "rewards/forensics_tiou_reward": 0.47549912333488464, + "step": 412 + }, + { + "completion_length": 25.484375, + "epoch": 1.2672811059907834, + "grad_norm": 5.835460186004639, + "kl": 0.330078125, + "learning_rate": 6.83076923076923e-07, + "loss": 0.0813, + "reward": 0.6996891498565674, + "reward_std": 0.07979121431708336, + "reward_std_all": 0.08797791227698326, + "rewards-max/forensics_sinkhorn_reward": 0.9669858515262604, + "rewards-max/forensics_strict_edge_reward": 0.4864512234926224, + "rewards-max/forensics_tiou_reward": 0.9420705735683441, + "rewards/forensics_sinkhorn_reward": 0.6084751486778259, + "rewards/forensics_strict_edge_reward": 0.034762414172291756, + "rewards/forensics_tiou_reward": 0.4213935285806656, + "step": 413 + }, + { + "completion_length": 25.265625, + "epoch": 1.2703533026113671, + "grad_norm": 6.753900051116943, + "kl": 0.248046875, + "learning_rate": 6.823076923076922e-07, + "loss": 0.0845, + "reward": 0.7030490338802338, + "reward_std": 0.05734294839203358, + "reward_std_all": 0.07764443289488554, + "rewards-max/forensics_sinkhorn_reward": 0.9635702073574066, + "rewards-max/forensics_strict_edge_reward": 0.46688374876976013, + "rewards-max/forensics_tiou_reward": 0.9339138567447662, + "rewards/forensics_sinkhorn_reward": 0.5936914086341858, + "rewards/forensics_strict_edge_reward": 0.02874341607093811, + "rewards/forensics_tiou_reward": 0.43242791295051575, + "step": 414 + }, + { + "completion_length": 27.828125, + "epoch": 1.273425499231951, + "grad_norm": 4.28060245513916, + "kl": 0.3046875, + "learning_rate": 6.815384615384615e-07, + "loss": 0.1261, + "reward": 0.7699584364891052, + "reward_std": 0.040607718750834465, + "reward_std_all": 0.0401036748662591, + "rewards-max/forensics_sinkhorn_reward": 0.9722249805927277, + "rewards-max/forensics_strict_edge_reward": 0.33149442076683044, + "rewards-max/forensics_tiou_reward": 0.9543291628360748, + "rewards/forensics_sinkhorn_reward": 0.6698369681835175, + "rewards/forensics_strict_edge_reward": 0.020150769036263227, + "rewards/forensics_tiou_reward": 0.48708197474479675, + "step": 415 + }, + { + "completion_length": 26.671875, + "epoch": 1.2764976958525347, + "grad_norm": 4.685543537139893, + "kl": 0.3359375, + "learning_rate": 6.807692307692307e-07, + "loss": 0.141, + "reward": 0.7830333709716797, + "reward_std": 0.028725741431117058, + "reward_std_all": 0.028220240958034992, + "rewards-max/forensics_sinkhorn_reward": 0.9295163452625275, + "rewards-max/forensics_strict_edge_reward": 0.16806571185588837, + "rewards-max/forensics_tiou_reward": 0.8553734719753265, + "rewards/forensics_sinkhorn_reward": 0.6097082197666168, + "rewards/forensics_strict_edge_reward": 0.013136795721948147, + "rewards/forensics_tiou_reward": 0.4715326130390167, + "step": 416 + }, + { + "completion_length": 24.359375, + "epoch": 1.2795698924731183, + "grad_norm": 5.353207588195801, + "kl": 0.2900390625, + "learning_rate": 6.800000000000001e-07, + "loss": 0.1045, + "reward": 0.7159186899662018, + "reward_std": 0.07540032267570496, + "reward_std_all": 0.08559341728687286, + "rewards-max/forensics_sinkhorn_reward": 0.9289256036281586, + "rewards-max/forensics_strict_edge_reward": 0.07834082318004221, + "rewards-max/forensics_tiou_reward": 0.852969616651535, + "rewards/forensics_sinkhorn_reward": 0.6482466459274292, + "rewards/forensics_strict_edge_reward": 0.004297031846363097, + "rewards/forensics_tiou_reward": 0.41722430288791656, + "step": 417 + }, + { + "completion_length": 24.875, + "epoch": 1.282642089093702, + "grad_norm": 5.768828868865967, + "kl": 0.392578125, + "learning_rate": 6.792307692307692e-07, + "loss": 0.0417, + "reward": 0.7375619113445282, + "reward_std": 0.0747501440346241, + "reward_std_all": 0.07265700586140156, + "rewards-max/forensics_sinkhorn_reward": 0.9503589570522308, + "rewards-max/forensics_strict_edge_reward": 0.22880059108138084, + "rewards-max/forensics_tiou_reward": 0.906076967716217, + "rewards/forensics_sinkhorn_reward": 0.6446171998977661, + "rewards/forensics_strict_edge_reward": 0.009710447513498366, + "rewards/forensics_tiou_reward": 0.4535926282405853, + "step": 418 + }, + { + "completion_length": 21.15625, + "epoch": 1.2857142857142856, + "grad_norm": 5.674266815185547, + "kl": 0.25732421875, + "learning_rate": 6.784615384615384e-07, + "loss": 0.0431, + "reward": 0.7672470211982727, + "reward_std": 0.04110979847609997, + "reward_std_all": 0.040855009108781815, + "rewards-max/forensics_sinkhorn_reward": 0.9673339426517487, + "rewards-max/forensics_strict_edge_reward": 0.5795845240354538, + "rewards-max/forensics_tiou_reward": 0.9489688277244568, + "rewards/forensics_sinkhorn_reward": 0.7181281447410583, + "rewards/forensics_strict_edge_reward": 0.030229171738028526, + "rewards/forensics_tiou_reward": 0.4860522300004959, + "step": 419 + }, + { + "completion_length": 27.609375, + "epoch": 1.2887864823348694, + "grad_norm": 5.729823589324951, + "kl": 0.521484375, + "learning_rate": 6.776923076923077e-07, + "loss": 0.0878, + "reward": 0.7121598124504089, + "reward_std": 0.04932445473968983, + "reward_std_all": 0.06789363734424114, + "rewards-max/forensics_sinkhorn_reward": 0.9481717348098755, + "rewards-max/forensics_strict_edge_reward": 0.18277469277381897, + "rewards-max/forensics_tiou_reward": 0.9032729268074036, + "rewards/forensics_sinkhorn_reward": 0.6069640517234802, + "rewards/forensics_strict_edge_reward": 0.013522106572054327, + "rewards/forensics_tiou_reward": 0.38059237599372864, + "step": 420 + }, + { + "completion_length": 26.25, + "epoch": 1.2918586789554531, + "grad_norm": 5.381641864776611, + "kl": 0.5087890625, + "learning_rate": 6.769230769230769e-07, + "loss": 0.0201, + "reward": 0.7811232507228851, + "reward_std": 0.03335283836349845, + "reward_std_all": 0.03209084179252386, + "rewards-max/forensics_sinkhorn_reward": 0.9740696251392365, + "rewards-max/forensics_strict_edge_reward": 0.40819549560546875, + "rewards-max/forensics_tiou_reward": 0.9614450633525848, + "rewards/forensics_sinkhorn_reward": 0.7177444696426392, + "rewards/forensics_strict_edge_reward": 0.031558332964777946, + "rewards/forensics_tiou_reward": 0.527801901102066, + "step": 421 + }, + { + "completion_length": 23.625, + "epoch": 1.294930875576037, + "grad_norm": 4.767314910888672, + "kl": 0.365234375, + "learning_rate": 6.761538461538461e-07, + "loss": 0.0672, + "reward": 0.7307125329971313, + "reward_std": 0.08404218032956123, + "reward_std_all": 0.08372307941317558, + "rewards-max/forensics_sinkhorn_reward": 0.9495011866092682, + "rewards-max/forensics_strict_edge_reward": 0.15408027567900717, + "rewards-max/forensics_tiou_reward": 0.9041654467582703, + "rewards/forensics_sinkhorn_reward": 0.6636297106742859, + "rewards/forensics_strict_edge_reward": 0.014954082434996963, + "rewards/forensics_tiou_reward": 0.45790886878967285, + "step": 422 + }, + { + "completion_length": 25.25, + "epoch": 1.2980030721966207, + "grad_norm": 4.727041244506836, + "kl": 0.3251953125, + "learning_rate": 6.753846153846153e-07, + "loss": 0.0662, + "reward": 0.7769486904144287, + "reward_std": 0.036377882584929466, + "reward_std_all": 0.035462613217532635, + "rewards-max/forensics_sinkhorn_reward": 0.9256923794746399, + "rewards-max/forensics_strict_edge_reward": 0.19933770969510078, + "rewards-max/forensics_tiou_reward": 0.8346715569496155, + "rewards/forensics_sinkhorn_reward": 0.7258942425251007, + "rewards/forensics_strict_edge_reward": 0.009357923408970237, + "rewards/forensics_tiou_reward": 0.49429789185523987, + "step": 423 + }, + { + "completion_length": 22.375, + "epoch": 1.3010752688172043, + "grad_norm": 5.0090861320495605, + "kl": 0.306640625, + "learning_rate": 6.746153846153845e-07, + "loss": 0.0231, + "reward": 0.7526135742664337, + "reward_std": 0.05381036177277565, + "reward_std_all": 0.05604194849729538, + "rewards-max/forensics_sinkhorn_reward": 0.9695771634578705, + "rewards-max/forensics_strict_edge_reward": 0.37933504581451416, + "rewards-max/forensics_tiou_reward": 0.9448734223842621, + "rewards/forensics_sinkhorn_reward": 0.7030771672725677, + "rewards/forensics_strict_edge_reward": 0.029106621630489826, + "rewards/forensics_tiou_reward": 0.5010954588651657, + "step": 424 + }, + { + "completion_length": 23.046875, + "epoch": 1.304147465437788, + "grad_norm": 4.560751914978027, + "kl": 0.3408203125, + "learning_rate": 6.738461538461539e-07, + "loss": 0.0157, + "reward": 0.7925205230712891, + "reward_std": 0.017921040707733482, + "reward_std_all": 0.0189276987221092, + "rewards-max/forensics_sinkhorn_reward": 0.9518267214298248, + "rewards-max/forensics_strict_edge_reward": 0.4158726632595062, + "rewards-max/forensics_tiou_reward": 0.8929089307785034, + "rewards/forensics_sinkhorn_reward": 0.7265910804271698, + "rewards/forensics_strict_edge_reward": 0.025643950328230858, + "rewards/forensics_tiou_reward": 0.502009242773056, + "step": 425 + }, + { + "completion_length": 22.078125, + "epoch": 1.3072196620583718, + "grad_norm": 6.125117301940918, + "kl": 0.32421875, + "learning_rate": 6.730769230769231e-07, + "loss": 0.0215, + "reward": 0.7722317576408386, + "reward_std": 0.03962369076907635, + "reward_std_all": 0.03874374181032181, + "rewards-max/forensics_sinkhorn_reward": 0.94842728972435, + "rewards-max/forensics_strict_edge_reward": 0.21929579600691795, + "rewards-max/forensics_tiou_reward": 0.8856640756130219, + "rewards/forensics_sinkhorn_reward": 0.7042472064495087, + "rewards/forensics_strict_edge_reward": 0.02522304281592369, + "rewards/forensics_tiou_reward": 0.4618227779865265, + "step": 426 + }, + { + "completion_length": 24.828125, + "epoch": 1.3102918586789554, + "grad_norm": 4.866631507873535, + "kl": 0.333984375, + "learning_rate": 6.723076923076922e-07, + "loss": -0.049, + "reward": 0.7949226796627045, + "reward_std": 0.017861640779301524, + "reward_std_all": 0.017819518921896815, + "rewards-max/forensics_sinkhorn_reward": 0.9703542590141296, + "rewards-max/forensics_strict_edge_reward": 0.6563799977302551, + "rewards-max/forensics_tiou_reward": 0.9526082873344421, + "rewards/forensics_sinkhorn_reward": 0.7744453251361847, + "rewards/forensics_strict_edge_reward": 0.04913933388888836, + "rewards/forensics_tiou_reward": 0.5713910162448883, + "step": 427 + }, + { + "completion_length": 22.46875, + "epoch": 1.3133640552995391, + "grad_norm": 4.7407636642456055, + "kl": 0.359375, + "learning_rate": 6.715384615384615e-07, + "loss": 0.1429, + "reward": 0.7753977179527283, + "reward_std": 0.0355221051722765, + "reward_std_all": 0.03479800559580326, + "rewards-max/forensics_sinkhorn_reward": 0.9813856482505798, + "rewards-max/forensics_strict_edge_reward": 0.7244620621204376, + "rewards-max/forensics_tiou_reward": 0.9825849533081055, + "rewards/forensics_sinkhorn_reward": 0.7353620231151581, + "rewards/forensics_strict_edge_reward": 0.060568081215023994, + "rewards/forensics_tiou_reward": 0.5297561138868332, + "step": 428 + }, + { + "completion_length": 24.25, + "epoch": 1.316436251920123, + "grad_norm": 5.674677848815918, + "kl": 0.466796875, + "learning_rate": 6.707692307692307e-07, + "loss": 0.0298, + "reward": 0.7574538290500641, + "reward_std": 0.03868460934609175, + "reward_std_all": 0.045343960635364056, + "rewards-max/forensics_sinkhorn_reward": 0.9426257610321045, + "rewards-max/forensics_strict_edge_reward": 0.14895004034042358, + "rewards-max/forensics_tiou_reward": 0.8860916495323181, + "rewards/forensics_sinkhorn_reward": 0.7229205369949341, + "rewards/forensics_strict_edge_reward": 0.016417210921645164, + "rewards/forensics_tiou_reward": 0.4848499596118927, + "step": 429 + }, + { + "completion_length": 28.15625, + "epoch": 1.3195084485407067, + "grad_norm": 4.068338871002197, + "kl": 0.349609375, + "learning_rate": 6.7e-07, + "loss": 0.0724, + "reward": 0.781827449798584, + "reward_std": 0.02907862374559045, + "reward_std_all": 0.029185870196670294, + "rewards-max/forensics_sinkhorn_reward": 0.9653440415859222, + "rewards-max/forensics_strict_edge_reward": 0.6087257117033005, + "rewards-max/forensics_tiou_reward": 0.9450405538082123, + "rewards/forensics_sinkhorn_reward": 0.6871474087238312, + "rewards/forensics_strict_edge_reward": 0.037202393636107445, + "rewards/forensics_tiou_reward": 0.5135585367679596, + "step": 430 + }, + { + "completion_length": 25.203125, + "epoch": 1.3225806451612903, + "grad_norm": 5.55728006362915, + "kl": 0.3544921875, + "learning_rate": 6.692307692307692e-07, + "loss": 0.0103, + "reward": 0.7618212997913361, + "reward_std": 0.0637886356562376, + "reward_std_all": 0.05869613029062748, + "rewards-max/forensics_sinkhorn_reward": 0.9196163415908813, + "rewards-max/forensics_strict_edge_reward": 0.4635082483291626, + "rewards-max/forensics_tiou_reward": 0.8481902480125427, + "rewards/forensics_sinkhorn_reward": 0.6862689256668091, + "rewards/forensics_strict_edge_reward": 0.05078628286719322, + "rewards/forensics_tiou_reward": 0.4504093825817108, + "step": 431 + }, + { + "completion_length": 24.625, + "epoch": 1.325652841781874, + "grad_norm": 6.3461222648620605, + "kl": 0.404296875, + "learning_rate": 6.684615384615384e-07, + "loss": 0.0544, + "reward": 0.7577340006828308, + "reward_std": 0.052714958786964417, + "reward_std_all": 0.051885636523365974, + "rewards-max/forensics_sinkhorn_reward": 0.96620312333107, + "rewards-max/forensics_strict_edge_reward": 0.24938895553350449, + "rewards-max/forensics_tiou_reward": 0.9352725148200989, + "rewards/forensics_sinkhorn_reward": 0.6759050488471985, + "rewards/forensics_strict_edge_reward": 0.0168400127440691, + "rewards/forensics_tiou_reward": 0.4743283540010452, + "step": 432 + }, + { + "completion_length": 23.59375, + "epoch": 1.3287250384024578, + "grad_norm": 5.812036037445068, + "kl": 0.345703125, + "learning_rate": 6.676923076923077e-07, + "loss": 0.052, + "reward": 0.7561515867710114, + "reward_std": 0.04574013315141201, + "reward_std_all": 0.05166122503578663, + "rewards-max/forensics_sinkhorn_reward": 0.9671885669231415, + "rewards-max/forensics_strict_edge_reward": 0.49185195565223694, + "rewards-max/forensics_tiou_reward": 0.9499830305576324, + "rewards/forensics_sinkhorn_reward": 0.6979030072689056, + "rewards/forensics_strict_edge_reward": 0.03723031096160412, + "rewards/forensics_tiou_reward": 0.5222612917423248, + "step": 433 + }, + { + "completion_length": 28.0625, + "epoch": 1.3317972350230414, + "grad_norm": 5.2144670486450195, + "kl": 0.349609375, + "learning_rate": 6.669230769230769e-07, + "loss": 0.1411, + "reward": 0.7470332980155945, + "reward_std": 0.06026553735136986, + "reward_std_all": 0.06127661652863026, + "rewards-max/forensics_sinkhorn_reward": 0.9681935012340546, + "rewards-max/forensics_strict_edge_reward": 0.3057021824643016, + "rewards-max/forensics_tiou_reward": 0.9484938979148865, + "rewards/forensics_sinkhorn_reward": 0.6485855877399445, + "rewards/forensics_strict_edge_reward": 0.013580216735135764, + "rewards/forensics_tiou_reward": 0.4694890081882477, + "step": 434 + }, + { + "completion_length": 27.28125, + "epoch": 1.3348694316436251, + "grad_norm": 4.855060577392578, + "kl": 0.3828125, + "learning_rate": 6.661538461538462e-07, + "loss": 0.2334, + "reward": 0.7356384694576263, + "reward_std": 0.06997047178447247, + "reward_std_all": 0.07083188556134701, + "rewards-max/forensics_sinkhorn_reward": 0.9494003653526306, + "rewards-max/forensics_strict_edge_reward": 0.18756617861799896, + "rewards-max/forensics_tiou_reward": 0.8956894278526306, + "rewards/forensics_sinkhorn_reward": 0.6483741402626038, + "rewards/forensics_strict_edge_reward": 0.020093464612727985, + "rewards/forensics_tiou_reward": 0.4388619214296341, + "step": 435 + }, + { + "completion_length": 23.90625, + "epoch": 1.337941628264209, + "grad_norm": 5.029098987579346, + "kl": 0.3349609375, + "learning_rate": 6.653846153846153e-07, + "loss": 0.0713, + "reward": 0.743558943271637, + "reward_std": 0.03460246976464987, + "reward_std_all": 0.0508514828979969, + "rewards-max/forensics_sinkhorn_reward": 0.9716473519802094, + "rewards-max/forensics_strict_edge_reward": 0.579467847943306, + "rewards-max/forensics_tiou_reward": 0.9571722745895386, + "rewards/forensics_sinkhorn_reward": 0.7061521112918854, + "rewards/forensics_strict_edge_reward": 0.05200577899813652, + "rewards/forensics_tiou_reward": 0.47701478004455566, + "step": 436 + }, + { + "completion_length": 27.765625, + "epoch": 1.3410138248847927, + "grad_norm": 4.948565483093262, + "kl": 0.4072265625, + "learning_rate": 6.646153846153845e-07, + "loss": 0.0299, + "reward": 0.7224324345588684, + "reward_std": 0.08775336667895317, + "reward_std_all": 0.08570332452654839, + "rewards-max/forensics_sinkhorn_reward": 0.9814459085464478, + "rewards-max/forensics_strict_edge_reward": 0.38415683805942535, + "rewards-max/forensics_tiou_reward": 0.979712188243866, + "rewards/forensics_sinkhorn_reward": 0.6685636341571808, + "rewards/forensics_strict_edge_reward": 0.021137861534953117, + "rewards/forensics_tiou_reward": 0.45956169068813324, + "step": 437 + }, + { + "completion_length": 22.421875, + "epoch": 1.3440860215053765, + "grad_norm": 6.0889153480529785, + "kl": 0.3525390625, + "learning_rate": 6.638461538461539e-07, + "loss": 0.0658, + "reward": 0.7027469277381897, + "reward_std": 0.07564619183540344, + "reward_std_all": 0.09058960527181625, + "rewards-max/forensics_sinkhorn_reward": 0.9416617751121521, + "rewards-max/forensics_strict_edge_reward": 0.32014521956443787, + "rewards-max/forensics_tiou_reward": 0.8817957639694214, + "rewards/forensics_sinkhorn_reward": 0.6110396981239319, + "rewards/forensics_strict_edge_reward": 0.033287267200648785, + "rewards/forensics_tiou_reward": 0.346816286444664, + "step": 438 + }, + { + "completion_length": 24.078125, + "epoch": 1.34715821812596, + "grad_norm": 6.163120746612549, + "kl": 0.322265625, + "learning_rate": 6.630769230769231e-07, + "loss": 0.0027, + "reward": 0.6755186915397644, + "reward_std": 0.06564166024327278, + "reward_std_all": 0.09313131868839264, + "rewards-max/forensics_sinkhorn_reward": 0.9868803322315216, + "rewards-max/forensics_strict_edge_reward": 0.9375866651535034, + "rewards-max/forensics_tiou_reward": 0.9948508143424988, + "rewards/forensics_sinkhorn_reward": 0.56498122215271, + "rewards/forensics_strict_edge_reward": 0.07791964896023273, + "rewards/forensics_tiou_reward": 0.3861975818872452, + "step": 439 + }, + { + "completion_length": 27.28125, + "epoch": 1.3502304147465438, + "grad_norm": 6.05726957321167, + "kl": 0.4443359375, + "learning_rate": 6.623076923076923e-07, + "loss": 0.0881, + "reward": 0.7050112187862396, + "reward_std": 0.11948167905211449, + "reward_std_all": 0.11288942396640778, + "rewards-max/forensics_sinkhorn_reward": 0.959216445684433, + "rewards-max/forensics_strict_edge_reward": 0.3376219682395458, + "rewards-max/forensics_tiou_reward": 0.9086765348911285, + "rewards/forensics_sinkhorn_reward": 0.6222354173660278, + "rewards/forensics_strict_edge_reward": 0.020780073711648583, + "rewards/forensics_tiou_reward": 0.4576815217733383, + "step": 440 + }, + { + "completion_length": 27.8125, + "epoch": 1.3533026113671274, + "grad_norm": 8.554096221923828, + "kl": 0.4140625, + "learning_rate": 6.615384615384615e-07, + "loss": -0.0087, + "reward": 0.6244472861289978, + "reward_std": 0.0890231691300869, + "reward_std_all": 0.13592581450939178, + "rewards-max/forensics_sinkhorn_reward": 0.936516672372818, + "rewards-max/forensics_strict_edge_reward": 0.09562498703598976, + "rewards-max/forensics_tiou_reward": 0.8623170852661133, + "rewards/forensics_sinkhorn_reward": 0.53032086789608, + "rewards/forensics_strict_edge_reward": 0.004871306242421269, + "rewards/forensics_tiou_reward": 0.3781268745660782, + "step": 441 + }, + { + "completion_length": 22.484375, + "epoch": 1.3563748079877112, + "grad_norm": 5.147723197937012, + "kl": 0.416015625, + "learning_rate": 6.607692307692307e-07, + "loss": -0.0145, + "reward": 0.7425350844860077, + "reward_std": 0.060664067044854164, + "reward_std_all": 0.061960699036717415, + "rewards-max/forensics_sinkhorn_reward": 0.9657419323921204, + "rewards-max/forensics_strict_edge_reward": 0.28648165985941887, + "rewards-max/forensics_tiou_reward": 0.9380285441875458, + "rewards/forensics_sinkhorn_reward": 0.6317103505134583, + "rewards/forensics_strict_edge_reward": 0.029111470328643918, + "rewards/forensics_tiou_reward": 0.4286251962184906, + "step": 442 + }, + { + "completion_length": 26.3125, + "epoch": 1.359447004608295, + "grad_norm": 6.786252975463867, + "kl": 0.6162109375, + "learning_rate": 6.6e-07, + "loss": 0.0978, + "reward": 0.7093783020973206, + "reward_std": 0.08183088526129723, + "reward_std_all": 0.08647667616605759, + "rewards-max/forensics_sinkhorn_reward": 0.9668999314308167, + "rewards-max/forensics_strict_edge_reward": 0.39643193781375885, + "rewards-max/forensics_tiou_reward": 0.944313645362854, + "rewards/forensics_sinkhorn_reward": 0.6692003011703491, + "rewards/forensics_strict_edge_reward": 0.037826843559741974, + "rewards/forensics_tiou_reward": 0.5002938210964203, + "step": 443 + }, + { + "completion_length": 22.78125, + "epoch": 1.3625192012288787, + "grad_norm": 4.2822723388671875, + "kl": 0.2783203125, + "learning_rate": 6.592307692307691e-07, + "loss": 0.0303, + "reward": 0.7478136420249939, + "reward_std": 0.03964866977185011, + "reward_std_all": 0.046503085643053055, + "rewards-max/forensics_sinkhorn_reward": 0.9654743671417236, + "rewards-max/forensics_strict_edge_reward": 0.30629871785640717, + "rewards-max/forensics_tiou_reward": 0.9347449839115143, + "rewards/forensics_sinkhorn_reward": 0.7323901057243347, + "rewards/forensics_strict_edge_reward": 0.032438986003398895, + "rewards/forensics_tiou_reward": 0.497765451669693, + "step": 444 + }, + { + "completion_length": 23.296875, + "epoch": 1.3655913978494625, + "grad_norm": 5.051908016204834, + "kl": 0.33203125, + "learning_rate": 6.584615384615384e-07, + "loss": 0.0826, + "reward": 0.733268678188324, + "reward_std": 0.055140696465969086, + "reward_std_all": 0.06641683913767338, + "rewards-max/forensics_sinkhorn_reward": 0.9786799848079681, + "rewards-max/forensics_strict_edge_reward": 0.5192448049783707, + "rewards-max/forensics_tiou_reward": 0.9698944091796875, + "rewards/forensics_sinkhorn_reward": 0.6918926537036896, + "rewards/forensics_strict_edge_reward": 0.041230857372283936, + "rewards/forensics_tiou_reward": 0.5128860175609589, + "step": 445 + }, + { + "completion_length": 22.328125, + "epoch": 1.368663594470046, + "grad_norm": 6.2024245262146, + "kl": 0.435546875, + "learning_rate": 6.576923076923077e-07, + "loss": 0.0338, + "reward": 0.778999000787735, + "reward_std": 0.04172813566401601, + "reward_std_all": 0.03864882327616215, + "rewards-max/forensics_sinkhorn_reward": 0.9806578755378723, + "rewards-max/forensics_strict_edge_reward": 0.6483739018440247, + "rewards-max/forensics_tiou_reward": 0.9782043099403381, + "rewards/forensics_sinkhorn_reward": 0.7536229193210602, + "rewards/forensics_strict_edge_reward": 0.05339074693620205, + "rewards/forensics_tiou_reward": 0.6074638068675995, + "step": 446 + }, + { + "completion_length": 25.140625, + "epoch": 1.3717357910906298, + "grad_norm": 5.210337162017822, + "kl": 0.279296875, + "learning_rate": 6.569230769230769e-07, + "loss": 0.2136, + "reward": 0.6988670527935028, + "reward_std": 0.093772292137146, + "reward_std_all": 0.10172762721776962, + "rewards-max/forensics_sinkhorn_reward": 0.9809786677360535, + "rewards-max/forensics_strict_edge_reward": 0.6620846390724182, + "rewards-max/forensics_tiou_reward": 0.9807533621788025, + "rewards/forensics_sinkhorn_reward": 0.5955335795879364, + "rewards/forensics_strict_edge_reward": 0.03783009760081768, + "rewards/forensics_tiou_reward": 0.4375161826610565, + "step": 447 + }, + { + "completion_length": 23.0625, + "epoch": 1.3748079877112136, + "grad_norm": 5.6276445388793945, + "kl": 0.318359375, + "learning_rate": 6.561538461538462e-07, + "loss": 0.0927, + "reward": 0.7387514710426331, + "reward_std": 0.05536365695297718, + "reward_std_all": 0.061642708256840706, + "rewards-max/forensics_sinkhorn_reward": 0.9275755286216736, + "rewards-max/forensics_strict_edge_reward": 0.13097119331359863, + "rewards-max/forensics_tiou_reward": 0.8587813079357147, + "rewards/forensics_sinkhorn_reward": 0.6209412813186646, + "rewards/forensics_strict_edge_reward": 0.008423977298662066, + "rewards/forensics_tiou_reward": 0.3658396601676941, + "step": 448 + }, + { + "completion_length": 24.046875, + "epoch": 1.3778801843317972, + "grad_norm": 6.476032733917236, + "kl": 0.3134765625, + "learning_rate": 6.553846153846153e-07, + "loss": 0.13, + "reward": 0.6832382678985596, + "reward_std": 0.10393213480710983, + "reward_std_all": 0.11528515815734863, + "rewards-max/forensics_sinkhorn_reward": 0.9803104102611542, + "rewards-max/forensics_strict_edge_reward": 0.4408009983599186, + "rewards-max/forensics_tiou_reward": 0.9798026084899902, + "rewards/forensics_sinkhorn_reward": 0.6010854244232178, + "rewards/forensics_strict_edge_reward": 0.024839706136845052, + "rewards/forensics_tiou_reward": 0.46117573976516724, + "step": 449 + }, + { + "completion_length": 29.484375, + "epoch": 1.380952380952381, + "grad_norm": 5.626120567321777, + "kl": 0.390625, + "learning_rate": 6.546153846153845e-07, + "loss": 0.0742, + "reward": 0.7211239635944366, + "reward_std": 0.07937836647033691, + "reward_std_all": 0.08032489195466042, + "rewards-max/forensics_sinkhorn_reward": 0.959830641746521, + "rewards-max/forensics_strict_edge_reward": 0.30083712935447693, + "rewards-max/forensics_tiou_reward": 0.9312293529510498, + "rewards/forensics_sinkhorn_reward": 0.5951717793941498, + "rewards/forensics_strict_edge_reward": 0.015342934988439083, + "rewards/forensics_tiou_reward": 0.42693139612674713, + "step": 450 + }, + { + "completion_length": 26.140625, + "epoch": 1.3840245775729647, + "grad_norm": 4.785362720489502, + "kl": 0.38671875, + "learning_rate": 6.538461538461538e-07, + "loss": 0.0182, + "reward": 0.7409034669399261, + "reward_std": 0.06017259135842323, + "reward_std_all": 0.06706545129418373, + "rewards-max/forensics_sinkhorn_reward": 0.944142609834671, + "rewards-max/forensics_strict_edge_reward": 0.0923546589910984, + "rewards-max/forensics_tiou_reward": 0.8715965151786804, + "rewards/forensics_sinkhorn_reward": 0.6731905937194824, + "rewards/forensics_strict_edge_reward": 0.006516557652503252, + "rewards/forensics_tiou_reward": 0.4642089307308197, + "step": 451 + }, + { + "completion_length": 22.3125, + "epoch": 1.3870967741935485, + "grad_norm": 5.636239051818848, + "kl": 0.373046875, + "learning_rate": 6.530769230769231e-07, + "loss": 0.1077, + "reward": 0.7265101075172424, + "reward_std": 0.08677293360233307, + "reward_std_all": 0.08418813720345497, + "rewards-max/forensics_sinkhorn_reward": 0.9288095831871033, + "rewards-max/forensics_strict_edge_reward": 0.11662979423999786, + "rewards-max/forensics_tiou_reward": 0.8539741635322571, + "rewards/forensics_sinkhorn_reward": 0.6627691686153412, + "rewards/forensics_strict_edge_reward": 0.007948391139507294, + "rewards/forensics_tiou_reward": 0.43070587515830994, + "step": 452 + }, + { + "completion_length": 21.859375, + "epoch": 1.390168970814132, + "grad_norm": 5.816046237945557, + "kl": 0.4794921875, + "learning_rate": 6.523076923076923e-07, + "loss": 0.1492, + "reward": 0.7569314241409302, + "reward_std": 0.07335085794329643, + "reward_std_all": 0.06675809249281883, + "rewards-max/forensics_sinkhorn_reward": 0.9469812214374542, + "rewards-max/forensics_strict_edge_reward": 0.30850303173065186, + "rewards-max/forensics_tiou_reward": 0.8957399129867554, + "rewards/forensics_sinkhorn_reward": 0.7248934209346771, + "rewards/forensics_strict_edge_reward": 0.022684145718812943, + "rewards/forensics_tiou_reward": 0.47660762071609497, + "step": 453 + }, + { + "completion_length": 22.953125, + "epoch": 1.3932411674347158, + "grad_norm": 7.3369340896606445, + "kl": 0.4873046875, + "learning_rate": 6.515384615384615e-07, + "loss": 0.1125, + "reward": 0.7743001878261566, + "reward_std": 0.04422800405882299, + "reward_std_all": 0.04174322611652315, + "rewards-max/forensics_sinkhorn_reward": 0.966646671295166, + "rewards-max/forensics_strict_edge_reward": 0.47147156298160553, + "rewards-max/forensics_tiou_reward": 0.950208306312561, + "rewards/forensics_sinkhorn_reward": 0.7212444841861725, + "rewards/forensics_strict_edge_reward": 0.03525327146053314, + "rewards/forensics_tiou_reward": 0.5172643810510635, + "step": 454 + }, + { + "completion_length": 22.046875, + "epoch": 1.3963133640552996, + "grad_norm": 4.917571067810059, + "kl": 0.3505859375, + "learning_rate": 6.507692307692307e-07, + "loss": 0.0633, + "reward": 0.7053132057189941, + "reward_std": 0.0632460005581379, + "reward_std_all": 0.08229347877204418, + "rewards-max/forensics_sinkhorn_reward": 0.972455769777298, + "rewards-max/forensics_strict_edge_reward": 0.5149057954549789, + "rewards-max/forensics_tiou_reward": 0.9546523988246918, + "rewards/forensics_sinkhorn_reward": 0.6667933464050293, + "rewards/forensics_strict_edge_reward": 0.0624527046456933, + "rewards/forensics_tiou_reward": 0.49671682715415955, + "step": 455 + }, + { + "completion_length": 23.578125, + "epoch": 1.3993855606758832, + "grad_norm": 5.738681316375732, + "kl": 0.4453125, + "learning_rate": 6.5e-07, + "loss": 0.0137, + "reward": 0.7241502106189728, + "reward_std": 0.05713896080851555, + "reward_std_all": 0.06993689388036728, + "rewards-max/forensics_sinkhorn_reward": 0.9872277677059174, + "rewards-max/forensics_strict_edge_reward": 0.6571269035339355, + "rewards-max/forensics_tiou_reward": 0.995356947183609, + "rewards/forensics_sinkhorn_reward": 0.656167209148407, + "rewards/forensics_strict_edge_reward": 0.05644359439611435, + "rewards/forensics_tiou_reward": 0.4847857803106308, + "step": 456 + }, + { + "completion_length": 23.0, + "epoch": 1.402457757296467, + "grad_norm": 5.361695289611816, + "kl": 0.3349609375, + "learning_rate": 6.492307692307692e-07, + "loss": 0.1086, + "reward": 0.7170824408531189, + "reward_std": 0.0852297805249691, + "reward_std_all": 0.08848215267062187, + "rewards-max/forensics_sinkhorn_reward": 0.9710935056209564, + "rewards-max/forensics_strict_edge_reward": 0.5523883104324341, + "rewards-max/forensics_tiou_reward": 0.9543154239654541, + "rewards/forensics_sinkhorn_reward": 0.621576726436615, + "rewards/forensics_strict_edge_reward": 0.06077345833182335, + "rewards/forensics_tiou_reward": 0.44074438512325287, + "step": 457 + }, + { + "completion_length": 23.796875, + "epoch": 1.4055299539170507, + "grad_norm": 6.5778937339782715, + "kl": 0.4462890625, + "learning_rate": 6.484615384615383e-07, + "loss": 0.1226, + "reward": 0.736170083284378, + "reward_std": 0.03480948507785797, + "reward_std_all": 0.052812207490205765, + "rewards-max/forensics_sinkhorn_reward": 0.9154790341854095, + "rewards-max/forensics_strict_edge_reward": 0.16250645741820335, + "rewards-max/forensics_tiou_reward": 0.8232504427433014, + "rewards/forensics_sinkhorn_reward": 0.6492131054401398, + "rewards/forensics_strict_edge_reward": 0.012555110966786742, + "rewards/forensics_tiou_reward": 0.416034072637558, + "step": 458 + }, + { + "completion_length": 22.5, + "epoch": 1.4086021505376345, + "grad_norm": 6.164571285247803, + "kl": 0.3076171875, + "learning_rate": 6.476923076923077e-07, + "loss": 0.0433, + "reward": 0.7470157444477081, + "reward_std": 0.05386284179985523, + "reward_std_all": 0.06067809276282787, + "rewards-max/forensics_sinkhorn_reward": 0.9872525930404663, + "rewards-max/forensics_strict_edge_reward": 0.8582656681537628, + "rewards-max/forensics_tiou_reward": 0.9949625730514526, + "rewards/forensics_sinkhorn_reward": 0.7491908073425293, + "rewards/forensics_strict_edge_reward": 0.05566227436065674, + "rewards/forensics_tiou_reward": 0.5759308636188507, + "step": 459 + }, + { + "completion_length": 22.671875, + "epoch": 1.411674347158218, + "grad_norm": 5.784832954406738, + "kl": 0.380859375, + "learning_rate": 6.469230769230769e-07, + "loss": 0.0757, + "reward": 0.7041272521018982, + "reward_std": 0.05320522282272577, + "reward_std_all": 0.07723213359713554, + "rewards-max/forensics_sinkhorn_reward": 0.9701016247272491, + "rewards-max/forensics_strict_edge_reward": 0.04700072109699249, + "rewards-max/forensics_tiou_reward": 0.9449842870235443, + "rewards/forensics_sinkhorn_reward": 0.6531829237937927, + "rewards/forensics_strict_edge_reward": 0.004054087912663817, + "rewards/forensics_tiou_reward": 0.4584432542324066, + "step": 460 + }, + { + "completion_length": 21.96875, + "epoch": 1.4147465437788018, + "grad_norm": 7.456137180328369, + "kl": 0.359375, + "learning_rate": 6.461538461538462e-07, + "loss": 0.0856, + "reward": 0.739008754491806, + "reward_std": 0.08596034720540047, + "reward_std_all": 0.08016225695610046, + "rewards-max/forensics_sinkhorn_reward": 0.9801914393901825, + "rewards-max/forensics_strict_edge_reward": 0.26341477781534195, + "rewards-max/forensics_tiou_reward": 0.9761781990528107, + "rewards/forensics_sinkhorn_reward": 0.677656501531601, + "rewards/forensics_strict_edge_reward": 0.015434302389621735, + "rewards/forensics_tiou_reward": 0.4445422440767288, + "step": 461 + }, + { + "completion_length": 23.171875, + "epoch": 1.4178187403993856, + "grad_norm": 5.182041645050049, + "kl": 0.3203125, + "learning_rate": 6.453846153846153e-07, + "loss": -0.0345, + "reward": 0.7638172507286072, + "reward_std": 0.03545411420054734, + "reward_std_all": 0.04152611084282398, + "rewards-max/forensics_sinkhorn_reward": 0.9561417698860168, + "rewards-max/forensics_strict_edge_reward": 0.49982161819934845, + "rewards-max/forensics_tiou_reward": 0.925298660993576, + "rewards/forensics_sinkhorn_reward": 0.732843428850174, + "rewards/forensics_strict_edge_reward": 0.03666591830551624, + "rewards/forensics_tiou_reward": 0.5417304039001465, + "step": 462 + }, + { + "completion_length": 25.078125, + "epoch": 1.4208909370199692, + "grad_norm": 4.597148418426514, + "kl": 0.36328125, + "learning_rate": 6.446153846153845e-07, + "loss": -0.0239, + "reward": 0.7401838600635529, + "reward_std": 0.07770612090826035, + "reward_std_all": 0.07468385621905327, + "rewards-max/forensics_sinkhorn_reward": 0.9678612053394318, + "rewards-max/forensics_strict_edge_reward": 0.6428841799497604, + "rewards-max/forensics_tiou_reward": 0.9519611895084381, + "rewards/forensics_sinkhorn_reward": 0.6730723679065704, + "rewards/forensics_strict_edge_reward": 0.0301281800493598, + "rewards/forensics_tiou_reward": 0.4818340241909027, + "step": 463 + }, + { + "completion_length": 23.6875, + "epoch": 1.423963133640553, + "grad_norm": 4.92365837097168, + "kl": 0.3681640625, + "learning_rate": 6.438461538461538e-07, + "loss": 0.0054, + "reward": 0.7132799923419952, + "reward_std": 0.10360073298215866, + "reward_std_all": 0.0990709736943245, + "rewards-max/forensics_sinkhorn_reward": 0.935204267501831, + "rewards-max/forensics_strict_edge_reward": 0.2571656070649624, + "rewards-max/forensics_tiou_reward": 0.8723755776882172, + "rewards/forensics_sinkhorn_reward": 0.6552889347076416, + "rewards/forensics_strict_edge_reward": 0.02378912433050573, + "rewards/forensics_tiou_reward": 0.4392145127058029, + "step": 464 + }, + { + "completion_length": 24.703125, + "epoch": 1.4270353302611367, + "grad_norm": 5.978327751159668, + "kl": 0.3427734375, + "learning_rate": 6.43076923076923e-07, + "loss": 0.0429, + "reward": 0.7522556185722351, + "reward_std": 0.036324949003756046, + "reward_std_all": 0.047290390357375145, + "rewards-max/forensics_sinkhorn_reward": 0.9817405939102173, + "rewards-max/forensics_strict_edge_reward": 0.529000461101532, + "rewards-max/forensics_tiou_reward": 0.9787469804286957, + "rewards/forensics_sinkhorn_reward": 0.6992334723472595, + "rewards/forensics_strict_edge_reward": 0.036634813994169235, + "rewards/forensics_tiou_reward": 0.5150368809700012, + "step": 465 + }, + { + "completion_length": 25.671875, + "epoch": 1.4301075268817205, + "grad_norm": 4.550301551818848, + "kl": 0.328125, + "learning_rate": 6.423076923076924e-07, + "loss": 0.1291, + "reward": 0.7104419469833374, + "reward_std": 0.07795684039592743, + "reward_std_all": 0.08975154533982277, + "rewards-max/forensics_sinkhorn_reward": 0.9593896269798279, + "rewards-max/forensics_strict_edge_reward": 0.44429154694080353, + "rewards-max/forensics_tiou_reward": 0.920745462179184, + "rewards/forensics_sinkhorn_reward": 0.5668835341930389, + "rewards/forensics_strict_edge_reward": 0.028809107840061188, + "rewards/forensics_tiou_reward": 0.42228811979293823, + "step": 466 + }, + { + "completion_length": 24.59375, + "epoch": 1.4331797235023043, + "grad_norm": 5.582429885864258, + "kl": 0.5615234375, + "learning_rate": 6.415384615384615e-07, + "loss": 0.146, + "reward": 0.7509828209877014, + "reward_std": 0.039717523381114006, + "reward_std_all": 0.046126301400363445, + "rewards-max/forensics_sinkhorn_reward": 0.9590809047222137, + "rewards-max/forensics_strict_edge_reward": 0.13876578491181135, + "rewards-max/forensics_tiou_reward": 0.8885507583618164, + "rewards/forensics_sinkhorn_reward": 0.6880316138267517, + "rewards/forensics_strict_edge_reward": 0.008835361048113555, + "rewards/forensics_tiou_reward": 0.39781762659549713, + "step": 467 + }, + { + "completion_length": 23.921875, + "epoch": 1.4362519201228878, + "grad_norm": 6.188971996307373, + "kl": 0.3603515625, + "learning_rate": 6.407692307692307e-07, + "loss": 0.0697, + "reward": 0.7180662751197815, + "reward_std": 0.0916469506919384, + "reward_std_all": 0.08923636004328728, + "rewards-max/forensics_sinkhorn_reward": 0.9754964709281921, + "rewards-max/forensics_strict_edge_reward": 0.6236754059791565, + "rewards-max/forensics_tiou_reward": 0.9653312861919403, + "rewards/forensics_sinkhorn_reward": 0.6157610416412354, + "rewards/forensics_strict_edge_reward": 0.03481929376721382, + "rewards/forensics_tiou_reward": 0.33457885682582855, + "step": 468 + }, + { + "completion_length": 24.125, + "epoch": 1.4393241167434716, + "grad_norm": 5.457013130187988, + "kl": 0.3173828125, + "learning_rate": 6.4e-07, + "loss": 0.1152, + "reward": 0.726161926984787, + "reward_std": 0.05928395688533783, + "reward_std_all": 0.06831883825361729, + "rewards-max/forensics_sinkhorn_reward": 0.984169989824295, + "rewards-max/forensics_strict_edge_reward": 0.3706423342227936, + "rewards-max/forensics_tiou_reward": 0.9844972789287567, + "rewards/forensics_sinkhorn_reward": 0.6416573524475098, + "rewards/forensics_strict_edge_reward": 0.028005107305943966, + "rewards/forensics_tiou_reward": 0.40465857088565826, + "step": 469 + }, + { + "completion_length": 23.109375, + "epoch": 1.4423963133640554, + "grad_norm": 4.3388824462890625, + "kl": 0.3232421875, + "learning_rate": 6.392307692307692e-07, + "loss": 0.0119, + "reward": 0.7596500813961029, + "reward_std": 0.025898978114128113, + "reward_std_all": 0.03927509253844619, + "rewards-max/forensics_sinkhorn_reward": 0.9842782020568848, + "rewards-max/forensics_strict_edge_reward": 0.8205508291721344, + "rewards-max/forensics_tiou_reward": 0.9884907007217407, + "rewards/forensics_sinkhorn_reward": 0.7290554046630859, + "rewards/forensics_strict_edge_reward": 0.08580052852630615, + "rewards/forensics_tiou_reward": 0.5530236661434174, + "step": 470 + }, + { + "completion_length": 23.234375, + "epoch": 1.445468509984639, + "grad_norm": 5.191361427307129, + "kl": 0.3662109375, + "learning_rate": 6.384615384615383e-07, + "loss": 0.0997, + "reward": 0.7424454092979431, + "reward_std": 0.031544082798063755, + "reward_std_all": 0.04761642590165138, + "rewards-max/forensics_sinkhorn_reward": 0.9646033644676208, + "rewards-max/forensics_strict_edge_reward": 0.4793061912059784, + "rewards-max/forensics_tiou_reward": 0.9399616122245789, + "rewards/forensics_sinkhorn_reward": 0.6946547031402588, + "rewards/forensics_strict_edge_reward": 0.04497721046209335, + "rewards/forensics_tiou_reward": 0.4812561124563217, + "step": 471 + }, + { + "completion_length": 23.96875, + "epoch": 1.4485407066052227, + "grad_norm": 5.307833671569824, + "kl": 0.4599609375, + "learning_rate": 6.376923076923077e-07, + "loss": 0.1104, + "reward": 0.7756361365318298, + "reward_std": 0.037045182660222054, + "reward_std_all": 0.0373455174267292, + "rewards-max/forensics_sinkhorn_reward": 0.9750222265720367, + "rewards-max/forensics_strict_edge_reward": 0.33995509147644043, + "rewards-max/forensics_tiou_reward": 0.9647020101547241, + "rewards/forensics_sinkhorn_reward": 0.7271770536899567, + "rewards/forensics_strict_edge_reward": 0.026834659511223435, + "rewards/forensics_tiou_reward": 0.5072776675224304, + "step": 472 + }, + { + "completion_length": 26.5625, + "epoch": 1.4516129032258065, + "grad_norm": 5.742395877838135, + "kl": 0.30859375, + "learning_rate": 6.369230769230769e-07, + "loss": 0.0753, + "reward": 0.7283064424991608, + "reward_std": 0.05294218845665455, + "reward_std_all": 0.06938163563609123, + "rewards-max/forensics_sinkhorn_reward": 0.9521133601665497, + "rewards-max/forensics_strict_edge_reward": 0.45573790371418, + "rewards-max/forensics_tiou_reward": 0.9139717221260071, + "rewards/forensics_sinkhorn_reward": 0.668056458234787, + "rewards/forensics_strict_edge_reward": 0.039805782958865166, + "rewards/forensics_tiou_reward": 0.47134363651275635, + "step": 473 + }, + { + "completion_length": 22.984375, + "epoch": 1.4546850998463903, + "grad_norm": 5.258537769317627, + "kl": 0.3095703125, + "learning_rate": 6.361538461538462e-07, + "loss": 0.0445, + "reward": 0.7074538469314575, + "reward_std": 0.0584727693349123, + "reward_std_all": 0.07953999564051628, + "rewards-max/forensics_sinkhorn_reward": 0.9755761921405792, + "rewards-max/forensics_strict_edge_reward": 0.44491301476955414, + "rewards-max/forensics_tiou_reward": 0.9638314545154572, + "rewards/forensics_sinkhorn_reward": 0.6928231716156006, + "rewards/forensics_strict_edge_reward": 0.04782150313258171, + "rewards/forensics_tiou_reward": 0.49341902136802673, + "step": 474 + }, + { + "completion_length": 24.375, + "epoch": 1.4577572964669738, + "grad_norm": 5.133844375610352, + "kl": 0.251953125, + "learning_rate": 6.353846153846154e-07, + "loss": 0.0043, + "reward": 0.7338152229785919, + "reward_std": 0.07787538692355156, + "reward_std_all": 0.07574865967035294, + "rewards-max/forensics_sinkhorn_reward": 0.9676687717437744, + "rewards-max/forensics_strict_edge_reward": 0.25333166122436523, + "rewards-max/forensics_tiou_reward": 0.9419251382350922, + "rewards/forensics_sinkhorn_reward": 0.6333504617214203, + "rewards/forensics_strict_edge_reward": 0.014691077172756195, + "rewards/forensics_tiou_reward": 0.40068066120147705, + "step": 475 + }, + { + "completion_length": 22.75, + "epoch": 1.4608294930875576, + "grad_norm": 4.2725749015808105, + "kl": 0.2900390625, + "learning_rate": 6.346153846153845e-07, + "loss": -0.0222, + "reward": 0.7727255523204803, + "reward_std": 0.024394381791353226, + "reward_std_all": 0.03043812606483698, + "rewards-max/forensics_sinkhorn_reward": 0.9483133256435394, + "rewards-max/forensics_strict_edge_reward": 0.3642716035246849, + "rewards-max/forensics_tiou_reward": 0.8963099122047424, + "rewards/forensics_sinkhorn_reward": 0.7132278382778168, + "rewards/forensics_strict_edge_reward": 0.023636675905436277, + "rewards/forensics_tiou_reward": 0.5025676041841507, + "step": 476 + }, + { + "completion_length": 24.34375, + "epoch": 1.4639016897081414, + "grad_norm": 5.418365955352783, + "kl": 0.2978515625, + "learning_rate": 6.338461538461538e-07, + "loss": -0.053, + "reward": 0.7671906650066376, + "reward_std": 0.042737243697047234, + "reward_std_all": 0.042540209367871284, + "rewards-max/forensics_sinkhorn_reward": 0.9703514873981476, + "rewards-max/forensics_strict_edge_reward": 0.5154883414506912, + "rewards-max/forensics_tiou_reward": 0.9480874538421631, + "rewards/forensics_sinkhorn_reward": 0.7030338048934937, + "rewards/forensics_strict_edge_reward": 0.039882381446659565, + "rewards/forensics_tiou_reward": 0.49448804557323456, + "step": 477 + }, + { + "completion_length": 24.578125, + "epoch": 1.466973886328725, + "grad_norm": 4.830808162689209, + "kl": 0.2958984375, + "learning_rate": 6.33076923076923e-07, + "loss": 0.0345, + "reward": 0.7702966034412384, + "reward_std": 0.04768069088459015, + "reward_std_all": 0.04529138654470444, + "rewards-max/forensics_sinkhorn_reward": 0.9788236618041992, + "rewards-max/forensics_strict_edge_reward": 0.7286747694015503, + "rewards-max/forensics_tiou_reward": 0.9743768870830536, + "rewards/forensics_sinkhorn_reward": 0.7326970398426056, + "rewards/forensics_strict_edge_reward": 0.07641799375414848, + "rewards/forensics_tiou_reward": 0.5642949938774109, + "step": 478 + }, + { + "completion_length": 22.875, + "epoch": 1.4700460829493087, + "grad_norm": 6.069859981536865, + "kl": 0.3857421875, + "learning_rate": 6.323076923076924e-07, + "loss": 0.072, + "reward": 0.7489524781703949, + "reward_std": 0.06797178089618683, + "reward_std_all": 0.0666382685303688, + "rewards-max/forensics_sinkhorn_reward": 0.9576573669910431, + "rewards-max/forensics_strict_edge_reward": 0.35254401341080666, + "rewards-max/forensics_tiou_reward": 0.9211035966873169, + "rewards/forensics_sinkhorn_reward": 0.6723623275756836, + "rewards/forensics_strict_edge_reward": 0.030636015813797712, + "rewards/forensics_tiou_reward": 0.4231562465429306, + "step": 479 + }, + { + "completion_length": 22.5625, + "epoch": 1.4731182795698925, + "grad_norm": 4.489915370941162, + "kl": 0.3212890625, + "learning_rate": 6.315384615384615e-07, + "loss": 0.1672, + "reward": 0.7541794180870056, + "reward_std": 0.0536740068346262, + "reward_std_all": 0.055162761360406876, + "rewards-max/forensics_sinkhorn_reward": 0.9275166094303131, + "rewards-max/forensics_strict_edge_reward": 0.30537426099181175, + "rewards-max/forensics_tiou_reward": 0.8492475152015686, + "rewards/forensics_sinkhorn_reward": 0.656732052564621, + "rewards/forensics_strict_edge_reward": 0.023789732484146953, + "rewards/forensics_tiou_reward": 0.44656240940093994, + "step": 480 + }, + { + "completion_length": 27.578125, + "epoch": 1.4761904761904763, + "grad_norm": 4.86618185043335, + "kl": 0.3916015625, + "learning_rate": 6.307692307692307e-07, + "loss": 0.0381, + "reward": 0.7716245055198669, + "reward_std": 0.0518017653375864, + "reward_std_all": 0.0475796926766634, + "rewards-max/forensics_sinkhorn_reward": 0.9733111262321472, + "rewards-max/forensics_strict_edge_reward": 0.45270346850156784, + "rewards-max/forensics_tiou_reward": 0.9587774574756622, + "rewards/forensics_sinkhorn_reward": 0.71368008852005, + "rewards/forensics_strict_edge_reward": 0.02986025344580412, + "rewards/forensics_tiou_reward": 0.5596306622028351, + "step": 481 + }, + { + "completion_length": 24.28125, + "epoch": 1.4792626728110598, + "grad_norm": 5.781702995300293, + "kl": 0.3349609375, + "learning_rate": 6.3e-07, + "loss": 0.0389, + "reward": 0.7248740792274475, + "reward_std": 0.07669033110141754, + "reward_std_all": 0.07794594764709473, + "rewards-max/forensics_sinkhorn_reward": 0.9531901478767395, + "rewards-max/forensics_strict_edge_reward": 0.5862891674041748, + "rewards-max/forensics_tiou_reward": 0.917053610086441, + "rewards/forensics_sinkhorn_reward": 0.6471811830997467, + "rewards/forensics_strict_edge_reward": 0.02822755742818117, + "rewards/forensics_tiou_reward": 0.40435758233070374, + "step": 482 + }, + { + "completion_length": 27.546875, + "epoch": 1.4823348694316436, + "grad_norm": 4.518404483795166, + "kl": 0.298828125, + "learning_rate": 6.292307692307692e-07, + "loss": 0.0651, + "reward": 0.7105937004089355, + "reward_std": 0.07835347950458527, + "reward_std_all": 0.08338989689946175, + "rewards-max/forensics_sinkhorn_reward": 0.9757735729217529, + "rewards-max/forensics_strict_edge_reward": 0.24938895553350449, + "rewards-max/forensics_tiou_reward": 0.9605361521244049, + "rewards/forensics_sinkhorn_reward": 0.645373523235321, + "rewards/forensics_strict_edge_reward": 0.022172103635966778, + "rewards/forensics_tiou_reward": 0.4667798578739166, + "step": 483 + }, + { + "completion_length": 25.0625, + "epoch": 1.4854070660522274, + "grad_norm": 5.159846305847168, + "kl": 0.3232421875, + "learning_rate": 6.284615384615385e-07, + "loss": 0.0459, + "reward": 0.7365825176239014, + "reward_std": 0.07627879828214645, + "reward_std_all": 0.0737566240131855, + "rewards-max/forensics_sinkhorn_reward": 0.9810838401317596, + "rewards-max/forensics_strict_edge_reward": 0.670350581407547, + "rewards-max/forensics_tiou_reward": 0.9783133268356323, + "rewards/forensics_sinkhorn_reward": 0.6942694783210754, + "rewards/forensics_strict_edge_reward": 0.05417372286319733, + "rewards/forensics_tiou_reward": 0.5168567299842834, + "step": 484 + }, + { + "completion_length": 26.625, + "epoch": 1.488479262672811, + "grad_norm": 4.768603801727295, + "kl": 0.30859375, + "learning_rate": 6.276923076923076e-07, + "loss": 0.051, + "reward": 0.7759858071804047, + "reward_std": 0.03415248799137771, + "reward_std_all": 0.036113624926656485, + "rewards-max/forensics_sinkhorn_reward": 0.9586979448795319, + "rewards-max/forensics_strict_edge_reward": 0.40824490785598755, + "rewards-max/forensics_tiou_reward": 0.9276515245437622, + "rewards/forensics_sinkhorn_reward": 0.6992605924606323, + "rewards/forensics_strict_edge_reward": 0.029227430000901222, + "rewards/forensics_tiou_reward": 0.5093288421630859, + "step": 485 + }, + { + "completion_length": 26.625, + "epoch": 1.4915514592933947, + "grad_norm": 8.034205436706543, + "kl": 0.3896484375, + "learning_rate": 6.269230769230769e-07, + "loss": -0.059, + "reward": 0.6763530969619751, + "reward_std": 0.1009986437857151, + "reward_std_all": 0.10954992100596428, + "rewards-max/forensics_sinkhorn_reward": 0.9226821064949036, + "rewards-max/forensics_strict_edge_reward": 0.3826866000890732, + "rewards-max/forensics_tiou_reward": 0.8406727910041809, + "rewards/forensics_sinkhorn_reward": 0.6100442409515381, + "rewards/forensics_strict_edge_reward": 0.049347303342074156, + "rewards/forensics_tiou_reward": 0.41899213194847107, + "step": 486 + }, + { + "completion_length": 25.9375, + "epoch": 1.4946236559139785, + "grad_norm": 4.328708648681641, + "kl": 0.35546875, + "learning_rate": 6.261538461538462e-07, + "loss": 0.0785, + "reward": 0.7595410943031311, + "reward_std": 0.04653232637792826, + "reward_std_all": 0.0474515063688159, + "rewards-max/forensics_sinkhorn_reward": 0.9751114547252655, + "rewards-max/forensics_strict_edge_reward": 0.48032720386981964, + "rewards-max/forensics_tiou_reward": 0.9620620012283325, + "rewards/forensics_sinkhorn_reward": 0.715812623500824, + "rewards/forensics_strict_edge_reward": 0.04521439969539642, + "rewards/forensics_tiou_reward": 0.48017077147960663, + "step": 487 + }, + { + "completion_length": 25.796875, + "epoch": 1.4976958525345623, + "grad_norm": 5.020196914672852, + "kl": 0.2783203125, + "learning_rate": 6.253846153846154e-07, + "loss": 0.0575, + "reward": 0.7764869332313538, + "reward_std": 0.03676389157772064, + "reward_std_all": 0.035727242939174175, + "rewards-max/forensics_sinkhorn_reward": 0.9706838726997375, + "rewards-max/forensics_strict_edge_reward": 0.33508339524269104, + "rewards-max/forensics_tiou_reward": 0.9530521333217621, + "rewards/forensics_sinkhorn_reward": 0.7126112878322601, + "rewards/forensics_strict_edge_reward": 0.03190525155514479, + "rewards/forensics_tiou_reward": 0.5242630690336227, + "step": 488 + }, + { + "completion_length": 28.515625, + "epoch": 1.500768049155146, + "grad_norm": 5.360908031463623, + "kl": 0.388671875, + "learning_rate": 6.246153846153845e-07, + "loss": 0.0758, + "reward": 0.7413786053657532, + "reward_std": 0.05984613858163357, + "reward_std_all": 0.061464132741093636, + "rewards-max/forensics_sinkhorn_reward": 0.9552640020847321, + "rewards-max/forensics_strict_edge_reward": 0.11075000744313002, + "rewards-max/forensics_tiou_reward": 0.9028996229171753, + "rewards/forensics_sinkhorn_reward": 0.6328535377979279, + "rewards/forensics_strict_edge_reward": 0.008197246090276167, + "rewards/forensics_tiou_reward": 0.46135711669921875, + "step": 489 + }, + { + "completion_length": 23.1875, + "epoch": 1.5038402457757296, + "grad_norm": 5.1095757484436035, + "kl": 0.458984375, + "learning_rate": 6.238461538461538e-07, + "loss": -0.0094, + "reward": 0.7562561929225922, + "reward_std": 0.033625880256295204, + "reward_std_all": 0.04239492956548929, + "rewards-max/forensics_sinkhorn_reward": 0.9631901681423187, + "rewards-max/forensics_strict_edge_reward": 0.49533946067094803, + "rewards-max/forensics_tiou_reward": 0.9409538507461548, + "rewards/forensics_sinkhorn_reward": 0.7172526717185974, + "rewards/forensics_strict_edge_reward": 0.05545667186379433, + "rewards/forensics_tiou_reward": 0.5191434025764465, + "step": 490 + }, + { + "completion_length": 21.90625, + "epoch": 1.5069124423963134, + "grad_norm": 5.494870185852051, + "kl": 0.361328125, + "learning_rate": 6.23076923076923e-07, + "loss": 0.1024, + "reward": 0.7582233548164368, + "reward_std": 0.051066527143120766, + "reward_std_all": 0.05098664294928312, + "rewards-max/forensics_sinkhorn_reward": 0.9619130790233612, + "rewards-max/forensics_strict_edge_reward": 0.31800276041030884, + "rewards-max/forensics_tiou_reward": 0.9333856105804443, + "rewards/forensics_sinkhorn_reward": 0.6775329113006592, + "rewards/forensics_strict_edge_reward": 0.026966837234795094, + "rewards/forensics_tiou_reward": 0.4922793507575989, + "step": 491 + }, + { + "completion_length": 23.46875, + "epoch": 1.509984639016897, + "grad_norm": 10.649685859680176, + "kl": 1.1279296875, + "learning_rate": 6.223076923076924e-07, + "loss": 0.0356, + "reward": 0.7623151540756226, + "reward_std": 0.019169215112924576, + "reward_std_all": 0.03292479366064072, + "rewards-max/forensics_sinkhorn_reward": 0.9739063680171967, + "rewards-max/forensics_strict_edge_reward": 0.2945442795753479, + "rewards-max/forensics_tiou_reward": 0.9596981406211853, + "rewards/forensics_sinkhorn_reward": 0.7214722037315369, + "rewards/forensics_strict_edge_reward": 0.02739203441888094, + "rewards/forensics_tiou_reward": 0.4964999556541443, + "step": 492 + }, + { + "completion_length": 24.34375, + "epoch": 1.5130568356374807, + "grad_norm": 6.633374214172363, + "kl": 0.5546875, + "learning_rate": 6.215384615384615e-07, + "loss": 0.1155, + "reward": 0.7757631540298462, + "reward_std": 0.03481292026117444, + "reward_std_all": 0.034498011227697134, + "rewards-max/forensics_sinkhorn_reward": 0.9722317457199097, + "rewards-max/forensics_strict_edge_reward": 0.28830963373184204, + "rewards-max/forensics_tiou_reward": 0.9541698396205902, + "rewards/forensics_sinkhorn_reward": 0.7207040190696716, + "rewards/forensics_strict_edge_reward": 0.023608386516571045, + "rewards/forensics_tiou_reward": 0.5181155800819397, + "step": 493 + }, + { + "completion_length": 25.65625, + "epoch": 1.5161290322580645, + "grad_norm": 4.71400260925293, + "kl": 0.26708984375, + "learning_rate": 6.207692307692307e-07, + "loss": 0.1163, + "reward": 0.763911098241806, + "reward_std": 0.050631213933229446, + "reward_std_all": 0.04860240779817104, + "rewards-max/forensics_sinkhorn_reward": 0.9666288495063782, + "rewards-max/forensics_strict_edge_reward": 0.0711783766746521, + "rewards-max/forensics_tiou_reward": 0.9366492331027985, + "rewards/forensics_sinkhorn_reward": 0.7370873391628265, + "rewards/forensics_strict_edge_reward": 0.007391275372356176, + "rewards/forensics_tiou_reward": 0.56361985206604, + "step": 494 + }, + { + "completion_length": 23.953125, + "epoch": 1.5192012288786483, + "grad_norm": 4.646671295166016, + "kl": 0.3583984375, + "learning_rate": 6.2e-07, + "loss": 0.1195, + "reward": 0.7769467234611511, + "reward_std": 0.040382156148552895, + "reward_std_all": 0.038143415935337543, + "rewards-max/forensics_sinkhorn_reward": 0.9755080342292786, + "rewards-max/forensics_strict_edge_reward": 0.3201892003417015, + "rewards-max/forensics_tiou_reward": 0.9583256244659424, + "rewards/forensics_sinkhorn_reward": 0.7440330684185028, + "rewards/forensics_strict_edge_reward": 0.0266883063595742, + "rewards/forensics_tiou_reward": 0.5558414310216904, + "step": 495 + }, + { + "completion_length": 22.703125, + "epoch": 1.522273425499232, + "grad_norm": 5.519537448883057, + "kl": 0.3974609375, + "learning_rate": 6.192307692307692e-07, + "loss": -0.0088, + "reward": 0.7573568224906921, + "reward_std": 0.04800181370228529, + "reward_std_all": 0.052286041900515556, + "rewards-max/forensics_sinkhorn_reward": 0.9609058797359467, + "rewards-max/forensics_strict_edge_reward": 0.20117726549506187, + "rewards-max/forensics_tiou_reward": 0.9282086193561554, + "rewards/forensics_sinkhorn_reward": 0.6873368620872498, + "rewards/forensics_strict_edge_reward": 0.010623745154589415, + "rewards/forensics_tiou_reward": 0.44760653376579285, + "step": 496 + }, + { + "completion_length": 26.015625, + "epoch": 1.5253456221198156, + "grad_norm": 8.185402870178223, + "kl": 0.57421875, + "learning_rate": 6.184615384615385e-07, + "loss": 0.164, + "reward": 0.7251850962638855, + "reward_std": 0.08146193623542786, + "reward_std_all": 0.08040110766887665, + "rewards-max/forensics_sinkhorn_reward": 0.9127284586429596, + "rewards-max/forensics_strict_edge_reward": 0.20886878482997417, + "rewards-max/forensics_tiou_reward": 0.8260747790336609, + "rewards/forensics_sinkhorn_reward": 0.6043882369995117, + "rewards/forensics_strict_edge_reward": 0.016876442066859454, + "rewards/forensics_tiou_reward": 0.3488151729106903, + "step": 497 + }, + { + "completion_length": 24.484375, + "epoch": 1.5284178187403994, + "grad_norm": 5.89113712310791, + "kl": 0.3544921875, + "learning_rate": 6.176923076923076e-07, + "loss": -0.0134, + "reward": 0.7645629048347473, + "reward_std": 0.054043060168623924, + "reward_std_all": 0.05127090774476528, + "rewards-max/forensics_sinkhorn_reward": 0.9623713493347168, + "rewards-max/forensics_strict_edge_reward": 0.5753957629203796, + "rewards-max/forensics_tiou_reward": 0.9300028085708618, + "rewards/forensics_sinkhorn_reward": 0.7211765050888062, + "rewards/forensics_strict_edge_reward": 0.05868748016655445, + "rewards/forensics_tiou_reward": 0.5226170122623444, + "step": 498 + }, + { + "completion_length": 24.09375, + "epoch": 1.531490015360983, + "grad_norm": 7.3733696937561035, + "kl": 0.3154296875, + "learning_rate": 6.169230769230768e-07, + "loss": 0.0516, + "reward": 0.7367986142635345, + "reward_std": 0.03749650623649359, + "reward_std_all": 0.05393903702497482, + "rewards-max/forensics_sinkhorn_reward": 0.9327062368392944, + "rewards-max/forensics_strict_edge_reward": 0.15077018178999424, + "rewards-max/forensics_tiou_reward": 0.8649499118328094, + "rewards/forensics_sinkhorn_reward": 0.5939809083938599, + "rewards/forensics_strict_edge_reward": 0.015000960323959589, + "rewards/forensics_tiou_reward": 0.3743569999933243, + "step": 499 + }, + { + "completion_length": 25.375, + "epoch": 1.5345622119815667, + "grad_norm": 5.140129089355469, + "kl": 0.4443359375, + "learning_rate": 6.161538461538462e-07, + "loss": 0.1701, + "reward": 0.7028290033340454, + "reward_std": 0.07713350467383862, + "reward_std_all": 0.09296931326389313, + "rewards-max/forensics_sinkhorn_reward": 0.9489244222640991, + "rewards-max/forensics_strict_edge_reward": 0.24428774137049913, + "rewards-max/forensics_tiou_reward": 0.8878953456878662, + "rewards/forensics_sinkhorn_reward": 0.6180340647697449, + "rewards/forensics_strict_edge_reward": 0.013652357971295714, + "rewards/forensics_tiou_reward": 0.39876240491867065, + "step": 500 + }, + { + "completion_length": 23.0625, + "epoch": 1.5376344086021505, + "grad_norm": 5.373363018035889, + "kl": 0.2802734375, + "learning_rate": 6.153846153846154e-07, + "loss": 0.0138, + "reward": 0.7011196911334991, + "reward_std": 0.08456357941031456, + "reward_std_all": 0.09953759238123894, + "rewards-max/forensics_sinkhorn_reward": 0.9626651108264923, + "rewards-max/forensics_strict_edge_reward": 0.29145028814673424, + "rewards-max/forensics_tiou_reward": 0.9337213933467865, + "rewards/forensics_sinkhorn_reward": 0.6085807085037231, + "rewards/forensics_strict_edge_reward": 0.02615108131431043, + "rewards/forensics_tiou_reward": 0.4335832893848419, + "step": 501 + }, + { + "completion_length": 27.5625, + "epoch": 1.5407066052227343, + "grad_norm": 6.52694034576416, + "kl": 0.3720703125, + "learning_rate": 6.146153846153845e-07, + "loss": 0.0871, + "reward": 0.7480640113353729, + "reward_std": 0.03789807856082916, + "reward_std_all": 0.048707811161875725, + "rewards-max/forensics_sinkhorn_reward": 0.9547587633132935, + "rewards-max/forensics_strict_edge_reward": 0.2268797680735588, + "rewards-max/forensics_tiou_reward": 0.90329709649086, + "rewards/forensics_sinkhorn_reward": 0.6888577342033386, + "rewards/forensics_strict_edge_reward": 0.01832688506692648, + "rewards/forensics_tiou_reward": 0.4806172549724579, + "step": 502 + }, + { + "completion_length": 30.359375, + "epoch": 1.543778801843318, + "grad_norm": 4.384621620178223, + "kl": 0.3544921875, + "learning_rate": 6.138461538461538e-07, + "loss": 0.0742, + "reward": 0.770114004611969, + "reward_std": 0.036283151246607304, + "reward_std_all": 0.037542106583714485, + "rewards-max/forensics_sinkhorn_reward": 0.9833572506904602, + "rewards-max/forensics_strict_edge_reward": 0.5060962587594986, + "rewards-max/forensics_tiou_reward": 0.9844174385070801, + "rewards/forensics_sinkhorn_reward": 0.7496690154075623, + "rewards/forensics_strict_edge_reward": 0.04348982870578766, + "rewards/forensics_tiou_reward": 0.600645124912262, + "step": 503 + }, + { + "completion_length": 26.890625, + "epoch": 1.5468509984639018, + "grad_norm": 5.30443000793457, + "kl": 0.3544921875, + "learning_rate": 6.13076923076923e-07, + "loss": 0.0891, + "reward": 0.746554434299469, + "reward_std": 0.04856792837381363, + "reward_std_all": 0.05436103418469429, + "rewards-max/forensics_sinkhorn_reward": 0.9751749932765961, + "rewards-max/forensics_strict_edge_reward": 0.28679482638835907, + "rewards-max/forensics_tiou_reward": 0.9611589014530182, + "rewards/forensics_sinkhorn_reward": 0.6721814274787903, + "rewards/forensics_strict_edge_reward": 0.01585921086370945, + "rewards/forensics_tiou_reward": 0.5060073733329773, + "step": 504 + }, + { + "completion_length": 23.328125, + "epoch": 1.5499231950844854, + "grad_norm": 5.5107645988464355, + "kl": 0.3515625, + "learning_rate": 6.123076923076923e-07, + "loss": -0.0111, + "reward": 0.7670088112354279, + "reward_std": 0.05537278205156326, + "reward_std_all": 0.05147700197994709, + "rewards-max/forensics_sinkhorn_reward": 0.9773460626602173, + "rewards-max/forensics_strict_edge_reward": 0.49555233120918274, + "rewards-max/forensics_tiou_reward": 0.9726996123790741, + "rewards/forensics_sinkhorn_reward": 0.7113810777664185, + "rewards/forensics_strict_edge_reward": 0.055409446358680725, + "rewards/forensics_tiou_reward": 0.4731361120939255, + "step": 505 + }, + { + "completion_length": 24.40625, + "epoch": 1.5529953917050692, + "grad_norm": 5.002136707305908, + "kl": 0.30078125, + "learning_rate": 6.115384615384616e-07, + "loss": 0.0752, + "reward": 0.7673086822032928, + "reward_std": 0.04364364966750145, + "reward_std_all": 0.04291127994656563, + "rewards-max/forensics_sinkhorn_reward": 0.9662081897258759, + "rewards-max/forensics_strict_edge_reward": 0.37411409616470337, + "rewards-max/forensics_tiou_reward": 0.9448645412921906, + "rewards/forensics_sinkhorn_reward": 0.7133854031562805, + "rewards/forensics_strict_edge_reward": 0.049122538417577744, + "rewards/forensics_tiou_reward": 0.5585246980190277, + "step": 506 + }, + { + "completion_length": 24.28125, + "epoch": 1.5560675883256527, + "grad_norm": 6.21854305267334, + "kl": 0.322265625, + "learning_rate": 6.107692307692307e-07, + "loss": 0.0921, + "reward": 0.7447168529033661, + "reward_std": 0.08156232163310051, + "reward_std_all": 0.0757378526031971, + "rewards-max/forensics_sinkhorn_reward": 0.9778252243995667, + "rewards-max/forensics_strict_edge_reward": 0.24876798689365387, + "rewards-max/forensics_tiou_reward": 0.9668658375740051, + "rewards/forensics_sinkhorn_reward": 0.681669682264328, + "rewards/forensics_strict_edge_reward": 0.023744708858430386, + "rewards/forensics_tiou_reward": 0.5046847760677338, + "step": 507 + }, + { + "completion_length": 29.140625, + "epoch": 1.5591397849462365, + "grad_norm": 4.647692680358887, + "kl": 0.3203125, + "learning_rate": 6.1e-07, + "loss": 0.0242, + "reward": 0.7793949544429779, + "reward_std": 0.03639431297779083, + "reward_std_all": 0.03519662469625473, + "rewards-max/forensics_sinkhorn_reward": 0.9119532704353333, + "rewards-max/forensics_strict_edge_reward": 0.22432304918766022, + "rewards-max/forensics_tiou_reward": 0.8343715369701385, + "rewards/forensics_sinkhorn_reward": 0.6707358956336975, + "rewards/forensics_strict_edge_reward": 0.02026007417589426, + "rewards/forensics_tiou_reward": 0.4815981984138489, + "step": 508 + }, + { + "completion_length": 23.859375, + "epoch": 1.5622119815668203, + "grad_norm": 4.934382915496826, + "kl": 0.3056640625, + "learning_rate": 6.092307692307692e-07, + "loss": -0.0063, + "reward": 0.7545538544654846, + "reward_std": 0.050114765763282776, + "reward_std_all": 0.05047007091343403, + "rewards-max/forensics_sinkhorn_reward": 0.9794407784938812, + "rewards-max/forensics_strict_edge_reward": 0.42959781736135483, + "rewards-max/forensics_tiou_reward": 0.9695360958576202, + "rewards/forensics_sinkhorn_reward": 0.7306766510009766, + "rewards/forensics_strict_edge_reward": 0.04221387207508087, + "rewards/forensics_tiou_reward": 0.5908980071544647, + "step": 509 + }, + { + "completion_length": 23.515625, + "epoch": 1.565284178187404, + "grad_norm": 5.671987533569336, + "kl": 0.349609375, + "learning_rate": 6.084615384615385e-07, + "loss": 0.1467, + "reward": 0.72670778632164, + "reward_std": 0.058765375055372715, + "reward_std_all": 0.06798937544226646, + "rewards-max/forensics_sinkhorn_reward": 0.9706606566905975, + "rewards-max/forensics_strict_edge_reward": 0.48720505833625793, + "rewards-max/forensics_tiou_reward": 0.955617368221283, + "rewards/forensics_sinkhorn_reward": 0.6466456949710846, + "rewards/forensics_strict_edge_reward": 0.028542978689074516, + "rewards/forensics_tiou_reward": 0.4352562725543976, + "step": 510 + }, + { + "completion_length": 25.328125, + "epoch": 1.5683563748079878, + "grad_norm": 5.315916061401367, + "kl": 0.5068359375, + "learning_rate": 6.076923076923076e-07, + "loss": 0.0567, + "reward": 0.7657158076763153, + "reward_std": 0.05078116850927472, + "reward_std_all": 0.048313082195818424, + "rewards-max/forensics_sinkhorn_reward": 0.9602363407611847, + "rewards-max/forensics_strict_edge_reward": 0.2790645956993103, + "rewards-max/forensics_tiou_reward": 0.9210772812366486, + "rewards/forensics_sinkhorn_reward": 0.684025377035141, + "rewards/forensics_strict_edge_reward": 0.024110733531415462, + "rewards/forensics_tiou_reward": 0.49045026302337646, + "step": 511 + }, + { + "completion_length": 23.34375, + "epoch": 1.5714285714285714, + "grad_norm": 4.820674896240234, + "kl": 0.34765625, + "learning_rate": 6.069230769230768e-07, + "loss": 0.0828, + "reward": 0.7495439052581787, + "reward_std": 0.06632681377232075, + "reward_std_all": 0.06388630904257298, + "rewards-max/forensics_sinkhorn_reward": 0.9692847728729248, + "rewards-max/forensics_strict_edge_reward": 0.4159500002861023, + "rewards-max/forensics_tiou_reward": 0.9493221342563629, + "rewards/forensics_sinkhorn_reward": 0.661889374256134, + "rewards/forensics_strict_edge_reward": 0.035616081207990646, + "rewards/forensics_tiou_reward": 0.4991934150457382, + "step": 512 + }, + { + "completion_length": 25.171875, + "epoch": 1.5745007680491552, + "grad_norm": 5.179501533508301, + "kl": 0.3671875, + "learning_rate": 6.061538461538462e-07, + "loss": 0.1637, + "reward": 0.7460094690322876, + "reward_std": 0.07700482476502657, + "reward_std_all": 0.07186134718358517, + "rewards-max/forensics_sinkhorn_reward": 0.9177014231681824, + "rewards-max/forensics_strict_edge_reward": 0.49514414370059967, + "rewards-max/forensics_tiou_reward": 0.8463487923145294, + "rewards/forensics_sinkhorn_reward": 0.6143040657043457, + "rewards/forensics_strict_edge_reward": 0.044272978557273746, + "rewards/forensics_tiou_reward": 0.46732620894908905, + "step": 513 + }, + { + "completion_length": 24.546875, + "epoch": 1.5775729646697387, + "grad_norm": 6.05659294128418, + "kl": 0.373046875, + "learning_rate": 6.053846153846154e-07, + "loss": 0.0028, + "reward": 0.7168903052806854, + "reward_std": 0.09480550140142441, + "reward_std_all": 0.09200622513890266, + "rewards-max/forensics_sinkhorn_reward": 0.9282847344875336, + "rewards-max/forensics_strict_edge_reward": 0.040140550408978015, + "rewards-max/forensics_tiou_reward": 0.8392235338687897, + "rewards/forensics_sinkhorn_reward": 0.6376811563968658, + "rewards/forensics_strict_edge_reward": 0.0039267137599381385, + "rewards/forensics_tiou_reward": 0.42962098121643066, + "step": 514 + }, + { + "completion_length": 27.234375, + "epoch": 1.5806451612903225, + "grad_norm": 5.3084821701049805, + "kl": 0.380859375, + "learning_rate": 6.046153846153847e-07, + "loss": 0.057, + "reward": 0.7259660959243774, + "reward_std": 0.06742462888360023, + "reward_std_all": 0.07296162657439709, + "rewards-max/forensics_sinkhorn_reward": 0.9658587872982025, + "rewards-max/forensics_strict_edge_reward": 0.2520219385623932, + "rewards-max/forensics_tiou_reward": 0.9378741085529327, + "rewards/forensics_sinkhorn_reward": 0.6331635415554047, + "rewards/forensics_strict_edge_reward": 0.012670232448726892, + "rewards/forensics_tiou_reward": 0.45931223034858704, + "step": 515 + }, + { + "completion_length": 24.53125, + "epoch": 1.5837173579109063, + "grad_norm": 4.752344131469727, + "kl": 0.3798828125, + "learning_rate": 6.038461538461538e-07, + "loss": 0.0066, + "reward": 0.7381918132305145, + "reward_std": 0.044284517178311944, + "reward_std_all": 0.053762183990329504, + "rewards-max/forensics_sinkhorn_reward": 0.9569794237613678, + "rewards-max/forensics_strict_edge_reward": 0.3037334904074669, + "rewards-max/forensics_tiou_reward": 0.9151637554168701, + "rewards/forensics_sinkhorn_reward": 0.6744257807731628, + "rewards/forensics_strict_edge_reward": 0.027345901355147362, + "rewards/forensics_tiou_reward": 0.5109153091907501, + "step": 516 + }, + { + "completion_length": 22.1875, + "epoch": 1.58678955453149, + "grad_norm": 5.191107749938965, + "kl": 0.3330078125, + "learning_rate": 6.03076923076923e-07, + "loss": -0.0447, + "reward": 0.7627894878387451, + "reward_std": 0.029761076904833317, + "reward_std_all": 0.038208214566111565, + "rewards-max/forensics_sinkhorn_reward": 0.9613608121871948, + "rewards-max/forensics_strict_edge_reward": 0.40700532495975494, + "rewards-max/forensics_tiou_reward": 0.931807279586792, + "rewards/forensics_sinkhorn_reward": 0.7006286382675171, + "rewards/forensics_strict_edge_reward": 0.0263626161031425, + "rewards/forensics_tiou_reward": 0.46603187918663025, + "step": 517 + }, + { + "completion_length": 25.625, + "epoch": 1.5898617511520738, + "grad_norm": 6.128625392913818, + "kl": 0.3212890625, + "learning_rate": 6.023076923076923e-07, + "loss": 0.1029, + "reward": 0.6750576198101044, + "reward_std": 0.05615561082959175, + "reward_std_all": 0.09081916511058807, + "rewards-max/forensics_sinkhorn_reward": 0.9456472992897034, + "rewards-max/forensics_strict_edge_reward": 0.2910424992442131, + "rewards-max/forensics_tiou_reward": 0.8947326838970184, + "rewards/forensics_sinkhorn_reward": 0.5653172731399536, + "rewards/forensics_strict_edge_reward": 0.020802832441404462, + "rewards/forensics_tiou_reward": 0.3614148795604706, + "step": 518 + }, + { + "completion_length": 22.609375, + "epoch": 1.5929339477726574, + "grad_norm": 5.590939998626709, + "kl": 0.3291015625, + "learning_rate": 6.015384615384615e-07, + "loss": 0.115, + "reward": 0.7543351054191589, + "reward_std": 0.04610149562358856, + "reward_std_all": 0.050968363881111145, + "rewards-max/forensics_sinkhorn_reward": 0.9722813367843628, + "rewards-max/forensics_strict_edge_reward": 0.4602832943201065, + "rewards-max/forensics_tiou_reward": 0.9575404822826385, + "rewards/forensics_sinkhorn_reward": 0.6578051149845123, + "rewards/forensics_strict_edge_reward": 0.02898648101836443, + "rewards/forensics_tiou_reward": 0.4844696819782257, + "step": 519 + }, + { + "completion_length": 24.03125, + "epoch": 1.5960061443932412, + "grad_norm": 5.525546073913574, + "kl": 0.38671875, + "learning_rate": 6.007692307692307e-07, + "loss": -0.0605, + "reward": 0.7455747723579407, + "reward_std": 0.06556405732408166, + "reward_std_all": 0.06550271157175303, + "rewards-max/forensics_sinkhorn_reward": 0.9497931003570557, + "rewards-max/forensics_strict_edge_reward": 0.4853668659925461, + "rewards-max/forensics_tiou_reward": 0.9105598032474518, + "rewards/forensics_sinkhorn_reward": 0.6830792129039764, + "rewards/forensics_strict_edge_reward": 0.033312054350972176, + "rewards/forensics_tiou_reward": 0.41553421318531036, + "step": 520 + }, + { + "completion_length": 26.546875, + "epoch": 1.5990783410138247, + "grad_norm": 6.2567667961120605, + "kl": 0.3408203125, + "learning_rate": 6e-07, + "loss": 0.0432, + "reward": 0.7485706210136414, + "reward_std": 0.06840904802083969, + "reward_std_all": 0.06725383549928665, + "rewards-max/forensics_sinkhorn_reward": 0.9815452992916107, + "rewards-max/forensics_strict_edge_reward": 0.29145028814673424, + "rewards-max/forensics_tiou_reward": 0.976487398147583, + "rewards/forensics_sinkhorn_reward": 0.6497615575790405, + "rewards/forensics_strict_edge_reward": 0.017269277945160866, + "rewards/forensics_tiou_reward": 0.41429612040519714, + "step": 521 + }, + { + "completion_length": 22.140625, + "epoch": 1.6021505376344085, + "grad_norm": 6.2587890625, + "kl": 0.3603515625, + "learning_rate": 5.992307692307692e-07, + "loss": 0.146, + "reward": 0.736611932516098, + "reward_std": 0.08018882945179939, + "reward_std_all": 0.07693652808666229, + "rewards-max/forensics_sinkhorn_reward": 0.9623714983463287, + "rewards-max/forensics_strict_edge_reward": 0.45348018780350685, + "rewards-max/forensics_tiou_reward": 0.9165148735046387, + "rewards/forensics_sinkhorn_reward": 0.6503269374370575, + "rewards/forensics_strict_edge_reward": 0.03836861485615373, + "rewards/forensics_tiou_reward": 0.4506777673959732, + "step": 522 + }, + { + "completion_length": 28.28125, + "epoch": 1.6052227342549923, + "grad_norm": 5.400631904602051, + "kl": 0.3603515625, + "learning_rate": 5.984615384615385e-07, + "loss": 0.1056, + "reward": 0.7091707289218903, + "reward_std": 0.057602012529969215, + "reward_std_all": 0.07769573293626308, + "rewards-max/forensics_sinkhorn_reward": 0.979341447353363, + "rewards-max/forensics_strict_edge_reward": 0.3243762031197548, + "rewards-max/forensics_tiou_reward": 0.9665104746818542, + "rewards/forensics_sinkhorn_reward": 0.6193398833274841, + "rewards/forensics_strict_edge_reward": 0.021065525244921446, + "rewards/forensics_tiou_reward": 0.5132315158843994, + "step": 523 + }, + { + "completion_length": 21.796875, + "epoch": 1.608294930875576, + "grad_norm": 5.795227527618408, + "kl": 0.29296875, + "learning_rate": 5.976923076923077e-07, + "loss": 0.0591, + "reward": 0.7741601765155792, + "reward_std": 0.0364743722602725, + "reward_std_all": 0.03615426830947399, + "rewards-max/forensics_sinkhorn_reward": 0.9719935655593872, + "rewards-max/forensics_strict_edge_reward": 0.4752257168292999, + "rewards-max/forensics_tiou_reward": 0.9621561765670776, + "rewards/forensics_sinkhorn_reward": 0.7357157170772552, + "rewards/forensics_strict_edge_reward": 0.026788045652210712, + "rewards/forensics_tiou_reward": 0.5239407122135162, + "step": 524 + }, + { + "completion_length": 22.109375, + "epoch": 1.6113671274961598, + "grad_norm": 5.567805290222168, + "kl": 0.3232421875, + "learning_rate": 5.969230769230768e-07, + "loss": 0.0699, + "reward": 0.7043308317661285, + "reward_std": 0.08255177736282349, + "reward_std_all": 0.08850781619548798, + "rewards-max/forensics_sinkhorn_reward": 0.9463789761066437, + "rewards-max/forensics_strict_edge_reward": 0.3407678585499525, + "rewards-max/forensics_tiou_reward": 0.8878128528594971, + "rewards/forensics_sinkhorn_reward": 0.5914224684238434, + "rewards/forensics_strict_edge_reward": 0.01646096573676914, + "rewards/forensics_tiou_reward": 0.34120115637779236, + "step": 525 + }, + { + "completion_length": 27.15625, + "epoch": 1.6144393241167436, + "grad_norm": 5.530982494354248, + "kl": 0.3955078125, + "learning_rate": 5.961538461538461e-07, + "loss": 0.0859, + "reward": 0.7791778147220612, + "reward_std": 0.035338070476427674, + "reward_std_all": 0.03538288036361337, + "rewards-max/forensics_sinkhorn_reward": 0.9464494287967682, + "rewards-max/forensics_strict_edge_reward": 0.25232090055942535, + "rewards-max/forensics_tiou_reward": 0.8939900398254395, + "rewards/forensics_sinkhorn_reward": 0.7222039699554443, + "rewards/forensics_strict_edge_reward": 0.014997826889157295, + "rewards/forensics_tiou_reward": 0.4805818647146225, + "step": 526 + }, + { + "completion_length": 23.5, + "epoch": 1.6175115207373272, + "grad_norm": 5.513359546661377, + "kl": 0.310546875, + "learning_rate": 5.953846153846154e-07, + "loss": 0.0467, + "reward": 0.7689100503921509, + "reward_std": 0.037710607051849365, + "reward_std_all": 0.03886691853404045, + "rewards-max/forensics_sinkhorn_reward": 0.9848390519618988, + "rewards-max/forensics_strict_edge_reward": 0.39243943989276886, + "rewards-max/forensics_tiou_reward": 0.985365241765976, + "rewards/forensics_sinkhorn_reward": 0.7086751163005829, + "rewards/forensics_strict_edge_reward": 0.049109588842839, + "rewards/forensics_tiou_reward": 0.5359969288110733, + "step": 527 + }, + { + "completion_length": 21.453125, + "epoch": 1.620583717357911, + "grad_norm": 6.0838541984558105, + "kl": 0.37890625, + "learning_rate": 5.946153846153847e-07, + "loss": 0.2117, + "reward": 0.7331224381923676, + "reward_std": 0.0564560629427433, + "reward_std_all": 0.06413859874010086, + "rewards-max/forensics_sinkhorn_reward": 0.9419687986373901, + "rewards-max/forensics_strict_edge_reward": 0.2874234840273857, + "rewards-max/forensics_tiou_reward": 0.8908880650997162, + "rewards/forensics_sinkhorn_reward": 0.650936096906662, + "rewards/forensics_strict_edge_reward": 0.0239476403221488, + "rewards/forensics_tiou_reward": 0.42105206847190857, + "step": 528 + }, + { + "completion_length": 24.1875, + "epoch": 1.6236559139784945, + "grad_norm": 5.930139064788818, + "kl": 0.310546875, + "learning_rate": 5.938461538461538e-07, + "loss": 0.069, + "reward": 0.7347880005836487, + "reward_std": 0.055159274488687515, + "reward_std_all": 0.06644688919186592, + "rewards-max/forensics_sinkhorn_reward": 0.958457738161087, + "rewards-max/forensics_strict_edge_reward": 0.6779094636440277, + "rewards-max/forensics_tiou_reward": 0.9333553314208984, + "rewards/forensics_sinkhorn_reward": 0.6495635509490967, + "rewards/forensics_strict_edge_reward": 0.043410140089690685, + "rewards/forensics_tiou_reward": 0.43173404037952423, + "step": 529 + }, + { + "completion_length": 24.203125, + "epoch": 1.6267281105990783, + "grad_norm": 5.237308979034424, + "kl": 0.35546875, + "learning_rate": 5.93076923076923e-07, + "loss": 0.0504, + "reward": 0.778364360332489, + "reward_std": 0.027102605439722538, + "reward_std_all": 0.02836840134114027, + "rewards-max/forensics_sinkhorn_reward": 0.9595597982406616, + "rewards-max/forensics_strict_edge_reward": 0.21445735730230808, + "rewards-max/forensics_tiou_reward": 0.9128767549991608, + "rewards/forensics_sinkhorn_reward": 0.7101091742515564, + "rewards/forensics_strict_edge_reward": 0.020434682141058147, + "rewards/forensics_tiou_reward": 0.4940802603960037, + "step": 530 + }, + { + "completion_length": 26.015625, + "epoch": 1.629800307219662, + "grad_norm": 10.139472007751465, + "kl": 0.8759765625, + "learning_rate": 5.923076923076923e-07, + "loss": 0.0104, + "reward": 0.73670294880867, + "reward_std": 0.05599525384604931, + "reward_std_all": 0.062414247542619705, + "rewards-max/forensics_sinkhorn_reward": 0.9678951799869537, + "rewards-max/forensics_strict_edge_reward": 0.4175232835114002, + "rewards-max/forensics_tiou_reward": 0.9420250356197357, + "rewards/forensics_sinkhorn_reward": 0.6188640296459198, + "rewards/forensics_strict_edge_reward": 0.028339712880551815, + "rewards/forensics_tiou_reward": 0.43172743916511536, + "step": 531 + }, + { + "completion_length": 26.328125, + "epoch": 1.6328725038402458, + "grad_norm": 5.705209732055664, + "kl": 0.39453125, + "learning_rate": 5.915384615384615e-07, + "loss": 0.0924, + "reward": 0.7419840395450592, + "reward_std": 0.04817580338567495, + "reward_std_all": 0.055812107399106026, + "rewards-max/forensics_sinkhorn_reward": 0.9428049623966217, + "rewards-max/forensics_strict_edge_reward": 0.39958298951387405, + "rewards-max/forensics_tiou_reward": 0.8940498530864716, + "rewards/forensics_sinkhorn_reward": 0.6483157575130463, + "rewards/forensics_strict_edge_reward": 0.02154468186199665, + "rewards/forensics_tiou_reward": 0.43728339672088623, + "step": 532 + }, + { + "completion_length": 26.359375, + "epoch": 1.6359447004608296, + "grad_norm": 6.183478355407715, + "kl": 0.3173828125, + "learning_rate": 5.907692307692307e-07, + "loss": 0.1246, + "reward": 0.7626746892929077, + "reward_std": 0.06075302138924599, + "reward_std_all": 0.056267622858285904, + "rewards-max/forensics_sinkhorn_reward": 0.9759305417537689, + "rewards-max/forensics_strict_edge_reward": 0.582625538110733, + "rewards-max/forensics_tiou_reward": 0.9663394391536713, + "rewards/forensics_sinkhorn_reward": 0.7133015394210815, + "rewards/forensics_strict_edge_reward": 0.04936795309185982, + "rewards/forensics_tiou_reward": 0.4997524619102478, + "step": 533 + }, + { + "completion_length": 21.765625, + "epoch": 1.6390168970814132, + "grad_norm": 6.991114616394043, + "kl": 0.3271484375, + "learning_rate": 5.9e-07, + "loss": -0.1011, + "reward": 0.7572098076343536, + "reward_std": 0.04326006583869457, + "reward_std_all": 0.04528186470270157, + "rewards-max/forensics_sinkhorn_reward": 0.946379691362381, + "rewards-max/forensics_strict_edge_reward": 0.14227638766169548, + "rewards-max/forensics_tiou_reward": 0.8661891520023346, + "rewards/forensics_sinkhorn_reward": 0.718252032995224, + "rewards/forensics_strict_edge_reward": 0.011070172768086195, + "rewards/forensics_tiou_reward": 0.46950410306453705, + "step": 534 + }, + { + "completion_length": 23.984375, + "epoch": 1.642089093701997, + "grad_norm": 6.663951396942139, + "kl": 0.2744140625, + "learning_rate": 5.892307692307692e-07, + "loss": 0.0652, + "reward": 0.7433145046234131, + "reward_std": 0.07462053932249546, + "reward_std_all": 0.07099074684083462, + "rewards-max/forensics_sinkhorn_reward": 0.9778324365615845, + "rewards-max/forensics_strict_edge_reward": 0.42705684900283813, + "rewards-max/forensics_tiou_reward": 0.9683302938938141, + "rewards/forensics_sinkhorn_reward": 0.665686696767807, + "rewards/forensics_strict_edge_reward": 0.03194698225706816, + "rewards/forensics_tiou_reward": 0.44251789152622223, + "step": 535 + }, + { + "completion_length": 24.75, + "epoch": 1.6451612903225805, + "grad_norm": 7.020610809326172, + "kl": 0.412109375, + "learning_rate": 5.884615384615385e-07, + "loss": 0.1029, + "reward": 0.7057770788669586, + "reward_std": 0.08902426436543465, + "reward_std_all": 0.0963219478726387, + "rewards-max/forensics_sinkhorn_reward": 0.9785006046295166, + "rewards-max/forensics_strict_edge_reward": 0.6096915602684021, + "rewards-max/forensics_tiou_reward": 0.973447322845459, + "rewards/forensics_sinkhorn_reward": 0.6600245833396912, + "rewards/forensics_strict_edge_reward": 0.06853906437754631, + "rewards/forensics_tiou_reward": 0.4587138146162033, + "step": 536 + }, + { + "completion_length": 23.96875, + "epoch": 1.6482334869431643, + "grad_norm": 5.632664203643799, + "kl": 0.3974609375, + "learning_rate": 5.876923076923077e-07, + "loss": 0.0093, + "reward": 0.7236527800559998, + "reward_std": 0.08085665479302406, + "reward_std_all": 0.08039616048336029, + "rewards-max/forensics_sinkhorn_reward": 0.9613861441612244, + "rewards-max/forensics_strict_edge_reward": 0.42500291764736176, + "rewards-max/forensics_tiou_reward": 0.9332343339920044, + "rewards/forensics_sinkhorn_reward": 0.6574849784374237, + "rewards/forensics_strict_edge_reward": 0.031050294172018766, + "rewards/forensics_tiou_reward": 0.48101571202278137, + "step": 537 + }, + { + "completion_length": 24.375, + "epoch": 1.651305683563748, + "grad_norm": 5.538454532623291, + "kl": 0.390625, + "learning_rate": 5.869230769230768e-07, + "loss": -0.0431, + "reward": 0.7661167979240417, + "reward_std": 0.04182830359786749, + "reward_std_all": 0.042108042165637016, + "rewards-max/forensics_sinkhorn_reward": 0.9399314224720001, + "rewards-max/forensics_strict_edge_reward": 0.38842640817165375, + "rewards-max/forensics_tiou_reward": 0.881689190864563, + "rewards/forensics_sinkhorn_reward": 0.7286085486412048, + "rewards/forensics_strict_edge_reward": 0.03232668340206146, + "rewards/forensics_tiou_reward": 0.5572628080844879, + "step": 538 + }, + { + "completion_length": 28.34375, + "epoch": 1.6543778801843319, + "grad_norm": 5.042006969451904, + "kl": 0.3974609375, + "learning_rate": 5.861538461538461e-07, + "loss": 0.0862, + "reward": 0.6963167786598206, + "reward_std": 0.08136139437556267, + "reward_std_all": 0.09017853811383247, + "rewards-max/forensics_sinkhorn_reward": 0.9611000120639801, + "rewards-max/forensics_strict_edge_reward": 0.16062496602535248, + "rewards-max/forensics_tiou_reward": 0.9186963438987732, + "rewards/forensics_sinkhorn_reward": 0.6451685726642609, + "rewards/forensics_strict_edge_reward": 0.015179506037384272, + "rewards/forensics_tiou_reward": 0.4417376220226288, + "step": 539 + }, + { + "completion_length": 28.21875, + "epoch": 1.6574500768049156, + "grad_norm": 5.426444053649902, + "kl": 0.4130859375, + "learning_rate": 5.853846153846154e-07, + "loss": 0.1367, + "reward": 0.7503541111946106, + "reward_std": 0.06850515305995941, + "reward_std_all": 0.06475573778152466, + "rewards-max/forensics_sinkhorn_reward": 0.9835838675498962, + "rewards-max/forensics_strict_edge_reward": 0.34148213267326355, + "rewards-max/forensics_tiou_reward": 0.9872311949729919, + "rewards/forensics_sinkhorn_reward": 0.7000914216041565, + "rewards/forensics_strict_edge_reward": 0.03876134194433689, + "rewards/forensics_tiou_reward": 0.5503258109092712, + "step": 540 + }, + { + "completion_length": 24.484375, + "epoch": 1.6605222734254992, + "grad_norm": 4.728396892547607, + "kl": 0.462890625, + "learning_rate": 5.846153846153847e-07, + "loss": 0.122, + "reward": 0.7633605301380157, + "reward_std": 0.05231539346277714, + "reward_std_all": 0.049897314980626106, + "rewards-max/forensics_sinkhorn_reward": 0.9880191385746002, + "rewards-max/forensics_strict_edge_reward": 0.7246644794940948, + "rewards-max/forensics_tiou_reward": 0.9968746602535248, + "rewards/forensics_sinkhorn_reward": 0.7220478355884552, + "rewards/forensics_strict_edge_reward": 0.06795640662312508, + "rewards/forensics_tiou_reward": 0.5527323782444, + "step": 541 + }, + { + "completion_length": 23.9375, + "epoch": 1.663594470046083, + "grad_norm": 4.962321758270264, + "kl": 0.5400390625, + "learning_rate": 5.838461538461538e-07, + "loss": 0.0425, + "reward": 0.7801069617271423, + "reward_std": 0.025057210121303797, + "reward_std_all": 0.02650881838053465, + "rewards-max/forensics_sinkhorn_reward": 0.9797424674034119, + "rewards-max/forensics_strict_edge_reward": 0.35115350782871246, + "rewards-max/forensics_tiou_reward": 0.9793243706226349, + "rewards/forensics_sinkhorn_reward": 0.7748618125915527, + "rewards/forensics_strict_edge_reward": 0.039338165894150734, + "rewards/forensics_tiou_reward": 0.5913439989089966, + "step": 542 + }, + { + "completion_length": 24.953125, + "epoch": 1.6666666666666665, + "grad_norm": 5.4287214279174805, + "kl": 0.35595703125, + "learning_rate": 5.83076923076923e-07, + "loss": 0.0776, + "reward": 0.7496063411235809, + "reward_std": 0.06627475470304489, + "reward_std_all": 0.06352391839027405, + "rewards-max/forensics_sinkhorn_reward": 0.9141665697097778, + "rewards-max/forensics_strict_edge_reward": 0.2533185724169016, + "rewards-max/forensics_tiou_reward": 0.8199860453605652, + "rewards/forensics_sinkhorn_reward": 0.6202640235424042, + "rewards/forensics_strict_edge_reward": 0.02225586853455752, + "rewards/forensics_tiou_reward": 0.4215851426124573, + "step": 543 + }, + { + "completion_length": 27.9375, + "epoch": 1.6697388632872503, + "grad_norm": 4.972827911376953, + "kl": 0.2685546875, + "learning_rate": 5.823076923076923e-07, + "loss": -0.0525, + "reward": 0.7278097569942474, + "reward_std": 0.03753626253455877, + "reward_std_all": 0.05660079047083855, + "rewards-max/forensics_sinkhorn_reward": 0.9356085360050201, + "rewards-max/forensics_strict_edge_reward": 0.39845556020736694, + "rewards-max/forensics_tiou_reward": 0.8691767454147339, + "rewards/forensics_sinkhorn_reward": 0.6633330881595612, + "rewards/forensics_strict_edge_reward": 0.03419914282858372, + "rewards/forensics_tiou_reward": 0.48144830763339996, + "step": 544 + }, + { + "completion_length": 25.03125, + "epoch": 1.672811059907834, + "grad_norm": 5.552694797515869, + "kl": 0.26513671875, + "learning_rate": 5.815384615384615e-07, + "loss": 0.0221, + "reward": 0.7548276484012604, + "reward_std": 0.0613184180110693, + "reward_std_all": 0.05861359089612961, + "rewards-max/forensics_sinkhorn_reward": 0.9728588759899139, + "rewards-max/forensics_strict_edge_reward": 0.5052543058991432, + "rewards-max/forensics_tiou_reward": 0.9590712785720825, + "rewards/forensics_sinkhorn_reward": 0.6661019921302795, + "rewards/forensics_strict_edge_reward": 0.030452082864940166, + "rewards/forensics_tiou_reward": 0.47656700015068054, + "step": 545 + }, + { + "completion_length": 23.59375, + "epoch": 1.6758832565284179, + "grad_norm": 5.086447715759277, + "kl": 0.3955078125, + "learning_rate": 5.807692307692307e-07, + "loss": 0.1023, + "reward": 0.7565587759017944, + "reward_std": 0.038892149925231934, + "reward_std_all": 0.04383665509521961, + "rewards-max/forensics_sinkhorn_reward": 0.9492134153842926, + "rewards-max/forensics_strict_edge_reward": 0.1868166420608759, + "rewards-max/forensics_tiou_reward": 0.8992453515529633, + "rewards/forensics_sinkhorn_reward": 0.6759867072105408, + "rewards/forensics_strict_edge_reward": 0.02166811702772975, + "rewards/forensics_tiou_reward": 0.5089436322450638, + "step": 546 + }, + { + "completion_length": 24.3125, + "epoch": 1.6789554531490016, + "grad_norm": 5.7509026527404785, + "kl": 0.625, + "learning_rate": 5.8e-07, + "loss": 0.1973, + "reward": 0.7291846573352814, + "reward_std": 0.07933191768825054, + "reward_std_all": 0.07760823518037796, + "rewards-max/forensics_sinkhorn_reward": 0.9665953814983368, + "rewards-max/forensics_strict_edge_reward": 0.37661072611808777, + "rewards-max/forensics_tiou_reward": 0.9418897032737732, + "rewards/forensics_sinkhorn_reward": 0.6208686828613281, + "rewards/forensics_strict_edge_reward": 0.03210534527897835, + "rewards/forensics_tiou_reward": 0.42203161120414734, + "step": 547 + }, + { + "completion_length": 22.84375, + "epoch": 1.6820276497695854, + "grad_norm": 5.1792120933532715, + "kl": 0.2998046875, + "learning_rate": 5.792307692307692e-07, + "loss": 0.1611, + "reward": 0.7220916152000427, + "reward_std": 0.08399283327162266, + "reward_std_all": 0.0867253839969635, + "rewards-max/forensics_sinkhorn_reward": 0.9161738753318787, + "rewards-max/forensics_strict_edge_reward": 0.03939043916761875, + "rewards-max/forensics_tiou_reward": 0.8251504004001617, + "rewards/forensics_sinkhorn_reward": 0.5946572721004486, + "rewards/forensics_strict_edge_reward": 0.00466740399133414, + "rewards/forensics_tiou_reward": 0.4031500518321991, + "step": 548 + }, + { + "completion_length": 24.234375, + "epoch": 1.685099846390169, + "grad_norm": 4.664536476135254, + "kl": 0.3662109375, + "learning_rate": 5.784615384615385e-07, + "loss": 0.0418, + "reward": 0.7449985444545746, + "reward_std": 0.026832707226276398, + "reward_std_all": 0.04653994366526604, + "rewards-max/forensics_sinkhorn_reward": 0.9716756343841553, + "rewards-max/forensics_strict_edge_reward": 0.3313090205192566, + "rewards-max/forensics_tiou_reward": 0.9541888535022736, + "rewards/forensics_sinkhorn_reward": 0.6895883679389954, + "rewards/forensics_strict_edge_reward": 0.03145594848319888, + "rewards/forensics_tiou_reward": 0.469440296292305, + "step": 549 + }, + { + "completion_length": 25.375, + "epoch": 1.6881720430107527, + "grad_norm": 5.748261451721191, + "kl": 0.248046875, + "learning_rate": 5.776923076923077e-07, + "loss": 0.107, + "reward": 0.7494222819805145, + "reward_std": 0.047576405107975006, + "reward_std_all": 0.05562121048569679, + "rewards-max/forensics_sinkhorn_reward": 0.9757571518421173, + "rewards-max/forensics_strict_edge_reward": 0.639672726392746, + "rewards-max/forensics_tiou_reward": 0.9657033383846283, + "rewards/forensics_sinkhorn_reward": 0.699874609708786, + "rewards/forensics_strict_edge_reward": 0.03836212772876024, + "rewards/forensics_tiou_reward": 0.463409960269928, + "step": 550 + }, + { + "completion_length": 22.078125, + "epoch": 1.6912442396313363, + "grad_norm": 5.105411052703857, + "kl": 0.396484375, + "learning_rate": 5.769230769230768e-07, + "loss": -0.0282, + "reward": 0.7428693771362305, + "reward_std": 0.05797100719064474, + "reward_std_all": 0.06062666326761246, + "rewards-max/forensics_sinkhorn_reward": 0.9778975248336792, + "rewards-max/forensics_strict_edge_reward": 0.6915034651756287, + "rewards-max/forensics_tiou_reward": 0.9730963408946991, + "rewards/forensics_sinkhorn_reward": 0.6921932399272919, + "rewards/forensics_strict_edge_reward": 0.056122712790966034, + "rewards/forensics_tiou_reward": 0.45982763171195984, + "step": 551 + }, + { + "completion_length": 24.890625, + "epoch": 1.69431643625192, + "grad_norm": 5.197232246398926, + "kl": 0.416015625, + "learning_rate": 5.761538461538461e-07, + "loss": -0.0821, + "reward": 0.7887932360172272, + "reward_std": 0.021886768750846386, + "reward_std_all": 0.022139960899949074, + "rewards-max/forensics_sinkhorn_reward": 0.9828408360481262, + "rewards-max/forensics_strict_edge_reward": 0.45597073435783386, + "rewards-max/forensics_tiou_reward": 0.9832615256309509, + "rewards/forensics_sinkhorn_reward": 0.7387051284313202, + "rewards/forensics_strict_edge_reward": 0.04457101132720709, + "rewards/forensics_tiou_reward": 0.5513371825218201, + "step": 552 + }, + { + "completion_length": 21.1875, + "epoch": 1.6973886328725039, + "grad_norm": 7.495517730712891, + "kl": 0.3232421875, + "learning_rate": 5.753846153846153e-07, + "loss": 0.0676, + "reward": 0.7847769856452942, + "reward_std": 0.02302721468731761, + "reward_std_all": 0.024275273084640503, + "rewards-max/forensics_sinkhorn_reward": 0.9602195918560028, + "rewards-max/forensics_strict_edge_reward": 0.32482436299324036, + "rewards-max/forensics_tiou_reward": 0.9289330840110779, + "rewards/forensics_sinkhorn_reward": 0.6998815536499023, + "rewards/forensics_strict_edge_reward": 0.03006044030189514, + "rewards/forensics_tiou_reward": 0.43033646047115326, + "step": 553 + }, + { + "completion_length": 26.234375, + "epoch": 1.7004608294930876, + "grad_norm": 5.037136554718018, + "kl": 0.3154296875, + "learning_rate": 5.746153846153847e-07, + "loss": -0.029, + "reward": 0.7845450341701508, + "reward_std": 0.029304989613592625, + "reward_std_all": 0.028387460857629776, + "rewards-max/forensics_sinkhorn_reward": 0.9595099091529846, + "rewards-max/forensics_strict_edge_reward": 0.42256641387939453, + "rewards-max/forensics_tiou_reward": 0.9294803142547607, + "rewards/forensics_sinkhorn_reward": 0.7541773915290833, + "rewards/forensics_strict_edge_reward": 0.055674225091934204, + "rewards/forensics_tiou_reward": 0.5327383875846863, + "step": 554 + }, + { + "completion_length": 25.390625, + "epoch": 1.7035330261136714, + "grad_norm": 5.416619777679443, + "kl": 0.27392578125, + "learning_rate": 5.738461538461539e-07, + "loss": 0.1352, + "reward": 0.7493935823440552, + "reward_std": 0.04856202658265829, + "reward_std_all": 0.052275207825005054, + "rewards-max/forensics_sinkhorn_reward": 0.9451321959495544, + "rewards-max/forensics_strict_edge_reward": 0.276227742433548, + "rewards-max/forensics_tiou_reward": 0.8989401161670685, + "rewards/forensics_sinkhorn_reward": 0.6738614141941071, + "rewards/forensics_strict_edge_reward": 0.026304252445697784, + "rewards/forensics_tiou_reward": 0.4619769752025604, + "step": 555 + }, + { + "completion_length": 24.296875, + "epoch": 1.706605222734255, + "grad_norm": 5.2177276611328125, + "kl": 0.2998046875, + "learning_rate": 5.73076923076923e-07, + "loss": 0.1247, + "reward": 0.723083883523941, + "reward_std": 0.06312699802219868, + "reward_std_all": 0.07168576773256063, + "rewards-max/forensics_sinkhorn_reward": 0.980072408914566, + "rewards-max/forensics_strict_edge_reward": 0.19567500162520446, + "rewards-max/forensics_tiou_reward": 0.9715146422386169, + "rewards/forensics_sinkhorn_reward": 0.5928591787815094, + "rewards/forensics_strict_edge_reward": 0.012397250364301726, + "rewards/forensics_tiou_reward": 0.3644546568393707, + "step": 556 + }, + { + "completion_length": 22.46875, + "epoch": 1.7096774193548387, + "grad_norm": 4.72487211227417, + "kl": 0.396484375, + "learning_rate": 5.723076923076923e-07, + "loss": 0.0682, + "reward": 0.7133220434188843, + "reward_std": 0.09636799804866314, + "reward_std_all": 0.09422561526298523, + "rewards-max/forensics_sinkhorn_reward": 0.9635346531867981, + "rewards-max/forensics_strict_edge_reward": 0.2741934359073639, + "rewards-max/forensics_tiou_reward": 0.9310682117938995, + "rewards/forensics_sinkhorn_reward": 0.6734890043735504, + "rewards/forensics_strict_edge_reward": 0.022947794757783413, + "rewards/forensics_tiou_reward": 0.44655542075634, + "step": 557 + }, + { + "completion_length": 22.671875, + "epoch": 1.7127496159754223, + "grad_norm": 5.0709404945373535, + "kl": 0.4794921875, + "learning_rate": 5.715384615384615e-07, + "loss": -0.021, + "reward": 0.7677647471427917, + "reward_std": 0.044337463565170765, + "reward_std_all": 0.0430673873052001, + "rewards-max/forensics_sinkhorn_reward": 0.9717667400836945, + "rewards-max/forensics_strict_edge_reward": 0.5441897064447403, + "rewards-max/forensics_tiou_reward": 0.9578465223312378, + "rewards/forensics_sinkhorn_reward": 0.7207820415496826, + "rewards/forensics_strict_edge_reward": 0.07269730418920517, + "rewards/forensics_tiou_reward": 0.5179539173841476, + "step": 558 + }, + { + "completion_length": 26.71875, + "epoch": 1.715821812596006, + "grad_norm": 4.491642475128174, + "kl": 0.4052734375, + "learning_rate": 5.707692307692307e-07, + "loss": 0.0629, + "reward": 0.7239814400672913, + "reward_std": 0.0577161218971014, + "reward_std_all": 0.07044491916894913, + "rewards-max/forensics_sinkhorn_reward": 0.9786421954631805, + "rewards-max/forensics_strict_edge_reward": 0.38679473102092743, + "rewards-max/forensics_tiou_reward": 0.9742474257946014, + "rewards/forensics_sinkhorn_reward": 0.6413726806640625, + "rewards/forensics_strict_edge_reward": 0.0396390575915575, + "rewards/forensics_tiou_reward": 0.484534427523613, + "step": 559 + }, + { + "completion_length": 27.375, + "epoch": 1.7188940092165899, + "grad_norm": 5.361271858215332, + "kl": 0.40234375, + "learning_rate": 5.699999999999999e-07, + "loss": 0.1534, + "reward": 0.710812896490097, + "reward_std": 0.08724047243595123, + "reward_std_all": 0.08801621571183205, + "rewards-max/forensics_sinkhorn_reward": 0.9380720853805542, + "rewards-max/forensics_strict_edge_reward": 0.2830090746283531, + "rewards-max/forensics_tiou_reward": 0.8822046518325806, + "rewards/forensics_sinkhorn_reward": 0.5942067801952362, + "rewards/forensics_strict_edge_reward": 0.03214069735258818, + "rewards/forensics_tiou_reward": 0.40580227971076965, + "step": 560 + }, + { + "completion_length": 24.140625, + "epoch": 1.7219662058371736, + "grad_norm": 6.0555548667907715, + "kl": 0.390625, + "learning_rate": 5.692307692307692e-07, + "loss": 0.1086, + "reward": 0.7797804176807404, + "reward_std": 0.03660887759178877, + "reward_std_all": 0.034842836670577526, + "rewards-max/forensics_sinkhorn_reward": 0.9610289037227631, + "rewards-max/forensics_strict_edge_reward": 0.3393824100494385, + "rewards-max/forensics_tiou_reward": 0.930375337600708, + "rewards/forensics_sinkhorn_reward": 0.7073453664779663, + "rewards/forensics_strict_edge_reward": 0.0328264981508255, + "rewards/forensics_tiou_reward": 0.5226892530918121, + "step": 561 + }, + { + "completion_length": 26.640625, + "epoch": 1.7250384024577574, + "grad_norm": 5.175234794616699, + "kl": 0.466796875, + "learning_rate": 5.684615384615385e-07, + "loss": -0.018, + "reward": 0.757578045129776, + "reward_std": 0.02709741471335292, + "reward_std_all": 0.03918516985140741, + "rewards-max/forensics_sinkhorn_reward": 0.961437463760376, + "rewards-max/forensics_strict_edge_reward": 0.20643453299999237, + "rewards-max/forensics_tiou_reward": 0.9344324171543121, + "rewards/forensics_sinkhorn_reward": 0.6787546575069427, + "rewards/forensics_strict_edge_reward": 0.014642433263361454, + "rewards/forensics_tiou_reward": 0.48429545760154724, + "step": 562 + }, + { + "completion_length": 26.109375, + "epoch": 1.728110599078341, + "grad_norm": 5.8121018409729, + "kl": 0.359375, + "learning_rate": 5.676923076923077e-07, + "loss": 0.0649, + "reward": 0.7300612926483154, + "reward_std": 0.05514442827552557, + "reward_std_all": 0.06375138461589813, + "rewards-max/forensics_sinkhorn_reward": 0.9761339426040649, + "rewards-max/forensics_strict_edge_reward": 0.6032288670539856, + "rewards-max/forensics_tiou_reward": 0.9675433933734894, + "rewards/forensics_sinkhorn_reward": 0.623483270406723, + "rewards/forensics_strict_edge_reward": 0.04778123460710049, + "rewards/forensics_tiou_reward": 0.43631306290626526, + "step": 563 + }, + { + "completion_length": 25.921875, + "epoch": 1.7311827956989247, + "grad_norm": 6.287751197814941, + "kl": 0.38671875, + "learning_rate": 5.669230769230769e-07, + "loss": 0.0833, + "reward": 0.731402724981308, + "reward_std": 0.04821506328880787, + "reward_std_all": 0.060618843883275986, + "rewards-max/forensics_sinkhorn_reward": 0.9595450758934021, + "rewards-max/forensics_strict_edge_reward": 0.4529753476381302, + "rewards-max/forensics_tiou_reward": 0.927112340927124, + "rewards/forensics_sinkhorn_reward": 0.6612853705883026, + "rewards/forensics_strict_edge_reward": 0.03954404219985008, + "rewards/forensics_tiou_reward": 0.4523973613977432, + "step": 564 + }, + { + "completion_length": 25.8125, + "epoch": 1.7342549923195083, + "grad_norm": 5.387674808502197, + "kl": 0.3525390625, + "learning_rate": 5.661538461538461e-07, + "loss": -0.0423, + "reward": 0.7584703862667084, + "reward_std": 0.015380330383777618, + "reward_std_all": 0.03311047679744661, + "rewards-max/forensics_sinkhorn_reward": 0.9761140048503876, + "rewards-max/forensics_strict_edge_reward": 0.5550253242254257, + "rewards-max/forensics_tiou_reward": 0.9650227427482605, + "rewards/forensics_sinkhorn_reward": 0.7292992472648621, + "rewards/forensics_strict_edge_reward": 0.03709491714835167, + "rewards/forensics_tiou_reward": 0.5755011141300201, + "step": 565 + }, + { + "completion_length": 23.828125, + "epoch": 1.737327188940092, + "grad_norm": 5.754950523376465, + "kl": 0.4111328125, + "learning_rate": 5.653846153846153e-07, + "loss": 0.1159, + "reward": 0.7314414978027344, + "reward_std": 0.057670269161462784, + "reward_std_all": 0.0661725103855133, + "rewards-max/forensics_sinkhorn_reward": 0.9599430561065674, + "rewards-max/forensics_strict_edge_reward": 0.2708844020962715, + "rewards-max/forensics_tiou_reward": 0.9215202927589417, + "rewards/forensics_sinkhorn_reward": 0.6576389968395233, + "rewards/forensics_strict_edge_reward": 0.01546678738668561, + "rewards/forensics_tiou_reward": 0.4462722837924957, + "step": 566 + }, + { + "completion_length": 23.328125, + "epoch": 1.7403993855606759, + "grad_norm": 6.875593185424805, + "kl": 0.27734375, + "learning_rate": 5.646153846153847e-07, + "loss": 0.1087, + "reward": 0.7376116514205933, + "reward_std": 0.06467229127883911, + "reward_std_all": 0.06942984834313393, + "rewards-max/forensics_sinkhorn_reward": 0.9830150604248047, + "rewards-max/forensics_strict_edge_reward": 0.5582164824008942, + "rewards-max/forensics_tiou_reward": 0.9830214083194733, + "rewards/forensics_sinkhorn_reward": 0.701793909072876, + "rewards/forensics_strict_edge_reward": 0.05729740113019943, + "rewards/forensics_tiou_reward": 0.5296697020530701, + "step": 567 + }, + { + "completion_length": 25.40625, + "epoch": 1.7434715821812596, + "grad_norm": 5.431262493133545, + "kl": 0.3056640625, + "learning_rate": 5.638461538461539e-07, + "loss": 0.0174, + "reward": 0.7862675487995148, + "reward_std": 0.01490045350510627, + "reward_std_all": 0.01878724154084921, + "rewards-max/forensics_sinkhorn_reward": 0.9690931439399719, + "rewards-max/forensics_strict_edge_reward": 0.06367684548604302, + "rewards-max/forensics_tiou_reward": 0.942233681678772, + "rewards/forensics_sinkhorn_reward": 0.7352264523506165, + "rewards/forensics_strict_edge_reward": 0.004350177070591599, + "rewards/forensics_tiou_reward": 0.5118397176265717, + "step": 568 + }, + { + "completion_length": 28.578125, + "epoch": 1.7465437788018434, + "grad_norm": 6.190098762512207, + "kl": 0.38671875, + "learning_rate": 5.63076923076923e-07, + "loss": 0.0501, + "reward": 0.7495759725570679, + "reward_std": 0.07091113924980164, + "reward_std_all": 0.066720400005579, + "rewards-max/forensics_sinkhorn_reward": 0.9647840857505798, + "rewards-max/forensics_strict_edge_reward": 0.4291405826807022, + "rewards-max/forensics_tiou_reward": 0.9372131526470184, + "rewards/forensics_sinkhorn_reward": 0.6889156699180603, + "rewards/forensics_strict_edge_reward": 0.03564881905913353, + "rewards/forensics_tiou_reward": 0.47598133981227875, + "step": 569 + }, + { + "completion_length": 26.265625, + "epoch": 1.7496159754224272, + "grad_norm": 4.500640869140625, + "kl": 0.3408203125, + "learning_rate": 5.623076923076923e-07, + "loss": -0.0635, + "reward": 0.7780423164367676, + "reward_std": 0.03938216343522072, + "reward_std_all": 0.037305038422346115, + "rewards-max/forensics_sinkhorn_reward": 0.979128509759903, + "rewards-max/forensics_strict_edge_reward": 0.5945845991373062, + "rewards-max/forensics_tiou_reward": 0.9720427691936493, + "rewards/forensics_sinkhorn_reward": 0.768778920173645, + "rewards/forensics_strict_edge_reward": 0.05251025687903166, + "rewards/forensics_tiou_reward": 0.5785402953624725, + "step": 570 + }, + { + "completion_length": 25.46875, + "epoch": 1.7526881720430108, + "grad_norm": 5.048827171325684, + "kl": 0.3994140625, + "learning_rate": 5.615384615384615e-07, + "loss": 0.1049, + "reward": 0.7910500168800354, + "reward_std": 0.01716168411076069, + "reward_std_all": 0.0179691044613719, + "rewards-max/forensics_sinkhorn_reward": 0.9775198996067047, + "rewards-max/forensics_strict_edge_reward": 0.34988541156053543, + "rewards-max/forensics_tiou_reward": 0.9672050774097443, + "rewards/forensics_sinkhorn_reward": 0.747468888759613, + "rewards/forensics_strict_edge_reward": 0.027708958368748426, + "rewards/forensics_tiou_reward": 0.5577297806739807, + "step": 571 + }, + { + "completion_length": 28.28125, + "epoch": 1.7557603686635943, + "grad_norm": 6.2897539138793945, + "kl": 0.4140625, + "learning_rate": 5.607692307692308e-07, + "loss": 0.0325, + "reward": 0.6958486437797546, + "reward_std": 0.05209849029779434, + "reward_std_all": 0.0778088141232729, + "rewards-max/forensics_sinkhorn_reward": 0.9642502665519714, + "rewards-max/forensics_strict_edge_reward": 0.3953459709882736, + "rewards-max/forensics_tiou_reward": 0.9354365468025208, + "rewards/forensics_sinkhorn_reward": 0.5962855517864227, + "rewards/forensics_strict_edge_reward": 0.038630323484539986, + "rewards/forensics_tiou_reward": 0.40085749328136444, + "step": 572 + }, + { + "completion_length": 28.65625, + "epoch": 1.758832565284178, + "grad_norm": 4.14538049697876, + "kl": 0.3525390625, + "learning_rate": 5.6e-07, + "loss": 0.1499, + "reward": 0.7442919611930847, + "reward_std": 0.05423400737345219, + "reward_std_all": 0.06039338931441307, + "rewards-max/forensics_sinkhorn_reward": 0.9813105165958405, + "rewards-max/forensics_strict_edge_reward": 0.7540311813354492, + "rewards-max/forensics_tiou_reward": 0.9812857806682587, + "rewards/forensics_sinkhorn_reward": 0.6639346182346344, + "rewards/forensics_strict_edge_reward": 0.061779312789440155, + "rewards/forensics_tiou_reward": 0.5022297650575638, + "step": 573 + }, + { + "completion_length": 24.71875, + "epoch": 1.7619047619047619, + "grad_norm": 5.501622200012207, + "kl": 0.5322265625, + "learning_rate": 5.592307692307691e-07, + "loss": -0.0364, + "reward": 0.7460019290447235, + "reward_std": 0.02134692668914795, + "reward_std_all": 0.042626990005373955, + "rewards-max/forensics_sinkhorn_reward": 0.9740312397480011, + "rewards-max/forensics_strict_edge_reward": 0.4445425719022751, + "rewards-max/forensics_tiou_reward": 0.9553318917751312, + "rewards/forensics_sinkhorn_reward": 0.6986662745475769, + "rewards/forensics_strict_edge_reward": 0.040351876989006996, + "rewards/forensics_tiou_reward": 0.4956386387348175, + "step": 574 + }, + { + "completion_length": 25.5, + "epoch": 1.7649769585253456, + "grad_norm": 4.417118549346924, + "kl": 0.3232421875, + "learning_rate": 5.584615384615385e-07, + "loss": 0.0085, + "reward": 0.7821722328662872, + "reward_std": 0.027604996226727962, + "reward_std_all": 0.028313027694821358, + "rewards-max/forensics_sinkhorn_reward": 0.9657443463802338, + "rewards-max/forensics_strict_edge_reward": 0.17047491669654846, + "rewards-max/forensics_tiou_reward": 0.943783164024353, + "rewards/forensics_sinkhorn_reward": 0.7338769435882568, + "rewards/forensics_strict_edge_reward": 0.008550613885745406, + "rewards/forensics_tiou_reward": 0.4836883544921875, + "step": 575 + }, + { + "completion_length": 25.21875, + "epoch": 1.7680491551459294, + "grad_norm": 6.83552885055542, + "kl": 0.373046875, + "learning_rate": 5.576923076923077e-07, + "loss": 0.0477, + "reward": 0.7121732234954834, + "reward_std": 0.09238898754119873, + "reward_std_all": 0.09164947643876076, + "rewards-max/forensics_sinkhorn_reward": 0.9799903929233551, + "rewards-max/forensics_strict_edge_reward": 0.6215263903141022, + "rewards-max/forensics_tiou_reward": 0.9756492972373962, + "rewards/forensics_sinkhorn_reward": 0.6588848531246185, + "rewards/forensics_strict_edge_reward": 0.055106922052800655, + "rewards/forensics_tiou_reward": 0.5142546445131302, + "step": 576 + }, + { + "completion_length": 25.375, + "epoch": 1.7711213517665132, + "grad_norm": 6.034354209899902, + "kl": 0.4013671875, + "learning_rate": 5.569230769230769e-07, + "loss": 0.0938, + "reward": 0.7104970812797546, + "reward_std": 0.10100976191461086, + "reward_std_all": 0.09778308495879173, + "rewards-max/forensics_sinkhorn_reward": 0.9804362952709198, + "rewards-max/forensics_strict_edge_reward": 0.43546412885189056, + "rewards-max/forensics_tiou_reward": 0.9743005335330963, + "rewards/forensics_sinkhorn_reward": 0.6154041588306427, + "rewards/forensics_strict_edge_reward": 0.04791307635605335, + "rewards/forensics_tiou_reward": 0.4388665407896042, + "step": 577 + }, + { + "completion_length": 25.8125, + "epoch": 1.7741935483870968, + "grad_norm": 4.129523754119873, + "kl": 0.412109375, + "learning_rate": 5.561538461538461e-07, + "loss": -0.0253, + "reward": 0.7728543281555176, + "reward_std": 0.02639874257147312, + "reward_std_all": 0.030682459473609924, + "rewards-max/forensics_sinkhorn_reward": 0.9507259428501129, + "rewards-max/forensics_strict_edge_reward": 0.2512741517275572, + "rewards-max/forensics_tiou_reward": 0.9060092866420746, + "rewards/forensics_sinkhorn_reward": 0.7101556658744812, + "rewards/forensics_strict_edge_reward": 0.03937900857999921, + "rewards/forensics_tiou_reward": 0.48527804017066956, + "step": 578 + }, + { + "completion_length": 21.828125, + "epoch": 1.7772657450076805, + "grad_norm": 6.193716526031494, + "kl": 0.4033203125, + "learning_rate": 5.553846153846153e-07, + "loss": 0.1417, + "reward": 0.7528194487094879, + "reward_std": 0.07137809321284294, + "reward_std_all": 0.06644294038414955, + "rewards-max/forensics_sinkhorn_reward": 0.9328470826148987, + "rewards-max/forensics_strict_edge_reward": 0.08681682433234528, + "rewards-max/forensics_tiou_reward": 0.8562348783016205, + "rewards/forensics_sinkhorn_reward": 0.6767893433570862, + "rewards/forensics_strict_edge_reward": 0.007831709561287425, + "rewards/forensics_tiou_reward": 0.4336331784725189, + "step": 579 + }, + { + "completion_length": 29.34375, + "epoch": 1.780337941628264, + "grad_norm": 4.585134983062744, + "kl": 0.3779296875, + "learning_rate": 5.546153846153846e-07, + "loss": 0.1498, + "reward": 0.7215433716773987, + "reward_std": 0.10045234113931656, + "reward_std_all": 0.09505646303296089, + "rewards-max/forensics_sinkhorn_reward": 0.9777180850505829, + "rewards-max/forensics_strict_edge_reward": 0.6236754059791565, + "rewards-max/forensics_tiou_reward": 0.9748754501342773, + "rewards/forensics_sinkhorn_reward": 0.6139738261699677, + "rewards/forensics_strict_edge_reward": 0.07064937427639961, + "rewards/forensics_tiou_reward": 0.48279422521591187, + "step": 580 + }, + { + "completion_length": 22.203125, + "epoch": 1.7834101382488479, + "grad_norm": 9.74008560180664, + "kl": 0.77734375, + "learning_rate": 5.538461538461539e-07, + "loss": 0.1146, + "reward": 0.7706001102924347, + "reward_std": 0.0501219742000103, + "reward_std_all": 0.04671099781990051, + "rewards-max/forensics_sinkhorn_reward": 0.9633859097957611, + "rewards-max/forensics_strict_edge_reward": 0.28244586661458015, + "rewards-max/forensics_tiou_reward": 0.9260033369064331, + "rewards/forensics_sinkhorn_reward": 0.6923078000545502, + "rewards/forensics_strict_edge_reward": 0.02706692391075194, + "rewards/forensics_tiou_reward": 0.5268197357654572, + "step": 581 + }, + { + "completion_length": 25.453125, + "epoch": 1.7864823348694316, + "grad_norm": 6.369254112243652, + "kl": 0.427734375, + "learning_rate": 5.53076923076923e-07, + "loss": 0.1743, + "reward": 0.7725760042667389, + "reward_std": 0.03194055240601301, + "reward_std_all": 0.03327557630836964, + "rewards-max/forensics_sinkhorn_reward": 0.9552589058876038, + "rewards-max/forensics_strict_edge_reward": 0.30757738649845123, + "rewards-max/forensics_tiou_reward": 0.918097198009491, + "rewards/forensics_sinkhorn_reward": 0.6808826923370361, + "rewards/forensics_strict_edge_reward": 0.02011042181402445, + "rewards/forensics_tiou_reward": 0.46772240102291107, + "step": 582 + }, + { + "completion_length": 24.890625, + "epoch": 1.7895545314900154, + "grad_norm": 6.086848735809326, + "kl": 0.4169921875, + "learning_rate": 5.523076923076923e-07, + "loss": -0.0403, + "reward": 0.7354951798915863, + "reward_std": 0.07464753091335297, + "reward_std_all": 0.07342062704265118, + "rewards-max/forensics_sinkhorn_reward": 0.9113297760486603, + "rewards-max/forensics_strict_edge_reward": 0.19538203813135624, + "rewards-max/forensics_tiou_reward": 0.8196347951889038, + "rewards/forensics_sinkhorn_reward": 0.6491197943687439, + "rewards/forensics_strict_edge_reward": 0.020956865744665265, + "rewards/forensics_tiou_reward": 0.4172540158033371, + "step": 583 + }, + { + "completion_length": 26.25, + "epoch": 1.7926267281105992, + "grad_norm": 5.028934001922607, + "kl": 0.3759765625, + "learning_rate": 5.515384615384615e-07, + "loss": 0.0735, + "reward": 0.772628515958786, + "reward_std": 0.0338821467012167, + "reward_std_all": 0.03495366312563419, + "rewards-max/forensics_sinkhorn_reward": 0.9817324876785278, + "rewards-max/forensics_strict_edge_reward": 0.4363861009478569, + "rewards-max/forensics_tiou_reward": 0.9799437522888184, + "rewards/forensics_sinkhorn_reward": 0.7156690359115601, + "rewards/forensics_strict_edge_reward": 0.02826809510588646, + "rewards/forensics_tiou_reward": 0.5566583871841431, + "step": 584 + }, + { + "completion_length": 23.375, + "epoch": 1.7956989247311828, + "grad_norm": 6.431771755218506, + "kl": 0.3212890625, + "learning_rate": 5.507692307692308e-07, + "loss": -0.0421, + "reward": 0.7414856255054474, + "reward_std": 0.045450933277606964, + "reward_std_all": 0.052894458174705505, + "rewards-max/forensics_sinkhorn_reward": 0.9699739515781403, + "rewards-max/forensics_strict_edge_reward": 0.3515331596136093, + "rewards-max/forensics_tiou_reward": 0.946127861738205, + "rewards/forensics_sinkhorn_reward": 0.6834854781627655, + "rewards/forensics_strict_edge_reward": 0.04061090759932995, + "rewards/forensics_tiou_reward": 0.45320621132850647, + "step": 585 + }, + { + "completion_length": 23.890625, + "epoch": 1.7987711213517665, + "grad_norm": 5.7963433265686035, + "kl": 0.333984375, + "learning_rate": 5.5e-07, + "loss": -0.0608, + "reward": 0.7422322630882263, + "reward_std": 0.0804678238928318, + "reward_std_all": 0.07638006657361984, + "rewards-max/forensics_sinkhorn_reward": 0.983826607465744, + "rewards-max/forensics_strict_edge_reward": 0.42959781736135483, + "rewards-max/forensics_tiou_reward": 0.9844392836093903, + "rewards/forensics_sinkhorn_reward": 0.6814270615577698, + "rewards/forensics_strict_edge_reward": 0.0400185976177454, + "rewards/forensics_tiou_reward": 0.5135847181081772, + "step": 586 + }, + { + "completion_length": 23.328125, + "epoch": 1.80184331797235, + "grad_norm": 7.8835577964782715, + "kl": 0.404296875, + "learning_rate": 5.492307692307691e-07, + "loss": 0.0045, + "reward": 0.7552317380905151, + "reward_std": 0.033486830070614815, + "reward_std_all": 0.0436101034283638, + "rewards-max/forensics_sinkhorn_reward": 0.9613529741764069, + "rewards-max/forensics_strict_edge_reward": 0.22912148758769035, + "rewards-max/forensics_tiou_reward": 0.9337884187698364, + "rewards/forensics_sinkhorn_reward": 0.6980872750282288, + "rewards/forensics_strict_edge_reward": 0.014158789534121752, + "rewards/forensics_tiou_reward": 0.5078323483467102, + "step": 587 + }, + { + "completion_length": 27.25, + "epoch": 1.8049155145929339, + "grad_norm": 5.268822193145752, + "kl": 0.3984375, + "learning_rate": 5.484615384615385e-07, + "loss": -0.0014, + "reward": 0.7849942743778229, + "reward_std": 0.024100484792143106, + "reward_std_all": 0.02471588458865881, + "rewards-max/forensics_sinkhorn_reward": 0.9675138294696808, + "rewards-max/forensics_strict_edge_reward": 0.24503491073846817, + "rewards-max/forensics_tiou_reward": 0.9471846222877502, + "rewards/forensics_sinkhorn_reward": 0.7296835780143738, + "rewards/forensics_strict_edge_reward": 0.020922791212797165, + "rewards/forensics_tiou_reward": 0.5448191165924072, + "step": 588 + }, + { + "completion_length": 25.765625, + "epoch": 1.8079877112135176, + "grad_norm": 8.079438209533691, + "kl": 0.4375, + "learning_rate": 5.476923076923077e-07, + "loss": 0.0589, + "reward": 0.7411144971847534, + "reward_std": 0.04476737044751644, + "reward_std_all": 0.05760115757584572, + "rewards-max/forensics_sinkhorn_reward": 0.9718131721019745, + "rewards-max/forensics_strict_edge_reward": 0.30345969647169113, + "rewards-max/forensics_tiou_reward": 0.9560493230819702, + "rewards/forensics_sinkhorn_reward": 0.6961013674736023, + "rewards/forensics_strict_edge_reward": 0.02531627658754587, + "rewards/forensics_tiou_reward": 0.472782239317894, + "step": 589 + }, + { + "completion_length": 26.59375, + "epoch": 1.8110599078341014, + "grad_norm": 5.168153285980225, + "kl": 0.4228515625, + "learning_rate": 5.469230769230769e-07, + "loss": 0.0456, + "reward": 0.7670029997825623, + "reward_std": 0.046495845541357994, + "reward_std_all": 0.04500264301896095, + "rewards-max/forensics_sinkhorn_reward": 0.9843006730079651, + "rewards-max/forensics_strict_edge_reward": 0.5628271400928497, + "rewards-max/forensics_tiou_reward": 0.9868544340133667, + "rewards/forensics_sinkhorn_reward": 0.7181132733821869, + "rewards/forensics_strict_edge_reward": 0.06444395333528519, + "rewards/forensics_tiou_reward": 0.5461876690387726, + "step": 590 + }, + { + "completion_length": 24.3125, + "epoch": 1.8141321044546852, + "grad_norm": 5.907680988311768, + "kl": 0.3486328125, + "learning_rate": 5.461538461538461e-07, + "loss": 0.016, + "reward": 0.7403899431228638, + "reward_std": 0.022013092879205942, + "reward_std_all": 0.04604298248887062, + "rewards-max/forensics_sinkhorn_reward": 0.9836739897727966, + "rewards-max/forensics_strict_edge_reward": 0.578471302986145, + "rewards-max/forensics_tiou_reward": 0.9874987602233887, + "rewards/forensics_sinkhorn_reward": 0.6738122999668121, + "rewards/forensics_strict_edge_reward": 0.05062001943588257, + "rewards/forensics_tiou_reward": 0.49878303706645966, + "step": 591 + }, + { + "completion_length": 27.8125, + "epoch": 1.817204301075269, + "grad_norm": 5.690028667449951, + "kl": 0.353515625, + "learning_rate": 5.453846153846153e-07, + "loss": 0.0122, + "reward": 0.7663296759128571, + "reward_std": 0.04921477474272251, + "reward_std_all": 0.04879486933350563, + "rewards-max/forensics_sinkhorn_reward": 0.9843803346157074, + "rewards-max/forensics_strict_edge_reward": 0.4809299036860466, + "rewards-max/forensics_tiou_reward": 0.9855283200740814, + "rewards/forensics_sinkhorn_reward": 0.6579211056232452, + "rewards/forensics_strict_edge_reward": 0.05427752807736397, + "rewards/forensics_tiou_reward": 0.5422329306602478, + "step": 592 + }, + { + "completion_length": 26.6875, + "epoch": 1.8202764976958525, + "grad_norm": 5.065611839294434, + "kl": 0.3583984375, + "learning_rate": 5.446153846153846e-07, + "loss": 0.0577, + "reward": 0.6799991130828857, + "reward_std": 0.054905897937715054, + "reward_std_all": 0.08723288774490356, + "rewards-max/forensics_sinkhorn_reward": 0.9638276696205139, + "rewards-max/forensics_strict_edge_reward": 0.2714046686887741, + "rewards-max/forensics_tiou_reward": 0.933608740568161, + "rewards/forensics_sinkhorn_reward": 0.599122166633606, + "rewards/forensics_strict_edge_reward": 0.028638084419071674, + "rewards/forensics_tiou_reward": 0.4552619457244873, + "step": 593 + }, + { + "completion_length": 23.25, + "epoch": 1.823348694316436, + "grad_norm": 4.831943511962891, + "kl": 0.29541015625, + "learning_rate": 5.438461538461538e-07, + "loss": -0.0043, + "reward": 0.7680230140686035, + "reward_std": 0.01670143147930503, + "reward_std_all": 0.028353244997560978, + "rewards-max/forensics_sinkhorn_reward": 0.9812044501304626, + "rewards-max/forensics_strict_edge_reward": 0.6032288670539856, + "rewards-max/forensics_tiou_reward": 0.976999819278717, + "rewards/forensics_sinkhorn_reward": 0.7171691954135895, + "rewards/forensics_strict_edge_reward": 0.06443838030099869, + "rewards/forensics_tiou_reward": 0.5105708539485931, + "step": 594 + }, + { + "completion_length": 24.859375, + "epoch": 1.8264208909370199, + "grad_norm": 5.989454746246338, + "kl": 0.4921875, + "learning_rate": 5.430769230769231e-07, + "loss": 0.094, + "reward": 0.7522427141666412, + "reward_std": 0.04110173601657152, + "reward_std_all": 0.047220559790730476, + "rewards-max/forensics_sinkhorn_reward": 0.9575738310813904, + "rewards-max/forensics_strict_edge_reward": 0.4608110338449478, + "rewards-max/forensics_tiou_reward": 0.9193870425224304, + "rewards/forensics_sinkhorn_reward": 0.6788308620452881, + "rewards/forensics_strict_edge_reward": 0.039544736966490746, + "rewards/forensics_tiou_reward": 0.4503500312566757, + "step": 595 + }, + { + "completion_length": 21.46875, + "epoch": 1.8294930875576036, + "grad_norm": 6.991167068481445, + "kl": 0.3544921875, + "learning_rate": 5.423076923076923e-07, + "loss": 0.0005, + "reward": 0.7149173617362976, + "reward_std": 0.08654254674911499, + "reward_std_all": 0.08733325079083443, + "rewards-max/forensics_sinkhorn_reward": 0.965874046087265, + "rewards-max/forensics_strict_edge_reward": 0.22623636573553085, + "rewards-max/forensics_tiou_reward": 0.9367735385894775, + "rewards/forensics_sinkhorn_reward": 0.6255174577236176, + "rewards/forensics_strict_edge_reward": 0.018999219872057438, + "rewards/forensics_tiou_reward": 0.4278578460216522, + "step": 596 + }, + { + "completion_length": 23.515625, + "epoch": 1.8325652841781874, + "grad_norm": 5.030428409576416, + "kl": 0.3037109375, + "learning_rate": 5.415384615384615e-07, + "loss": 0.0802, + "reward": 0.7652357220649719, + "reward_std": 0.04746108874678612, + "reward_std_all": 0.046186549589037895, + "rewards-max/forensics_sinkhorn_reward": 0.9691379368305206, + "rewards-max/forensics_strict_edge_reward": 0.35375696420669556, + "rewards-max/forensics_tiou_reward": 0.9440010488033295, + "rewards/forensics_sinkhorn_reward": 0.6820848882198334, + "rewards/forensics_strict_edge_reward": 0.0437578521668911, + "rewards/forensics_tiou_reward": 0.5250290632247925, + "step": 597 + }, + { + "completion_length": 21.8125, + "epoch": 1.8356374807987712, + "grad_norm": 6.3237624168396, + "kl": 0.341796875, + "learning_rate": 5.407692307692308e-07, + "loss": 0.1334, + "reward": 0.729026734828949, + "reward_std": 0.0822637751698494, + "reward_std_all": 0.08017803728580475, + "rewards-max/forensics_sinkhorn_reward": 0.9300952553749084, + "rewards-max/forensics_strict_edge_reward": 0.5055906474590302, + "rewards-max/forensics_tiou_reward": 0.8417621552944183, + "rewards/forensics_sinkhorn_reward": 0.6574225425720215, + "rewards/forensics_strict_edge_reward": 0.05621209554374218, + "rewards/forensics_tiou_reward": 0.4586372375488281, + "step": 598 + }, + { + "completion_length": 24.953125, + "epoch": 1.838709677419355, + "grad_norm": 6.511962413787842, + "kl": 0.3349609375, + "learning_rate": 5.4e-07, + "loss": 0.0541, + "reward": 0.7211836278438568, + "reward_std": 0.07978511601686478, + "reward_std_all": 0.08050877973437309, + "rewards-max/forensics_sinkhorn_reward": 0.9572245180606842, + "rewards-max/forensics_strict_edge_reward": 0.3448660671710968, + "rewards-max/forensics_tiou_reward": 0.9095878601074219, + "rewards/forensics_sinkhorn_reward": 0.5976914167404175, + "rewards/forensics_strict_edge_reward": 0.02403881959617138, + "rewards/forensics_tiou_reward": 0.3615196645259857, + "step": 599 + }, + { + "completion_length": 25.875, + "epoch": 1.8417818740399385, + "grad_norm": 5.105074882507324, + "kl": 0.3486328125, + "learning_rate": 5.392307692307691e-07, + "loss": 0.0483, + "reward": 0.7752358019351959, + "reward_std": 0.049046154133975506, + "reward_std_all": 0.044846028089523315, + "rewards-max/forensics_sinkhorn_reward": 0.9734774529933929, + "rewards-max/forensics_strict_edge_reward": 0.4358413964509964, + "rewards-max/forensics_tiou_reward": 0.9594458639621735, + "rewards/forensics_sinkhorn_reward": 0.6963736116886139, + "rewards/forensics_strict_edge_reward": 0.039889080449938774, + "rewards/forensics_tiou_reward": 0.5208384394645691, + "step": 600 + }, + { + "completion_length": 23.421875, + "epoch": 1.8448540706605223, + "grad_norm": 5.840621471405029, + "kl": 0.3935546875, + "learning_rate": 5.384615384615384e-07, + "loss": 0.104, + "reward": 0.7430493831634521, + "reward_std": 0.07641010358929634, + "reward_std_all": 0.07247218117117882, + "rewards-max/forensics_sinkhorn_reward": 0.9840989112854004, + "rewards-max/forensics_strict_edge_reward": 0.6221541315317154, + "rewards-max/forensics_tiou_reward": 0.9852614998817444, + "rewards/forensics_sinkhorn_reward": 0.6978519856929779, + "rewards/forensics_strict_edge_reward": 0.029825027100741863, + "rewards/forensics_tiou_reward": 0.5167160332202911, + "step": 601 + }, + { + "completion_length": 24.53125, + "epoch": 1.8479262672811059, + "grad_norm": 5.7805328369140625, + "kl": 0.603515625, + "learning_rate": 5.376923076923077e-07, + "loss": -0.086, + "reward": 0.7743142247200012, + "reward_std": 0.04469004645943642, + "reward_std_all": 0.04202026687562466, + "rewards-max/forensics_sinkhorn_reward": 0.9731510877609253, + "rewards-max/forensics_strict_edge_reward": 0.4918200522661209, + "rewards-max/forensics_tiou_reward": 0.9555519819259644, + "rewards/forensics_sinkhorn_reward": 0.767023891210556, + "rewards/forensics_strict_edge_reward": 0.05873502045869827, + "rewards/forensics_tiou_reward": 0.5815615952014923, + "step": 602 + }, + { + "completion_length": 23.78125, + "epoch": 1.8509984639016897, + "grad_norm": 5.888652801513672, + "kl": 0.3515625, + "learning_rate": 5.369230769230769e-07, + "loss": -0.0237, + "reward": 0.7834444046020508, + "reward_std": 0.02739805751480162, + "reward_std_all": 0.02751619415357709, + "rewards-max/forensics_sinkhorn_reward": 0.9684267044067383, + "rewards-max/forensics_strict_edge_reward": 0.4661732092499733, + "rewards-max/forensics_tiou_reward": 0.946815550327301, + "rewards/forensics_sinkhorn_reward": 0.6982356011867523, + "rewards/forensics_strict_edge_reward": 0.03183696744963527, + "rewards/forensics_tiou_reward": 0.49253514409065247, + "step": 603 + }, + { + "completion_length": 24.1875, + "epoch": 1.8540706605222734, + "grad_norm": 6.101314544677734, + "kl": 0.3076171875, + "learning_rate": 5.361538461538462e-07, + "loss": 0.0112, + "reward": 0.7941874861717224, + "reward_std": 0.019006403861567378, + "reward_std_all": 0.01899424660950899, + "rewards-max/forensics_sinkhorn_reward": 0.9798674881458282, + "rewards-max/forensics_strict_edge_reward": 0.29716574400663376, + "rewards-max/forensics_tiou_reward": 0.9733864665031433, + "rewards/forensics_sinkhorn_reward": 0.76324862241745, + "rewards/forensics_strict_edge_reward": 0.020714220590889454, + "rewards/forensics_tiou_reward": 0.5720234662294388, + "step": 604 + }, + { + "completion_length": 21.875, + "epoch": 1.8571428571428572, + "grad_norm": 5.751452922821045, + "kl": 0.4560546875, + "learning_rate": 5.353846153846153e-07, + "loss": 0.0529, + "reward": 0.7721838355064392, + "reward_std": 0.058847338892519474, + "reward_std_all": 0.05270326044410467, + "rewards-max/forensics_sinkhorn_reward": 0.9863570034503937, + "rewards-max/forensics_strict_edge_reward": 0.776590883731842, + "rewards-max/forensics_tiou_reward": 0.9925832152366638, + "rewards/forensics_sinkhorn_reward": 0.780449628829956, + "rewards/forensics_strict_edge_reward": 0.05447379872202873, + "rewards/forensics_tiou_reward": 0.5939993262290955, + "step": 605 + }, + { + "completion_length": 25.109375, + "epoch": 1.860215053763441, + "grad_norm": 5.711493492126465, + "kl": 0.34375, + "learning_rate": 5.346153846153846e-07, + "loss": 0.0455, + "reward": 0.6905909478664398, + "reward_std": 0.0636001918464899, + "reward_std_all": 0.08500491082668304, + "rewards-max/forensics_sinkhorn_reward": 0.910767674446106, + "rewards-max/forensics_strict_edge_reward": 0.05229995836270973, + "rewards-max/forensics_tiou_reward": 0.8050245344638824, + "rewards/forensics_sinkhorn_reward": 0.6341371238231659, + "rewards/forensics_strict_edge_reward": 0.004535214873612858, + "rewards/forensics_tiou_reward": 0.40586498379707336, + "step": 606 + }, + { + "completion_length": 25.1875, + "epoch": 1.8632872503840245, + "grad_norm": 5.657567501068115, + "kl": 0.27734375, + "learning_rate": 5.338461538461538e-07, + "loss": 0.0253, + "reward": 0.7658738195896149, + "reward_std": 0.04172534681856632, + "reward_std_all": 0.04232605267316103, + "rewards-max/forensics_sinkhorn_reward": 0.9734235405921936, + "rewards-max/forensics_strict_edge_reward": 0.28095155395567417, + "rewards-max/forensics_tiou_reward": 0.949713796377182, + "rewards/forensics_sinkhorn_reward": 0.7140320539474487, + "rewards/forensics_strict_edge_reward": 0.03212853893637657, + "rewards/forensics_tiou_reward": 0.536904588341713, + "step": 607 + }, + { + "completion_length": 26.15625, + "epoch": 1.8663594470046083, + "grad_norm": 5.0274977684021, + "kl": 0.40625, + "learning_rate": 5.33076923076923e-07, + "loss": -0.062, + "reward": 0.7631745636463165, + "reward_std": 0.02741054631769657, + "reward_std_all": 0.033645499497652054, + "rewards-max/forensics_sinkhorn_reward": 0.9836375117301941, + "rewards-max/forensics_strict_edge_reward": 0.7229099273681641, + "rewards-max/forensics_tiou_reward": 0.9855357110500336, + "rewards/forensics_sinkhorn_reward": 0.7275741100311279, + "rewards/forensics_strict_edge_reward": 0.058262262493371964, + "rewards/forensics_tiou_reward": 0.5646414458751678, + "step": 608 + }, + { + "completion_length": 25.984375, + "epoch": 1.8694316436251919, + "grad_norm": 5.636667251586914, + "kl": 0.328125, + "learning_rate": 5.323076923076923e-07, + "loss": 0.0418, + "reward": 0.7398293018341064, + "reward_std": 0.05162312090396881, + "reward_std_all": 0.05677555501461029, + "rewards-max/forensics_sinkhorn_reward": 0.9810977280139923, + "rewards-max/forensics_strict_edge_reward": 0.5393132865428925, + "rewards-max/forensics_tiou_reward": 0.9806086122989655, + "rewards/forensics_sinkhorn_reward": 0.6798232793807983, + "rewards/forensics_strict_edge_reward": 0.025514845736324787, + "rewards/forensics_tiou_reward": 0.48091651499271393, + "step": 609 + }, + { + "completion_length": 24.40625, + "epoch": 1.8725038402457757, + "grad_norm": 4.943045139312744, + "kl": 0.4912109375, + "learning_rate": 5.315384615384615e-07, + "loss": -0.0688, + "reward": 0.7612939476966858, + "reward_std": 0.023435962153598666, + "reward_std_all": 0.03301523998379707, + "rewards-max/forensics_sinkhorn_reward": 0.9683073461055756, + "rewards-max/forensics_strict_edge_reward": 0.16023516654968262, + "rewards-max/forensics_tiou_reward": 0.9491676390171051, + "rewards/forensics_sinkhorn_reward": 0.7230098247528076, + "rewards/forensics_strict_edge_reward": 0.01164174871519208, + "rewards/forensics_tiou_reward": 0.4981095790863037, + "step": 610 + }, + { + "completion_length": 21.40625, + "epoch": 1.8755760368663594, + "grad_norm": 5.308543682098389, + "kl": 0.3935546875, + "learning_rate": 5.307692307692308e-07, + "loss": 0.0386, + "reward": 0.7702213525772095, + "reward_std": 0.043860155157744884, + "reward_std_all": 0.042366085574030876, + "rewards-max/forensics_sinkhorn_reward": 0.9793702960014343, + "rewards-max/forensics_strict_edge_reward": 0.744143158197403, + "rewards-max/forensics_tiou_reward": 0.9731321632862091, + "rewards/forensics_sinkhorn_reward": 0.7421000897884369, + "rewards/forensics_strict_edge_reward": 0.05158636532723904, + "rewards/forensics_tiou_reward": 0.5324647724628448, + "step": 611 + }, + { + "completion_length": 21.140625, + "epoch": 1.8786482334869432, + "grad_norm": 5.965699195861816, + "kl": 0.54296875, + "learning_rate": 5.3e-07, + "loss": 0.0395, + "reward": 0.7328146696090698, + "reward_std": 0.07583276554942131, + "reward_std_all": 0.07475805282592773, + "rewards-max/forensics_sinkhorn_reward": 0.9786544442176819, + "rewards-max/forensics_strict_edge_reward": 0.4962490648031235, + "rewards-max/forensics_tiou_reward": 0.9714171588420868, + "rewards/forensics_sinkhorn_reward": 0.6619664430618286, + "rewards/forensics_strict_edge_reward": 0.031965525820851326, + "rewards/forensics_tiou_reward": 0.4624319225549698, + "step": 612 + }, + { + "completion_length": 22.765625, + "epoch": 1.881720430107527, + "grad_norm": 6.499261856079102, + "kl": 0.3330078125, + "learning_rate": 5.292307692307692e-07, + "loss": 0.0994, + "reward": 0.7469267249107361, + "reward_std": 0.033345695585012436, + "reward_std_all": 0.04867058806121349, + "rewards-max/forensics_sinkhorn_reward": 0.9626556038856506, + "rewards-max/forensics_strict_edge_reward": 0.28461217135190964, + "rewards-max/forensics_tiou_reward": 0.9290959239006042, + "rewards/forensics_sinkhorn_reward": 0.74301677942276, + "rewards/forensics_strict_edge_reward": 0.02366661000996828, + "rewards/forensics_tiou_reward": 0.5512682050466537, + "step": 613 + }, + { + "completion_length": 24.3125, + "epoch": 1.8847926267281108, + "grad_norm": 7.170101642608643, + "kl": 0.66015625, + "learning_rate": 5.284615384615384e-07, + "loss": 0.0883, + "reward": 0.740513026714325, + "reward_std": 0.058655524626374245, + "reward_std_all": 0.06149721331894398, + "rewards-max/forensics_sinkhorn_reward": 0.9561789929866791, + "rewards-max/forensics_strict_edge_reward": 0.3092816695570946, + "rewards-max/forensics_tiou_reward": 0.9163690209388733, + "rewards/forensics_sinkhorn_reward": 0.6536187529563904, + "rewards/forensics_strict_edge_reward": 0.018866792088374496, + "rewards/forensics_tiou_reward": 0.4364764541387558, + "step": 614 + }, + { + "completion_length": 25.03125, + "epoch": 1.8878648233486943, + "grad_norm": 5.653330326080322, + "kl": 0.3349609375, + "learning_rate": 5.276923076923076e-07, + "loss": 0.1065, + "reward": 0.7450312674045563, + "reward_std": 0.06947849504649639, + "reward_std_all": 0.06983917579054832, + "rewards-max/forensics_sinkhorn_reward": 0.9569355845451355, + "rewards-max/forensics_strict_edge_reward": 0.09881237661466002, + "rewards-max/forensics_tiou_reward": 0.912970244884491, + "rewards/forensics_sinkhorn_reward": 0.6855892837047577, + "rewards/forensics_strict_edge_reward": 0.006216441513970494, + "rewards/forensics_tiou_reward": 0.4225170314311981, + "step": 615 + }, + { + "completion_length": 26.875, + "epoch": 1.8909370199692779, + "grad_norm": 5.1502814292907715, + "kl": 0.3447265625, + "learning_rate": 5.269230769230769e-07, + "loss": -0.0631, + "reward": 0.7652747333049774, + "reward_std": 0.026394234038889408, + "reward_std_all": 0.03213966265320778, + "rewards-max/forensics_sinkhorn_reward": 0.9761615693569183, + "rewards-max/forensics_strict_edge_reward": 0.435970202088356, + "rewards-max/forensics_tiou_reward": 0.9643551409244537, + "rewards/forensics_sinkhorn_reward": 0.7495043873786926, + "rewards/forensics_strict_edge_reward": 0.03580884449183941, + "rewards/forensics_tiou_reward": 0.5484146177768707, + "step": 616 + }, + { + "completion_length": 23.59375, + "epoch": 1.8940092165898617, + "grad_norm": 5.102884292602539, + "kl": 0.48046875, + "learning_rate": 5.261538461538462e-07, + "loss": 0.1019, + "reward": 0.7399138510227203, + "reward_std": 0.041515031829476357, + "reward_std_all": 0.054914938285946846, + "rewards-max/forensics_sinkhorn_reward": 0.9660635590553284, + "rewards-max/forensics_strict_edge_reward": 0.3503752648830414, + "rewards-max/forensics_tiou_reward": 0.9395694136619568, + "rewards/forensics_sinkhorn_reward": 0.7033115029335022, + "rewards/forensics_strict_edge_reward": 0.029858529567718506, + "rewards/forensics_tiou_reward": 0.5306717157363892, + "step": 617 + }, + { + "completion_length": 26.46875, + "epoch": 1.8970814132104454, + "grad_norm": 5.177690029144287, + "kl": 0.4267578125, + "learning_rate": 5.253846153846153e-07, + "loss": 0.0995, + "reward": 0.7732460498809814, + "reward_std": 0.042038616724312305, + "reward_std_all": 0.040186161175370216, + "rewards-max/forensics_sinkhorn_reward": 0.9741469621658325, + "rewards-max/forensics_strict_edge_reward": 0.35325733572244644, + "rewards-max/forensics_tiou_reward": 0.9581886827945709, + "rewards/forensics_sinkhorn_reward": 0.7393041253089905, + "rewards/forensics_strict_edge_reward": 0.037057007662951946, + "rewards/forensics_tiou_reward": 0.5380967855453491, + "step": 618 + }, + { + "completion_length": 22.421875, + "epoch": 1.9001536098310292, + "grad_norm": 6.185136318206787, + "kl": 0.44140625, + "learning_rate": 5.246153846153846e-07, + "loss": 0.1105, + "reward": 0.7223977446556091, + "reward_std": 0.047134583815932274, + "reward_std_all": 0.06361955776810646, + "rewards-max/forensics_sinkhorn_reward": 0.9826741218566895, + "rewards-max/forensics_strict_edge_reward": 0.5238595008850098, + "rewards-max/forensics_tiou_reward": 0.9793796241283417, + "rewards/forensics_sinkhorn_reward": 0.6690913438796997, + "rewards/forensics_strict_edge_reward": 0.03866763785481453, + "rewards/forensics_tiou_reward": 0.4545610696077347, + "step": 619 + }, + { + "completion_length": 23.8125, + "epoch": 1.903225806451613, + "grad_norm": 4.865814208984375, + "kl": 0.3349609375, + "learning_rate": 5.238461538461538e-07, + "loss": 0.1224, + "reward": 0.7477460503578186, + "reward_std": 0.0731564462184906, + "reward_std_all": 0.06903308536857367, + "rewards-max/forensics_sinkhorn_reward": 0.9721863269805908, + "rewards-max/forensics_strict_edge_reward": 0.42271359264850616, + "rewards-max/forensics_tiou_reward": 0.944068044424057, + "rewards/forensics_sinkhorn_reward": 0.6747253835201263, + "rewards/forensics_strict_edge_reward": 0.02900643623434007, + "rewards/forensics_tiou_reward": 0.47072839736938477, + "step": 620 + }, + { + "completion_length": 24.40625, + "epoch": 1.9062980030721968, + "grad_norm": 4.480029106140137, + "kl": 0.3134765625, + "learning_rate": 5.23076923076923e-07, + "loss": 0.1135, + "reward": 0.7634137272834778, + "reward_std": 0.03667036537081003, + "reward_std_all": 0.04186460003256798, + "rewards-max/forensics_sinkhorn_reward": 0.9734474718570709, + "rewards-max/forensics_strict_edge_reward": 0.4600999802350998, + "rewards-max/forensics_tiou_reward": 0.957137793302536, + "rewards/forensics_sinkhorn_reward": 0.6964722573757172, + "rewards/forensics_strict_edge_reward": 0.028813225217163563, + "rewards/forensics_tiou_reward": 0.5069783627986908, + "step": 621 + }, + { + "completion_length": 23.65625, + "epoch": 1.9093701996927803, + "grad_norm": 5.5179972648620605, + "kl": 0.33203125, + "learning_rate": 5.223076923076924e-07, + "loss": 0.0093, + "reward": 0.7388364672660828, + "reward_std": 0.04533689469099045, + "reward_std_all": 0.055582256987690926, + "rewards-max/forensics_sinkhorn_reward": 0.9511003494262695, + "rewards-max/forensics_strict_edge_reward": 0.3238218426704407, + "rewards-max/forensics_tiou_reward": 0.8995179533958435, + "rewards/forensics_sinkhorn_reward": 0.7111693024635315, + "rewards/forensics_strict_edge_reward": 0.023890452459454536, + "rewards/forensics_tiou_reward": 0.5112289786338806, + "step": 622 + }, + { + "completion_length": 27.28125, + "epoch": 1.912442396313364, + "grad_norm": 5.882576942443848, + "kl": 0.4013671875, + "learning_rate": 5.215384615384615e-07, + "loss": 0.0877, + "reward": 0.7690408825874329, + "reward_std": 0.039212255738675594, + "reward_std_all": 0.04210742376744747, + "rewards-max/forensics_sinkhorn_reward": 0.9663542807102203, + "rewards-max/forensics_strict_edge_reward": 0.16356412693858147, + "rewards-max/forensics_tiou_reward": 0.9438900947570801, + "rewards/forensics_sinkhorn_reward": 0.7277884483337402, + "rewards/forensics_strict_edge_reward": 0.009701048373244703, + "rewards/forensics_tiou_reward": 0.5593758821487427, + "step": 623 + }, + { + "completion_length": 26.5625, + "epoch": 1.9155145929339477, + "grad_norm": 6.661516189575195, + "kl": 0.3798828125, + "learning_rate": 5.207692307692308e-07, + "loss": 0.0033, + "reward": 0.7306167781352997, + "reward_std": 0.08299195393919945, + "reward_std_all": 0.08074808120727539, + "rewards-max/forensics_sinkhorn_reward": 0.9660055041313171, + "rewards-max/forensics_strict_edge_reward": 0.18334465473890305, + "rewards-max/forensics_tiou_reward": 0.9400967955589294, + "rewards/forensics_sinkhorn_reward": 0.6759775578975677, + "rewards/forensics_strict_edge_reward": 0.0098533951677382, + "rewards/forensics_tiou_reward": 0.46746405959129333, + "step": 624 + }, + { + "completion_length": 28.6875, + "epoch": 1.9185867895545314, + "grad_norm": 5.556920528411865, + "kl": 0.3525390625, + "learning_rate": 5.2e-07, + "loss": 0.0091, + "reward": 0.7114897668361664, + "reward_std": 0.0657455250620842, + "reward_std_all": 0.08122213929891586, + "rewards-max/forensics_sinkhorn_reward": 0.9730722010135651, + "rewards-max/forensics_strict_edge_reward": 0.5699801295995712, + "rewards-max/forensics_tiou_reward": 0.9633668661117554, + "rewards/forensics_sinkhorn_reward": 0.6303678750991821, + "rewards/forensics_strict_edge_reward": 0.05769503489136696, + "rewards/forensics_tiou_reward": 0.47913509607315063, + "step": 625 + }, + { + "completion_length": 26.515625, + "epoch": 1.9216589861751152, + "grad_norm": 5.125434875488281, + "kl": 0.36328125, + "learning_rate": 5.192307692307692e-07, + "loss": 0.1138, + "reward": 0.7791858613491058, + "reward_std": 0.04602894186973572, + "reward_std_all": 0.04144911840558052, + "rewards-max/forensics_sinkhorn_reward": 0.9758956730365753, + "rewards-max/forensics_strict_edge_reward": 0.42571647465229034, + "rewards-max/forensics_tiou_reward": 0.9676813781261444, + "rewards/forensics_sinkhorn_reward": 0.7123987078666687, + "rewards/forensics_strict_edge_reward": 0.039022691547870636, + "rewards/forensics_tiou_reward": 0.5913100242614746, + "step": 626 + }, + { + "completion_length": 24.453125, + "epoch": 1.924731182795699, + "grad_norm": 4.94476318359375, + "kl": 0.306640625, + "learning_rate": 5.184615384615384e-07, + "loss": -0.062, + "reward": 0.7747500836849213, + "reward_std": 0.03781030233949423, + "reward_std_all": 0.03713074140250683, + "rewards-max/forensics_sinkhorn_reward": 0.9671790897846222, + "rewards-max/forensics_strict_edge_reward": 0.2723527029156685, + "rewards-max/forensics_tiou_reward": 0.9466696679592133, + "rewards/forensics_sinkhorn_reward": 0.734552264213562, + "rewards/forensics_strict_edge_reward": 0.03409457951784134, + "rewards/forensics_tiou_reward": 0.6198121011257172, + "step": 627 + }, + { + "completion_length": 26.46875, + "epoch": 1.9278033794162828, + "grad_norm": 5.37254524230957, + "kl": 0.380859375, + "learning_rate": 5.176923076923076e-07, + "loss": 0.1529, + "reward": 0.7627968490123749, + "reward_std": 0.05227663740515709, + "reward_std_all": 0.05134426988661289, + "rewards-max/forensics_sinkhorn_reward": 0.9725364744663239, + "rewards-max/forensics_strict_edge_reward": 0.41768769919872284, + "rewards-max/forensics_tiou_reward": 0.9576447010040283, + "rewards/forensics_sinkhorn_reward": 0.7149617075920105, + "rewards/forensics_strict_edge_reward": 0.03662115894258022, + "rewards/forensics_tiou_reward": 0.5388768017292023, + "step": 628 + }, + { + "completion_length": 24.15625, + "epoch": 1.9308755760368663, + "grad_norm": 5.94468355178833, + "kl": 0.482421875, + "learning_rate": 5.16923076923077e-07, + "loss": -0.0631, + "reward": 0.744875967502594, + "reward_std": 0.04236606881022453, + "reward_std_all": 0.05066908709704876, + "rewards-max/forensics_sinkhorn_reward": 0.9629655480384827, + "rewards-max/forensics_strict_edge_reward": 0.6525696814060211, + "rewards-max/forensics_tiou_reward": 0.9392847418785095, + "rewards/forensics_sinkhorn_reward": 0.67636638879776, + "rewards/forensics_strict_edge_reward": 0.043895239010453224, + "rewards/forensics_tiou_reward": 0.48487982153892517, + "step": 629 + }, + { + "completion_length": 27.203125, + "epoch": 1.93394777265745, + "grad_norm": 4.870296478271484, + "kl": 0.3564453125, + "learning_rate": 5.161538461538462e-07, + "loss": 0.0721, + "reward": 0.7268745601177216, + "reward_std": 0.047240521758794785, + "reward_std_all": 0.060977187007665634, + "rewards-max/forensics_sinkhorn_reward": 0.9606619477272034, + "rewards-max/forensics_strict_edge_reward": 0.19725025910884142, + "rewards-max/forensics_tiou_reward": 0.9284578263759613, + "rewards/forensics_sinkhorn_reward": 0.6300396025180817, + "rewards/forensics_strict_edge_reward": 0.02590177138336003, + "rewards/forensics_tiou_reward": 0.4628843516111374, + "step": 630 + }, + { + "completion_length": 25.734375, + "epoch": 1.9370199692780337, + "grad_norm": 6.886788845062256, + "kl": 0.369140625, + "learning_rate": 5.153846153846153e-07, + "loss": 0.0185, + "reward": 0.7352387607097626, + "reward_std": 0.03432759770657867, + "reward_std_all": 0.05342390900477767, + "rewards-max/forensics_sinkhorn_reward": 0.9756678640842438, + "rewards-max/forensics_strict_edge_reward": 0.2704370878636837, + "rewards-max/forensics_tiou_reward": 0.9608010351657867, + "rewards/forensics_sinkhorn_reward": 0.6995783150196075, + "rewards/forensics_strict_edge_reward": 0.01481637149117887, + "rewards/forensics_tiou_reward": 0.5304332077503204, + "step": 631 + }, + { + "completion_length": 23.28125, + "epoch": 1.9400921658986174, + "grad_norm": 5.233556747436523, + "kl": 0.3369140625, + "learning_rate": 5.146153846153846e-07, + "loss": 0.0383, + "reward": 0.7377089858055115, + "reward_std": 0.04837712459266186, + "reward_std_all": 0.06170194037258625, + "rewards-max/forensics_sinkhorn_reward": 0.9473349153995514, + "rewards-max/forensics_strict_edge_reward": 0.2776384875178337, + "rewards-max/forensics_tiou_reward": 0.9052645564079285, + "rewards/forensics_sinkhorn_reward": 0.7048996686935425, + "rewards/forensics_strict_edge_reward": 0.028192407451570034, + "rewards/forensics_tiou_reward": 0.49505244195461273, + "step": 632 + }, + { + "completion_length": 24.21875, + "epoch": 1.9431643625192012, + "grad_norm": 5.809961795806885, + "kl": 0.3798828125, + "learning_rate": 5.138461538461538e-07, + "loss": 0.0508, + "reward": 0.6680876314640045, + "reward_std": 0.07415610179305077, + "reward_std_all": 0.10430119931697845, + "rewards-max/forensics_sinkhorn_reward": 0.9623730182647705, + "rewards-max/forensics_strict_edge_reward": 0.2665891721844673, + "rewards-max/forensics_tiou_reward": 0.9269484877586365, + "rewards/forensics_sinkhorn_reward": 0.5504414141178131, + "rewards/forensics_strict_edge_reward": 0.024096434703096747, + "rewards/forensics_tiou_reward": 0.3800092935562134, + "step": 633 + }, + { + "completion_length": 24.296875, + "epoch": 1.946236559139785, + "grad_norm": 5.040389060974121, + "kl": 0.5283203125, + "learning_rate": 5.13076923076923e-07, + "loss": 0.1696, + "reward": 0.7724615335464478, + "reward_std": 0.04532196559011936, + "reward_std_all": 0.0436449246481061, + "rewards-max/forensics_sinkhorn_reward": 0.9465733468532562, + "rewards-max/forensics_strict_edge_reward": 0.3271569311618805, + "rewards-max/forensics_tiou_reward": 0.8922423720359802, + "rewards/forensics_sinkhorn_reward": 0.7178484201431274, + "rewards/forensics_strict_edge_reward": 0.03450045734643936, + "rewards/forensics_tiou_reward": 0.5285492539405823, + "step": 634 + }, + { + "completion_length": 26.71875, + "epoch": 1.9493087557603688, + "grad_norm": 4.781694412231445, + "kl": 0.4072265625, + "learning_rate": 5.123076923076923e-07, + "loss": 0.0035, + "reward": 0.7385417819023132, + "reward_std": 0.04245607554912567, + "reward_std_all": 0.055616334080696106, + "rewards-max/forensics_sinkhorn_reward": 0.9141646027565002, + "rewards-max/forensics_strict_edge_reward": 0.09774813894182444, + "rewards-max/forensics_tiou_reward": 0.820354163646698, + "rewards/forensics_sinkhorn_reward": 0.6623766720294952, + "rewards/forensics_strict_edge_reward": 0.0075291653629392385, + "rewards/forensics_tiou_reward": 0.3994193971157074, + "step": 635 + }, + { + "completion_length": 21.96875, + "epoch": 1.9523809523809523, + "grad_norm": 7.370574474334717, + "kl": 0.6689453125, + "learning_rate": 5.115384615384615e-07, + "loss": 0.0881, + "reward": 0.7585614323616028, + "reward_std": 0.0671574305742979, + "reward_std_all": 0.06180337816476822, + "rewards-max/forensics_sinkhorn_reward": 0.978480875492096, + "rewards-max/forensics_strict_edge_reward": 0.3112511858344078, + "rewards-max/forensics_tiou_reward": 0.9658916890621185, + "rewards/forensics_sinkhorn_reward": 0.7213587462902069, + "rewards/forensics_strict_edge_reward": 0.02267326321452856, + "rewards/forensics_tiou_reward": 0.5454666614532471, + "step": 636 + }, + { + "completion_length": 27.609375, + "epoch": 1.955453149001536, + "grad_norm": 5.102722644805908, + "kl": 0.39453125, + "learning_rate": 5.107692307692308e-07, + "loss": 0.1442, + "reward": 0.7513884007930756, + "reward_std": 0.07708413898944855, + "reward_std_all": 0.07101251557469368, + "rewards-max/forensics_sinkhorn_reward": 0.9499833285808563, + "rewards-max/forensics_strict_edge_reward": 0.26961492095142603, + "rewards-max/forensics_tiou_reward": 0.9034731388092041, + "rewards/forensics_sinkhorn_reward": 0.6551804542541504, + "rewards/forensics_strict_edge_reward": 0.014990385505370796, + "rewards/forensics_tiou_reward": 0.4208073765039444, + "step": 637 + }, + { + "completion_length": 28.359375, + "epoch": 1.9585253456221197, + "grad_norm": 5.1786322593688965, + "kl": 0.59765625, + "learning_rate": 5.1e-07, + "loss": -0.0162, + "reward": 0.7985113561153412, + "reward_std": 0.006474902620539069, + "reward_std_all": 0.0089728650636971, + "rewards-max/forensics_sinkhorn_reward": 0.9438004791736603, + "rewards-max/forensics_strict_edge_reward": 0.46505312621593475, + "rewards-max/forensics_tiou_reward": 0.8722963631153107, + "rewards/forensics_sinkhorn_reward": 0.7132900059223175, + "rewards/forensics_strict_edge_reward": 0.03632282093167305, + "rewards/forensics_tiou_reward": 0.510730966925621, + "step": 638 + }, + { + "completion_length": 23.859375, + "epoch": 1.9615975422427034, + "grad_norm": 5.050290584564209, + "kl": 0.4951171875, + "learning_rate": 5.092307692307692e-07, + "loss": -0.0776, + "reward": 0.7818458378314972, + "reward_std": 0.02975472714751959, + "reward_std_all": 0.02947266586124897, + "rewards-max/forensics_sinkhorn_reward": 0.9762164056301117, + "rewards-max/forensics_strict_edge_reward": 0.6097898483276367, + "rewards-max/forensics_tiou_reward": 0.9719885587692261, + "rewards/forensics_sinkhorn_reward": 0.7485518753528595, + "rewards/forensics_strict_edge_reward": 0.06237337738275528, + "rewards/forensics_tiou_reward": 0.5233940482139587, + "step": 639 + }, + { + "completion_length": 20.8125, + "epoch": 1.9646697388632872, + "grad_norm": 6.463298320770264, + "kl": 0.5068359375, + "learning_rate": 5.084615384615384e-07, + "loss": 0.0033, + "reward": 0.773612380027771, + "reward_std": 0.04314960166811943, + "reward_std_all": 0.04069879092276096, + "rewards-max/forensics_sinkhorn_reward": 0.9678695499897003, + "rewards-max/forensics_strict_edge_reward": 0.3967413753271103, + "rewards-max/forensics_tiou_reward": 0.94504514336586, + "rewards/forensics_sinkhorn_reward": 0.7550867199897766, + "rewards/forensics_strict_edge_reward": 0.054056013002991676, + "rewards/forensics_tiou_reward": 0.5177852213382721, + "step": 640 + }, + { + "completion_length": 24.125, + "epoch": 1.967741935483871, + "grad_norm": 6.4950947761535645, + "kl": 0.48046875, + "learning_rate": 5.076923076923076e-07, + "loss": -0.0088, + "reward": 0.7703179121017456, + "reward_std": 0.04137042537331581, + "reward_std_all": 0.04023071378469467, + "rewards-max/forensics_sinkhorn_reward": 0.9719627797603607, + "rewards-max/forensics_strict_edge_reward": 0.4869672954082489, + "rewards-max/forensics_tiou_reward": 0.9596629738807678, + "rewards/forensics_sinkhorn_reward": 0.7299100458621979, + "rewards/forensics_strict_edge_reward": 0.044059653766453266, + "rewards/forensics_tiou_reward": 0.5477142930030823, + "step": 641 + }, + { + "completion_length": 24.078125, + "epoch": 1.9708141321044548, + "grad_norm": 6.717779159545898, + "kl": 0.36328125, + "learning_rate": 5.06923076923077e-07, + "loss": 0.0026, + "reward": 0.7729805111885071, + "reward_std": 0.031584097538143396, + "reward_std_all": 0.03346183896064758, + "rewards-max/forensics_sinkhorn_reward": 0.9788838028907776, + "rewards-max/forensics_strict_edge_reward": 0.5303425639867783, + "rewards-max/forensics_tiou_reward": 0.9755562245845795, + "rewards/forensics_sinkhorn_reward": 0.7504407465457916, + "rewards/forensics_strict_edge_reward": 0.031124088913202286, + "rewards/forensics_tiou_reward": 0.5478724539279938, + "step": 642 + }, + { + "completion_length": 22.34375, + "epoch": 1.9738863287250386, + "grad_norm": 6.48579216003418, + "kl": 0.3740234375, + "learning_rate": 5.061538461538462e-07, + "loss": -0.0326, + "reward": 0.7401922941207886, + "reward_std": 0.08455813676118851, + "reward_std_all": 0.07880595698952675, + "rewards-max/forensics_sinkhorn_reward": 0.9726841449737549, + "rewards-max/forensics_strict_edge_reward": 0.6711390465497971, + "rewards-max/forensics_tiou_reward": 0.9606525897979736, + "rewards/forensics_sinkhorn_reward": 0.6527254581451416, + "rewards/forensics_strict_edge_reward": 0.06425142008811235, + "rewards/forensics_tiou_reward": 0.44374214112758636, + "step": 643 + }, + { + "completion_length": 23.171875, + "epoch": 1.976958525345622, + "grad_norm": 5.434587478637695, + "kl": 0.40625, + "learning_rate": 5.053846153846154e-07, + "loss": 0.0101, + "reward": 0.7653914093971252, + "reward_std": 0.028746013529598713, + "reward_std_all": 0.03367048129439354, + "rewards-max/forensics_sinkhorn_reward": 0.974780946969986, + "rewards-max/forensics_strict_edge_reward": 0.3048876151442528, + "rewards-max/forensics_tiou_reward": 0.9561154842376709, + "rewards/forensics_sinkhorn_reward": 0.713382214307785, + "rewards/forensics_strict_edge_reward": 0.013860448962077498, + "rewards/forensics_tiou_reward": 0.5073128342628479, + "step": 644 + }, + { + "completion_length": 25.84375, + "epoch": 1.9800307219662059, + "grad_norm": 5.502074241638184, + "kl": 0.365234375, + "learning_rate": 5.046153846153846e-07, + "loss": 0.0979, + "reward": 0.7749045491218567, + "reward_std": 0.03878786787390709, + "reward_std_all": 0.03743756748735905, + "rewards-max/forensics_sinkhorn_reward": 0.9634632170200348, + "rewards-max/forensics_strict_edge_reward": 0.33247560262680054, + "rewards-max/forensics_tiou_reward": 0.933941662311554, + "rewards/forensics_sinkhorn_reward": 0.6939164698123932, + "rewards/forensics_strict_edge_reward": 0.04136356059461832, + "rewards/forensics_tiou_reward": 0.5544522106647491, + "step": 645 + }, + { + "completion_length": 28.3125, + "epoch": 1.9831029185867894, + "grad_norm": 4.44268798828125, + "kl": 0.34765625, + "learning_rate": 5.038461538461538e-07, + "loss": 0.1082, + "reward": 0.7504468262195587, + "reward_std": 0.04415997862815857, + "reward_std_all": 0.05093568190932274, + "rewards-max/forensics_sinkhorn_reward": 0.9734644293785095, + "rewards-max/forensics_strict_edge_reward": 0.5498051345348358, + "rewards-max/forensics_tiou_reward": 0.9571230709552765, + "rewards/forensics_sinkhorn_reward": 0.677417129278183, + "rewards/forensics_strict_edge_reward": 0.049122804775834084, + "rewards/forensics_tiou_reward": 0.5240612328052521, + "step": 646 + }, + { + "completion_length": 25.515625, + "epoch": 1.9861751152073732, + "grad_norm": 5.161639213562012, + "kl": 0.29345703125, + "learning_rate": 5.03076923076923e-07, + "loss": 0.0893, + "reward": 0.7730439007282257, + "reward_std": 0.03644918277859688, + "reward_std_all": 0.036236001178622246, + "rewards-max/forensics_sinkhorn_reward": 0.9745086431503296, + "rewards-max/forensics_strict_edge_reward": 0.37411409616470337, + "rewards-max/forensics_tiou_reward": 0.9613242447376251, + "rewards/forensics_sinkhorn_reward": 0.7022208273410797, + "rewards/forensics_strict_edge_reward": 0.04001761972904205, + "rewards/forensics_tiou_reward": 0.5726175010204315, + "step": 647 + }, + { + "completion_length": 27.0, + "epoch": 1.989247311827957, + "grad_norm": 5.521492004394531, + "kl": 0.3583984375, + "learning_rate": 5.023076923076923e-07, + "loss": 0.1319, + "reward": 0.7422921061515808, + "reward_std": 0.06440624035894871, + "reward_std_all": 0.06688807532191277, + "rewards-max/forensics_sinkhorn_reward": 0.9585190117359161, + "rewards-max/forensics_strict_edge_reward": 0.2649490535259247, + "rewards-max/forensics_tiou_reward": 0.9222863912582397, + "rewards/forensics_sinkhorn_reward": 0.6415693461894989, + "rewards/forensics_strict_edge_reward": 0.023560460656881332, + "rewards/forensics_tiou_reward": 0.48517388105392456, + "step": 648 + }, + { + "completion_length": 27.625, + "epoch": 1.9923195084485408, + "grad_norm": 5.0271735191345215, + "kl": 0.33203125, + "learning_rate": 5.015384615384614e-07, + "loss": -0.0102, + "reward": 0.7230153679847717, + "reward_std": 0.040419770404696465, + "reward_std_all": 0.06208283081650734, + "rewards-max/forensics_sinkhorn_reward": 0.9696246385574341, + "rewards-max/forensics_strict_edge_reward": 0.36567531526088715, + "rewards-max/forensics_tiou_reward": 0.9502800703048706, + "rewards/forensics_sinkhorn_reward": 0.6477339565753937, + "rewards/forensics_strict_edge_reward": 0.03657035995274782, + "rewards/forensics_tiou_reward": 0.504833847284317, + "step": 649 + }, + { + "completion_length": 21.625, + "epoch": 1.9953917050691246, + "grad_norm": 6.259210109710693, + "kl": 0.4130859375, + "learning_rate": 5.007692307692308e-07, + "loss": 0.0306, + "reward": 0.7330491244792938, + "reward_std": 0.05959041230380535, + "reward_std_all": 0.0651077888906002, + "rewards-max/forensics_sinkhorn_reward": 0.9612421691417694, + "rewards-max/forensics_strict_edge_reward": 0.3429277576506138, + "rewards-max/forensics_tiou_reward": 0.9220106303691864, + "rewards/forensics_sinkhorn_reward": 0.6957896649837494, + "rewards/forensics_strict_edge_reward": 0.029821678064763546, + "rewards/forensics_tiou_reward": 0.49755752086639404, + "step": 650 + }, + { + "completion_length": 25.1875, + "epoch": 1.998463901689708, + "grad_norm": 6.991339206695557, + "kl": 0.369140625, + "learning_rate": 5e-07, + "loss": 0.0665, + "reward": 0.7681192755699158, + "reward_std": 0.051359549164772034, + "reward_std_all": 0.04811317380517721, + "rewards-max/forensics_sinkhorn_reward": 0.9698388278484344, + "rewards-max/forensics_strict_edge_reward": 0.3079311400651932, + "rewards-max/forensics_tiou_reward": 0.9428881704807281, + "rewards/forensics_sinkhorn_reward": 0.732396811246872, + "rewards/forensics_strict_edge_reward": 0.029853339307010174, + "rewards/forensics_tiou_reward": 0.5536199510097504, + "step": 651 + }, + { + "completion_length": 27.600000381469727, + "epoch": 2.0, + "grad_norm": 3.078295946121216, + "kl": 0.5234375, + "learning_rate": 4.992307692307692e-07, + "loss": -0.0003, + "reward": 0.7660355567932129, + "reward_std": 0.047264087945222855, + "reward_std_all": 0.06307307630777359, + "rewards-max/forensics_sinkhorn_reward": 0.955263614654541, + "rewards-max/forensics_strict_edge_reward": 0.46455901861190796, + "rewards-max/forensics_tiou_reward": 0.9230045080184937, + "rewards/forensics_sinkhorn_reward": 0.7170583009719849, + "rewards/forensics_strict_edge_reward": 0.1499072015285492, + "rewards/forensics_tiou_reward": 0.6374408602714539, + "step": 652 + }, + { + "completion_length": 22.46875, + "epoch": 2.003072196620584, + "grad_norm": 4.984899520874023, + "kl": 0.3984375, + "learning_rate": 4.984615384615384e-07, + "loss": -0.0008, + "reward": 0.7770887911319733, + "reward_std": 0.03774287551641464, + "reward_std_all": 0.03639345429837704, + "rewards-max/forensics_sinkhorn_reward": 0.9791255295276642, + "rewards-max/forensics_strict_edge_reward": 0.527929812669754, + "rewards-max/forensics_tiou_reward": 0.9710459113121033, + "rewards/forensics_sinkhorn_reward": 0.6976928114891052, + "rewards/forensics_strict_edge_reward": 0.05411408469080925, + "rewards/forensics_tiou_reward": 0.5378062129020691, + "step": 653 + }, + { + "completion_length": 23.75, + "epoch": 2.0061443932411676, + "grad_norm": 7.2914533615112305, + "kl": 0.458984375, + "learning_rate": 4.976923076923077e-07, + "loss": 0.0258, + "reward": 0.7614496350288391, + "reward_std": 0.04789363406598568, + "reward_std_all": 0.04891776666045189, + "rewards-max/forensics_sinkhorn_reward": 0.9858510196208954, + "rewards-max/forensics_strict_edge_reward": 0.5438897460699081, + "rewards-max/forensics_tiou_reward": 0.9894065260887146, + "rewards/forensics_sinkhorn_reward": 0.7349505126476288, + "rewards/forensics_strict_edge_reward": 0.06641360279172659, + "rewards/forensics_tiou_reward": 0.5193451046943665, + "step": 654 + }, + { + "completion_length": 25.6875, + "epoch": 2.0092165898617513, + "grad_norm": 5.32430362701416, + "kl": 0.681640625, + "learning_rate": 4.969230769230769e-07, + "loss": 0.0366, + "reward": 0.7631988227367401, + "reward_std": 0.04806523583829403, + "reward_std_all": 0.04766762163490057, + "rewards-max/forensics_sinkhorn_reward": 0.9774624407291412, + "rewards-max/forensics_strict_edge_reward": 0.2916923835873604, + "rewards-max/forensics_tiou_reward": 0.9668953120708466, + "rewards/forensics_sinkhorn_reward": 0.7057908177375793, + "rewards/forensics_strict_edge_reward": 0.02471893187612295, + "rewards/forensics_tiou_reward": 0.516510546207428, + "step": 655 + }, + { + "completion_length": 25.296875, + "epoch": 2.0122887864823347, + "grad_norm": 5.320306777954102, + "kl": 0.328125, + "learning_rate": 4.961538461538462e-07, + "loss": 0.1282, + "reward": 0.765594869852066, + "reward_std": 0.051925092935562134, + "reward_std_all": 0.049152811989188194, + "rewards-max/forensics_sinkhorn_reward": 0.9781385958194733, + "rewards-max/forensics_strict_edge_reward": 0.5599738955497742, + "rewards-max/forensics_tiou_reward": 0.9733920097351074, + "rewards/forensics_sinkhorn_reward": 0.6688685715198517, + "rewards/forensics_strict_edge_reward": 0.05759049020707607, + "rewards/forensics_tiou_reward": 0.5414765179157257, + "step": 656 + }, + { + "completion_length": 27.796875, + "epoch": 2.0153609831029184, + "grad_norm": 5.138331413269043, + "kl": 0.3427734375, + "learning_rate": 4.953846153846154e-07, + "loss": 0.0568, + "reward": 0.7398295998573303, + "reward_std": 0.0573818339034915, + "reward_std_all": 0.06520435586571693, + "rewards-max/forensics_sinkhorn_reward": 0.9773712158203125, + "rewards-max/forensics_strict_edge_reward": 0.4679330885410309, + "rewards-max/forensics_tiou_reward": 0.9708761274814606, + "rewards/forensics_sinkhorn_reward": 0.661060094833374, + "rewards/forensics_strict_edge_reward": 0.07112237624824047, + "rewards/forensics_tiou_reward": 0.582648903131485, + "step": 657 + }, + { + "completion_length": 26.53125, + "epoch": 2.0184331797235022, + "grad_norm": 5.277169704437256, + "kl": 0.4853515625, + "learning_rate": 4.946153846153846e-07, + "loss": 0.1337, + "reward": 0.7254293859004974, + "reward_std": 0.07609516941010952, + "reward_std_all": 0.07703211158514023, + "rewards-max/forensics_sinkhorn_reward": 0.9679920375347137, + "rewards-max/forensics_strict_edge_reward": 0.31976478546857834, + "rewards-max/forensics_tiou_reward": 0.9409500956535339, + "rewards/forensics_sinkhorn_reward": 0.6561278700828552, + "rewards/forensics_strict_edge_reward": 0.031956094317138195, + "rewards/forensics_tiou_reward": 0.45933006703853607, + "step": 658 + }, + { + "completion_length": 24.078125, + "epoch": 2.021505376344086, + "grad_norm": 6.753081798553467, + "kl": 0.455078125, + "learning_rate": 4.938461538461538e-07, + "loss": -0.0156, + "reward": 0.7534638941287994, + "reward_std": 0.06467434391379356, + "reward_std_all": 0.06130855344235897, + "rewards-max/forensics_sinkhorn_reward": 0.9778205752372742, + "rewards-max/forensics_strict_edge_reward": 0.32197293639183044, + "rewards-max/forensics_tiou_reward": 0.9685251414775848, + "rewards/forensics_sinkhorn_reward": 0.7009556293487549, + "rewards/forensics_strict_edge_reward": 0.031794275157153606, + "rewards/forensics_tiou_reward": 0.5444931983947754, + "step": 659 + }, + { + "completion_length": 22.5, + "epoch": 2.02457757296467, + "grad_norm": 5.527687072753906, + "kl": 0.486328125, + "learning_rate": 4.93076923076923e-07, + "loss": 0.0914, + "reward": 0.7379494309425354, + "reward_std": 0.0642994586378336, + "reward_std_all": 0.06645663641393185, + "rewards-max/forensics_sinkhorn_reward": 0.9609747529029846, + "rewards-max/forensics_strict_edge_reward": 0.37511126697063446, + "rewards-max/forensics_tiou_reward": 0.9340502917766571, + "rewards/forensics_sinkhorn_reward": 0.7014955580234528, + "rewards/forensics_strict_edge_reward": 0.04713607393205166, + "rewards/forensics_tiou_reward": 0.49122607707977295, + "step": 660 + }, + { + "completion_length": 27.28125, + "epoch": 2.0276497695852536, + "grad_norm": 6.4447503089904785, + "kl": 0.4453125, + "learning_rate": 4.923076923076923e-07, + "loss": -0.0272, + "reward": 0.7604208588600159, + "reward_std": 0.031950043980032206, + "reward_std_all": 0.0410389993339777, + "rewards-max/forensics_sinkhorn_reward": 0.9610336422920227, + "rewards-max/forensics_strict_edge_reward": 0.659381777048111, + "rewards-max/forensics_tiou_reward": 0.9299975335597992, + "rewards/forensics_sinkhorn_reward": 0.745299220085144, + "rewards/forensics_strict_edge_reward": 0.03234424069523811, + "rewards/forensics_tiou_reward": 0.5237585306167603, + "step": 661 + }, + { + "completion_length": 24.015625, + "epoch": 2.0307219662058373, + "grad_norm": 6.262659072875977, + "kl": 0.341796875, + "learning_rate": 4.915384615384615e-07, + "loss": -0.062, + "reward": 0.7969932556152344, + "reward_std": 0.014439215185120702, + "reward_std_all": 0.014765687752515078, + "rewards-max/forensics_sinkhorn_reward": 0.9743944108486176, + "rewards-max/forensics_strict_edge_reward": 0.5803665816783905, + "rewards-max/forensics_tiou_reward": 0.9636817872524261, + "rewards/forensics_sinkhorn_reward": 0.7837757468223572, + "rewards/forensics_strict_edge_reward": 0.05575949884951115, + "rewards/forensics_tiou_reward": 0.5967521071434021, + "step": 662 + }, + { + "completion_length": 25.859375, + "epoch": 2.033794162826421, + "grad_norm": 5.485055446624756, + "kl": 0.439453125, + "learning_rate": 4.907692307692308e-07, + "loss": 0.0353, + "reward": 0.7936726212501526, + "reward_std": 0.014250247273594141, + "reward_std_all": 0.015433603432029486, + "rewards-max/forensics_sinkhorn_reward": 0.9661868810653687, + "rewards-max/forensics_strict_edge_reward": 0.34904851019382477, + "rewards-max/forensics_tiou_reward": 0.9381239116191864, + "rewards/forensics_sinkhorn_reward": 0.7404677271842957, + "rewards/forensics_strict_edge_reward": 0.03316880762577057, + "rewards/forensics_tiou_reward": 0.5903203189373016, + "step": 663 + }, + { + "completion_length": 24.859375, + "epoch": 2.0368663594470044, + "grad_norm": 5.81436014175415, + "kl": 0.3837890625, + "learning_rate": 4.9e-07, + "loss": 0.0756, + "reward": 0.7658198177814484, + "reward_std": 0.04256442189216614, + "reward_std_all": 0.04287124238908291, + "rewards-max/forensics_sinkhorn_reward": 0.9573721587657928, + "rewards-max/forensics_strict_edge_reward": 0.619256280362606, + "rewards-max/forensics_tiou_reward": 0.9286632239818573, + "rewards/forensics_sinkhorn_reward": 0.6924113035202026, + "rewards/forensics_strict_edge_reward": 0.05587754026055336, + "rewards/forensics_tiou_reward": 0.4542410671710968, + "step": 664 + }, + { + "completion_length": 26.03125, + "epoch": 2.0399385560675882, + "grad_norm": 5.03870964050293, + "kl": 0.43359375, + "learning_rate": 4.892307692307692e-07, + "loss": -0.0032, + "reward": 0.7989506721496582, + "reward_std": 0.007624780526384711, + "reward_std_all": 0.009352662949822843, + "rewards-max/forensics_sinkhorn_reward": 0.962538480758667, + "rewards-max/forensics_strict_edge_reward": 0.41973960399627686, + "rewards-max/forensics_tiou_reward": 0.932870090007782, + "rewards/forensics_sinkhorn_reward": 0.7714612782001495, + "rewards/forensics_strict_edge_reward": 0.032597252167761326, + "rewards/forensics_tiou_reward": 0.6073543131351471, + "step": 665 + }, + { + "completion_length": 23.28125, + "epoch": 2.043010752688172, + "grad_norm": 4.692725658416748, + "kl": 0.2890625, + "learning_rate": 4.884615384615384e-07, + "loss": 0.0264, + "reward": 0.7893881797790527, + "reward_std": 0.020789492409676313, + "reward_std_all": 0.021249283803626895, + "rewards-max/forensics_sinkhorn_reward": 0.9595426321029663, + "rewards-max/forensics_strict_edge_reward": 0.32841892540454865, + "rewards-max/forensics_tiou_reward": 0.9310119152069092, + "rewards/forensics_sinkhorn_reward": 0.7420113682746887, + "rewards/forensics_strict_edge_reward": 0.028075307607650757, + "rewards/forensics_tiou_reward": 0.5619712471961975, + "step": 666 + }, + { + "completion_length": 25.5625, + "epoch": 2.046082949308756, + "grad_norm": 5.645880699157715, + "kl": 0.28662109375, + "learning_rate": 4.876923076923077e-07, + "loss": -0.0422, + "reward": 0.7587090730667114, + "reward_std": 0.02390260621905327, + "reward_std_all": 0.03655601851642132, + "rewards-max/forensics_sinkhorn_reward": 0.9613346755504608, + "rewards-max/forensics_strict_edge_reward": 0.276407852768898, + "rewards-max/forensics_tiou_reward": 0.9236113727092743, + "rewards/forensics_sinkhorn_reward": 0.7103707492351532, + "rewards/forensics_strict_edge_reward": 0.021538853645324707, + "rewards/forensics_tiou_reward": 0.5163833796977997, + "step": 667 + }, + { + "completion_length": 23.4375, + "epoch": 2.0491551459293396, + "grad_norm": 5.229628086090088, + "kl": 0.3857421875, + "learning_rate": 4.869230769230769e-07, + "loss": -0.0131, + "reward": 0.7404862940311432, + "reward_std": 0.040457936469465494, + "reward_std_all": 0.05478437710553408, + "rewards-max/forensics_sinkhorn_reward": 0.9865265488624573, + "rewards-max/forensics_strict_edge_reward": 0.7748240828514099, + "rewards-max/forensics_tiou_reward": 0.9906382262706757, + "rewards/forensics_sinkhorn_reward": 0.7212371826171875, + "rewards/forensics_strict_edge_reward": 0.0859333798289299, + "rewards/forensics_tiou_reward": 0.6080426275730133, + "step": 668 + }, + { + "completion_length": 24.21875, + "epoch": 2.0522273425499233, + "grad_norm": 5.558378219604492, + "kl": 0.3740234375, + "learning_rate": 4.861538461538461e-07, + "loss": 0.1126, + "reward": 0.7631588578224182, + "reward_std": 0.04381410591304302, + "reward_std_all": 0.044075967743992805, + "rewards-max/forensics_sinkhorn_reward": 0.9648683965206146, + "rewards-max/forensics_strict_edge_reward": 0.2700234577059746, + "rewards-max/forensics_tiou_reward": 0.9316367506980896, + "rewards/forensics_sinkhorn_reward": 0.7247692048549652, + "rewards/forensics_strict_edge_reward": 0.03805666323751211, + "rewards/forensics_tiou_reward": 0.5746113061904907, + "step": 669 + }, + { + "completion_length": 24.671875, + "epoch": 2.055299539170507, + "grad_norm": 5.077936172485352, + "kl": 0.3369140625, + "learning_rate": 4.853846153846154e-07, + "loss": -0.0304, + "reward": 0.7488957047462463, + "reward_std": 0.032187025994062424, + "reward_std_all": 0.043259941041469574, + "rewards-max/forensics_sinkhorn_reward": 0.9759686887264252, + "rewards-max/forensics_strict_edge_reward": 0.47831887006759644, + "rewards-max/forensics_tiou_reward": 0.9677560031414032, + "rewards/forensics_sinkhorn_reward": 0.7239613234996796, + "rewards/forensics_strict_edge_reward": 0.036068037152290344, + "rewards/forensics_tiou_reward": 0.533881664276123, + "step": 670 + }, + { + "completion_length": 22.796875, + "epoch": 2.0583717357910905, + "grad_norm": 6.097135543823242, + "kl": 0.380859375, + "learning_rate": 4.846153846153846e-07, + "loss": 0.0386, + "reward": 0.7534038424491882, + "reward_std": 0.04497038386762142, + "reward_std_all": 0.04789008013904095, + "rewards-max/forensics_sinkhorn_reward": 0.9878675043582916, + "rewards-max/forensics_strict_edge_reward": 0.732279509305954, + "rewards-max/forensics_tiou_reward": 0.9965423345565796, + "rewards/forensics_sinkhorn_reward": 0.7018986642360687, + "rewards/forensics_strict_edge_reward": 0.06779919378459454, + "rewards/forensics_tiou_reward": 0.5309574007987976, + "step": 671 + }, + { + "completion_length": 30.078125, + "epoch": 2.0614439324116742, + "grad_norm": 5.3568878173828125, + "kl": 0.3291015625, + "learning_rate": 4.838461538461539e-07, + "loss": 0.0145, + "reward": 0.7637292742729187, + "reward_std": 0.038757557049393654, + "reward_std_all": 0.04229099489748478, + "rewards-max/forensics_sinkhorn_reward": 0.9748923182487488, + "rewards-max/forensics_strict_edge_reward": 0.5550394356250763, + "rewards-max/forensics_tiou_reward": 0.9616222083568573, + "rewards/forensics_sinkhorn_reward": 0.6582933664321899, + "rewards/forensics_strict_edge_reward": 0.0939637552946806, + "rewards/forensics_tiou_reward": 0.5775187611579895, + "step": 672 + }, + { + "completion_length": 29.09375, + "epoch": 2.064516129032258, + "grad_norm": 5.398927688598633, + "kl": 0.517578125, + "learning_rate": 4.83076923076923e-07, + "loss": 0.0851, + "reward": 0.7658984661102295, + "reward_std": 0.039484988898038864, + "reward_std_all": 0.03989916667342186, + "rewards-max/forensics_sinkhorn_reward": 0.9849700629711151, + "rewards-max/forensics_strict_edge_reward": 0.5043367967009544, + "rewards-max/forensics_tiou_reward": 0.987720251083374, + "rewards/forensics_sinkhorn_reward": 0.7188616394996643, + "rewards/forensics_strict_edge_reward": 0.04807480238378048, + "rewards/forensics_tiou_reward": 0.5883000195026398, + "step": 673 + }, + { + "completion_length": 22.328125, + "epoch": 2.067588325652842, + "grad_norm": 6.00490140914917, + "kl": 0.39453125, + "learning_rate": 4.823076923076923e-07, + "loss": 0.0269, + "reward": 0.7613888382911682, + "reward_std": 0.05571551062166691, + "reward_std_all": 0.0526999831199646, + "rewards-max/forensics_sinkhorn_reward": 0.978728711605072, + "rewards-max/forensics_strict_edge_reward": 0.48898807168006897, + "rewards-max/forensics_tiou_reward": 0.9731834232807159, + "rewards/forensics_sinkhorn_reward": 0.7403281629085541, + "rewards/forensics_strict_edge_reward": 0.059624120593070984, + "rewards/forensics_tiou_reward": 0.5625602900981903, + "step": 674 + }, + { + "completion_length": 26.875, + "epoch": 2.0706605222734256, + "grad_norm": 5.874098777770996, + "kl": 0.3583984375, + "learning_rate": 4.815384615384615e-07, + "loss": 0.0291, + "reward": 0.7837272584438324, + "reward_std": 0.026541434228420258, + "reward_std_all": 0.027629205957055092, + "rewards-max/forensics_sinkhorn_reward": 0.9563733339309692, + "rewards-max/forensics_strict_edge_reward": 0.4168103337287903, + "rewards-max/forensics_tiou_reward": 0.9162804782390594, + "rewards/forensics_sinkhorn_reward": 0.7086935639381409, + "rewards/forensics_strict_edge_reward": 0.03596939146518707, + "rewards/forensics_tiou_reward": 0.549528107047081, + "step": 675 + }, + { + "completion_length": 23.796875, + "epoch": 2.0737327188940093, + "grad_norm": 5.231691360473633, + "kl": 0.41796875, + "learning_rate": 4.807692307692307e-07, + "loss": 0.0792, + "reward": 0.772780567407608, + "reward_std": 0.01825902354903519, + "reward_std_all": 0.026776541024446487, + "rewards-max/forensics_sinkhorn_reward": 0.9610042572021484, + "rewards-max/forensics_strict_edge_reward": 0.3850686699151993, + "rewards-max/forensics_tiou_reward": 0.910067230463028, + "rewards/forensics_sinkhorn_reward": 0.7474187612533569, + "rewards/forensics_strict_edge_reward": 0.0277265808545053, + "rewards/forensics_tiou_reward": 0.577692300081253, + "step": 676 + }, + { + "completion_length": 24.53125, + "epoch": 2.076804915514593, + "grad_norm": 5.549280643463135, + "kl": 0.3408203125, + "learning_rate": 4.8e-07, + "loss": 0.0173, + "reward": 0.7762488722801208, + "reward_std": 0.013947087107226253, + "reward_std_all": 0.02430864702910185, + "rewards-max/forensics_sinkhorn_reward": 0.963206946849823, + "rewards-max/forensics_strict_edge_reward": 0.31600624322891235, + "rewards-max/forensics_tiou_reward": 0.9364263117313385, + "rewards/forensics_sinkhorn_reward": 0.7434293925762177, + "rewards/forensics_strict_edge_reward": 0.025065378285944462, + "rewards/forensics_tiou_reward": 0.5503588616847992, + "step": 677 + }, + { + "completion_length": 26.046875, + "epoch": 2.0798771121351765, + "grad_norm": 6.492523193359375, + "kl": 0.2841796875, + "learning_rate": 4.792307692307692e-07, + "loss": -0.0932, + "reward": 0.7914744019508362, + "reward_std": 0.013213529484346509, + "reward_std_all": 0.01529751531779766, + "rewards-max/forensics_sinkhorn_reward": 0.9675123393535614, + "rewards-max/forensics_strict_edge_reward": 0.41483041644096375, + "rewards-max/forensics_tiou_reward": 0.9433929622173309, + "rewards/forensics_sinkhorn_reward": 0.7944866120815277, + "rewards/forensics_strict_edge_reward": 0.047452098689973354, + "rewards/forensics_tiou_reward": 0.6091650426387787, + "step": 678 + }, + { + "completion_length": 24.34375, + "epoch": 2.0829493087557602, + "grad_norm": 8.278010368347168, + "kl": 0.330078125, + "learning_rate": 4.784615384615384e-07, + "loss": 0.0129, + "reward": 0.7044212818145752, + "reward_std": 0.071627676486969, + "reward_std_all": 0.08825267106294632, + "rewards-max/forensics_sinkhorn_reward": 0.9429738223552704, + "rewards-max/forensics_strict_edge_reward": 0.09177047130651772, + "rewards-max/forensics_tiou_reward": 0.8873647153377533, + "rewards/forensics_sinkhorn_reward": 0.6062094569206238, + "rewards/forensics_strict_edge_reward": 0.004513018066063523, + "rewards/forensics_tiou_reward": 0.3779052793979645, + "step": 679 + }, + { + "completion_length": 28.859375, + "epoch": 2.086021505376344, + "grad_norm": 6.229989528656006, + "kl": 0.482421875, + "learning_rate": 4.776923076923077e-07, + "loss": 0.0062, + "reward": 0.7365435361862183, + "reward_std": 0.03348943404853344, + "reward_std_all": 0.052378229796886444, + "rewards-max/forensics_sinkhorn_reward": 0.9708774387836456, + "rewards-max/forensics_strict_edge_reward": 0.4934397041797638, + "rewards-max/forensics_tiou_reward": 0.9497570395469666, + "rewards/forensics_sinkhorn_reward": 0.6910877525806427, + "rewards/forensics_strict_edge_reward": 0.05239330232143402, + "rewards/forensics_tiou_reward": 0.5383935570716858, + "step": 680 + }, + { + "completion_length": 25.28125, + "epoch": 2.089093701996928, + "grad_norm": 5.508248329162598, + "kl": 0.3779296875, + "learning_rate": 4.769230769230769e-07, + "loss": -0.0238, + "reward": 0.772395670413971, + "reward_std": 0.02479209564626217, + "reward_std_all": 0.029515312053263187, + "rewards-max/forensics_sinkhorn_reward": 0.9840191304683685, + "rewards-max/forensics_strict_edge_reward": 0.580880731344223, + "rewards-max/forensics_tiou_reward": 0.9854491353034973, + "rewards/forensics_sinkhorn_reward": 0.7787067294120789, + "rewards/forensics_strict_edge_reward": 0.09473518840968609, + "rewards/forensics_tiou_reward": 0.6086982190608978, + "step": 681 + }, + { + "completion_length": 24.671875, + "epoch": 2.0921658986175116, + "grad_norm": 6.416445255279541, + "kl": 0.3603515625, + "learning_rate": 4.7615384615384613e-07, + "loss": 0.0772, + "reward": 0.7683956921100616, + "reward_std": 0.05560258775949478, + "reward_std_all": 0.05159697122871876, + "rewards-max/forensics_sinkhorn_reward": 0.9714371263980865, + "rewards-max/forensics_strict_edge_reward": 0.6573919653892517, + "rewards-max/forensics_tiou_reward": 0.9517997205257416, + "rewards/forensics_sinkhorn_reward": 0.6918400526046753, + "rewards/forensics_strict_edge_reward": 0.0525917150080204, + "rewards/forensics_tiou_reward": 0.5011998414993286, + "step": 682 + }, + { + "completion_length": 23.203125, + "epoch": 2.0952380952380953, + "grad_norm": 5.09495735168457, + "kl": 0.3916015625, + "learning_rate": 4.753846153846154e-07, + "loss": -0.0028, + "reward": 0.7937218546867371, + "reward_std": 0.02019249740988016, + "reward_std_all": 0.01983673544600606, + "rewards-max/forensics_sinkhorn_reward": 0.9797657430171967, + "rewards-max/forensics_strict_edge_reward": 0.7165313363075256, + "rewards-max/forensics_tiou_reward": 0.9777747392654419, + "rewards/forensics_sinkhorn_reward": 0.7716014683246613, + "rewards/forensics_strict_edge_reward": 0.05666573718190193, + "rewards/forensics_tiou_reward": 0.5705891251564026, + "step": 683 + }, + { + "completion_length": 24.515625, + "epoch": 2.098310291858679, + "grad_norm": 8.49365234375, + "kl": 0.5, + "learning_rate": 4.7461538461538456e-07, + "loss": -0.0348, + "reward": 0.7443823218345642, + "reward_std": 0.03728369250893593, + "reward_std_all": 0.04922807961702347, + "rewards-max/forensics_sinkhorn_reward": 0.9732044339179993, + "rewards-max/forensics_strict_edge_reward": 0.3197720944881439, + "rewards-max/forensics_tiou_reward": 0.9613916277885437, + "rewards/forensics_sinkhorn_reward": 0.7216855585575104, + "rewards/forensics_strict_edge_reward": 0.032479808665812016, + "rewards/forensics_tiou_reward": 0.5617590248584747, + "step": 684 + }, + { + "completion_length": 24.53125, + "epoch": 2.1013824884792625, + "grad_norm": 6.468053817749023, + "kl": 0.34765625, + "learning_rate": 4.738461538461538e-07, + "loss": 0.021, + "reward": 0.7428143322467804, + "reward_std": 0.0413881354033947, + "reward_std_all": 0.05271955393254757, + "rewards-max/forensics_sinkhorn_reward": 0.9851444065570831, + "rewards-max/forensics_strict_edge_reward": 0.8106843829154968, + "rewards-max/forensics_tiou_reward": 0.9905862510204315, + "rewards/forensics_sinkhorn_reward": 0.7347453236579895, + "rewards/forensics_strict_edge_reward": 0.05187332257628441, + "rewards/forensics_tiou_reward": 0.5281263291835785, + "step": 685 + }, + { + "completion_length": 24.875, + "epoch": 2.1044546850998462, + "grad_norm": 6.46860933303833, + "kl": 0.392578125, + "learning_rate": 4.7307692307692304e-07, + "loss": -0.0192, + "reward": 0.78496915102005, + "reward_std": 0.02310040406882763, + "reward_std_all": 0.024214346893131733, + "rewards-max/forensics_sinkhorn_reward": 0.9769213795661926, + "rewards-max/forensics_strict_edge_reward": 0.3885071277618408, + "rewards-max/forensics_tiou_reward": 0.9666200280189514, + "rewards/forensics_sinkhorn_reward": 0.7236601412296295, + "rewards/forensics_strict_edge_reward": 0.025664580054581165, + "rewards/forensics_tiou_reward": 0.5474785417318344, + "step": 686 + }, + { + "completion_length": 24.734375, + "epoch": 2.10752688172043, + "grad_norm": 5.8924455642700195, + "kl": 0.41015625, + "learning_rate": 4.723076923076923e-07, + "loss": 0.0253, + "reward": 0.7700722515583038, + "reward_std": 0.02929825522005558, + "reward_std_all": 0.03350266348570585, + "rewards-max/forensics_sinkhorn_reward": 0.9632160067558289, + "rewards-max/forensics_strict_edge_reward": 0.15171722322702408, + "rewards-max/forensics_tiou_reward": 0.9348468482494354, + "rewards/forensics_sinkhorn_reward": 0.7306445837020874, + "rewards/forensics_strict_edge_reward": 0.013819635845720768, + "rewards/forensics_tiou_reward": 0.5800689458847046, + "step": 687 + }, + { + "completion_length": 26.828125, + "epoch": 2.110599078341014, + "grad_norm": 5.327317714691162, + "kl": 0.373046875, + "learning_rate": 4.715384615384615e-07, + "loss": 0.0157, + "reward": 0.7811760008335114, + "reward_std": 0.03517324477434158, + "reward_std_all": 0.0334488395601511, + "rewards-max/forensics_sinkhorn_reward": 0.9714031219482422, + "rewards-max/forensics_strict_edge_reward": 0.3215830326080322, + "rewards-max/forensics_tiou_reward": 0.9548098742961884, + "rewards/forensics_sinkhorn_reward": 0.7769201397895813, + "rewards/forensics_strict_edge_reward": 0.05218565743416548, + "rewards/forensics_tiou_reward": 0.6193576455116272, + "step": 688 + }, + { + "completion_length": 25.46875, + "epoch": 2.1136712749615976, + "grad_norm": 6.726799011230469, + "kl": 0.3203125, + "learning_rate": 4.7076923076923073e-07, + "loss": -0.0081, + "reward": 0.6718283593654633, + "reward_std": 0.02755017578601837, + "reward_std_all": 0.07987875863909721, + "rewards-max/forensics_sinkhorn_reward": 0.9817833304405212, + "rewards-max/forensics_strict_edge_reward": 0.6697178184986115, + "rewards-max/forensics_tiou_reward": 0.9796128273010254, + "rewards/forensics_sinkhorn_reward": 0.6319010853767395, + "rewards/forensics_strict_edge_reward": 0.05168118514120579, + "rewards/forensics_tiou_reward": 0.5079610645771027, + "step": 689 + }, + { + "completion_length": 24.21875, + "epoch": 2.1167434715821813, + "grad_norm": 6.295464515686035, + "kl": 0.4912109375, + "learning_rate": 4.6999999999999995e-07, + "loss": -0.014, + "reward": 0.769388347864151, + "reward_std": 0.012662941822782159, + "reward_std_all": 0.02675600815564394, + "rewards-max/forensics_sinkhorn_reward": 0.9624500870704651, + "rewards-max/forensics_strict_edge_reward": 0.4651319086551666, + "rewards-max/forensics_tiou_reward": 0.9317935109138489, + "rewards/forensics_sinkhorn_reward": 0.7430203258991241, + "rewards/forensics_strict_edge_reward": 0.022434556856751442, + "rewards/forensics_tiou_reward": 0.516635000705719, + "step": 690 + }, + { + "completion_length": 21.8125, + "epoch": 2.119815668202765, + "grad_norm": 7.738192558288574, + "kl": 0.447265625, + "learning_rate": 4.692307692307692e-07, + "loss": -0.0807, + "reward": 0.800405889749527, + "reward_std": 0.00795107102021575, + "reward_std_all": 0.009165479335933924, + "rewards-max/forensics_sinkhorn_reward": 0.9862470328807831, + "rewards-max/forensics_strict_edge_reward": 0.8053101599216461, + "rewards-max/forensics_tiou_reward": 0.9910825490951538, + "rewards/forensics_sinkhorn_reward": 0.8161455392837524, + "rewards/forensics_strict_edge_reward": 0.0758228525519371, + "rewards/forensics_tiou_reward": 0.6486692130565643, + "step": 691 + }, + { + "completion_length": 25.9375, + "epoch": 2.122887864823349, + "grad_norm": 5.798412799835205, + "kl": 0.4501953125, + "learning_rate": 4.684615384615385e-07, + "loss": 0.0137, + "reward": 0.7324484586715698, + "reward_std": 0.031696492806077003, + "reward_std_all": 0.05192261561751366, + "rewards-max/forensics_sinkhorn_reward": 0.9619776010513306, + "rewards-max/forensics_strict_edge_reward": 0.5090932846069336, + "rewards-max/forensics_tiou_reward": 0.9344067871570587, + "rewards/forensics_sinkhorn_reward": 0.7198157608509064, + "rewards/forensics_strict_edge_reward": 0.06189640611410141, + "rewards/forensics_tiou_reward": 0.5433489382266998, + "step": 692 + }, + { + "completion_length": 22.984375, + "epoch": 2.1259600614439322, + "grad_norm": 6.214157581329346, + "kl": 0.345703125, + "learning_rate": 4.6769230769230764e-07, + "loss": -0.0249, + "reward": 0.792157381772995, + "reward_std": 0.013900456135161221, + "reward_std_all": 0.016683738445863128, + "rewards-max/forensics_sinkhorn_reward": 0.9526170194149017, + "rewards-max/forensics_strict_edge_reward": 0.29145028814673424, + "rewards-max/forensics_tiou_reward": 0.9148283302783966, + "rewards/forensics_sinkhorn_reward": 0.7656548619270325, + "rewards/forensics_strict_edge_reward": 0.02424107724800706, + "rewards/forensics_tiou_reward": 0.5485257655382156, + "step": 693 + }, + { + "completion_length": 24.796875, + "epoch": 2.129032258064516, + "grad_norm": 5.900240898132324, + "kl": 0.4228515625, + "learning_rate": 4.669230769230769e-07, + "loss": -0.0851, + "reward": 0.7617016136646271, + "reward_std": 0.011526591144502163, + "reward_std_all": 0.03019883669912815, + "rewards-max/forensics_sinkhorn_reward": 0.9671953022480011, + "rewards-max/forensics_strict_edge_reward": 0.46730002760887146, + "rewards-max/forensics_tiou_reward": 0.9464133083820343, + "rewards/forensics_sinkhorn_reward": 0.6824314892292023, + "rewards/forensics_strict_edge_reward": 0.033177874982357025, + "rewards/forensics_tiou_reward": 0.5074077546596527, + "step": 694 + }, + { + "completion_length": 23.15625, + "epoch": 2.1321044546851, + "grad_norm": 6.047443866729736, + "kl": 0.322265625, + "learning_rate": 4.661538461538461e-07, + "loss": 0.0126, + "reward": 0.7737704217433929, + "reward_std": 0.03416871000081301, + "reward_std_all": 0.034974764101207256, + "rewards-max/forensics_sinkhorn_reward": 0.9475935697555542, + "rewards-max/forensics_strict_edge_reward": 0.2945736087858677, + "rewards-max/forensics_tiou_reward": 0.8973881006240845, + "rewards/forensics_sinkhorn_reward": 0.7041601538658142, + "rewards/forensics_strict_edge_reward": 0.031223497353494167, + "rewards/forensics_tiou_reward": 0.5188332796096802, + "step": 695 + }, + { + "completion_length": 24.625, + "epoch": 2.1351766513056836, + "grad_norm": 5.725533962249756, + "kl": 0.5419921875, + "learning_rate": 4.653846153846154e-07, + "loss": -0.0389, + "reward": 0.7656659781932831, + "reward_std": 0.05008135363459587, + "reward_std_all": 0.04777183569967747, + "rewards-max/forensics_sinkhorn_reward": 0.9783750474452972, + "rewards-max/forensics_strict_edge_reward": 0.32197293639183044, + "rewards-max/forensics_tiou_reward": 0.9707237184047699, + "rewards/forensics_sinkhorn_reward": 0.6911157369613647, + "rewards/forensics_strict_edge_reward": 0.038739053532481194, + "rewards/forensics_tiou_reward": 0.47750021517276764, + "step": 696 + }, + { + "completion_length": 26.140625, + "epoch": 2.1382488479262673, + "grad_norm": 6.794408321380615, + "kl": 0.39453125, + "learning_rate": 4.646153846153846e-07, + "loss": 0.0067, + "reward": 0.7828114330768585, + "reward_std": 0.010725522181019187, + "reward_std_all": 0.01797992456704378, + "rewards-max/forensics_sinkhorn_reward": 0.9806063175201416, + "rewards-max/forensics_strict_edge_reward": 0.615243673324585, + "rewards-max/forensics_tiou_reward": 0.9776318669319153, + "rewards/forensics_sinkhorn_reward": 0.7757332623004913, + "rewards/forensics_strict_edge_reward": 0.03777717053890228, + "rewards/forensics_tiou_reward": 0.6025871932506561, + "step": 697 + }, + { + "completion_length": 23.0625, + "epoch": 2.141321044546851, + "grad_norm": 6.169040679931641, + "kl": 0.4111328125, + "learning_rate": 4.638461538461538e-07, + "loss": 0.0129, + "reward": 0.745408833026886, + "reward_std": 0.06878466531634331, + "reward_std_all": 0.06642778404057026, + "rewards-max/forensics_sinkhorn_reward": 0.96633580327034, + "rewards-max/forensics_strict_edge_reward": 0.40488630533218384, + "rewards-max/forensics_tiou_reward": 0.9354164302349091, + "rewards/forensics_sinkhorn_reward": 0.735577255487442, + "rewards/forensics_strict_edge_reward": 0.037846408784389496, + "rewards/forensics_tiou_reward": 0.557290181517601, + "step": 698 + }, + { + "completion_length": 26.59375, + "epoch": 2.144393241167435, + "grad_norm": 5.392300128936768, + "kl": 0.3603515625, + "learning_rate": 4.6307692307692303e-07, + "loss": -0.0224, + "reward": 0.7897111773490906, + "reward_std": 0.028838252648711205, + "reward_std_all": 0.026791268959641457, + "rewards-max/forensics_sinkhorn_reward": 0.9831128716468811, + "rewards-max/forensics_strict_edge_reward": 0.756708562374115, + "rewards-max/forensics_tiou_reward": 0.9835533797740936, + "rewards/forensics_sinkhorn_reward": 0.7908482253551483, + "rewards/forensics_strict_edge_reward": 0.07679399475455284, + "rewards/forensics_tiou_reward": 0.6278956830501556, + "step": 699 + }, + { + "completion_length": 23.765625, + "epoch": 2.1474654377880182, + "grad_norm": 4.637333869934082, + "kl": 0.404296875, + "learning_rate": 4.623076923076923e-07, + "loss": 0.0871, + "reward": 0.7589786052703857, + "reward_std": 0.05284947715699673, + "reward_std_all": 0.05134493298828602, + "rewards-max/forensics_sinkhorn_reward": 0.9835439622402191, + "rewards-max/forensics_strict_edge_reward": 0.5891991257667542, + "rewards-max/forensics_tiou_reward": 0.9877057373523712, + "rewards/forensics_sinkhorn_reward": 0.7423050999641418, + "rewards/forensics_strict_edge_reward": 0.07251342013478279, + "rewards/forensics_tiou_reward": 0.5912423133850098, + "step": 700 + }, + { + "completion_length": 22.359375, + "epoch": 2.150537634408602, + "grad_norm": 5.946938991546631, + "kl": 0.3408203125, + "learning_rate": 4.6153846153846156e-07, + "loss": 0.0133, + "reward": 0.7687299251556396, + "reward_std": 0.04894193820655346, + "reward_std_all": 0.04707274865359068, + "rewards-max/forensics_sinkhorn_reward": 0.9793740510940552, + "rewards-max/forensics_strict_edge_reward": 0.6515887677669525, + "rewards-max/forensics_tiou_reward": 0.9754754602909088, + "rewards/forensics_sinkhorn_reward": 0.7589655220508575, + "rewards/forensics_strict_edge_reward": 0.07448435947299004, + "rewards/forensics_tiou_reward": 0.579008936882019, + "step": 701 + }, + { + "completion_length": 26.90625, + "epoch": 2.153609831029186, + "grad_norm": 5.280116081237793, + "kl": 0.4140625, + "learning_rate": 4.607692307692307e-07, + "loss": 0.0479, + "reward": 0.7497719824314117, + "reward_std": 0.05079376697540283, + "reward_std_all": 0.05277908779680729, + "rewards-max/forensics_sinkhorn_reward": 0.9587827324867249, + "rewards-max/forensics_strict_edge_reward": 0.3011625409126282, + "rewards-max/forensics_tiou_reward": 0.9226366877555847, + "rewards/forensics_sinkhorn_reward": 0.7251823246479034, + "rewards/forensics_strict_edge_reward": 0.03939848393201828, + "rewards/forensics_tiou_reward": 0.5697295665740967, + "step": 702 + }, + { + "completion_length": 27.71875, + "epoch": 2.1566820276497696, + "grad_norm": 6.161871910095215, + "kl": 0.359375, + "learning_rate": 4.6e-07, + "loss": 0.0408, + "reward": 0.7349808514118195, + "reward_std": 0.060146089643239975, + "reward_std_all": 0.06518200784921646, + "rewards-max/forensics_sinkhorn_reward": 0.9717740714550018, + "rewards-max/forensics_strict_edge_reward": 0.43265749141573906, + "rewards-max/forensics_tiou_reward": 0.9532176852226257, + "rewards/forensics_sinkhorn_reward": 0.649215817451477, + "rewards/forensics_strict_edge_reward": 0.019083015038631856, + "rewards/forensics_tiou_reward": 0.44779375195503235, + "step": 703 + }, + { + "completion_length": 24.3125, + "epoch": 2.1597542242703534, + "grad_norm": 6.956363201141357, + "kl": 0.4033203125, + "learning_rate": 4.592307692307692e-07, + "loss": 0.0323, + "reward": 0.7844191491603851, + "reward_std": 0.026563692837953568, + "reward_std_all": 0.026363695971667767, + "rewards-max/forensics_sinkhorn_reward": 0.9733544290065765, + "rewards-max/forensics_strict_edge_reward": 0.45270346850156784, + "rewards-max/forensics_tiou_reward": 0.9547078907489777, + "rewards/forensics_sinkhorn_reward": 0.770147442817688, + "rewards/forensics_strict_edge_reward": 0.03492050152271986, + "rewards/forensics_tiou_reward": 0.5633323788642883, + "step": 704 + }, + { + "completion_length": 22.5625, + "epoch": 2.162826420890937, + "grad_norm": 5.899792671203613, + "kl": 0.462890625, + "learning_rate": 4.5846153846153847e-07, + "loss": 0.0912, + "reward": 0.764108270406723, + "reward_std": 0.04501354042440653, + "reward_std_all": 0.04479947127401829, + "rewards-max/forensics_sinkhorn_reward": 0.9745583236217499, + "rewards-max/forensics_strict_edge_reward": 0.5147627294063568, + "rewards-max/forensics_tiou_reward": 0.9600739777088165, + "rewards/forensics_sinkhorn_reward": 0.7351852059364319, + "rewards/forensics_strict_edge_reward": 0.03059791773557663, + "rewards/forensics_tiou_reward": 0.5307484269142151, + "step": 705 + }, + { + "completion_length": 22.875, + "epoch": 2.165898617511521, + "grad_norm": 6.80149507522583, + "kl": 0.521484375, + "learning_rate": 4.5769230769230763e-07, + "loss": 0.1354, + "reward": 0.7387205362319946, + "reward_std": 0.06904629990458488, + "reward_std_all": 0.07120996341109276, + "rewards-max/forensics_sinkhorn_reward": 0.9850908815860748, + "rewards-max/forensics_strict_edge_reward": 0.6408846378326416, + "rewards-max/forensics_tiou_reward": 0.9878459870815277, + "rewards/forensics_sinkhorn_reward": 0.7100822925567627, + "rewards/forensics_strict_edge_reward": 0.06896941550076008, + "rewards/forensics_tiou_reward": 0.5170905888080597, + "step": 706 + }, + { + "completion_length": 22.25, + "epoch": 2.1689708141321047, + "grad_norm": 7.475020408630371, + "kl": 0.4912109375, + "learning_rate": 4.569230769230769e-07, + "loss": -0.0363, + "reward": 0.7741946578025818, + "reward_std": 0.04486513137817383, + "reward_std_all": 0.041950102895498276, + "rewards-max/forensics_sinkhorn_reward": 0.9801156520843506, + "rewards-max/forensics_strict_edge_reward": 0.7309097647666931, + "rewards-max/forensics_tiou_reward": 0.9769026935100555, + "rewards/forensics_sinkhorn_reward": 0.7765782475471497, + "rewards/forensics_strict_edge_reward": 0.1171765848994255, + "rewards/forensics_tiou_reward": 0.596532940864563, + "step": 707 + }, + { + "completion_length": 25.203125, + "epoch": 2.172043010752688, + "grad_norm": 8.73770523071289, + "kl": 0.51171875, + "learning_rate": 4.561538461538461e-07, + "loss": 0.009, + "reward": 0.7744385302066803, + "reward_std": 0.03190326411277056, + "reward_std_all": 0.032512277364730835, + "rewards-max/forensics_sinkhorn_reward": 0.9820402264595032, + "rewards-max/forensics_strict_edge_reward": 0.5630159378051758, + "rewards-max/forensics_tiou_reward": 0.9807697534561157, + "rewards/forensics_sinkhorn_reward": 0.7496543824672699, + "rewards/forensics_strict_edge_reward": 0.0719999186694622, + "rewards/forensics_tiou_reward": 0.5553011894226074, + "step": 708 + }, + { + "completion_length": 24.84375, + "epoch": 2.175115207373272, + "grad_norm": 5.245757579803467, + "kl": 0.373046875, + "learning_rate": 4.553846153846154e-07, + "loss": 0.0054, + "reward": 0.7929134964942932, + "reward_std": 0.013561944011598825, + "reward_std_all": 0.015519884414970875, + "rewards-max/forensics_sinkhorn_reward": 0.9836527705192566, + "rewards-max/forensics_strict_edge_reward": 0.6079075410962105, + "rewards-max/forensics_tiou_reward": 0.9872867465019226, + "rewards/forensics_sinkhorn_reward": 0.7403464019298553, + "rewards/forensics_strict_edge_reward": 0.03683145344257355, + "rewards/forensics_tiou_reward": 0.5358504354953766, + "step": 709 + }, + { + "completion_length": 22.984375, + "epoch": 2.1781874039938556, + "grad_norm": 7.474020957946777, + "kl": 0.3935546875, + "learning_rate": 4.546153846153846e-07, + "loss": -0.0332, + "reward": 0.7833370864391327, + "reward_std": 0.027451408095657825, + "reward_std_all": 0.028009748086333275, + "rewards-max/forensics_sinkhorn_reward": 0.9800626933574677, + "rewards-max/forensics_strict_edge_reward": 0.6915990114212036, + "rewards-max/forensics_tiou_reward": 0.9762522876262665, + "rewards/forensics_sinkhorn_reward": 0.7762906849384308, + "rewards/forensics_strict_edge_reward": 0.04669388011097908, + "rewards/forensics_tiou_reward": 0.574819028377533, + "step": 710 + }, + { + "completion_length": 24.421875, + "epoch": 2.1812596006144394, + "grad_norm": 5.536338806152344, + "kl": 0.3037109375, + "learning_rate": 4.538461538461538e-07, + "loss": 0.0385, + "reward": 0.758892834186554, + "reward_std": 0.03235485544428229, + "reward_std_all": 0.039277184288948774, + "rewards-max/forensics_sinkhorn_reward": 0.9710147678852081, + "rewards-max/forensics_strict_edge_reward": 0.3885537385940552, + "rewards-max/forensics_tiou_reward": 0.9557822048664093, + "rewards/forensics_sinkhorn_reward": 0.7564109563827515, + "rewards/forensics_strict_edge_reward": 0.028140073642134666, + "rewards/forensics_tiou_reward": 0.6059346497058868, + "step": 711 + }, + { + "completion_length": 24.984375, + "epoch": 2.184331797235023, + "grad_norm": 7.574134349822998, + "kl": 0.45703125, + "learning_rate": 4.5307692307692307e-07, + "loss": -0.0002, + "reward": 0.7387458086013794, + "reward_std": 0.05782304238528013, + "reward_std_all": 0.061272745952010155, + "rewards-max/forensics_sinkhorn_reward": 0.9728027582168579, + "rewards-max/forensics_strict_edge_reward": 0.5669038891792297, + "rewards-max/forensics_tiou_reward": 0.9622681140899658, + "rewards/forensics_sinkhorn_reward": 0.7235863208770752, + "rewards/forensics_strict_edge_reward": 0.04998741205781698, + "rewards/forensics_tiou_reward": 0.5278221368789673, + "step": 712 + }, + { + "completion_length": 21.4375, + "epoch": 2.187403993855607, + "grad_norm": 6.663130283355713, + "kl": 0.3193359375, + "learning_rate": 4.523076923076923e-07, + "loss": 0.0595, + "reward": 0.7899585068225861, + "reward_std": 0.027412394527345896, + "reward_std_all": 0.02595120808109641, + "rewards-max/forensics_sinkhorn_reward": 0.9836727678775787, + "rewards-max/forensics_strict_edge_reward": 0.6076286435127258, + "rewards-max/forensics_tiou_reward": 0.9868063926696777, + "rewards/forensics_sinkhorn_reward": 0.7509654462337494, + "rewards/forensics_strict_edge_reward": 0.07596669346094131, + "rewards/forensics_tiou_reward": 0.572786271572113, + "step": 713 + }, + { + "completion_length": 24.84375, + "epoch": 2.1904761904761907, + "grad_norm": 7.342912673950195, + "kl": 0.3720703125, + "learning_rate": 4.5153846153846155e-07, + "loss": -0.007, + "reward": 0.7983587086200714, + "reward_std": 0.006524552241899073, + "reward_std_all": 0.009084954392164946, + "rewards-max/forensics_sinkhorn_reward": 0.95006263256073, + "rewards-max/forensics_strict_edge_reward": 0.2642662450671196, + "rewards-max/forensics_tiou_reward": 0.9078772664070129, + "rewards/forensics_sinkhorn_reward": 0.7538780868053436, + "rewards/forensics_strict_edge_reward": 0.020924996584653854, + "rewards/forensics_tiou_reward": 0.5602317750453949, + "step": 714 + }, + { + "completion_length": 23.875, + "epoch": 2.193548387096774, + "grad_norm": 9.680782318115234, + "kl": 0.66015625, + "learning_rate": 4.507692307692307e-07, + "loss": 0.092, + "reward": 0.7847253978252411, + "reward_std": 0.026598386466503143, + "reward_std_all": 0.026302436366677284, + "rewards-max/forensics_sinkhorn_reward": 0.9698033332824707, + "rewards-max/forensics_strict_edge_reward": 0.4216773808002472, + "rewards-max/forensics_tiou_reward": 0.952979564666748, + "rewards/forensics_sinkhorn_reward": 0.7066980302333832, + "rewards/forensics_strict_edge_reward": 0.05418165400624275, + "rewards/forensics_tiou_reward": 0.5006006211042404, + "step": 715 + }, + { + "completion_length": 26.140625, + "epoch": 2.196620583717358, + "grad_norm": 7.462852478027344, + "kl": 0.400390625, + "learning_rate": 4.5e-07, + "loss": 0.0931, + "reward": 0.7752244174480438, + "reward_std": 0.04038657806813717, + "reward_std_all": 0.038678696379065514, + "rewards-max/forensics_sinkhorn_reward": 0.9770551919937134, + "rewards-max/forensics_strict_edge_reward": 0.3386465162038803, + "rewards-max/forensics_tiou_reward": 0.960583508014679, + "rewards/forensics_sinkhorn_reward": 0.6730057895183563, + "rewards/forensics_strict_edge_reward": 0.02254038630053401, + "rewards/forensics_tiou_reward": 0.5261787474155426, + "step": 716 + }, + { + "completion_length": 28.296875, + "epoch": 2.1996927803379416, + "grad_norm": 6.09303617477417, + "kl": 0.390625, + "learning_rate": 4.492307692307692e-07, + "loss": -0.118, + "reward": 0.7569536864757538, + "reward_std": 0.049995333421975374, + "reward_std_all": 0.050492807291448116, + "rewards-max/forensics_sinkhorn_reward": 0.9825370907783508, + "rewards-max/forensics_strict_edge_reward": 0.44510695338249207, + "rewards-max/forensics_tiou_reward": 0.9820595681667328, + "rewards/forensics_sinkhorn_reward": 0.7520547211170197, + "rewards/forensics_strict_edge_reward": 0.049091240391135216, + "rewards/forensics_tiou_reward": 0.6323433816432953, + "step": 717 + }, + { + "completion_length": 25.359375, + "epoch": 2.2027649769585254, + "grad_norm": 7.446804046630859, + "kl": 0.3330078125, + "learning_rate": 4.4846153846153846e-07, + "loss": 0.0352, + "reward": 0.7480127215385437, + "reward_std": 0.06424971669912338, + "reward_std_all": 0.062021274119615555, + "rewards-max/forensics_sinkhorn_reward": 0.9715258181095123, + "rewards-max/forensics_strict_edge_reward": 0.49097171425819397, + "rewards-max/forensics_tiou_reward": 0.95653235912323, + "rewards/forensics_sinkhorn_reward": 0.7233502268791199, + "rewards/forensics_strict_edge_reward": 0.06515983305871487, + "rewards/forensics_tiou_reward": 0.5462104082107544, + "step": 718 + }, + { + "completion_length": 24.203125, + "epoch": 2.205837173579109, + "grad_norm": 6.329360485076904, + "kl": 0.4326171875, + "learning_rate": 4.476923076923076e-07, + "loss": 0.0381, + "reward": 0.7828371226787567, + "reward_std": 0.033130343072116375, + "reward_std_all": 0.032613617833703756, + "rewards-max/forensics_sinkhorn_reward": 0.9775526821613312, + "rewards-max/forensics_strict_edge_reward": 0.46963123977184296, + "rewards-max/forensics_tiou_reward": 0.9697850346565247, + "rewards/forensics_sinkhorn_reward": 0.7770123481750488, + "rewards/forensics_strict_edge_reward": 0.049366796389222145, + "rewards/forensics_tiou_reward": 0.5956729352474213, + "step": 719 + }, + { + "completion_length": 24.3125, + "epoch": 2.208909370199693, + "grad_norm": 7.438921928405762, + "kl": 0.4208984375, + "learning_rate": 4.469230769230769e-07, + "loss": 0.0423, + "reward": 0.7861864268779755, + "reward_std": 0.027895900420844555, + "reward_std_all": 0.027235519140958786, + "rewards-max/forensics_sinkhorn_reward": 0.9879120886325836, + "rewards-max/forensics_strict_edge_reward": 0.6852048337459564, + "rewards-max/forensics_tiou_reward": 0.995987743139267, + "rewards/forensics_sinkhorn_reward": 0.7803033590316772, + "rewards/forensics_strict_edge_reward": 0.043976882472634315, + "rewards/forensics_tiou_reward": 0.6144044995307922, + "step": 720 + }, + { + "completion_length": 23.59375, + "epoch": 2.2119815668202767, + "grad_norm": 6.401350975036621, + "kl": 0.4560546875, + "learning_rate": 4.4615384615384615e-07, + "loss": 0.0885, + "reward": 0.7775987088680267, + "reward_std": 0.05014978349208832, + "reward_std_all": 0.04512340575456619, + "rewards-max/forensics_sinkhorn_reward": 0.9789757430553436, + "rewards-max/forensics_strict_edge_reward": 0.6777393221855164, + "rewards-max/forensics_tiou_reward": 0.9748037457466125, + "rewards/forensics_sinkhorn_reward": 0.7877773642539978, + "rewards/forensics_strict_edge_reward": 0.07892107963562012, + "rewards/forensics_tiou_reward": 0.657611072063446, + "step": 721 + }, + { + "completion_length": 24.96875, + "epoch": 2.21505376344086, + "grad_norm": 6.792756080627441, + "kl": 0.392578125, + "learning_rate": 4.4538461538461537e-07, + "loss": 0.1735, + "reward": 0.7531148493289948, + "reward_std": 0.05588800832629204, + "reward_std_all": 0.054723260924220085, + "rewards-max/forensics_sinkhorn_reward": 0.9847131967544556, + "rewards-max/forensics_strict_edge_reward": 0.8142738044261932, + "rewards-max/forensics_tiou_reward": 0.9883554577827454, + "rewards/forensics_sinkhorn_reward": 0.7584319710731506, + "rewards/forensics_strict_edge_reward": 0.09066040441393852, + "rewards/forensics_tiou_reward": 0.6016945242881775, + "step": 722 + }, + { + "completion_length": 24.921875, + "epoch": 2.218125960061444, + "grad_norm": 6.730426788330078, + "kl": 0.306640625, + "learning_rate": 4.4461538461538463e-07, + "loss": 0.0374, + "reward": 0.7708479166030884, + "reward_std": 0.06034024676773697, + "reward_std_all": 0.054173279320821166, + "rewards-max/forensics_sinkhorn_reward": 0.9818449020385742, + "rewards-max/forensics_strict_edge_reward": 0.7361255586147308, + "rewards-max/forensics_tiou_reward": 0.9792197048664093, + "rewards/forensics_sinkhorn_reward": 0.7409418225288391, + "rewards/forensics_strict_edge_reward": 0.07703805714845657, + "rewards/forensics_tiou_reward": 0.5725553333759308, + "step": 723 + }, + { + "completion_length": 23.390625, + "epoch": 2.2211981566820276, + "grad_norm": 7.591517925262451, + "kl": 0.3076171875, + "learning_rate": 4.438461538461538e-07, + "loss": -0.0117, + "reward": 0.8010360300540924, + "reward_std": 0.007899535354226828, + "reward_std_all": 0.009133111918345094, + "rewards-max/forensics_sinkhorn_reward": 0.9763643145561218, + "rewards-max/forensics_strict_edge_reward": 0.6828135848045349, + "rewards-max/forensics_tiou_reward": 0.9681789875030518, + "rewards/forensics_sinkhorn_reward": 0.7784507274627686, + "rewards/forensics_strict_edge_reward": 0.06376455537974834, + "rewards/forensics_tiou_reward": 0.5787244737148285, + "step": 724 + }, + { + "completion_length": 28.0, + "epoch": 2.2242703533026114, + "grad_norm": 6.755563259124756, + "kl": 0.3388671875, + "learning_rate": 4.4307692307692306e-07, + "loss": -0.023, + "reward": 0.7455935180187225, + "reward_std": 0.03257137071341276, + "reward_std_all": 0.04908361053094268, + "rewards-max/forensics_sinkhorn_reward": 0.9799639582633972, + "rewards-max/forensics_strict_edge_reward": 0.5669038891792297, + "rewards-max/forensics_tiou_reward": 0.9763113856315613, + "rewards/forensics_sinkhorn_reward": 0.7091036438941956, + "rewards/forensics_strict_edge_reward": 0.06591028347611427, + "rewards/forensics_tiou_reward": 0.5802802443504333, + "step": 725 + }, + { + "completion_length": 25.828125, + "epoch": 2.227342549923195, + "grad_norm": 5.723032474517822, + "kl": 0.326171875, + "learning_rate": 4.423076923076923e-07, + "loss": -0.0177, + "reward": 0.7910906672477722, + "reward_std": 0.02442821953445673, + "reward_std_all": 0.022986582946032286, + "rewards-max/forensics_sinkhorn_reward": 0.9849808216094971, + "rewards-max/forensics_strict_edge_reward": 0.7837068438529968, + "rewards-max/forensics_tiou_reward": 0.9889810681343079, + "rewards/forensics_sinkhorn_reward": 0.7983037233352661, + "rewards/forensics_strict_edge_reward": 0.0857611708343029, + "rewards/forensics_tiou_reward": 0.6750200688838959, + "step": 726 + }, + { + "completion_length": 26.765625, + "epoch": 2.230414746543779, + "grad_norm": 5.264748573303223, + "kl": 0.4150390625, + "learning_rate": 4.4153846153846154e-07, + "loss": 0.0232, + "reward": 0.7783485352993011, + "reward_std": 0.04370126314461231, + "reward_std_all": 0.04019947350025177, + "rewards-max/forensics_sinkhorn_reward": 0.9792002439498901, + "rewards-max/forensics_strict_edge_reward": 0.4812743440270424, + "rewards-max/forensics_tiou_reward": 0.9707167148590088, + "rewards/forensics_sinkhorn_reward": 0.7731989622116089, + "rewards/forensics_strict_edge_reward": 0.036008844152092934, + "rewards/forensics_tiou_reward": 0.6258479356765747, + "step": 727 + }, + { + "completion_length": 24.21875, + "epoch": 2.2334869431643627, + "grad_norm": 6.8697967529296875, + "kl": 0.3623046875, + "learning_rate": 4.407692307692307e-07, + "loss": -0.0162, + "reward": 0.7265647947788239, + "reward_std": 0.05814873427152634, + "reward_std_all": 0.06486839056015015, + "rewards-max/forensics_sinkhorn_reward": 0.9637841582298279, + "rewards-max/forensics_strict_edge_reward": 0.15324781462550163, + "rewards-max/forensics_tiou_reward": 0.9200968146324158, + "rewards/forensics_sinkhorn_reward": 0.6503405869007111, + "rewards/forensics_strict_edge_reward": 0.015131374704651535, + "rewards/forensics_tiou_reward": 0.44821640849113464, + "step": 728 + }, + { + "completion_length": 26.6875, + "epoch": 2.236559139784946, + "grad_norm": 6.706305503845215, + "kl": 0.396484375, + "learning_rate": 4.3999999999999997e-07, + "loss": 0.068, + "reward": 0.7253423929214478, + "reward_std": 0.0731030460447073, + "reward_std_all": 0.07622448727488518, + "rewards-max/forensics_sinkhorn_reward": 0.9658357203006744, + "rewards-max/forensics_strict_edge_reward": 0.4335642233490944, + "rewards-max/forensics_tiou_reward": 0.9475966393947601, + "rewards/forensics_sinkhorn_reward": 0.6020602881908417, + "rewards/forensics_strict_edge_reward": 0.038784705102443695, + "rewards/forensics_tiou_reward": 0.49586932361125946, + "step": 729 + }, + { + "completion_length": 28.125, + "epoch": 2.23963133640553, + "grad_norm": 5.630795001983643, + "kl": 0.5185546875, + "learning_rate": 4.3923076923076924e-07, + "loss": 0.1078, + "reward": 0.7631775736808777, + "reward_std": 0.05965850688517094, + "reward_std_all": 0.055480342358350754, + "rewards-max/forensics_sinkhorn_reward": 0.9356436729431152, + "rewards-max/forensics_strict_edge_reward": 0.3622763156890869, + "rewards-max/forensics_tiou_reward": 0.8728826642036438, + "rewards/forensics_sinkhorn_reward": 0.7335335612297058, + "rewards/forensics_strict_edge_reward": 0.03639189712703228, + "rewards/forensics_tiou_reward": 0.5265336036682129, + "step": 730 + }, + { + "completion_length": 24.15625, + "epoch": 2.2427035330261136, + "grad_norm": 7.254637241363525, + "kl": 0.4404296875, + "learning_rate": 4.3846153846153845e-07, + "loss": 0.0911, + "reward": 0.7622174024581909, + "reward_std": 0.061735594645142555, + "reward_std_all": 0.05748094618320465, + "rewards-max/forensics_sinkhorn_reward": 0.9858232140541077, + "rewards-max/forensics_strict_edge_reward": 0.6363584399223328, + "rewards-max/forensics_tiou_reward": 0.9912474751472473, + "rewards/forensics_sinkhorn_reward": 0.7304835915565491, + "rewards/forensics_strict_edge_reward": 0.05755751207470894, + "rewards/forensics_tiou_reward": 0.5325828492641449, + "step": 731 + }, + { + "completion_length": 24.4375, + "epoch": 2.2457757296466974, + "grad_norm": 6.3879313468933105, + "kl": 0.400390625, + "learning_rate": 4.376923076923077e-07, + "loss": 0.0221, + "reward": 0.7459053695201874, + "reward_std": 0.020613530417904258, + "reward_std_all": 0.041766206617467105, + "rewards-max/forensics_sinkhorn_reward": 0.9522124826908112, + "rewards-max/forensics_strict_edge_reward": 0.3576568216085434, + "rewards-max/forensics_tiou_reward": 0.9148176312446594, + "rewards/forensics_sinkhorn_reward": 0.7338735461235046, + "rewards/forensics_strict_edge_reward": 0.03509945049881935, + "rewards/forensics_tiou_reward": 0.5405923128128052, + "step": 732 + }, + { + "completion_length": 26.421875, + "epoch": 2.248847926267281, + "grad_norm": 5.149492263793945, + "kl": 0.3720703125, + "learning_rate": 4.369230769230769e-07, + "loss": 0.0779, + "reward": 0.7802770137786865, + "reward_std": 0.016618174966424704, + "reward_std_all": 0.02253572642803192, + "rewards-max/forensics_sinkhorn_reward": 0.9776129722595215, + "rewards-max/forensics_strict_edge_reward": 0.4381614923477173, + "rewards-max/forensics_tiou_reward": 0.9689783155918121, + "rewards/forensics_sinkhorn_reward": 0.7242906391620636, + "rewards/forensics_strict_edge_reward": 0.03266576677560806, + "rewards/forensics_tiou_reward": 0.5265071988105774, + "step": 733 + }, + { + "completion_length": 24.875, + "epoch": 2.251920122887865, + "grad_norm": 5.568782329559326, + "kl": 0.4716796875, + "learning_rate": 4.3615384615384614e-07, + "loss": 0.0577, + "reward": 0.7732200622558594, + "reward_std": 0.04683924466371536, + "reward_std_all": 0.04351932555437088, + "rewards-max/forensics_sinkhorn_reward": 0.9858946204185486, + "rewards-max/forensics_strict_edge_reward": 0.8490802645683289, + "rewards-max/forensics_tiou_reward": 0.9932274520397186, + "rewards/forensics_sinkhorn_reward": 0.7323247790336609, + "rewards/forensics_strict_edge_reward": 0.06968307495117188, + "rewards/forensics_tiou_reward": 0.5878783762454987, + "step": 734 + }, + { + "completion_length": 26.578125, + "epoch": 2.2549923195084487, + "grad_norm": 5.926815509796143, + "kl": 0.5380859375, + "learning_rate": 4.3538461538461536e-07, + "loss": -0.0512, + "reward": 0.7624160945415497, + "reward_std": 0.04261538293212652, + "reward_std_all": 0.04348237067461014, + "rewards-max/forensics_sinkhorn_reward": 0.969332367181778, + "rewards-max/forensics_strict_edge_reward": 0.5062990635633469, + "rewards-max/forensics_tiou_reward": 0.9544171988964081, + "rewards/forensics_sinkhorn_reward": 0.7469671964645386, + "rewards/forensics_strict_edge_reward": 0.04501892998814583, + "rewards/forensics_tiou_reward": 0.6041957587003708, + "step": 735 + }, + { + "completion_length": 30.03125, + "epoch": 2.258064516129032, + "grad_norm": 4.713029861450195, + "kl": 0.4306640625, + "learning_rate": 4.346153846153846e-07, + "loss": 0.047, + "reward": 0.7646119892597198, + "reward_std": 0.03578048758208752, + "reward_std_all": 0.04214655049145222, + "rewards-max/forensics_sinkhorn_reward": 0.9653494358062744, + "rewards-max/forensics_strict_edge_reward": 0.36536598205566406, + "rewards-max/forensics_tiou_reward": 0.9434733688831329, + "rewards/forensics_sinkhorn_reward": 0.7739454209804535, + "rewards/forensics_strict_edge_reward": 0.023420199286192656, + "rewards/forensics_tiou_reward": 0.6318226158618927, + "step": 736 + }, + { + "completion_length": 27.078125, + "epoch": 2.261136712749616, + "grad_norm": 5.964945316314697, + "kl": 0.439453125, + "learning_rate": 4.338461538461538e-07, + "loss": 0.0263, + "reward": 0.7874762117862701, + "reward_std": 0.03266288060694933, + "reward_std_all": 0.030134101398289204, + "rewards-max/forensics_sinkhorn_reward": 0.9791562855243683, + "rewards-max/forensics_strict_edge_reward": 0.5100615918636322, + "rewards-max/forensics_tiou_reward": 0.967829704284668, + "rewards/forensics_sinkhorn_reward": 0.8010477125644684, + "rewards/forensics_strict_edge_reward": 0.0515255331993103, + "rewards/forensics_tiou_reward": 0.6632020473480225, + "step": 737 + }, + { + "completion_length": 24.078125, + "epoch": 2.2642089093701996, + "grad_norm": 7.276305198669434, + "kl": 0.37109375, + "learning_rate": 4.3307692307692305e-07, + "loss": 0.0563, + "reward": 0.764601081609726, + "reward_std": 0.04837730806320906, + "reward_std_all": 0.048093993216753006, + "rewards-max/forensics_sinkhorn_reward": 0.9771761894226074, + "rewards-max/forensics_strict_edge_reward": 0.5834526717662811, + "rewards-max/forensics_tiou_reward": 0.9697117507457733, + "rewards/forensics_sinkhorn_reward": 0.6879191696643829, + "rewards/forensics_strict_edge_reward": 0.04668193683028221, + "rewards/forensics_tiou_reward": 0.5189540684223175, + "step": 738 + }, + { + "completion_length": 23.5, + "epoch": 2.2672811059907834, + "grad_norm": 7.908443450927734, + "kl": 0.33203125, + "learning_rate": 4.3230769230769227e-07, + "loss": 0.0463, + "reward": 0.7680227756500244, + "reward_std": 0.03777000308036804, + "reward_std_all": 0.03845663368701935, + "rewards-max/forensics_sinkhorn_reward": 0.9845527708530426, + "rewards-max/forensics_strict_edge_reward": 0.6286447793245316, + "rewards-max/forensics_tiou_reward": 0.9866110980510712, + "rewards/forensics_sinkhorn_reward": 0.6781820058822632, + "rewards/forensics_strict_edge_reward": 0.06267168186604977, + "rewards/forensics_tiou_reward": 0.5123487710952759, + "step": 739 + }, + { + "completion_length": 25.640625, + "epoch": 2.270353302611367, + "grad_norm": 8.998656272888184, + "kl": 0.59375, + "learning_rate": 4.3153846153846153e-07, + "loss": -0.0138, + "reward": 0.7685128152370453, + "reward_std": 0.021963659208267927, + "reward_std_all": 0.028843540232628584, + "rewards-max/forensics_sinkhorn_reward": 0.9482085406780243, + "rewards-max/forensics_strict_edge_reward": 0.24432732909917831, + "rewards-max/forensics_tiou_reward": 0.9024804532527924, + "rewards/forensics_sinkhorn_reward": 0.6830233633518219, + "rewards/forensics_strict_edge_reward": 0.012080001179128885, + "rewards/forensics_tiou_reward": 0.43657416105270386, + "step": 740 + }, + { + "completion_length": 26.578125, + "epoch": 2.273425499231951, + "grad_norm": 5.416785717010498, + "kl": 0.408203125, + "learning_rate": 4.307692307692308e-07, + "loss": 0.0558, + "reward": 0.7690993845462799, + "reward_std": 0.05659852549433708, + "reward_std_all": 0.05168608948588371, + "rewards-max/forensics_sinkhorn_reward": 0.9719912111759186, + "rewards-max/forensics_strict_edge_reward": 0.5933051109313965, + "rewards-max/forensics_tiou_reward": 0.9615885615348816, + "rewards/forensics_sinkhorn_reward": 0.7317586243152618, + "rewards/forensics_strict_edge_reward": 0.07027804106473923, + "rewards/forensics_tiou_reward": 0.6046760678291321, + "step": 741 + }, + { + "completion_length": 22.390625, + "epoch": 2.2764976958525347, + "grad_norm": 6.254802703857422, + "kl": 0.46484375, + "learning_rate": 4.2999999999999996e-07, + "loss": 0.0758, + "reward": 0.754294216632843, + "reward_std": 0.0651151267811656, + "reward_std_all": 0.06172904837876558, + "rewards-max/forensics_sinkhorn_reward": 0.982520341873169, + "rewards-max/forensics_strict_edge_reward": 0.39297446608543396, + "rewards-max/forensics_tiou_reward": 0.9863837659358978, + "rewards/forensics_sinkhorn_reward": 0.6897626519203186, + "rewards/forensics_strict_edge_reward": 0.04304446931928396, + "rewards/forensics_tiou_reward": 0.5005372017621994, + "step": 742 + }, + { + "completion_length": 25.546875, + "epoch": 2.279569892473118, + "grad_norm": 7.55299186706543, + "kl": 0.666015625, + "learning_rate": 4.292307692307692e-07, + "loss": -0.029, + "reward": 0.7700681984424591, + "reward_std": 0.03007388673722744, + "reward_std_all": 0.0359498206526041, + "rewards-max/forensics_sinkhorn_reward": 0.9796070158481598, + "rewards-max/forensics_strict_edge_reward": 0.6188668608665466, + "rewards-max/forensics_tiou_reward": 0.975229024887085, + "rewards/forensics_sinkhorn_reward": 0.7727098166942596, + "rewards/forensics_strict_edge_reward": 0.06272147223353386, + "rewards/forensics_tiou_reward": 0.5754256695508957, + "step": 743 + }, + { + "completion_length": 24.5, + "epoch": 2.282642089093702, + "grad_norm": 8.29050064086914, + "kl": 0.380859375, + "learning_rate": 4.2846153846153844e-07, + "loss": 0.0247, + "reward": 0.7536006569862366, + "reward_std": 0.026671165600419044, + "reward_std_all": 0.037803564220666885, + "rewards-max/forensics_sinkhorn_reward": 0.9780663549900055, + "rewards-max/forensics_strict_edge_reward": 0.6165179014205933, + "rewards-max/forensics_tiou_reward": 0.9706372022628784, + "rewards/forensics_sinkhorn_reward": 0.6843658983707428, + "rewards/forensics_strict_edge_reward": 0.05927508510649204, + "rewards/forensics_tiou_reward": 0.5087405145168304, + "step": 744 + }, + { + "completion_length": 24.75, + "epoch": 2.2857142857142856, + "grad_norm": 6.267192840576172, + "kl": 0.4462890625, + "learning_rate": 4.276923076923077e-07, + "loss": 0.0175, + "reward": 0.7603665292263031, + "reward_std": 0.04327333066612482, + "reward_std_all": 0.046679241582751274, + "rewards-max/forensics_sinkhorn_reward": 0.9701814949512482, + "rewards-max/forensics_strict_edge_reward": 0.6167194992303848, + "rewards-max/forensics_tiou_reward": 0.9592825770378113, + "rewards/forensics_sinkhorn_reward": 0.7289853394031525, + "rewards/forensics_strict_edge_reward": 0.045316241681575775, + "rewards/forensics_tiou_reward": 0.5613197982311249, + "step": 745 + }, + { + "completion_length": 24.4375, + "epoch": 2.2887864823348694, + "grad_norm": 5.805148601531982, + "kl": 0.30859375, + "learning_rate": 4.2692307692307687e-07, + "loss": -0.0986, + "reward": 0.7562980949878693, + "reward_std": 0.01733211288228631, + "reward_std_all": 0.03577358089387417, + "rewards-max/forensics_sinkhorn_reward": 0.9803239405155182, + "rewards-max/forensics_strict_edge_reward": 0.6718102097511292, + "rewards-max/forensics_tiou_reward": 0.9744550287723541, + "rewards/forensics_sinkhorn_reward": 0.7718496322631836, + "rewards/forensics_strict_edge_reward": 0.08519332110881805, + "rewards/forensics_tiou_reward": 0.6832853555679321, + "step": 746 + }, + { + "completion_length": 24.265625, + "epoch": 2.291858678955453, + "grad_norm": 6.07654333114624, + "kl": 0.3466796875, + "learning_rate": 4.2615384615384613e-07, + "loss": 0.1027, + "reward": 0.7637794315814972, + "reward_std": 0.019089669920504093, + "reward_std_all": 0.03233557753264904, + "rewards-max/forensics_sinkhorn_reward": 0.9762187302112579, + "rewards-max/forensics_strict_edge_reward": 0.3288491666316986, + "rewards-max/forensics_tiou_reward": 0.9639020264148712, + "rewards/forensics_sinkhorn_reward": 0.6963525116443634, + "rewards/forensics_strict_edge_reward": 0.026682967320084572, + "rewards/forensics_tiou_reward": 0.5020130723714828, + "step": 747 + }, + { + "completion_length": 24.125, + "epoch": 2.294930875576037, + "grad_norm": 6.269800186157227, + "kl": 0.3876953125, + "learning_rate": 4.2538461538461535e-07, + "loss": 0.0471, + "reward": 0.7737952470779419, + "reward_std": 0.04357750993221998, + "reward_std_all": 0.04116677399724722, + "rewards-max/forensics_sinkhorn_reward": 0.9669908285140991, + "rewards-max/forensics_strict_edge_reward": 0.3617415875196457, + "rewards-max/forensics_tiou_reward": 0.9402584433555603, + "rewards/forensics_sinkhorn_reward": 0.692416787147522, + "rewards/forensics_strict_edge_reward": 0.03698070114478469, + "rewards/forensics_tiou_reward": 0.5283448696136475, + "step": 748 + }, + { + "completion_length": 22.515625, + "epoch": 2.2980030721966207, + "grad_norm": 6.677371978759766, + "kl": 0.419921875, + "learning_rate": 4.246153846153846e-07, + "loss": 0.0654, + "reward": 0.7812253534793854, + "reward_std": 0.03661278169602156, + "reward_std_all": 0.03425737842917442, + "rewards-max/forensics_sinkhorn_reward": 0.9700014591217041, + "rewards-max/forensics_strict_edge_reward": 0.3613134175539017, + "rewards-max/forensics_tiou_reward": 0.9456477463245392, + "rewards/forensics_sinkhorn_reward": 0.7861519157886505, + "rewards/forensics_strict_edge_reward": 0.05205694027245045, + "rewards/forensics_tiou_reward": 0.6089498400688171, + "step": 749 + }, + { + "completion_length": 25.453125, + "epoch": 2.3010752688172045, + "grad_norm": 7.304887294769287, + "kl": 0.501953125, + "learning_rate": 4.2384615384615383e-07, + "loss": -0.0457, + "reward": 0.7412298619747162, + "reward_std": 0.030573212541639805, + "reward_std_all": 0.04810722451657057, + "rewards-max/forensics_sinkhorn_reward": 0.9719862937927246, + "rewards-max/forensics_strict_edge_reward": 0.288056418299675, + "rewards-max/forensics_tiou_reward": 0.944981038570404, + "rewards/forensics_sinkhorn_reward": 0.7116765081882477, + "rewards/forensics_strict_edge_reward": 0.04589930875226855, + "rewards/forensics_tiou_reward": 0.5391895473003387, + "step": 750 + }, + { + "completion_length": 27.390625, + "epoch": 2.3041474654377883, + "grad_norm": 5.968143463134766, + "kl": 0.4658203125, + "learning_rate": 4.2307692307692304e-07, + "loss": 0.1317, + "reward": 0.6991592049598694, + "reward_std": 0.07953848131000996, + "reward_std_all": 0.09779030829668045, + "rewards-max/forensics_sinkhorn_reward": 0.9707363843917847, + "rewards-max/forensics_strict_edge_reward": 0.31011533737182617, + "rewards-max/forensics_tiou_reward": 0.9581063687801361, + "rewards/forensics_sinkhorn_reward": 0.6397085189819336, + "rewards/forensics_strict_edge_reward": 0.03424426494166255, + "rewards/forensics_tiou_reward": 0.44738367199897766, + "step": 751 + }, + { + "completion_length": 23.0625, + "epoch": 2.3072196620583716, + "grad_norm": 7.075549125671387, + "kl": 0.3583984375, + "learning_rate": 4.223076923076923e-07, + "loss": 0.1037, + "reward": 0.7687821984291077, + "reward_std": 0.03639502450823784, + "reward_std_all": 0.038051556795835495, + "rewards-max/forensics_sinkhorn_reward": 0.9752824008464813, + "rewards-max/forensics_strict_edge_reward": 0.39267270267009735, + "rewards-max/forensics_tiou_reward": 0.9620945155620575, + "rewards/forensics_sinkhorn_reward": 0.7326294183731079, + "rewards/forensics_strict_edge_reward": 0.03276716358959675, + "rewards/forensics_tiou_reward": 0.514166384935379, + "step": 752 + }, + { + "completion_length": 23.984375, + "epoch": 2.3102918586789554, + "grad_norm": 7.758074760437012, + "kl": 0.4267578125, + "learning_rate": 4.215384615384615e-07, + "loss": 0.0456, + "reward": 0.7926494181156158, + "reward_std": 0.018824162892997265, + "reward_std_all": 0.018761634826660156, + "rewards-max/forensics_sinkhorn_reward": 0.9679445028305054, + "rewards-max/forensics_strict_edge_reward": 0.3836938589811325, + "rewards-max/forensics_tiou_reward": 0.9465572834014893, + "rewards/forensics_sinkhorn_reward": 0.7669362723827362, + "rewards/forensics_strict_edge_reward": 0.05059189535677433, + "rewards/forensics_tiou_reward": 0.5606124252080917, + "step": 753 + }, + { + "completion_length": 25.453125, + "epoch": 2.313364055299539, + "grad_norm": 6.5054402351379395, + "kl": 0.44140625, + "learning_rate": 4.207692307692308e-07, + "loss": 0.0216, + "reward": 0.7907362282276154, + "reward_std": 0.02582496590912342, + "reward_std_all": 0.024604848586022854, + "rewards-max/forensics_sinkhorn_reward": 0.975559264421463, + "rewards-max/forensics_strict_edge_reward": 0.4078541025519371, + "rewards-max/forensics_tiou_reward": 0.9651375710964203, + "rewards/forensics_sinkhorn_reward": 0.8091331124305725, + "rewards/forensics_strict_edge_reward": 0.03193904645740986, + "rewards/forensics_tiou_reward": 0.6234780550003052, + "step": 754 + }, + { + "completion_length": 23.15625, + "epoch": 2.316436251920123, + "grad_norm": 7.781042575836182, + "kl": 0.369140625, + "learning_rate": 4.1999999999999995e-07, + "loss": 0.0248, + "reward": 0.7493765354156494, + "reward_std": 0.014426817651838064, + "reward_std_all": 0.03740727063268423, + "rewards-max/forensics_sinkhorn_reward": 0.9776692688465118, + "rewards-max/forensics_strict_edge_reward": 0.5221183001995087, + "rewards-max/forensics_tiou_reward": 0.9631204605102539, + "rewards/forensics_sinkhorn_reward": 0.7252317070960999, + "rewards/forensics_strict_edge_reward": 0.04432966746389866, + "rewards/forensics_tiou_reward": 0.5573337078094482, + "step": 755 + }, + { + "completion_length": 23.421875, + "epoch": 2.3195084485407067, + "grad_norm": 5.16398811340332, + "kl": 0.4208984375, + "learning_rate": 4.192307692307692e-07, + "loss": 0.0559, + "reward": 0.7707920074462891, + "reward_std": 0.04004095308482647, + "reward_std_all": 0.03932562842965126, + "rewards-max/forensics_sinkhorn_reward": 0.9512929618358612, + "rewards-max/forensics_strict_edge_reward": 0.5195324569940567, + "rewards-max/forensics_tiou_reward": 0.8974450528621674, + "rewards/forensics_sinkhorn_reward": 0.7941771447658539, + "rewards/forensics_strict_edge_reward": 0.05961194820702076, + "rewards/forensics_tiou_reward": 0.5975734293460846, + "step": 756 + }, + { + "completion_length": 24.078125, + "epoch": 2.3225806451612905, + "grad_norm": 6.616155624389648, + "kl": 0.3125, + "learning_rate": 4.1846153846153843e-07, + "loss": 0.0242, + "reward": 0.781044065952301, + "reward_std": 0.03438358847051859, + "reward_std_all": 0.032890379428863525, + "rewards-max/forensics_sinkhorn_reward": 0.9701143503189087, + "rewards-max/forensics_strict_edge_reward": 0.35114636272192, + "rewards-max/forensics_tiou_reward": 0.9523217380046844, + "rewards/forensics_sinkhorn_reward": 0.6988787055015564, + "rewards/forensics_strict_edge_reward": 0.03489538421854377, + "rewards/forensics_tiou_reward": 0.48768429458141327, + "step": 757 + }, + { + "completion_length": 20.890625, + "epoch": 2.3256528417818743, + "grad_norm": 6.93781042098999, + "kl": 0.5234375, + "learning_rate": 4.176923076923077e-07, + "loss": -0.0125, + "reward": 0.7844986617565155, + "reward_std": 0.03601703327149153, + "reward_std_all": 0.03307235799729824, + "rewards-max/forensics_sinkhorn_reward": 0.979862242937088, + "rewards-max/forensics_strict_edge_reward": 0.495777428150177, + "rewards-max/forensics_tiou_reward": 0.972205400466919, + "rewards/forensics_sinkhorn_reward": 0.7564972043037415, + "rewards/forensics_strict_edge_reward": 0.0658287052065134, + "rewards/forensics_tiou_reward": 0.5471718311309814, + "step": 758 + }, + { + "completion_length": 25.84375, + "epoch": 2.3287250384024576, + "grad_norm": 6.0598835945129395, + "kl": 0.3173828125, + "learning_rate": 4.1692307692307686e-07, + "loss": -0.0271, + "reward": 0.7725448906421661, + "reward_std": 0.04445421323180199, + "reward_std_all": 0.041961291804909706, + "rewards-max/forensics_sinkhorn_reward": 0.9614136517047882, + "rewards-max/forensics_strict_edge_reward": 0.3464784771203995, + "rewards-max/forensics_tiou_reward": 0.9307974874973297, + "rewards/forensics_sinkhorn_reward": 0.7655898034572601, + "rewards/forensics_strict_edge_reward": 0.04160379245877266, + "rewards/forensics_tiou_reward": 0.5916070342063904, + "step": 759 + }, + { + "completion_length": 25.4375, + "epoch": 2.3317972350230414, + "grad_norm": 6.378767967224121, + "kl": 0.3701171875, + "learning_rate": 4.161538461538461e-07, + "loss": 0.0518, + "reward": 0.7245976328849792, + "reward_std": 0.0722469910979271, + "reward_std_all": 0.07871188223361969, + "rewards-max/forensics_sinkhorn_reward": 0.9647018909454346, + "rewards-max/forensics_strict_edge_reward": 0.6091909855604172, + "rewards-max/forensics_tiou_reward": 0.9387378394603729, + "rewards/forensics_sinkhorn_reward": 0.6246817708015442, + "rewards/forensics_strict_edge_reward": 0.04689801577478647, + "rewards/forensics_tiou_reward": 0.45795394480228424, + "step": 760 + }, + { + "completion_length": 25.640625, + "epoch": 2.334869431643625, + "grad_norm": 7.38823127746582, + "kl": 0.3701171875, + "learning_rate": 4.153846153846154e-07, + "loss": -0.0177, + "reward": 0.7795301377773285, + "reward_std": 0.028036711271852255, + "reward_std_all": 0.028603955637663603, + "rewards-max/forensics_sinkhorn_reward": 0.9801433086395264, + "rewards-max/forensics_strict_edge_reward": 0.5185287445783615, + "rewards-max/forensics_tiou_reward": 0.9665087461471558, + "rewards/forensics_sinkhorn_reward": 0.755378395318985, + "rewards/forensics_strict_edge_reward": 0.04379677586257458, + "rewards/forensics_tiou_reward": 0.5839545130729675, + "step": 761 + }, + { + "completion_length": 28.75, + "epoch": 2.337941628264209, + "grad_norm": 5.9269514083862305, + "kl": 0.53125, + "learning_rate": 4.146153846153846e-07, + "loss": 0.0179, + "reward": 0.7902986407279968, + "reward_std": 0.025036781444214284, + "reward_std_all": 0.02395475679077208, + "rewards-max/forensics_sinkhorn_reward": 0.9848293662071228, + "rewards-max/forensics_strict_edge_reward": 0.7586064040660858, + "rewards-max/forensics_tiou_reward": 0.9903435409069061, + "rewards/forensics_sinkhorn_reward": 0.7426815927028656, + "rewards/forensics_strict_edge_reward": 0.08786278031766415, + "rewards/forensics_tiou_reward": 0.6289886832237244, + "step": 762 + }, + { + "completion_length": 26.671875, + "epoch": 2.3410138248847927, + "grad_norm": 5.759289264678955, + "kl": 0.4248046875, + "learning_rate": 4.1384615384615387e-07, + "loss": 0.0049, + "reward": 0.7826065421104431, + "reward_std": 0.02601452823728323, + "reward_std_all": 0.026280401274561882, + "rewards-max/forensics_sinkhorn_reward": 0.9822535216808319, + "rewards-max/forensics_strict_edge_reward": 0.5829301476478577, + "rewards-max/forensics_tiou_reward": 0.98060542345047, + "rewards/forensics_sinkhorn_reward": 0.7914519906044006, + "rewards/forensics_strict_edge_reward": 0.08703035488724709, + "rewards/forensics_tiou_reward": 0.6704067587852478, + "step": 763 + }, + { + "completion_length": 22.796875, + "epoch": 2.3440860215053765, + "grad_norm": 6.361464977264404, + "kl": 0.3720703125, + "learning_rate": 4.1307692307692303e-07, + "loss": 0.0033, + "reward": 0.7946406602859497, + "reward_std": 0.01781142596155405, + "reward_std_all": 0.01751338178291917, + "rewards-max/forensics_sinkhorn_reward": 0.9756240546703339, + "rewards-max/forensics_strict_edge_reward": 0.3251207396388054, + "rewards-max/forensics_tiou_reward": 0.9578942358493805, + "rewards/forensics_sinkhorn_reward": 0.7856821417808533, + "rewards/forensics_strict_edge_reward": 0.046050455421209335, + "rewards/forensics_tiou_reward": 0.5948314368724823, + "step": 764 + }, + { + "completion_length": 22.90625, + "epoch": 2.3471582181259603, + "grad_norm": 6.41117000579834, + "kl": 0.3095703125, + "learning_rate": 4.123076923076923e-07, + "loss": -0.0475, + "reward": 0.7888893485069275, + "reward_std": 0.004372448369394988, + "reward_std_all": 0.012006772449240088, + "rewards-max/forensics_sinkhorn_reward": 0.9636898636817932, + "rewards-max/forensics_strict_edge_reward": 0.5900419056415558, + "rewards-max/forensics_tiou_reward": 0.9183594286441803, + "rewards/forensics_sinkhorn_reward": 0.7597065567970276, + "rewards/forensics_strict_edge_reward": 0.03636910393834114, + "rewards/forensics_tiou_reward": 0.6117382645606995, + "step": 765 + }, + { + "completion_length": 26.359375, + "epoch": 2.3502304147465436, + "grad_norm": 6.282617092132568, + "kl": 0.3642578125, + "learning_rate": 4.115384615384615e-07, + "loss": -0.0357, + "reward": 0.788669764995575, + "reward_std": 0.01899230945855379, + "reward_std_all": 0.020116212777793407, + "rewards-max/forensics_sinkhorn_reward": 0.9512825906276703, + "rewards-max/forensics_strict_edge_reward": 0.4165525734424591, + "rewards-max/forensics_tiou_reward": 0.9118566811084747, + "rewards/forensics_sinkhorn_reward": 0.7355777621269226, + "rewards/forensics_strict_edge_reward": 0.0372901875525713, + "rewards/forensics_tiou_reward": 0.5668210983276367, + "step": 766 + }, + { + "completion_length": 21.15625, + "epoch": 2.3533026113671274, + "grad_norm": 6.489022254943848, + "kl": 0.48681640625, + "learning_rate": 4.107692307692308e-07, + "loss": 0.0958, + "reward": 0.7263367176055908, + "reward_std": 0.045337166637182236, + "reward_std_all": 0.06683262065052986, + "rewards-max/forensics_sinkhorn_reward": 0.984161764383316, + "rewards-max/forensics_strict_edge_reward": 0.5562380701303482, + "rewards-max/forensics_tiou_reward": 0.9868301749229431, + "rewards/forensics_sinkhorn_reward": 0.6915860772132874, + "rewards/forensics_strict_edge_reward": 0.040400709956884384, + "rewards/forensics_tiou_reward": 0.5099973678588867, + "step": 767 + }, + { + "completion_length": 24.25, + "epoch": 2.356374807987711, + "grad_norm": 5.824685096740723, + "kl": 0.45703125, + "learning_rate": 4.0999999999999994e-07, + "loss": -0.0295, + "reward": 0.7875359952449799, + "reward_std": 0.01853019359987229, + "reward_std_all": 0.020114415790885687, + "rewards-max/forensics_sinkhorn_reward": 0.9743295907974243, + "rewards-max/forensics_strict_edge_reward": 0.5311143696308136, + "rewards-max/forensics_tiou_reward": 0.9603853225708008, + "rewards/forensics_sinkhorn_reward": 0.7378832697868347, + "rewards/forensics_strict_edge_reward": 0.03457252494990826, + "rewards/forensics_tiou_reward": 0.5707235932350159, + "step": 768 + }, + { + "completion_length": 26.296875, + "epoch": 2.359447004608295, + "grad_norm": 6.169503688812256, + "kl": 0.40234375, + "learning_rate": 4.092307692307692e-07, + "loss": -0.0234, + "reward": 0.7767764031887054, + "reward_std": 0.035515098832547665, + "reward_std_all": 0.03437824733555317, + "rewards-max/forensics_sinkhorn_reward": 0.9824678301811218, + "rewards-max/forensics_strict_edge_reward": 0.6868928670883179, + "rewards-max/forensics_tiou_reward": 0.9845196008682251, + "rewards/forensics_sinkhorn_reward": 0.7579677700996399, + "rewards/forensics_strict_edge_reward": 0.08101681061089039, + "rewards/forensics_tiou_reward": 0.6351465284824371, + "step": 769 + }, + { + "completion_length": 25.03125, + "epoch": 2.3625192012288787, + "grad_norm": 6.857965469360352, + "kl": 0.65625, + "learning_rate": 4.0846153846153847e-07, + "loss": 0.0706, + "reward": 0.7289094626903534, + "reward_std": 0.048444801941514015, + "reward_std_all": 0.06332480162382126, + "rewards-max/forensics_sinkhorn_reward": 0.98341104388237, + "rewards-max/forensics_strict_edge_reward": 0.5648730993270874, + "rewards-max/forensics_tiou_reward": 0.9851242303848267, + "rewards/forensics_sinkhorn_reward": 0.7213006317615509, + "rewards/forensics_strict_edge_reward": 0.047186246141791344, + "rewards/forensics_tiou_reward": 0.5868126153945923, + "step": 770 + }, + { + "completion_length": 25.359375, + "epoch": 2.3655913978494625, + "grad_norm": 7.471019268035889, + "kl": 0.390625, + "learning_rate": 4.076923076923077e-07, + "loss": -0.0012, + "reward": 0.7843662798404694, + "reward_std": 0.035379850305616856, + "reward_std_all": 0.03268601559102535, + "rewards-max/forensics_sinkhorn_reward": 0.9802578389644623, + "rewards-max/forensics_strict_edge_reward": 0.7401526570320129, + "rewards-max/forensics_tiou_reward": 0.9775150716304779, + "rewards/forensics_sinkhorn_reward": 0.7639409899711609, + "rewards/forensics_strict_edge_reward": 0.06964445114135742, + "rewards/forensics_tiou_reward": 0.5778488218784332, + "step": 771 + }, + { + "completion_length": 25.0625, + "epoch": 2.3686635944700463, + "grad_norm": 6.244054317474365, + "kl": 0.48828125, + "learning_rate": 4.069230769230769e-07, + "loss": 0.0634, + "reward": 0.7659077048301697, + "reward_std": 0.04021010734140873, + "reward_std_all": 0.04111408069729805, + "rewards-max/forensics_sinkhorn_reward": 0.9687690734863281, + "rewards-max/forensics_strict_edge_reward": 0.44795428216457367, + "rewards-max/forensics_tiou_reward": 0.9508981108665466, + "rewards/forensics_sinkhorn_reward": 0.7476207613945007, + "rewards/forensics_strict_edge_reward": 0.07426626607775688, + "rewards/forensics_tiou_reward": 0.6364002227783203, + "step": 772 + }, + { + "completion_length": 22.796875, + "epoch": 2.3717357910906296, + "grad_norm": 7.7798590660095215, + "kl": 0.28369140625, + "learning_rate": 4.061538461538461e-07, + "loss": 0.0396, + "reward": 0.7692146003246307, + "reward_std": 0.03667108900845051, + "reward_std_all": 0.03766019642353058, + "rewards-max/forensics_sinkhorn_reward": 0.9691981077194214, + "rewards-max/forensics_strict_edge_reward": 0.508862778544426, + "rewards-max/forensics_tiou_reward": 0.9492851197719574, + "rewards/forensics_sinkhorn_reward": 0.7403272390365601, + "rewards/forensics_strict_edge_reward": 0.03669141232967377, + "rewards/forensics_tiou_reward": 0.4887873977422714, + "step": 773 + }, + { + "completion_length": 25.296875, + "epoch": 2.3748079877112134, + "grad_norm": 7.024359226226807, + "kl": 0.51171875, + "learning_rate": 4.053846153846154e-07, + "loss": 0.014, + "reward": 0.7804504930973053, + "reward_std": 0.03472375124692917, + "reward_std_all": 0.03341609891504049, + "rewards-max/forensics_sinkhorn_reward": 0.9827175736427307, + "rewards-max/forensics_strict_edge_reward": 0.6428828537464142, + "rewards-max/forensics_tiou_reward": 0.9850525557994843, + "rewards/forensics_sinkhorn_reward": 0.7260873913764954, + "rewards/forensics_strict_edge_reward": 0.04927929490804672, + "rewards/forensics_tiou_reward": 0.5500498414039612, + "step": 774 + }, + { + "completion_length": 24.546875, + "epoch": 2.377880184331797, + "grad_norm": 5.837653636932373, + "kl": 0.3828125, + "learning_rate": 4.046153846153846e-07, + "loss": 0.0115, + "reward": 0.7881634831428528, + "reward_std": 0.020040566101670265, + "reward_std_all": 0.02100904146209359, + "rewards-max/forensics_sinkhorn_reward": 0.9769043028354645, + "rewards-max/forensics_strict_edge_reward": 0.3142388630658388, + "rewards-max/forensics_tiou_reward": 0.9634552896022797, + "rewards/forensics_sinkhorn_reward": 0.7697728276252747, + "rewards/forensics_strict_edge_reward": 0.038139954674988985, + "rewards/forensics_tiou_reward": 0.6100263893604279, + "step": 775 + }, + { + "completion_length": 24.328125, + "epoch": 2.380952380952381, + "grad_norm": 6.893193244934082, + "kl": 0.390625, + "learning_rate": 4.0384615384615386e-07, + "loss": 0.0133, + "reward": 0.7847957909107208, + "reward_std": 0.011546931928023696, + "reward_std_all": 0.018575387075543404, + "rewards-max/forensics_sinkhorn_reward": 0.9710734188556671, + "rewards-max/forensics_strict_edge_reward": 0.43846702575683594, + "rewards-max/forensics_tiou_reward": 0.9523822367191315, + "rewards/forensics_sinkhorn_reward": 0.7334648668766022, + "rewards/forensics_strict_edge_reward": 0.0484066940844059, + "rewards/forensics_tiou_reward": 0.5514161884784698, + "step": 776 + }, + { + "completion_length": 23.421875, + "epoch": 2.3840245775729647, + "grad_norm": 5.241590976715088, + "kl": 0.3828125, + "learning_rate": 4.03076923076923e-07, + "loss": -0.0744, + "reward": 0.794640988111496, + "reward_std": 0.020039235707372427, + "reward_std_all": 0.01938386308029294, + "rewards-max/forensics_sinkhorn_reward": 0.9850110411643982, + "rewards-max/forensics_strict_edge_reward": 0.5702531039714813, + "rewards-max/forensics_tiou_reward": 0.9897157251834869, + "rewards/forensics_sinkhorn_reward": 0.8018274903297424, + "rewards/forensics_strict_edge_reward": 0.07070156186819077, + "rewards/forensics_tiou_reward": 0.6700010895729065, + "step": 777 + }, + { + "completion_length": 25.5, + "epoch": 2.3870967741935485, + "grad_norm": 7.16496467590332, + "kl": 0.4306640625, + "learning_rate": 4.023076923076923e-07, + "loss": -0.0508, + "reward": 0.7635147869586945, + "reward_std": 0.0536231454461813, + "reward_std_all": 0.05093280132859945, + "rewards-max/forensics_sinkhorn_reward": 0.9723012745380402, + "rewards-max/forensics_strict_edge_reward": 0.4634254425764084, + "rewards-max/forensics_tiou_reward": 0.9590187966823578, + "rewards/forensics_sinkhorn_reward": 0.777599424123764, + "rewards/forensics_strict_edge_reward": 0.052456725388765335, + "rewards/forensics_tiou_reward": 0.5877580046653748, + "step": 778 + }, + { + "completion_length": 30.40625, + "epoch": 2.3901689708141323, + "grad_norm": 5.985849857330322, + "kl": 0.33984375, + "learning_rate": 4.015384615384615e-07, + "loss": 0.0516, + "reward": 0.7883173525333405, + "reward_std": 0.029150167480111122, + "reward_std_all": 0.027506913989782333, + "rewards-max/forensics_sinkhorn_reward": 0.9573664963245392, + "rewards-max/forensics_strict_edge_reward": 0.2947033792734146, + "rewards-max/forensics_tiou_reward": 0.9197766780853271, + "rewards/forensics_sinkhorn_reward": 0.7402832508087158, + "rewards/forensics_strict_edge_reward": 0.026206214912235737, + "rewards/forensics_tiou_reward": 0.5578165948390961, + "step": 779 + }, + { + "completion_length": 24.328125, + "epoch": 2.3932411674347156, + "grad_norm": 5.282867908477783, + "kl": 0.435546875, + "learning_rate": 4.0076923076923077e-07, + "loss": -0.0484, + "reward": 0.7865588963031769, + "reward_std": 0.03300078958272934, + "reward_std_all": 0.030484323389828205, + "rewards-max/forensics_sinkhorn_reward": 0.9788622856140137, + "rewards-max/forensics_strict_edge_reward": 0.43586206436157227, + "rewards-max/forensics_tiou_reward": 0.9675098657608032, + "rewards/forensics_sinkhorn_reward": 0.7858506739139557, + "rewards/forensics_strict_edge_reward": 0.053425952792167664, + "rewards/forensics_tiou_reward": 0.6092122793197632, + "step": 780 + }, + { + "completion_length": 23.9375, + "epoch": 2.3963133640552994, + "grad_norm": 5.122067451477051, + "kl": 0.470703125, + "learning_rate": 4e-07, + "loss": -0.0268, + "reward": 0.8010702133178711, + "reward_std": 0.007227290887385607, + "reward_std_all": 0.008776572067290545, + "rewards-max/forensics_sinkhorn_reward": 0.9816615283489227, + "rewards-max/forensics_strict_edge_reward": 0.6526533365249634, + "rewards-max/forensics_tiou_reward": 0.9810990393161774, + "rewards/forensics_sinkhorn_reward": 0.7391491234302521, + "rewards/forensics_strict_edge_reward": 0.061425428837537766, + "rewards/forensics_tiou_reward": 0.5668059885501862, + "step": 781 + }, + { + "completion_length": 25.578125, + "epoch": 2.399385560675883, + "grad_norm": 5.671462535858154, + "kl": 0.3623046875, + "learning_rate": 3.992307692307692e-07, + "loss": 0.0765, + "reward": 0.7389175593852997, + "reward_std": 0.031384995207190514, + "reward_std_all": 0.0499042384326458, + "rewards-max/forensics_sinkhorn_reward": 0.9722169637680054, + "rewards-max/forensics_strict_edge_reward": 0.4974842667579651, + "rewards-max/forensics_tiou_reward": 0.9506614506244659, + "rewards/forensics_sinkhorn_reward": 0.7259567975997925, + "rewards/forensics_strict_edge_reward": 0.06066280975937843, + "rewards/forensics_tiou_reward": 0.5465178489685059, + "step": 782 + }, + { + "completion_length": 27.234375, + "epoch": 2.402457757296467, + "grad_norm": 6.221834182739258, + "kl": 0.4189453125, + "learning_rate": 3.9846153846153846e-07, + "loss": 0.0774, + "reward": 0.7567814588546753, + "reward_std": 0.04973486252129078, + "reward_std_all": 0.05214885622262955, + "rewards-max/forensics_sinkhorn_reward": 0.9645478427410126, + "rewards-max/forensics_strict_edge_reward": 0.45597073435783386, + "rewards-max/forensics_tiou_reward": 0.9410525560379028, + "rewards/forensics_sinkhorn_reward": 0.7255075871944427, + "rewards/forensics_strict_edge_reward": 0.052513038739562035, + "rewards/forensics_tiou_reward": 0.5806978046894073, + "step": 783 + }, + { + "completion_length": 24.828125, + "epoch": 2.4055299539170507, + "grad_norm": 6.534727096557617, + "kl": 0.51171875, + "learning_rate": 3.976923076923077e-07, + "loss": -0.0276, + "reward": 0.796675056219101, + "reward_std": 0.01600096235051751, + "reward_std_all": 0.01585371233522892, + "rewards-max/forensics_sinkhorn_reward": 0.984133243560791, + "rewards-max/forensics_strict_edge_reward": 0.5875949561595917, + "rewards-max/forensics_tiou_reward": 0.987378716468811, + "rewards/forensics_sinkhorn_reward": 0.7740021347999573, + "rewards/forensics_strict_edge_reward": 0.07836876437067986, + "rewards/forensics_tiou_reward": 0.5840589106082916, + "step": 784 + }, + { + "completion_length": 25.125, + "epoch": 2.4086021505376345, + "grad_norm": 6.755589008331299, + "kl": 0.4140625, + "learning_rate": 3.969230769230769e-07, + "loss": -0.0484, + "reward": 0.7931357324123383, + "reward_std": 0.008696500677615404, + "reward_std_all": 0.012043702881783247, + "rewards-max/forensics_sinkhorn_reward": 0.9826288223266602, + "rewards-max/forensics_strict_edge_reward": 0.5382090210914612, + "rewards-max/forensics_tiou_reward": 0.9766133725643158, + "rewards/forensics_sinkhorn_reward": 0.7624220848083496, + "rewards/forensics_strict_edge_reward": 0.0503434669226408, + "rewards/forensics_tiou_reward": 0.5887962877750397, + "step": 785 + }, + { + "completion_length": 23.125, + "epoch": 2.4116743471582183, + "grad_norm": 6.339681148529053, + "kl": 0.451171875, + "learning_rate": 3.961538461538461e-07, + "loss": 0.0171, + "reward": 0.7775366604328156, + "reward_std": 0.04119174741208553, + "reward_std_all": 0.038732776418328285, + "rewards-max/forensics_sinkhorn_reward": 0.9529368281364441, + "rewards-max/forensics_strict_edge_reward": 0.36077773571014404, + "rewards-max/forensics_tiou_reward": 0.9052765071392059, + "rewards/forensics_sinkhorn_reward": 0.7607143521308899, + "rewards/forensics_strict_edge_reward": 0.031577530317008495, + "rewards/forensics_tiou_reward": 0.5562276840209961, + "step": 786 + }, + { + "completion_length": 21.4375, + "epoch": 2.4147465437788016, + "grad_norm": 6.420955181121826, + "kl": 0.544921875, + "learning_rate": 3.9538461538461537e-07, + "loss": -0.0193, + "reward": 0.7883682250976562, + "reward_std": 0.016532147768884897, + "reward_std_all": 0.01832450134679675, + "rewards-max/forensics_sinkhorn_reward": 0.9789117276668549, + "rewards-max/forensics_strict_edge_reward": 0.3836973309516907, + "rewards-max/forensics_tiou_reward": 0.9686753749847412, + "rewards/forensics_sinkhorn_reward": 0.7667727768421173, + "rewards/forensics_strict_edge_reward": 0.05539967119693756, + "rewards/forensics_tiou_reward": 0.6053674519062042, + "step": 787 + }, + { + "completion_length": 24.296875, + "epoch": 2.4178187403993854, + "grad_norm": 6.67878532409668, + "kl": 0.375, + "learning_rate": 3.946153846153846e-07, + "loss": -0.0516, + "reward": 0.7744750082492828, + "reward_std": 0.030921852216124535, + "reward_std_all": 0.032427554950118065, + "rewards-max/forensics_sinkhorn_reward": 0.979940801858902, + "rewards-max/forensics_strict_edge_reward": 0.5933051109313965, + "rewards-max/forensics_tiou_reward": 0.9807069301605225, + "rewards/forensics_sinkhorn_reward": 0.7794949412345886, + "rewards/forensics_strict_edge_reward": 0.06223030388355255, + "rewards/forensics_tiou_reward": 0.64970463514328, + "step": 788 + }, + { + "completion_length": 22.59375, + "epoch": 2.420890937019969, + "grad_norm": 6.463642120361328, + "kl": 0.3369140625, + "learning_rate": 3.9384615384615385e-07, + "loss": 0.0351, + "reward": 0.7076573669910431, + "reward_std": 0.035081653855741024, + "reward_std_all": 0.06829461827874184, + "rewards-max/forensics_sinkhorn_reward": 0.976889044046402, + "rewards-max/forensics_strict_edge_reward": 0.5127807669341564, + "rewards-max/forensics_tiou_reward": 0.9682060480117798, + "rewards/forensics_sinkhorn_reward": 0.6724041402339935, + "rewards/forensics_strict_edge_reward": 0.04096541414037347, + "rewards/forensics_tiou_reward": 0.5252754390239716, + "step": 789 + }, + { + "completion_length": 23.765625, + "epoch": 2.423963133640553, + "grad_norm": 7.467545986175537, + "kl": 0.466796875, + "learning_rate": 3.9307692307692306e-07, + "loss": -0.0061, + "reward": 0.7670712471008301, + "reward_std": 0.031652385368943214, + "reward_std_all": 0.037926970049738884, + "rewards-max/forensics_sinkhorn_reward": 0.9776599407196045, + "rewards-max/forensics_strict_edge_reward": 0.5500316619873047, + "rewards-max/forensics_tiou_reward": 0.9706194698810577, + "rewards/forensics_sinkhorn_reward": 0.7808772325515747, + "rewards/forensics_strict_edge_reward": 0.07032894901931286, + "rewards/forensics_tiou_reward": 0.5745702087879181, + "step": 790 + }, + { + "completion_length": 23.03125, + "epoch": 2.4270353302611367, + "grad_norm": 6.7222466468811035, + "kl": 0.3671875, + "learning_rate": 3.923076923076923e-07, + "loss": 0.0286, + "reward": 0.7751204967498779, + "reward_std": 0.039914124412462115, + "reward_std_all": 0.03819071757607162, + "rewards-max/forensics_sinkhorn_reward": 0.974338948726654, + "rewards-max/forensics_strict_edge_reward": 0.3800070434808731, + "rewards-max/forensics_tiou_reward": 0.9631209969520569, + "rewards/forensics_sinkhorn_reward": 0.7364145517349243, + "rewards/forensics_strict_edge_reward": 0.027046428993344307, + "rewards/forensics_tiou_reward": 0.49342796206474304, + "step": 791 + }, + { + "completion_length": 25.453125, + "epoch": 2.4301075268817205, + "grad_norm": 6.2685546875, + "kl": 0.560546875, + "learning_rate": 3.9153846153846154e-07, + "loss": -0.0534, + "reward": 0.7832753658294678, + "reward_std": 0.023363440297544003, + "reward_std_all": 0.02436706959269941, + "rewards-max/forensics_sinkhorn_reward": 0.9822677075862885, + "rewards-max/forensics_strict_edge_reward": 0.6340298652648926, + "rewards-max/forensics_tiou_reward": 0.9844098389148712, + "rewards/forensics_sinkhorn_reward": 0.7748926281929016, + "rewards/forensics_strict_edge_reward": 0.07446641102433205, + "rewards/forensics_tiou_reward": 0.6406400203704834, + "step": 792 + }, + { + "completion_length": 22.9375, + "epoch": 2.4331797235023043, + "grad_norm": 7.000774383544922, + "kl": 0.3857421875, + "learning_rate": 3.9076923076923076e-07, + "loss": 0.0247, + "reward": 0.7665632367134094, + "reward_std": 0.014107932802289724, + "reward_std_all": 0.028416733257472515, + "rewards-max/forensics_sinkhorn_reward": 0.9817626774311066, + "rewards-max/forensics_strict_edge_reward": 0.5605136454105377, + "rewards-max/forensics_tiou_reward": 0.9830788373947144, + "rewards/forensics_sinkhorn_reward": 0.7638078927993774, + "rewards/forensics_strict_edge_reward": 0.06298883631825447, + "rewards/forensics_tiou_reward": 0.6085794866085052, + "step": 793 + }, + { + "completion_length": 25.90625, + "epoch": 2.436251920122888, + "grad_norm": 7.811422348022461, + "kl": 0.4287109375, + "learning_rate": 3.8999999999999997e-07, + "loss": -0.0543, + "reward": 0.7789576351642609, + "reward_std": 0.02712376043200493, + "reward_std_all": 0.028682011179625988, + "rewards-max/forensics_sinkhorn_reward": 0.9763719141483307, + "rewards-max/forensics_strict_edge_reward": 0.47703270614147186, + "rewards-max/forensics_tiou_reward": 0.9641472101211548, + "rewards/forensics_sinkhorn_reward": 0.782854288816452, + "rewards/forensics_strict_edge_reward": 0.06550781056284904, + "rewards/forensics_tiou_reward": 0.636762946844101, + "step": 794 + }, + { + "completion_length": 28.546875, + "epoch": 2.439324116743472, + "grad_norm": 7.834274768829346, + "kl": 0.662109375, + "learning_rate": 3.892307692307692e-07, + "loss": 0.0059, + "reward": 0.7579160928726196, + "reward_std": 0.06248605251312256, + "reward_std_all": 0.05891132727265358, + "rewards-max/forensics_sinkhorn_reward": 0.9786604940891266, + "rewards-max/forensics_strict_edge_reward": 0.48898807168006897, + "rewards-max/forensics_tiou_reward": 0.9725932776927948, + "rewards/forensics_sinkhorn_reward": 0.7148405611515045, + "rewards/forensics_strict_edge_reward": 0.03279256634414196, + "rewards/forensics_tiou_reward": 0.5268677920103073, + "step": 795 + }, + { + "completion_length": 25.46875, + "epoch": 2.442396313364055, + "grad_norm": 7.140836715698242, + "kl": 0.494140625, + "learning_rate": 3.8846153846153845e-07, + "loss": -0.064, + "reward": 0.7778553366661072, + "reward_std": 0.03897637501358986, + "reward_std_all": 0.037096427753567696, + "rewards-max/forensics_sinkhorn_reward": 0.9845065176486969, + "rewards-max/forensics_strict_edge_reward": 0.5738379806280136, + "rewards-max/forensics_tiou_reward": 0.9869652092456818, + "rewards/forensics_sinkhorn_reward": 0.7509939074516296, + "rewards/forensics_strict_edge_reward": 0.07506437227129936, + "rewards/forensics_tiou_reward": 0.6102192401885986, + "step": 796 + }, + { + "completion_length": 22.046875, + "epoch": 2.445468509984639, + "grad_norm": 7.281030178070068, + "kl": 0.3330078125, + "learning_rate": 3.8769230769230767e-07, + "loss": -0.0608, + "reward": 0.7800403833389282, + "reward_std": 0.011377835180610418, + "reward_std_all": 0.01891251257620752, + "rewards-max/forensics_sinkhorn_reward": 0.9807054102420807, + "rewards-max/forensics_strict_edge_reward": 0.5434179902076721, + "rewards-max/forensics_tiou_reward": 0.9776399731636047, + "rewards/forensics_sinkhorn_reward": 0.753045529127121, + "rewards/forensics_strict_edge_reward": 0.055341679602861404, + "rewards/forensics_tiou_reward": 0.6000082194805145, + "step": 797 + }, + { + "completion_length": 23.609375, + "epoch": 2.4485407066052227, + "grad_norm": 7.003307342529297, + "kl": 0.556640625, + "learning_rate": 3.8692307692307693e-07, + "loss": 0.0887, + "reward": 0.7644140124320984, + "reward_std": 0.05934968125075102, + "reward_std_all": 0.05464575346559286, + "rewards-max/forensics_sinkhorn_reward": 0.9859167039394379, + "rewards-max/forensics_strict_edge_reward": 0.6364382356405258, + "rewards-max/forensics_tiou_reward": 0.9926697611808777, + "rewards/forensics_sinkhorn_reward": 0.738589346408844, + "rewards/forensics_strict_edge_reward": 0.05673602409660816, + "rewards/forensics_tiou_reward": 0.5862677693367004, + "step": 798 + }, + { + "completion_length": 27.140625, + "epoch": 2.4516129032258065, + "grad_norm": 6.358275890350342, + "kl": 0.388671875, + "learning_rate": 3.8615384615384615e-07, + "loss": -0.0514, + "reward": 0.7935861945152283, + "reward_std": 0.014401530381292105, + "reward_std_all": 0.015503572300076485, + "rewards-max/forensics_sinkhorn_reward": 0.9882140159606934, + "rewards-max/forensics_strict_edge_reward": 0.7482926547527313, + "rewards-max/forensics_tiou_reward": 0.9973760545253754, + "rewards/forensics_sinkhorn_reward": 0.7769932746887207, + "rewards/forensics_strict_edge_reward": 0.06436197645962238, + "rewards/forensics_tiou_reward": 0.6308706998825073, + "step": 799 + }, + { + "completion_length": 24.265625, + "epoch": 2.4546850998463903, + "grad_norm": 7.10965633392334, + "kl": 0.345703125, + "learning_rate": 3.8538461538461536e-07, + "loss": -0.0071, + "reward": 0.7928215265274048, + "reward_std": 0.02227146946825087, + "reward_std_all": 0.021339445374906063, + "rewards-max/forensics_sinkhorn_reward": 0.9768257439136505, + "rewards-max/forensics_strict_edge_reward": 0.6816803216934204, + "rewards-max/forensics_tiou_reward": 0.9715116620063782, + "rewards/forensics_sinkhorn_reward": 0.7519191801548004, + "rewards/forensics_strict_edge_reward": 0.07561255246400833, + "rewards/forensics_tiou_reward": 0.5962653756141663, + "step": 800 + }, + { + "completion_length": 25.671875, + "epoch": 2.457757296466974, + "grad_norm": 6.773257255554199, + "kl": 0.40625, + "learning_rate": 3.8461538461538463e-07, + "loss": 0.0332, + "reward": 0.7749206125736237, + "reward_std": 0.02785471733659506, + "reward_std_all": 0.03072180412709713, + "rewards-max/forensics_sinkhorn_reward": 0.9585212469100952, + "rewards-max/forensics_strict_edge_reward": 0.3814268186688423, + "rewards-max/forensics_tiou_reward": 0.904000461101532, + "rewards/forensics_sinkhorn_reward": 0.7301734089851379, + "rewards/forensics_strict_edge_reward": 0.03305635298602283, + "rewards/forensics_tiou_reward": 0.5666487216949463, + "step": 801 + }, + { + "completion_length": 24.265625, + "epoch": 2.460829493087558, + "grad_norm": 6.846908092498779, + "kl": 0.4677734375, + "learning_rate": 3.8384615384615384e-07, + "loss": 0.1086, + "reward": 0.7149408459663391, + "reward_std": 0.05882071703672409, + "reward_std_all": 0.07852260023355484, + "rewards-max/forensics_sinkhorn_reward": 0.9392060339450836, + "rewards-max/forensics_strict_edge_reward": 0.37195122241973877, + "rewards-max/forensics_tiou_reward": 0.888547271490097, + "rewards/forensics_sinkhorn_reward": 0.6642253994941711, + "rewards/forensics_strict_edge_reward": 0.03132256027311087, + "rewards/forensics_tiou_reward": 0.45826423168182373, + "step": 802 + }, + { + "completion_length": 27.53125, + "epoch": 2.463901689708141, + "grad_norm": 6.320784568786621, + "kl": 0.4716796875, + "learning_rate": 3.8307692307692305e-07, + "loss": -0.0359, + "reward": 0.7636202275753021, + "reward_std": 0.0254535349085927, + "reward_std_all": 0.032316128723323345, + "rewards-max/forensics_sinkhorn_reward": 0.9658175110816956, + "rewards-max/forensics_strict_edge_reward": 0.3311092481017113, + "rewards-max/forensics_tiou_reward": 0.9486771821975708, + "rewards/forensics_sinkhorn_reward": 0.7025554478168488, + "rewards/forensics_strict_edge_reward": 0.025183741934597492, + "rewards/forensics_tiou_reward": 0.5360242128372192, + "step": 803 + }, + { + "completion_length": 23.09375, + "epoch": 2.466973886328725, + "grad_norm": 6.661530494689941, + "kl": 0.353515625, + "learning_rate": 3.8230769230769227e-07, + "loss": 0.0045, + "reward": 0.7593104243278503, + "reward_std": 0.0424200976267457, + "reward_std_all": 0.04407305084168911, + "rewards-max/forensics_sinkhorn_reward": 0.9674746990203857, + "rewards-max/forensics_strict_edge_reward": 0.26538098976016045, + "rewards-max/forensics_tiou_reward": 0.9409023225307465, + "rewards/forensics_sinkhorn_reward": 0.7218763828277588, + "rewards/forensics_strict_edge_reward": 0.01841265382245183, + "rewards/forensics_tiou_reward": 0.5604367256164551, + "step": 804 + }, + { + "completion_length": 27.21875, + "epoch": 2.4700460829493087, + "grad_norm": 7.1031813621521, + "kl": 0.33203125, + "learning_rate": 3.8153846153846153e-07, + "loss": -0.0133, + "reward": 0.7643081843852997, + "reward_std": 0.017227524425834417, + "reward_std_all": 0.030679361429065466, + "rewards-max/forensics_sinkhorn_reward": 0.9549521803855896, + "rewards-max/forensics_strict_edge_reward": 0.15038888528943062, + "rewards-max/forensics_tiou_reward": 0.9101027548313141, + "rewards/forensics_sinkhorn_reward": 0.6975952684879303, + "rewards/forensics_strict_edge_reward": 0.014586982317268848, + "rewards/forensics_tiou_reward": 0.5214828103780746, + "step": 805 + }, + { + "completion_length": 22.53125, + "epoch": 2.4731182795698925, + "grad_norm": 8.013806343078613, + "kl": 0.3896484375, + "learning_rate": 3.8076923076923075e-07, + "loss": -0.041, + "reward": 0.783587783575058, + "reward_std": 0.03752307966351509, + "reward_std_all": 0.034599632024765015, + "rewards-max/forensics_sinkhorn_reward": 0.9569399654865265, + "rewards-max/forensics_strict_edge_reward": 0.2972450405359268, + "rewards-max/forensics_tiou_reward": 0.9186632931232452, + "rewards/forensics_sinkhorn_reward": 0.7597678899765015, + "rewards/forensics_strict_edge_reward": 0.026577227748930454, + "rewards/forensics_tiou_reward": 0.5368186831474304, + "step": 806 + }, + { + "completion_length": 24.765625, + "epoch": 2.4761904761904763, + "grad_norm": 7.578762054443359, + "kl": 0.4287109375, + "learning_rate": 3.7999999999999996e-07, + "loss": 0.0773, + "reward": 0.7574267983436584, + "reward_std": 0.036398012191057205, + "reward_std_all": 0.044385312125086784, + "rewards-max/forensics_sinkhorn_reward": 0.9493989944458008, + "rewards-max/forensics_strict_edge_reward": 0.4010457918047905, + "rewards-max/forensics_tiou_reward": 0.9029246866703033, + "rewards/forensics_sinkhorn_reward": 0.6583662331104279, + "rewards/forensics_strict_edge_reward": 0.03639122238382697, + "rewards/forensics_tiou_reward": 0.446416899561882, + "step": 807 + }, + { + "completion_length": 25.90625, + "epoch": 2.47926267281106, + "grad_norm": 6.359163761138916, + "kl": 0.458984375, + "learning_rate": 3.792307692307692e-07, + "loss": 0.0554, + "reward": 0.7821519672870636, + "reward_std": 0.027643797919154167, + "reward_std_all": 0.027435213327407837, + "rewards-max/forensics_sinkhorn_reward": 0.9839100539684296, + "rewards-max/forensics_strict_edge_reward": 0.8582656681537628, + "rewards-max/forensics_tiou_reward": 0.9868742227554321, + "rewards/forensics_sinkhorn_reward": 0.75785031914711, + "rewards/forensics_strict_edge_reward": 0.07747631520032883, + "rewards/forensics_tiou_reward": 0.6133842766284943, + "step": 808 + }, + { + "completion_length": 27.515625, + "epoch": 2.482334869431644, + "grad_norm": 5.382237434387207, + "kl": 0.443359375, + "learning_rate": 3.7846153846153844e-07, + "loss": 0.0547, + "reward": 0.7907643914222717, + "reward_std": 0.009039907483384013, + "reward_std_all": 0.013031183741986752, + "rewards-max/forensics_sinkhorn_reward": 0.9857043027877808, + "rewards-max/forensics_strict_edge_reward": 0.627477690577507, + "rewards-max/forensics_tiou_reward": 0.9916409850120544, + "rewards/forensics_sinkhorn_reward": 0.7034316956996918, + "rewards/forensics_strict_edge_reward": 0.04751308634877205, + "rewards/forensics_tiou_reward": 0.5777519047260284, + "step": 809 + }, + { + "completion_length": 28.21875, + "epoch": 2.485407066052227, + "grad_norm": 6.263173580169678, + "kl": 0.45703125, + "learning_rate": 3.776923076923077e-07, + "loss": -0.0283, + "reward": 0.7522140145301819, + "reward_std": 0.04885079897940159, + "reward_std_all": 0.05171041376888752, + "rewards-max/forensics_sinkhorn_reward": 0.9778223037719727, + "rewards-max/forensics_strict_edge_reward": 0.48625826835632324, + "rewards-max/forensics_tiou_reward": 0.9701011776924133, + "rewards/forensics_sinkhorn_reward": 0.7382667362689972, + "rewards/forensics_strict_edge_reward": 0.05322050489485264, + "rewards/forensics_tiou_reward": 0.5771219432353973, + "step": 810 + }, + { + "completion_length": 25.734375, + "epoch": 2.488479262672811, + "grad_norm": 7.091055870056152, + "kl": 0.341796875, + "learning_rate": 3.769230769230769e-07, + "loss": -0.0208, + "reward": 0.7952180206775665, + "reward_std": 0.018608586862683296, + "reward_std_all": 0.01816120371222496, + "rewards-max/forensics_sinkhorn_reward": 0.9850998222827911, + "rewards-max/forensics_strict_edge_reward": 0.7073339223861694, + "rewards-max/forensics_tiou_reward": 0.9871665835380554, + "rewards/forensics_sinkhorn_reward": 0.7866807579994202, + "rewards/forensics_strict_edge_reward": 0.04285075142979622, + "rewards/forensics_tiou_reward": 0.6341026127338409, + "step": 811 + }, + { + "completion_length": 24.6875, + "epoch": 2.4915514592933947, + "grad_norm": 6.0576348304748535, + "kl": 0.4765625, + "learning_rate": 3.7615384615384614e-07, + "loss": 0.0771, + "reward": 0.7373994588851929, + "reward_std": 0.06300079450011253, + "reward_std_all": 0.06473284214735031, + "rewards-max/forensics_sinkhorn_reward": 0.9081458449363708, + "rewards-max/forensics_strict_edge_reward": 0.22905148565769196, + "rewards-max/forensics_tiou_reward": 0.7958471775054932, + "rewards/forensics_sinkhorn_reward": 0.6314757466316223, + "rewards/forensics_strict_edge_reward": 0.017936676740646362, + "rewards/forensics_tiou_reward": 0.44363991916179657, + "step": 812 + }, + { + "completion_length": 28.328125, + "epoch": 2.4946236559139785, + "grad_norm": 6.193422794342041, + "kl": 0.4921875, + "learning_rate": 3.7538461538461535e-07, + "loss": -0.0359, + "reward": 0.7907372713088989, + "reward_std": 0.01996220159344375, + "reward_std_all": 0.019897747552022338, + "rewards-max/forensics_sinkhorn_reward": 0.984478086233139, + "rewards-max/forensics_strict_edge_reward": 0.7744058072566986, + "rewards-max/forensics_tiou_reward": 0.9869520664215088, + "rewards/forensics_sinkhorn_reward": 0.794945627450943, + "rewards/forensics_strict_edge_reward": 0.09045860543847084, + "rewards/forensics_tiou_reward": 0.6632903814315796, + "step": 813 + }, + { + "completion_length": 30.796875, + "epoch": 2.4976958525345623, + "grad_norm": 6.104927062988281, + "kl": 0.4052734375, + "learning_rate": 3.746153846153846e-07, + "loss": -0.0059, + "reward": 0.802700936794281, + "reward_std": 0.005224293447099626, + "reward_std_all": 0.00692856777459383, + "rewards-max/forensics_sinkhorn_reward": 0.9821536839008331, + "rewards-max/forensics_strict_edge_reward": 0.5020824074745178, + "rewards-max/forensics_tiou_reward": 0.9796343147754669, + "rewards/forensics_sinkhorn_reward": 0.7549826502799988, + "rewards/forensics_strict_edge_reward": 0.06285673938691616, + "rewards/forensics_tiou_reward": 0.6374652087688446, + "step": 814 + }, + { + "completion_length": 23.09375, + "epoch": 2.500768049155146, + "grad_norm": 8.403552055358887, + "kl": 0.5693359375, + "learning_rate": 3.7384615384615383e-07, + "loss": 0.0316, + "reward": 0.7613557875156403, + "reward_std": 0.028248930582776666, + "reward_std_all": 0.03553465148434043, + "rewards-max/forensics_sinkhorn_reward": 0.9711772501468658, + "rewards-max/forensics_strict_edge_reward": 0.6794509887695312, + "rewards-max/forensics_tiou_reward": 0.9557134211063385, + "rewards/forensics_sinkhorn_reward": 0.7219778895378113, + "rewards/forensics_strict_edge_reward": 0.0767788216471672, + "rewards/forensics_tiou_reward": 0.5260272175073624, + "step": 815 + }, + { + "completion_length": 24.25, + "epoch": 2.50384024577573, + "grad_norm": 6.213759899139404, + "kl": 0.4052734375, + "learning_rate": 3.7307692307692304e-07, + "loss": 0.0331, + "reward": 0.7942076921463013, + "reward_std": 0.020804871572181582, + "reward_std_all": 0.020061413291841745, + "rewards-max/forensics_sinkhorn_reward": 0.9826128482818604, + "rewards-max/forensics_strict_edge_reward": 0.8224053680896759, + "rewards-max/forensics_tiou_reward": 0.9839906692504883, + "rewards/forensics_sinkhorn_reward": 0.7890541851520538, + "rewards/forensics_strict_edge_reward": 0.08939750120043755, + "rewards/forensics_tiou_reward": 0.6228189766407013, + "step": 816 + }, + { + "completion_length": 24.46875, + "epoch": 2.506912442396313, + "grad_norm": 8.997374534606934, + "kl": 0.435546875, + "learning_rate": 3.7230769230769226e-07, + "loss": 0.121, + "reward": 0.7516139447689056, + "reward_std": 0.07254199683666229, + "reward_std_all": 0.06757008098065853, + "rewards-max/forensics_sinkhorn_reward": 0.9807422757148743, + "rewards-max/forensics_strict_edge_reward": 0.6032288670539856, + "rewards-max/forensics_tiou_reward": 0.9785342216491699, + "rewards/forensics_sinkhorn_reward": 0.6956429481506348, + "rewards/forensics_strict_edge_reward": 0.05542505346238613, + "rewards/forensics_tiou_reward": 0.48356790840625763, + "step": 817 + }, + { + "completion_length": 24.984375, + "epoch": 2.509984639016897, + "grad_norm": 5.722530841827393, + "kl": 0.4599609375, + "learning_rate": 3.715384615384615e-07, + "loss": -0.0252, + "reward": 0.7691306471824646, + "reward_std": 0.03668949939310551, + "reward_std_all": 0.037481119856238365, + "rewards-max/forensics_sinkhorn_reward": 0.9751776158809662, + "rewards-max/forensics_strict_edge_reward": 0.39542338252067566, + "rewards-max/forensics_tiou_reward": 0.9665651321411133, + "rewards/forensics_sinkhorn_reward": 0.7283697426319122, + "rewards/forensics_strict_edge_reward": 0.03886373154819012, + "rewards/forensics_tiou_reward": 0.5405797958374023, + "step": 818 + }, + { + "completion_length": 28.015625, + "epoch": 2.5130568356374807, + "grad_norm": 7.120983123779297, + "kl": 0.4521484375, + "learning_rate": 3.707692307692308e-07, + "loss": 0.0396, + "reward": 0.7959195971488953, + "reward_std": 0.014371179509907961, + "reward_std_all": 0.014758008997887373, + "rewards-max/forensics_sinkhorn_reward": 0.9686266481876373, + "rewards-max/forensics_strict_edge_reward": 0.37504860758781433, + "rewards-max/forensics_tiou_reward": 0.9484849572181702, + "rewards/forensics_sinkhorn_reward": 0.7892411351203918, + "rewards/forensics_strict_edge_reward": 0.03147326782345772, + "rewards/forensics_tiou_reward": 0.6495320200920105, + "step": 819 + }, + { + "completion_length": 27.0625, + "epoch": 2.5161290322580645, + "grad_norm": 6.866775989532471, + "kl": 0.4345703125, + "learning_rate": 3.7e-07, + "loss": 0.0123, + "reward": 0.7605257034301758, + "reward_std": 0.04245265945792198, + "reward_std_all": 0.044174203649163246, + "rewards-max/forensics_sinkhorn_reward": 0.9749686419963837, + "rewards-max/forensics_strict_edge_reward": 0.43558479845523834, + "rewards-max/forensics_tiou_reward": 0.9672808051109314, + "rewards/forensics_sinkhorn_reward": 0.7075295448303223, + "rewards/forensics_strict_edge_reward": 0.03900930378586054, + "rewards/forensics_tiou_reward": 0.5912329256534576, + "step": 820 + }, + { + "completion_length": 26.328125, + "epoch": 2.5192012288786483, + "grad_norm": 5.300475120544434, + "kl": 0.4736328125, + "learning_rate": 3.692307692307692e-07, + "loss": 0.0897, + "reward": 0.7723266780376434, + "reward_std": 0.045296257361769676, + "reward_std_all": 0.04284686781466007, + "rewards-max/forensics_sinkhorn_reward": 0.9767089486122131, + "rewards-max/forensics_strict_edge_reward": 0.34962454438209534, + "rewards-max/forensics_tiou_reward": 0.9691410660743713, + "rewards/forensics_sinkhorn_reward": 0.7579787075519562, + "rewards/forensics_strict_edge_reward": 0.05936518497765064, + "rewards/forensics_tiou_reward": 0.6268594861030579, + "step": 821 + }, + { + "completion_length": 25.984375, + "epoch": 2.522273425499232, + "grad_norm": 5.900044918060303, + "kl": 0.453125, + "learning_rate": 3.6846153846153843e-07, + "loss": -0.0214, + "reward": 0.7845708727836609, + "reward_std": 0.016546144615858793, + "reward_std_all": 0.021075174678117037, + "rewards-max/forensics_sinkhorn_reward": 0.9835158884525299, + "rewards-max/forensics_strict_edge_reward": 0.5550410598516464, + "rewards-max/forensics_tiou_reward": 0.9849149286746979, + "rewards/forensics_sinkhorn_reward": 0.7762205302715302, + "rewards/forensics_strict_edge_reward": 0.10557475592941046, + "rewards/forensics_tiou_reward": 0.6578579545021057, + "step": 822 + }, + { + "completion_length": 25.34375, + "epoch": 2.525345622119816, + "grad_norm": 5.782595157623291, + "kl": 0.3544921875, + "learning_rate": 3.676923076923077e-07, + "loss": 0.0554, + "reward": 0.7816283106803894, + "reward_std": 0.03152594622224569, + "reward_std_all": 0.030865728855133057, + "rewards-max/forensics_sinkhorn_reward": 0.9734396636486053, + "rewards-max/forensics_strict_edge_reward": 0.5221183001995087, + "rewards-max/forensics_tiou_reward": 0.961205780506134, + "rewards/forensics_sinkhorn_reward": 0.7622427046298981, + "rewards/forensics_strict_edge_reward": 0.07617010921239853, + "rewards/forensics_tiou_reward": 0.5589803457260132, + "step": 823 + }, + { + "completion_length": 26.953125, + "epoch": 2.528417818740399, + "grad_norm": 6.439636707305908, + "kl": 0.39453125, + "learning_rate": 3.669230769230769e-07, + "loss": 0.0061, + "reward": 0.7386824488639832, + "reward_std": 0.06857050955295563, + "reward_std_all": 0.07116994634270668, + "rewards-max/forensics_sinkhorn_reward": 0.9653129577636719, + "rewards-max/forensics_strict_edge_reward": 0.574172168970108, + "rewards-max/forensics_tiou_reward": 0.9408120512962341, + "rewards/forensics_sinkhorn_reward": 0.6656751334667206, + "rewards/forensics_strict_edge_reward": 0.0644789095968008, + "rewards/forensics_tiou_reward": 0.48573026061058044, + "step": 824 + }, + { + "completion_length": 25.84375, + "epoch": 2.531490015360983, + "grad_norm": 7.054378509521484, + "kl": 0.501953125, + "learning_rate": 3.661538461538461e-07, + "loss": 0.0009, + "reward": 0.7608863413333893, + "reward_std": 0.02766991686075926, + "reward_std_all": 0.034715376794338226, + "rewards-max/forensics_sinkhorn_reward": 0.9619752168655396, + "rewards-max/forensics_strict_edge_reward": 0.3433353379368782, + "rewards-max/forensics_tiou_reward": 0.9336590766906738, + "rewards/forensics_sinkhorn_reward": 0.7322061657905579, + "rewards/forensics_strict_edge_reward": 0.02747264876961708, + "rewards/forensics_tiou_reward": 0.5436652898788452, + "step": 825 + }, + { + "completion_length": 25.421875, + "epoch": 2.5345622119815667, + "grad_norm": 6.932606220245361, + "kl": 0.4609375, + "learning_rate": 3.6538461538461534e-07, + "loss": 0.1463, + "reward": 0.736841470003128, + "reward_std": 0.055988138541579247, + "reward_std_all": 0.0599849708378315, + "rewards-max/forensics_sinkhorn_reward": 0.984451949596405, + "rewards-max/forensics_strict_edge_reward": 0.7069477140903473, + "rewards-max/forensics_tiou_reward": 0.9832234680652618, + "rewards/forensics_sinkhorn_reward": 0.6786315441131592, + "rewards/forensics_strict_edge_reward": 0.07182791084051132, + "rewards/forensics_tiou_reward": 0.5083149075508118, + "step": 826 + }, + { + "completion_length": 22.390625, + "epoch": 2.5376344086021505, + "grad_norm": 5.296435832977295, + "kl": 0.4453125, + "learning_rate": 3.646153846153846e-07, + "loss": -0.0228, + "reward": 0.779155433177948, + "reward_std": 0.04162340238690376, + "reward_std_all": 0.039380090311169624, + "rewards-max/forensics_sinkhorn_reward": 0.9736393094062805, + "rewards-max/forensics_strict_edge_reward": 0.37785640358924866, + "rewards-max/forensics_tiou_reward": 0.9609012007713318, + "rewards/forensics_sinkhorn_reward": 0.7845551073551178, + "rewards/forensics_strict_edge_reward": 0.03316263295710087, + "rewards/forensics_tiou_reward": 0.5795008838176727, + "step": 827 + }, + { + "completion_length": 24.28125, + "epoch": 2.5407066052227343, + "grad_norm": 6.492927074432373, + "kl": 0.423828125, + "learning_rate": 3.638461538461538e-07, + "loss": 0.0566, + "reward": 0.7458577454090118, + "reward_std": 0.06785129941999912, + "reward_std_all": 0.06531267799437046, + "rewards-max/forensics_sinkhorn_reward": 0.9837607741355896, + "rewards-max/forensics_strict_edge_reward": 0.6799494326114655, + "rewards-max/forensics_tiou_reward": 0.9843749105930328, + "rewards/forensics_sinkhorn_reward": 0.7169949114322662, + "rewards/forensics_strict_edge_reward": 0.11014138534665108, + "rewards/forensics_tiou_reward": 0.5255610197782516, + "step": 828 + }, + { + "completion_length": 23.453125, + "epoch": 2.543778801843318, + "grad_norm": 6.137836456298828, + "kl": 0.357421875, + "learning_rate": 3.6307692307692303e-07, + "loss": -0.041, + "reward": 0.7586809992790222, + "reward_std": 0.0380465853959322, + "reward_std_all": 0.042482515797019005, + "rewards-max/forensics_sinkhorn_reward": 0.9823822975158691, + "rewards-max/forensics_strict_edge_reward": 0.38961950689554214, + "rewards-max/forensics_tiou_reward": 0.9780723750591278, + "rewards/forensics_sinkhorn_reward": 0.7394784390926361, + "rewards/forensics_strict_edge_reward": 0.0577357797883451, + "rewards/forensics_tiou_reward": 0.5334494113922119, + "step": 829 + }, + { + "completion_length": 25.4375, + "epoch": 2.546850998463902, + "grad_norm": 7.293638706207275, + "kl": 0.4375, + "learning_rate": 3.623076923076923e-07, + "loss": 0.1818, + "reward": 0.7774915993213654, + "reward_std": 0.035421499982476234, + "reward_std_all": 0.03413262218236923, + "rewards-max/forensics_sinkhorn_reward": 0.9762759208679199, + "rewards-max/forensics_strict_edge_reward": 0.4481906443834305, + "rewards-max/forensics_tiou_reward": 0.9712826013565063, + "rewards/forensics_sinkhorn_reward": 0.7076990902423859, + "rewards/forensics_strict_edge_reward": 0.054458631202578545, + "rewards/forensics_tiou_reward": 0.5465341210365295, + "step": 830 + }, + { + "completion_length": 26.40625, + "epoch": 2.549923195084485, + "grad_norm": 5.825774669647217, + "kl": 0.388671875, + "learning_rate": 3.615384615384615e-07, + "loss": -0.0111, + "reward": 0.7555584013462067, + "reward_std": 0.05103447940200567, + "reward_std_all": 0.05170431826263666, + "rewards-max/forensics_sinkhorn_reward": 0.9449826180934906, + "rewards-max/forensics_strict_edge_reward": 0.4786291643977165, + "rewards-max/forensics_tiou_reward": 0.8977908790111542, + "rewards/forensics_sinkhorn_reward": 0.7051900029182434, + "rewards/forensics_strict_edge_reward": 0.025656028650701046, + "rewards/forensics_tiou_reward": 0.5149735510349274, + "step": 831 + }, + { + "completion_length": 25.421875, + "epoch": 2.5529953917050694, + "grad_norm": 5.749354362487793, + "kl": 0.34765625, + "learning_rate": 3.607692307692308e-07, + "loss": -0.0157, + "reward": 0.804946780204773, + "reward_std": 0.0025039647007361054, + "reward_std_all": 0.004246314405463636, + "rewards-max/forensics_sinkhorn_reward": 0.9789007902145386, + "rewards-max/forensics_strict_edge_reward": 0.3108609616756439, + "rewards-max/forensics_tiou_reward": 0.9776366055011749, + "rewards/forensics_sinkhorn_reward": 0.8339241147041321, + "rewards/forensics_strict_edge_reward": 0.030895203351974487, + "rewards/forensics_tiou_reward": 0.7206573188304901, + "step": 832 + }, + { + "completion_length": 24.15625, + "epoch": 2.5560675883256527, + "grad_norm": 4.930218696594238, + "kl": 0.5283203125, + "learning_rate": 3.6e-07, + "loss": -0.0567, + "reward": 0.7731287479400635, + "reward_std": 0.04284028150141239, + "reward_std_all": 0.04148236662149429, + "rewards-max/forensics_sinkhorn_reward": 0.9789729118347168, + "rewards-max/forensics_strict_edge_reward": 0.5223433673381805, + "rewards-max/forensics_tiou_reward": 0.9741690158843994, + "rewards/forensics_sinkhorn_reward": 0.7795485258102417, + "rewards/forensics_strict_edge_reward": 0.06280533783137798, + "rewards/forensics_tiou_reward": 0.5954228341579437, + "step": 833 + }, + { + "completion_length": 22.65625, + "epoch": 2.5591397849462365, + "grad_norm": 6.966974258422852, + "kl": 0.34765625, + "learning_rate": 3.592307692307692e-07, + "loss": -0.0037, + "reward": 0.758352667093277, + "reward_std": 0.036419604904949665, + "reward_std_all": 0.0413212226703763, + "rewards-max/forensics_sinkhorn_reward": 0.9734753966331482, + "rewards-max/forensics_strict_edge_reward": 0.6083495318889618, + "rewards-max/forensics_tiou_reward": 0.9634304642677307, + "rewards/forensics_sinkhorn_reward": 0.7076051235198975, + "rewards/forensics_strict_edge_reward": 0.054428476840257645, + "rewards/forensics_tiou_reward": 0.5396531671285629, + "step": 834 + }, + { + "completion_length": 23.84375, + "epoch": 2.5622119815668203, + "grad_norm": 7.668595314025879, + "kl": 0.375, + "learning_rate": 3.584615384615384e-07, + "loss": 0.0811, + "reward": 0.778303861618042, + "reward_std": 0.014471592381596565, + "reward_std_all": 0.023473684675991535, + "rewards-max/forensics_sinkhorn_reward": 0.9455870687961578, + "rewards-max/forensics_strict_edge_reward": 0.3365553840994835, + "rewards-max/forensics_tiou_reward": 0.8982122838497162, + "rewards/forensics_sinkhorn_reward": 0.7035733461380005, + "rewards/forensics_strict_edge_reward": 0.0317225344479084, + "rewards/forensics_tiou_reward": 0.4663502871990204, + "step": 835 + }, + { + "completion_length": 25.296875, + "epoch": 2.565284178187404, + "grad_norm": 7.090236186981201, + "kl": 0.3525390625, + "learning_rate": 3.576923076923077e-07, + "loss": 0.1746, + "reward": 0.7275021076202393, + "reward_std": 0.05979816988110542, + "reward_std_all": 0.07134384289383888, + "rewards-max/forensics_sinkhorn_reward": 0.968740701675415, + "rewards-max/forensics_strict_edge_reward": 0.162779089063406, + "rewards-max/forensics_tiou_reward": 0.949434757232666, + "rewards/forensics_sinkhorn_reward": 0.6412437558174133, + "rewards/forensics_strict_edge_reward": 0.010339327622205019, + "rewards/forensics_tiou_reward": 0.4230858236551285, + "step": 836 + }, + { + "completion_length": 21.4375, + "epoch": 2.568356374807988, + "grad_norm": 6.586565017700195, + "kl": 0.3916015625, + "learning_rate": 3.569230769230769e-07, + "loss": 0.1199, + "reward": 0.7662412822246552, + "reward_std": 0.054641177877783775, + "reward_std_all": 0.051072023808956146, + "rewards-max/forensics_sinkhorn_reward": 0.9758242070674896, + "rewards-max/forensics_strict_edge_reward": 0.6442374289035797, + "rewards-max/forensics_tiou_reward": 0.9652436077594757, + "rewards/forensics_sinkhorn_reward": 0.7564217150211334, + "rewards/forensics_strict_edge_reward": 0.06623771600425243, + "rewards/forensics_tiou_reward": 0.5902126431465149, + "step": 837 + }, + { + "completion_length": 24.328125, + "epoch": 2.571428571428571, + "grad_norm": 7.787164211273193, + "kl": 0.4765625, + "learning_rate": 3.561538461538461e-07, + "loss": 0.025, + "reward": 0.7790207266807556, + "reward_std": 0.03406599164009094, + "reward_std_all": 0.03329949639737606, + "rewards-max/forensics_sinkhorn_reward": 0.95942422747612, + "rewards-max/forensics_strict_edge_reward": 0.22191835194826126, + "rewards-max/forensics_tiou_reward": 0.9264968037605286, + "rewards/forensics_sinkhorn_reward": 0.8008589744567871, + "rewards/forensics_strict_edge_reward": 0.02534575155004859, + "rewards/forensics_tiou_reward": 0.6101330518722534, + "step": 838 + }, + { + "completion_length": 29.875, + "epoch": 2.5745007680491554, + "grad_norm": 5.923499584197998, + "kl": 0.3720703125, + "learning_rate": 3.553846153846154e-07, + "loss": 0.0636, + "reward": 0.7130700051784515, + "reward_std": 0.05711756832897663, + "reward_std_all": 0.07362736575305462, + "rewards-max/forensics_sinkhorn_reward": 0.9573047161102295, + "rewards-max/forensics_strict_edge_reward": 0.1432615965604782, + "rewards-max/forensics_tiou_reward": 0.9319983422756195, + "rewards/forensics_sinkhorn_reward": 0.6314919888973236, + "rewards/forensics_strict_edge_reward": 0.007087146630510688, + "rewards/forensics_tiou_reward": 0.4371260404586792, + "step": 839 + }, + { + "completion_length": 23.359375, + "epoch": 2.5775729646697387, + "grad_norm": 6.4723029136657715, + "kl": 0.3388671875, + "learning_rate": 3.546153846153846e-07, + "loss": -0.0359, + "reward": 0.7915073335170746, + "reward_std": 0.020815234165638685, + "reward_std_all": 0.020893884357064962, + "rewards-max/forensics_sinkhorn_reward": 0.9804112017154694, + "rewards-max/forensics_strict_edge_reward": 0.6176230013370514, + "rewards-max/forensics_tiou_reward": 0.9742584526538849, + "rewards/forensics_sinkhorn_reward": 0.770399659872055, + "rewards/forensics_strict_edge_reward": 0.06155207008123398, + "rewards/forensics_tiou_reward": 0.5981162786483765, + "step": 840 + }, + { + "completion_length": 24.328125, + "epoch": 2.5806451612903225, + "grad_norm": 7.06697940826416, + "kl": 0.55859375, + "learning_rate": 3.5384615384615386e-07, + "loss": -0.0359, + "reward": 0.7943960428237915, + "reward_std": 0.015099340118467808, + "reward_std_all": 0.015954179223626852, + "rewards-max/forensics_sinkhorn_reward": 0.9696870446205139, + "rewards-max/forensics_strict_edge_reward": 0.5038523152470589, + "rewards-max/forensics_tiou_reward": 0.9496658146381378, + "rewards/forensics_sinkhorn_reward": 0.7812225520610809, + "rewards/forensics_strict_edge_reward": 0.05970700364559889, + "rewards/forensics_tiou_reward": 0.5829589366912842, + "step": 841 + }, + { + "completion_length": 21.734375, + "epoch": 2.5837173579109063, + "grad_norm": 6.37322998046875, + "kl": 0.4072265625, + "learning_rate": 3.530769230769231e-07, + "loss": 0.0468, + "reward": 0.7328825294971466, + "reward_std": 0.048178382217884064, + "reward_std_all": 0.06434142962098122, + "rewards-max/forensics_sinkhorn_reward": 0.9727118909358978, + "rewards-max/forensics_strict_edge_reward": 0.5686750710010529, + "rewards-max/forensics_tiou_reward": 0.9597460925579071, + "rewards/forensics_sinkhorn_reward": 0.705258309841156, + "rewards/forensics_strict_edge_reward": 0.033105358481407166, + "rewards/forensics_tiou_reward": 0.4935488998889923, + "step": 842 + }, + { + "completion_length": 26.171875, + "epoch": 2.58678955453149, + "grad_norm": 6.859733581542969, + "kl": 0.494140625, + "learning_rate": 3.523076923076923e-07, + "loss": 0.0086, + "reward": 0.7890040576457977, + "reward_std": 0.02088549267500639, + "reward_std_all": 0.021528422832489014, + "rewards-max/forensics_sinkhorn_reward": 0.9719007611274719, + "rewards-max/forensics_strict_edge_reward": 0.38680407404899597, + "rewards-max/forensics_tiou_reward": 0.9569825828075409, + "rewards/forensics_sinkhorn_reward": 0.7392701506614685, + "rewards/forensics_strict_edge_reward": 0.03694800566881895, + "rewards/forensics_tiou_reward": 0.5462543666362762, + "step": 843 + }, + { + "completion_length": 25.625, + "epoch": 2.589861751152074, + "grad_norm": 6.689237117767334, + "kl": 0.3232421875, + "learning_rate": 3.515384615384615e-07, + "loss": -0.0349, + "reward": 0.7908851504325867, + "reward_std": 0.026146601419895887, + "reward_std_all": 0.02479495434090495, + "rewards-max/forensics_sinkhorn_reward": 0.9760258197784424, + "rewards-max/forensics_strict_edge_reward": 0.580880731344223, + "rewards-max/forensics_tiou_reward": 0.9665283560752869, + "rewards/forensics_sinkhorn_reward": 0.8166374564170837, + "rewards/forensics_strict_edge_reward": 0.04629876650869846, + "rewards/forensics_tiou_reward": 0.6610249578952789, + "step": 844 + }, + { + "completion_length": 26.71875, + "epoch": 2.592933947772657, + "grad_norm": 9.641650199890137, + "kl": 0.40625, + "learning_rate": 3.5076923076923077e-07, + "loss": -0.0048, + "reward": 0.7374522387981415, + "reward_std": 0.08674923330545425, + "reward_std_all": 0.0808412991464138, + "rewards-max/forensics_sinkhorn_reward": 0.9797419607639313, + "rewards-max/forensics_strict_edge_reward": 0.6515887677669525, + "rewards-max/forensics_tiou_reward": 0.9762495756149292, + "rewards/forensics_sinkhorn_reward": 0.6609614789485931, + "rewards/forensics_strict_edge_reward": 0.05098961479961872, + "rewards/forensics_tiou_reward": 0.51381716132164, + "step": 845 + }, + { + "completion_length": 27.921875, + "epoch": 2.5960061443932414, + "grad_norm": 7.121694564819336, + "kl": 0.3798828125, + "learning_rate": 3.5e-07, + "loss": 0.0001, + "reward": 0.7963550686836243, + "reward_std": 0.014640015317127109, + "reward_std_all": 0.014628632692620158, + "rewards-max/forensics_sinkhorn_reward": 0.9736022651195526, + "rewards-max/forensics_strict_edge_reward": 0.5760182738304138, + "rewards-max/forensics_tiou_reward": 0.9618919789791107, + "rewards/forensics_sinkhorn_reward": 0.8366203904151917, + "rewards/forensics_strict_edge_reward": 0.09553047642111778, + "rewards/forensics_tiou_reward": 0.7060232162475586, + "step": 846 + }, + { + "completion_length": 24.359375, + "epoch": 2.5990783410138247, + "grad_norm": 6.656067371368408, + "kl": 0.3701171875, + "learning_rate": 3.492307692307692e-07, + "loss": -0.016, + "reward": 0.8027531802654266, + "reward_std": 0.005216121440753341, + "reward_std_all": 0.0066857957281172276, + "rewards-max/forensics_sinkhorn_reward": 0.9761429727077484, + "rewards-max/forensics_strict_edge_reward": 0.5363326966762543, + "rewards-max/forensics_tiou_reward": 0.9640330672264099, + "rewards/forensics_sinkhorn_reward": 0.7786404490470886, + "rewards/forensics_strict_edge_reward": 0.07183107547461987, + "rewards/forensics_tiou_reward": 0.6016801595687866, + "step": 847 + }, + { + "completion_length": 25.671875, + "epoch": 2.6021505376344085, + "grad_norm": 7.109375476837158, + "kl": 0.3642578125, + "learning_rate": 3.4846153846153846e-07, + "loss": 0.0411, + "reward": 0.7716737389564514, + "reward_std": 0.024789869785308838, + "reward_std_all": 0.030590735375881195, + "rewards-max/forensics_sinkhorn_reward": 0.9702657461166382, + "rewards-max/forensics_strict_edge_reward": 0.3090272545814514, + "rewards-max/forensics_tiou_reward": 0.953179806470871, + "rewards/forensics_sinkhorn_reward": 0.7247061133384705, + "rewards/forensics_strict_edge_reward": 0.0311491871252656, + "rewards/forensics_tiou_reward": 0.48142723739147186, + "step": 848 + }, + { + "completion_length": 23.296875, + "epoch": 2.6052227342549923, + "grad_norm": 6.3198466300964355, + "kl": 0.451171875, + "learning_rate": 3.476923076923077e-07, + "loss": -0.0048, + "reward": 0.7593534588813782, + "reward_std": 0.03833837783895433, + "reward_std_all": 0.04715851601213217, + "rewards-max/forensics_sinkhorn_reward": 0.9828177094459534, + "rewards-max/forensics_strict_edge_reward": 0.412410169839859, + "rewards-max/forensics_tiou_reward": 0.9854567050933838, + "rewards/forensics_sinkhorn_reward": 0.7615951895713806, + "rewards/forensics_strict_edge_reward": 0.041187530383467674, + "rewards/forensics_tiou_reward": 0.6030591130256653, + "step": 849 + }, + { + "completion_length": 21.578125, + "epoch": 2.608294930875576, + "grad_norm": 8.590364456176758, + "kl": 0.462890625, + "learning_rate": 3.4692307692307694e-07, + "loss": 0.0822, + "reward": 0.7840879261493683, + "reward_std": 0.02567419968545437, + "reward_std_all": 0.025715709663927555, + "rewards-max/forensics_sinkhorn_reward": 0.9857580661773682, + "rewards-max/forensics_strict_edge_reward": 0.6134152412414551, + "rewards-max/forensics_tiou_reward": 0.9911632239818573, + "rewards/forensics_sinkhorn_reward": 0.7445399165153503, + "rewards/forensics_strict_edge_reward": 0.06341077759861946, + "rewards/forensics_tiou_reward": 0.5895500183105469, + "step": 850 + }, + { + "completion_length": 22.015625, + "epoch": 2.61136712749616, + "grad_norm": 7.47346830368042, + "kl": 0.314453125, + "learning_rate": 3.461538461538461e-07, + "loss": -0.0489, + "reward": 0.7927356958389282, + "reward_std": 0.020939966663718224, + "reward_std_all": 0.020790446549654007, + "rewards-max/forensics_sinkhorn_reward": 0.9705038070678711, + "rewards-max/forensics_strict_edge_reward": 0.3375737890601158, + "rewards-max/forensics_tiou_reward": 0.9467691779136658, + "rewards/forensics_sinkhorn_reward": 0.7462933659553528, + "rewards/forensics_strict_edge_reward": 0.020996264182031155, + "rewards/forensics_tiou_reward": 0.5955460071563721, + "step": 851 + }, + { + "completion_length": 27.65625, + "epoch": 2.6144393241167436, + "grad_norm": 5.847975254058838, + "kl": 0.3388671875, + "learning_rate": 3.4538461538461537e-07, + "loss": 0.0306, + "reward": 0.777643620967865, + "reward_std": 0.04291294701397419, + "reward_std_all": 0.040006003342568874, + "rewards-max/forensics_sinkhorn_reward": 0.9742127656936646, + "rewards-max/forensics_strict_edge_reward": 0.32063715159893036, + "rewards-max/forensics_tiou_reward": 0.9612336158752441, + "rewards/forensics_sinkhorn_reward": 0.6996363997459412, + "rewards/forensics_strict_edge_reward": 0.03289967216551304, + "rewards/forensics_tiou_reward": 0.5461217761039734, + "step": 852 + }, + { + "completion_length": 24.984375, + "epoch": 2.6175115207373274, + "grad_norm": 6.970930099487305, + "kl": 0.40625, + "learning_rate": 3.446153846153846e-07, + "loss": -0.0008, + "reward": 0.7685919404029846, + "reward_std": 0.04816648177802563, + "reward_std_all": 0.04579739645123482, + "rewards-max/forensics_sinkhorn_reward": 0.9832101464271545, + "rewards-max/forensics_strict_edge_reward": 0.32200682908296585, + "rewards-max/forensics_tiou_reward": 0.9835412800312042, + "rewards/forensics_sinkhorn_reward": 0.7085474133491516, + "rewards/forensics_strict_edge_reward": 0.026620659744367003, + "rewards/forensics_tiou_reward": 0.5491476356983185, + "step": 853 + }, + { + "completion_length": 26.59375, + "epoch": 2.6205837173579107, + "grad_norm": 5.972087383270264, + "kl": 0.482421875, + "learning_rate": 3.4384615384615385e-07, + "loss": -0.0894, + "reward": 0.7768661379814148, + "reward_std": 0.013904381077736616, + "reward_std_all": 0.022301192861050367, + "rewards-max/forensics_sinkhorn_reward": 0.9756748676300049, + "rewards-max/forensics_strict_edge_reward": 0.4365982860326767, + "rewards-max/forensics_tiou_reward": 0.966264545917511, + "rewards/forensics_sinkhorn_reward": 0.7867585122585297, + "rewards/forensics_strict_edge_reward": 0.07007031887769699, + "rewards/forensics_tiou_reward": 0.6122787594795227, + "step": 854 + }, + { + "completion_length": 23.234375, + "epoch": 2.6236559139784945, + "grad_norm": 7.069056034088135, + "kl": 0.49609375, + "learning_rate": 3.4307692307692307e-07, + "loss": -0.0636, + "reward": 0.7879879772663116, + "reward_std": 0.023755179718136787, + "reward_std_all": 0.023619075305759907, + "rewards-max/forensics_sinkhorn_reward": 0.9821069240570068, + "rewards-max/forensics_strict_edge_reward": 0.8228278160095215, + "rewards-max/forensics_tiou_reward": 0.9834983050823212, + "rewards/forensics_sinkhorn_reward": 0.7814497351646423, + "rewards/forensics_strict_edge_reward": 0.07409258000552654, + "rewards/forensics_tiou_reward": 0.6060217022895813, + "step": 855 + }, + { + "completion_length": 25.828125, + "epoch": 2.6267281105990783, + "grad_norm": 9.332493782043457, + "kl": 0.390625, + "learning_rate": 3.423076923076923e-07, + "loss": -0.0248, + "reward": 0.797133058309555, + "reward_std": 0.01269736047834158, + "reward_std_all": 0.013789710588753223, + "rewards-max/forensics_sinkhorn_reward": 0.9511360824108124, + "rewards-max/forensics_strict_edge_reward": 0.36305589973926544, + "rewards-max/forensics_tiou_reward": 0.9115357398986816, + "rewards/forensics_sinkhorn_reward": 0.735031247138977, + "rewards/forensics_strict_edge_reward": 0.030704746954143047, + "rewards/forensics_tiou_reward": 0.5260164439678192, + "step": 856 + }, + { + "completion_length": 24.265625, + "epoch": 2.629800307219662, + "grad_norm": 8.772136688232422, + "kl": 0.470703125, + "learning_rate": 3.415384615384615e-07, + "loss": 0.0036, + "reward": 0.7742701470851898, + "reward_std": 0.04169499687850475, + "reward_std_all": 0.03979635052382946, + "rewards-max/forensics_sinkhorn_reward": 0.9769640266895294, + "rewards-max/forensics_strict_edge_reward": 0.5221183001995087, + "rewards-max/forensics_tiou_reward": 0.965259462594986, + "rewards/forensics_sinkhorn_reward": 0.760795384645462, + "rewards/forensics_strict_edge_reward": 0.056399717926979065, + "rewards/forensics_tiou_reward": 0.5965477228164673, + "step": 857 + }, + { + "completion_length": 27.46875, + "epoch": 2.632872503840246, + "grad_norm": 6.7635064125061035, + "kl": 0.38671875, + "learning_rate": 3.4076923076923076e-07, + "loss": 0.0006, + "reward": 0.7283392548561096, + "reward_std": 0.018189790192991495, + "reward_std_all": 0.04998227162286639, + "rewards-max/forensics_sinkhorn_reward": 0.9758968353271484, + "rewards-max/forensics_strict_edge_reward": 0.6352929472923279, + "rewards-max/forensics_tiou_reward": 0.9679467082023621, + "rewards/forensics_sinkhorn_reward": 0.6627627611160278, + "rewards/forensics_strict_edge_reward": 0.04841746389865875, + "rewards/forensics_tiou_reward": 0.49025946855545044, + "step": 858 + }, + { + "completion_length": 24.125, + "epoch": 2.6359447004608296, + "grad_norm": 11.085958480834961, + "kl": 0.59375, + "learning_rate": 3.4000000000000003e-07, + "loss": 0.0194, + "reward": 0.7730315625667572, + "reward_std": 0.018297167494893074, + "reward_std_all": 0.02835651207715273, + "rewards-max/forensics_sinkhorn_reward": 0.9567814469337463, + "rewards-max/forensics_strict_edge_reward": 0.3998229056596756, + "rewards-max/forensics_tiou_reward": 0.9263356626033783, + "rewards/forensics_sinkhorn_reward": 0.7347574532032013, + "rewards/forensics_strict_edge_reward": 0.061435832642018795, + "rewards/forensics_tiou_reward": 0.5620869398117065, + "step": 859 + }, + { + "completion_length": 26.90625, + "epoch": 2.6390168970814134, + "grad_norm": 5.611695766448975, + "kl": 0.4296875, + "learning_rate": 3.392307692307692e-07, + "loss": -0.1008, + "reward": 0.7564755082130432, + "reward_std": 0.02157843456370756, + "reward_std_all": 0.03311857534572482, + "rewards-max/forensics_sinkhorn_reward": 0.970152884721756, + "rewards-max/forensics_strict_edge_reward": 0.4481906443834305, + "rewards-max/forensics_tiou_reward": 0.9417329132556915, + "rewards/forensics_sinkhorn_reward": 0.7339467406272888, + "rewards/forensics_strict_edge_reward": 0.051006877794861794, + "rewards/forensics_tiou_reward": 0.6035532057285309, + "step": 860 + }, + { + "completion_length": 24.421875, + "epoch": 2.6420890937019967, + "grad_norm": 6.860748291015625, + "kl": 0.3193359375, + "learning_rate": 3.3846153846153845e-07, + "loss": -0.0156, + "reward": 0.7971347272396088, + "reward_std": 0.016120933811180294, + "reward_std_all": 0.015827556140720844, + "rewards-max/forensics_sinkhorn_reward": 0.984807163476944, + "rewards-max/forensics_strict_edge_reward": 0.39882805943489075, + "rewards-max/forensics_tiou_reward": 0.9902848601341248, + "rewards/forensics_sinkhorn_reward": 0.8113097250461578, + "rewards/forensics_strict_edge_reward": 0.05808861181139946, + "rewards/forensics_tiou_reward": 0.674191802740097, + "step": 861 + }, + { + "completion_length": 23.703125, + "epoch": 2.6451612903225805, + "grad_norm": 7.017099857330322, + "kl": 0.3828125, + "learning_rate": 3.3769230769230767e-07, + "loss": -0.0331, + "reward": 0.7572720050811768, + "reward_std": 0.048353323712944984, + "reward_std_all": 0.049741292372345924, + "rewards-max/forensics_sinkhorn_reward": 0.981103390455246, + "rewards-max/forensics_strict_edge_reward": 0.5836419016122818, + "rewards-max/forensics_tiou_reward": 0.980021744966507, + "rewards/forensics_sinkhorn_reward": 0.7316840291023254, + "rewards/forensics_strict_edge_reward": 0.07636168040335178, + "rewards/forensics_tiou_reward": 0.5601291060447693, + "step": 862 + }, + { + "completion_length": 23.75, + "epoch": 2.6482334869431643, + "grad_norm": 8.615148544311523, + "kl": 0.544921875, + "learning_rate": 3.3692307692307693e-07, + "loss": 0.036, + "reward": 0.7788849174976349, + "reward_std": 0.02406853681895882, + "reward_std_all": 0.026208346243947744, + "rewards-max/forensics_sinkhorn_reward": 0.9746116399765015, + "rewards-max/forensics_strict_edge_reward": 0.3288491666316986, + "rewards-max/forensics_tiou_reward": 0.9632731676101685, + "rewards/forensics_sinkhorn_reward": 0.7630689740180969, + "rewards/forensics_strict_edge_reward": 0.045157287269830704, + "rewards/forensics_tiou_reward": 0.6362279951572418, + "step": 863 + }, + { + "completion_length": 26.453125, + "epoch": 2.651305683563748, + "grad_norm": 6.095434188842773, + "kl": 0.44140625, + "learning_rate": 3.361538461538461e-07, + "loss": -0.1052, + "reward": 0.805259644985199, + "reward_std": 0.0018422940047457814, + "reward_std_all": 0.0036872305208817124, + "rewards-max/forensics_sinkhorn_reward": 0.9830528199672699, + "rewards-max/forensics_strict_edge_reward": 0.6566827446222305, + "rewards-max/forensics_tiou_reward": 0.9852166473865509, + "rewards/forensics_sinkhorn_reward": 0.8098962903022766, + "rewards/forensics_strict_edge_reward": 0.08767824620008469, + "rewards/forensics_tiou_reward": 0.6649159491062164, + "step": 864 + }, + { + "completion_length": 23.671875, + "epoch": 2.654377880184332, + "grad_norm": 9.789863586425781, + "kl": 0.4482421875, + "learning_rate": 3.3538461538461536e-07, + "loss": -0.0227, + "reward": 0.7677407264709473, + "reward_std": 0.03627347759902477, + "reward_std_all": 0.03828069567680359, + "rewards-max/forensics_sinkhorn_reward": 0.9769181609153748, + "rewards-max/forensics_strict_edge_reward": 0.20735950767993927, + "rewards-max/forensics_tiou_reward": 0.963148832321167, + "rewards/forensics_sinkhorn_reward": 0.7468448579311371, + "rewards/forensics_strict_edge_reward": 0.012425552122294903, + "rewards/forensics_tiou_reward": 0.5382567346096039, + "step": 865 + }, + { + "completion_length": 26.359375, + "epoch": 2.6574500768049156, + "grad_norm": 5.032227039337158, + "kl": 0.375, + "learning_rate": 3.346153846153846e-07, + "loss": -0.049, + "reward": 0.7979452013969421, + "reward_std": 0.008132825721986592, + "reward_std_all": 0.010019640321843326, + "rewards-max/forensics_sinkhorn_reward": 0.9426282346248627, + "rewards-max/forensics_strict_edge_reward": 0.27318618819117546, + "rewards-max/forensics_tiou_reward": 0.8956197500228882, + "rewards/forensics_sinkhorn_reward": 0.7965320646762848, + "rewards/forensics_strict_edge_reward": 0.05337767582386732, + "rewards/forensics_tiou_reward": 0.6738943457603455, + "step": 866 + }, + { + "completion_length": 24.9375, + "epoch": 2.6605222734254994, + "grad_norm": 6.859870910644531, + "kl": 0.40625, + "learning_rate": 3.3384615384615384e-07, + "loss": -0.0055, + "reward": 0.7444214224815369, + "reward_std": 0.02981998398900032, + "reward_std_all": 0.04654546082019806, + "rewards-max/forensics_sinkhorn_reward": 0.9767985641956329, + "rewards-max/forensics_strict_edge_reward": 0.5524591207504272, + "rewards-max/forensics_tiou_reward": 0.967007040977478, + "rewards/forensics_sinkhorn_reward": 0.7024702727794647, + "rewards/forensics_strict_edge_reward": 0.049588607624173164, + "rewards/forensics_tiou_reward": 0.4987686276435852, + "step": 867 + }, + { + "completion_length": 26.84375, + "epoch": 2.6635944700460827, + "grad_norm": 8.161059379577637, + "kl": 0.3134765625, + "learning_rate": 3.330769230769231e-07, + "loss": -0.0541, + "reward": 0.7856091260910034, + "reward_std": 0.019366029649972916, + "reward_std_all": 0.02140975371003151, + "rewards-max/forensics_sinkhorn_reward": 0.9791068434715271, + "rewards-max/forensics_strict_edge_reward": 0.5289513915777206, + "rewards-max/forensics_tiou_reward": 0.97907954454422, + "rewards/forensics_sinkhorn_reward": 0.7747041881084442, + "rewards/forensics_strict_edge_reward": 0.0515125235542655, + "rewards/forensics_tiou_reward": 0.5942785143852234, + "step": 868 + }, + { + "completion_length": 24.84375, + "epoch": 2.6666666666666665, + "grad_norm": 6.624024391174316, + "kl": 0.54296875, + "learning_rate": 3.3230769230769227e-07, + "loss": -0.012, + "reward": 0.7749400436878204, + "reward_std": 0.028041524230502546, + "reward_std_all": 0.029922109097242355, + "rewards-max/forensics_sinkhorn_reward": 0.9795363247394562, + "rewards-max/forensics_strict_edge_reward": 0.5422053933143616, + "rewards-max/forensics_tiou_reward": 0.9739231765270233, + "rewards/forensics_sinkhorn_reward": 0.8010041117668152, + "rewards/forensics_strict_edge_reward": 0.05411214008927345, + "rewards/forensics_tiou_reward": 0.6312893033027649, + "step": 869 + }, + { + "completion_length": 24.25, + "epoch": 2.6697388632872503, + "grad_norm": 8.459493637084961, + "kl": 0.625, + "learning_rate": 3.3153846153846154e-07, + "loss": -0.0216, + "reward": 0.8000553250312805, + "reward_std": 0.007066772552207112, + "reward_std_all": 0.00865980121307075, + "rewards-max/forensics_sinkhorn_reward": 0.9763144552707672, + "rewards-max/forensics_strict_edge_reward": 0.6703200340270996, + "rewards-max/forensics_tiou_reward": 0.9680817425251007, + "rewards/forensics_sinkhorn_reward": 0.7893724143505096, + "rewards/forensics_strict_edge_reward": 0.05924396030604839, + "rewards/forensics_tiou_reward": 0.6372163593769073, + "step": 870 + }, + { + "completion_length": 27.3125, + "epoch": 2.672811059907834, + "grad_norm": 7.290034770965576, + "kl": 0.4873046875, + "learning_rate": 3.3076923076923075e-07, + "loss": 0.1819, + "reward": 0.7160261869430542, + "reward_std": 0.06177986413240433, + "reward_std_all": 0.07439340464770794, + "rewards-max/forensics_sinkhorn_reward": 0.9798939228057861, + "rewards-max/forensics_strict_edge_reward": 0.49058565497398376, + "rewards-max/forensics_tiou_reward": 0.9725542068481445, + "rewards/forensics_sinkhorn_reward": 0.6367878913879395, + "rewards/forensics_strict_edge_reward": 0.057540347799658775, + "rewards/forensics_tiou_reward": 0.49423477053642273, + "step": 871 + }, + { + "completion_length": 21.96875, + "epoch": 2.675883256528418, + "grad_norm": 7.2137675285339355, + "kl": 0.3896484375, + "learning_rate": 3.3e-07, + "loss": -0.0, + "reward": 0.7325122058391571, + "reward_std": 0.04528358019888401, + "reward_std_all": 0.05867660790681839, + "rewards-max/forensics_sinkhorn_reward": 0.950052946805954, + "rewards-max/forensics_strict_edge_reward": 0.509157819673419, + "rewards-max/forensics_tiou_reward": 0.9154482781887054, + "rewards/forensics_sinkhorn_reward": 0.6783251166343689, + "rewards/forensics_strict_edge_reward": 0.06767853832570836, + "rewards/forensics_tiou_reward": 0.5003963857889175, + "step": 872 + }, + { + "completion_length": 24.328125, + "epoch": 2.6789554531490016, + "grad_norm": 5.901169776916504, + "kl": 0.3505859375, + "learning_rate": 3.292307692307692e-07, + "loss": -0.0252, + "reward": 0.794581264257431, + "reward_std": 0.02027737209573388, + "reward_std_all": 0.019615577068179846, + "rewards-max/forensics_sinkhorn_reward": 0.9836860001087189, + "rewards-max/forensics_strict_edge_reward": 0.513702392578125, + "rewards-max/forensics_tiou_reward": 0.987257570028305, + "rewards/forensics_sinkhorn_reward": 0.781656801700592, + "rewards/forensics_strict_edge_reward": 0.07634145766496658, + "rewards/forensics_tiou_reward": 0.6791847944259644, + "step": 873 + }, + { + "completion_length": 24.875, + "epoch": 2.6820276497695854, + "grad_norm": 7.598845958709717, + "kl": 0.4326171875, + "learning_rate": 3.2846153846153844e-07, + "loss": 0.0376, + "reward": 0.7582902014255524, + "reward_std": 0.05922648590058088, + "reward_std_all": 0.055987953674048185, + "rewards-max/forensics_sinkhorn_reward": 0.9820885062217712, + "rewards-max/forensics_strict_edge_reward": 0.595625251531601, + "rewards-max/forensics_tiou_reward": 0.9822944104671478, + "rewards/forensics_sinkhorn_reward": 0.7166301608085632, + "rewards/forensics_strict_edge_reward": 0.05798969976603985, + "rewards/forensics_tiou_reward": 0.5824785530567169, + "step": 874 + }, + { + "completion_length": 26.21875, + "epoch": 2.6850998463901687, + "grad_norm": 5.886633396148682, + "kl": 0.37890625, + "learning_rate": 3.2769230769230766e-07, + "loss": 0.0452, + "reward": 0.7843543291091919, + "reward_std": 0.033154924400150776, + "reward_std_all": 0.031211266294121742, + "rewards-max/forensics_sinkhorn_reward": 0.9773791432380676, + "rewards-max/forensics_strict_edge_reward": 0.4523152709007263, + "rewards-max/forensics_tiou_reward": 0.970008134841919, + "rewards/forensics_sinkhorn_reward": 0.747808188199997, + "rewards/forensics_strict_edge_reward": 0.04834204353392124, + "rewards/forensics_tiou_reward": 0.6020872890949249, + "step": 875 + }, + { + "completion_length": 28.46875, + "epoch": 2.688172043010753, + "grad_norm": 5.848746299743652, + "kl": 0.4130859375, + "learning_rate": 3.269230769230769e-07, + "loss": 0.1013, + "reward": 0.7345671653747559, + "reward_std": 0.07659811526536942, + "reward_std_all": 0.07770013809204102, + "rewards-max/forensics_sinkhorn_reward": 0.986422210931778, + "rewards-max/forensics_strict_edge_reward": 0.4157680794596672, + "rewards-max/forensics_tiou_reward": 0.9908285439014435, + "rewards/forensics_sinkhorn_reward": 0.6860559284687042, + "rewards/forensics_strict_edge_reward": 0.022066884441301227, + "rewards/forensics_tiou_reward": 0.5334259271621704, + "step": 876 + }, + { + "completion_length": 23.015625, + "epoch": 2.6912442396313363, + "grad_norm": 7.381026744842529, + "kl": 0.48828125, + "learning_rate": 3.2615384615384614e-07, + "loss": 0.0017, + "reward": 0.7893298268318176, + "reward_std": 0.02033443097025156, + "reward_std_all": 0.021090778522193432, + "rewards-max/forensics_sinkhorn_reward": 0.982476681470871, + "rewards-max/forensics_strict_edge_reward": 0.5972513854503632, + "rewards-max/forensics_tiou_reward": 0.9795298278331757, + "rewards/forensics_sinkhorn_reward": 0.7539986968040466, + "rewards/forensics_strict_edge_reward": 0.06286872364580631, + "rewards/forensics_tiou_reward": 0.6159608960151672, + "step": 877 + }, + { + "completion_length": 21.078125, + "epoch": 2.69431643625192, + "grad_norm": 6.701380252838135, + "kl": 0.3994140625, + "learning_rate": 3.2538461538461535e-07, + "loss": -0.0619, + "reward": 0.7962626218795776, + "reward_std": 0.015952790388837457, + "reward_std_all": 0.016168768517673016, + "rewards-max/forensics_sinkhorn_reward": 0.9761609137058258, + "rewards-max/forensics_strict_edge_reward": 0.44892527163028717, + "rewards-max/forensics_tiou_reward": 0.9650090634822845, + "rewards/forensics_sinkhorn_reward": 0.7873976826667786, + "rewards/forensics_strict_edge_reward": 0.03595034405589104, + "rewards/forensics_tiou_reward": 0.6214311122894287, + "step": 878 + }, + { + "completion_length": 24.34375, + "epoch": 2.697388632872504, + "grad_norm": 6.0258307456970215, + "kl": 0.376953125, + "learning_rate": 3.246153846153846e-07, + "loss": 0.1115, + "reward": 0.7890635132789612, + "reward_std": 0.029372187331318855, + "reward_std_all": 0.027310652658343315, + "rewards-max/forensics_sinkhorn_reward": 0.9772146046161652, + "rewards-max/forensics_strict_edge_reward": 0.5898899734020233, + "rewards-max/forensics_tiou_reward": 0.9677414000034332, + "rewards/forensics_sinkhorn_reward": 0.7771452069282532, + "rewards/forensics_strict_edge_reward": 0.05422712862491608, + "rewards/forensics_tiou_reward": 0.5965283513069153, + "step": 879 + }, + { + "completion_length": 24.34375, + "epoch": 2.7004608294930876, + "grad_norm": 7.462799549102783, + "kl": 0.3994140625, + "learning_rate": 3.2384615384615383e-07, + "loss": -0.0165, + "reward": 0.7723336815834045, + "reward_std": 0.0309678353369236, + "reward_std_all": 0.03366899862885475, + "rewards-max/forensics_sinkhorn_reward": 0.9760593175888062, + "rewards-max/forensics_strict_edge_reward": 0.4986550956964493, + "rewards-max/forensics_tiou_reward": 0.9667498767375946, + "rewards/forensics_sinkhorn_reward": 0.7565020322799683, + "rewards/forensics_strict_edge_reward": 0.050186218693852425, + "rewards/forensics_tiou_reward": 0.5687236189842224, + "step": 880 + }, + { + "completion_length": 22.9375, + "epoch": 2.7035330261136714, + "grad_norm": 5.686988353729248, + "kl": 0.361328125, + "learning_rate": 3.230769230769231e-07, + "loss": -0.0506, + "reward": 0.7957293391227722, + "reward_std": 0.013710550498217344, + "reward_std_all": 0.014659441076219082, + "rewards-max/forensics_sinkhorn_reward": 0.9525602757930756, + "rewards-max/forensics_strict_edge_reward": 0.4558599591255188, + "rewards-max/forensics_tiou_reward": 0.914629727602005, + "rewards/forensics_sinkhorn_reward": 0.8015460073947906, + "rewards/forensics_strict_edge_reward": 0.04290045704692602, + "rewards/forensics_tiou_reward": 0.5976487696170807, + "step": 881 + }, + { + "completion_length": 26.71875, + "epoch": 2.7066052227342547, + "grad_norm": 5.665462970733643, + "kl": 0.384765625, + "learning_rate": 3.2230769230769226e-07, + "loss": 0.0719, + "reward": 0.7357989549636841, + "reward_std": 0.06865421310067177, + "reward_std_all": 0.07114376127719879, + "rewards-max/forensics_sinkhorn_reward": 0.9771660268306732, + "rewards-max/forensics_strict_edge_reward": 0.6419725120067596, + "rewards-max/forensics_tiou_reward": 0.971006453037262, + "rewards/forensics_sinkhorn_reward": 0.7475114464759827, + "rewards/forensics_strict_edge_reward": 0.08434883505105972, + "rewards/forensics_tiou_reward": 0.6144094467163086, + "step": 882 + }, + { + "completion_length": 26.21875, + "epoch": 2.709677419354839, + "grad_norm": 6.3659868240356445, + "kl": 0.5546875, + "learning_rate": 3.215384615384615e-07, + "loss": 0.0454, + "reward": 0.7741868495941162, + "reward_std": 0.03274735528975725, + "reward_std_all": 0.03340739291161299, + "rewards-max/forensics_sinkhorn_reward": 0.9823004603385925, + "rewards-max/forensics_strict_edge_reward": 0.7032277882099152, + "rewards-max/forensics_tiou_reward": 0.9858101904392242, + "rewards/forensics_sinkhorn_reward": 0.7661237418651581, + "rewards/forensics_strict_edge_reward": 0.0707860067486763, + "rewards/forensics_tiou_reward": 0.6395784318447113, + "step": 883 + }, + { + "completion_length": 27.875, + "epoch": 2.7127496159754223, + "grad_norm": 6.493981838226318, + "kl": 0.4697265625, + "learning_rate": 3.2076923076923074e-07, + "loss": -0.0888, + "reward": 0.7727367877960205, + "reward_std": 0.006670567439869046, + "reward_std_all": 0.020316811627708375, + "rewards-max/forensics_sinkhorn_reward": 0.9696038961410522, + "rewards-max/forensics_strict_edge_reward": 0.5659221410751343, + "rewards-max/forensics_tiou_reward": 0.9530363082885742, + "rewards/forensics_sinkhorn_reward": 0.7711009085178375, + "rewards/forensics_strict_edge_reward": 0.06659053638577461, + "rewards/forensics_tiou_reward": 0.6040042638778687, + "step": 884 + }, + { + "completion_length": 28.46875, + "epoch": 2.715821812596006, + "grad_norm": 6.521970272064209, + "kl": 0.470703125, + "learning_rate": 3.2e-07, + "loss": 0.1108, + "reward": 0.7538996040821075, + "reward_std": 0.06645601242780685, + "reward_std_all": 0.062237344682216644, + "rewards-max/forensics_sinkhorn_reward": 0.9857624769210815, + "rewards-max/forensics_strict_edge_reward": 0.6889291703701019, + "rewards-max/forensics_tiou_reward": 0.9886612892150879, + "rewards/forensics_sinkhorn_reward": 0.692498117685318, + "rewards/forensics_strict_edge_reward": 0.07028679549694061, + "rewards/forensics_tiou_reward": 0.5386016666889191, + "step": 885 + }, + { + "completion_length": 24.703125, + "epoch": 2.71889400921659, + "grad_norm": 7.014496326446533, + "kl": 0.3974609375, + "learning_rate": 3.1923076923076917e-07, + "loss": 0.048, + "reward": 0.7693270146846771, + "reward_std": 0.04744795709848404, + "reward_std_all": 0.04511306807398796, + "rewards-max/forensics_sinkhorn_reward": 0.9734239876270294, + "rewards-max/forensics_strict_edge_reward": 0.4686945676803589, + "rewards-max/forensics_tiou_reward": 0.9582484364509583, + "rewards/forensics_sinkhorn_reward": 0.7266882359981537, + "rewards/forensics_strict_edge_reward": 0.02665764931589365, + "rewards/forensics_tiou_reward": 0.5179785490036011, + "step": 886 + }, + { + "completion_length": 23.578125, + "epoch": 2.7219662058371736, + "grad_norm": 5.783538341522217, + "kl": 0.3818359375, + "learning_rate": 3.1846153846153843e-07, + "loss": -0.0571, + "reward": 0.784652054309845, + "reward_std": 0.026985248550772667, + "reward_std_all": 0.02674742043018341, + "rewards-max/forensics_sinkhorn_reward": 0.9671872854232788, + "rewards-max/forensics_strict_edge_reward": 0.28160209581255913, + "rewards-max/forensics_tiou_reward": 0.9475110173225403, + "rewards/forensics_sinkhorn_reward": 0.7488041520118713, + "rewards/forensics_strict_edge_reward": 0.028653954854235053, + "rewards/forensics_tiou_reward": 0.552339643239975, + "step": 887 + }, + { + "completion_length": 22.40625, + "epoch": 2.7250384024577574, + "grad_norm": 5.2812819480896, + "kl": 0.390625, + "learning_rate": 3.176923076923077e-07, + "loss": -0.0287, + "reward": 0.7605755925178528, + "reward_std": 0.020971191814169288, + "reward_std_all": 0.03473680093884468, + "rewards-max/forensics_sinkhorn_reward": 0.9701745808124542, + "rewards-max/forensics_strict_edge_reward": 0.49436186254024506, + "rewards-max/forensics_tiou_reward": 0.9524593949317932, + "rewards/forensics_sinkhorn_reward": 0.753139317035675, + "rewards/forensics_strict_edge_reward": 0.05115099623799324, + "rewards/forensics_tiou_reward": 0.5878059566020966, + "step": 888 + }, + { + "completion_length": 24.46875, + "epoch": 2.7281105990783407, + "grad_norm": 6.110366344451904, + "kl": 0.3486328125, + "learning_rate": 3.169230769230769e-07, + "loss": 0.1425, + "reward": 0.7845446169376373, + "reward_std": 0.025197434006258845, + "reward_std_all": 0.0260664289817214, + "rewards-max/forensics_sinkhorn_reward": 0.9771140217781067, + "rewards-max/forensics_strict_edge_reward": 0.5521003231406212, + "rewards-max/forensics_tiou_reward": 0.9701131582260132, + "rewards/forensics_sinkhorn_reward": 0.7317254841327667, + "rewards/forensics_strict_edge_reward": 0.05003669252619147, + "rewards/forensics_tiou_reward": 0.5316431820392609, + "step": 889 + }, + { + "completion_length": 23.265625, + "epoch": 2.731182795698925, + "grad_norm": 7.655444145202637, + "kl": 0.28564453125, + "learning_rate": 3.161538461538462e-07, + "loss": -0.1075, + "reward": 0.788133829832077, + "reward_std": 0.019806533440714702, + "reward_std_all": 0.021088186651468277, + "rewards-max/forensics_sinkhorn_reward": 0.9640635251998901, + "rewards-max/forensics_strict_edge_reward": 0.30541160702705383, + "rewards-max/forensics_tiou_reward": 0.9411134123802185, + "rewards/forensics_sinkhorn_reward": 0.7604788243770599, + "rewards/forensics_strict_edge_reward": 0.03718572575598955, + "rewards/forensics_tiou_reward": 0.5355812609195709, + "step": 890 + }, + { + "completion_length": 25.90625, + "epoch": 2.7342549923195083, + "grad_norm": 5.370682716369629, + "kl": 0.3876953125, + "learning_rate": 3.1538461538461534e-07, + "loss": -0.0325, + "reward": 0.7662183046340942, + "reward_std": 0.03558311238884926, + "reward_std_all": 0.03813569433987141, + "rewards-max/forensics_sinkhorn_reward": 0.9788040518760681, + "rewards-max/forensics_strict_edge_reward": 0.579896405339241, + "rewards-max/forensics_tiou_reward": 0.9729216396808624, + "rewards/forensics_sinkhorn_reward": 0.7865541279315948, + "rewards/forensics_strict_edge_reward": 0.09010985819622874, + "rewards/forensics_tiou_reward": 0.6314694285392761, + "step": 891 + }, + { + "completion_length": 23.859375, + "epoch": 2.737327188940092, + "grad_norm": 6.765510082244873, + "kl": 0.3916015625, + "learning_rate": 3.146153846153846e-07, + "loss": 0.0222, + "reward": 0.7832672297954559, + "reward_std": 0.014666533097624779, + "reward_std_all": 0.021480157040059566, + "rewards-max/forensics_sinkhorn_reward": 0.980329692363739, + "rewards-max/forensics_strict_edge_reward": 0.525025486946106, + "rewards-max/forensics_tiou_reward": 0.9753759801387787, + "rewards/forensics_sinkhorn_reward": 0.7846992909908295, + "rewards/forensics_strict_edge_reward": 0.04758155718445778, + "rewards/forensics_tiou_reward": 0.5972308814525604, + "step": 892 + }, + { + "completion_length": 21.453125, + "epoch": 2.740399385560676, + "grad_norm": 7.889683723449707, + "kl": 0.4091796875, + "learning_rate": 3.138461538461538e-07, + "loss": 0.1234, + "reward": 0.7773095369338989, + "reward_std": 0.051542021334171295, + "reward_std_all": 0.04592550918459892, + "rewards-max/forensics_sinkhorn_reward": 0.985921174287796, + "rewards-max/forensics_strict_edge_reward": 0.8106843829154968, + "rewards-max/forensics_tiou_reward": 0.9903466403484344, + "rewards/forensics_sinkhorn_reward": 0.7887414395809174, + "rewards/forensics_strict_edge_reward": 0.11903358995914459, + "rewards/forensics_tiou_reward": 0.6273476183414459, + "step": 893 + }, + { + "completion_length": 26.59375, + "epoch": 2.7434715821812596, + "grad_norm": 5.565340995788574, + "kl": 0.5263671875, + "learning_rate": 3.130769230769231e-07, + "loss": 0.0416, + "reward": 0.7700815498828888, + "reward_std": 0.02169164828956127, + "reward_std_all": 0.030142979230731726, + "rewards-max/forensics_sinkhorn_reward": 0.9867188632488251, + "rewards-max/forensics_strict_edge_reward": 0.863698273897171, + "rewards-max/forensics_tiou_reward": 0.9944442808628082, + "rewards/forensics_sinkhorn_reward": 0.7276377081871033, + "rewards/forensics_strict_edge_reward": 0.08192289248108864, + "rewards/forensics_tiou_reward": 0.5820942968130112, + "step": 894 + }, + { + "completion_length": 26.234375, + "epoch": 2.7465437788018434, + "grad_norm": 5.905716896057129, + "kl": 0.4736328125, + "learning_rate": 3.1230769230769225e-07, + "loss": 0.071, + "reward": 0.7745249569416046, + "reward_std": 0.035963188856840134, + "reward_std_all": 0.03563530999235809, + "rewards-max/forensics_sinkhorn_reward": 0.9812942147254944, + "rewards-max/forensics_strict_edge_reward": 0.6629511117935181, + "rewards-max/forensics_tiou_reward": 0.9807871282100677, + "rewards/forensics_sinkhorn_reward": 0.7345840632915497, + "rewards/forensics_strict_edge_reward": 0.05798712745308876, + "rewards/forensics_tiou_reward": 0.5489002019166946, + "step": 895 + }, + { + "completion_length": 26.171875, + "epoch": 2.749615975422427, + "grad_norm": 5.9007768630981445, + "kl": 0.3994140625, + "learning_rate": 3.115384615384615e-07, + "loss": 0.0351, + "reward": 0.7888968884944916, + "reward_std": 0.03066906053572893, + "reward_std_all": 0.028395328670740128, + "rewards-max/forensics_sinkhorn_reward": 0.9823899865150452, + "rewards-max/forensics_strict_edge_reward": 0.6326714754104614, + "rewards-max/forensics_tiou_reward": 0.9819422960281372, + "rewards/forensics_sinkhorn_reward": 0.7855269312858582, + "rewards/forensics_strict_edge_reward": 0.08873138949275017, + "rewards/forensics_tiou_reward": 0.6598725914955139, + "step": 896 + }, + { + "completion_length": 25.125, + "epoch": 2.752688172043011, + "grad_norm": 6.826524257659912, + "kl": 0.3193359375, + "learning_rate": 3.1076923076923073e-07, + "loss": -0.0444, + "reward": 0.7922731637954712, + "reward_std": 0.01733515551313758, + "reward_std_all": 0.0179355920990929, + "rewards-max/forensics_sinkhorn_reward": 0.9813687801361084, + "rewards-max/forensics_strict_edge_reward": 0.7101751863956451, + "rewards-max/forensics_tiou_reward": 0.9721888601779938, + "rewards/forensics_sinkhorn_reward": 0.8436138331890106, + "rewards/forensics_strict_edge_reward": 0.07307614758610725, + "rewards/forensics_tiou_reward": 0.7083620429039001, + "step": 897 + }, + { + "completion_length": 24.578125, + "epoch": 2.7557603686635943, + "grad_norm": 7.32127046585083, + "kl": 0.52734375, + "learning_rate": 3.1e-07, + "loss": -0.0289, + "reward": 0.7605628669261932, + "reward_std": 0.04746425896883011, + "reward_std_all": 0.04746427200734615, + "rewards-max/forensics_sinkhorn_reward": 0.9828169345855713, + "rewards-max/forensics_strict_edge_reward": 0.6091753393411636, + "rewards-max/forensics_tiou_reward": 0.9832561314105988, + "rewards/forensics_sinkhorn_reward": 0.7437838315963745, + "rewards/forensics_strict_edge_reward": 0.07564761489629745, + "rewards/forensics_tiou_reward": 0.5766122341156006, + "step": 898 + }, + { + "completion_length": 24.515625, + "epoch": 2.758832565284178, + "grad_norm": 7.051201820373535, + "kl": 0.3330078125, + "learning_rate": 3.0923076923076926e-07, + "loss": -0.0391, + "reward": 0.7457412779331207, + "reward_std": 0.02984016586560756, + "reward_std_all": 0.046873068204149604, + "rewards-max/forensics_sinkhorn_reward": 0.9741529226303101, + "rewards-max/forensics_strict_edge_reward": 0.22394196968525648, + "rewards-max/forensics_tiou_reward": 0.9587434232234955, + "rewards/forensics_sinkhorn_reward": 0.6935702562332153, + "rewards/forensics_strict_edge_reward": 0.013815690530464053, + "rewards/forensics_tiou_reward": 0.524777963757515, + "step": 899 + }, + { + "completion_length": 26.140625, + "epoch": 2.761904761904762, + "grad_norm": 6.630404472351074, + "kl": 0.462890625, + "learning_rate": 3.084615384615384e-07, + "loss": 0.0188, + "reward": 0.7795208394527435, + "reward_std": 0.03385939635336399, + "reward_std_all": 0.03282160125672817, + "rewards-max/forensics_sinkhorn_reward": 0.97467240691185, + "rewards-max/forensics_strict_edge_reward": 0.3975096046924591, + "rewards-max/forensics_tiou_reward": 0.962945431470871, + "rewards/forensics_sinkhorn_reward": 0.7594824433326721, + "rewards/forensics_strict_edge_reward": 0.06241784989833832, + "rewards/forensics_tiou_reward": 0.6087248027324677, + "step": 900 + }, + { + "completion_length": 25.640625, + "epoch": 2.7649769585253456, + "grad_norm": 7.562358379364014, + "kl": 0.5205078125, + "learning_rate": 3.076923076923077e-07, + "loss": 0.0635, + "reward": 0.7703008353710175, + "reward_std": 0.036275072023272514, + "reward_std_all": 0.03660730645060539, + "rewards-max/forensics_sinkhorn_reward": 0.9812738299369812, + "rewards-max/forensics_strict_edge_reward": 0.5693852305412292, + "rewards-max/forensics_tiou_reward": 0.9781332612037659, + "rewards/forensics_sinkhorn_reward": 0.7589929103851318, + "rewards/forensics_strict_edge_reward": 0.05185735039412975, + "rewards/forensics_tiou_reward": 0.6070349812507629, + "step": 901 + }, + { + "completion_length": 25.28125, + "epoch": 2.7680491551459294, + "grad_norm": 6.774842739105225, + "kl": 0.3701171875, + "learning_rate": 3.069230769230769e-07, + "loss": 0.0628, + "reward": 0.7659330368041992, + "reward_std": 0.05732523277401924, + "reward_std_all": 0.05309445224702358, + "rewards-max/forensics_sinkhorn_reward": 0.9786677062511444, + "rewards-max/forensics_strict_edge_reward": 0.23136208578944206, + "rewards-max/forensics_tiou_reward": 0.9677570760250092, + "rewards/forensics_sinkhorn_reward": 0.7338471114635468, + "rewards/forensics_strict_edge_reward": 0.025275129359215498, + "rewards/forensics_tiou_reward": 0.5676960349082947, + "step": 902 + }, + { + "completion_length": 27.84375, + "epoch": 2.771121351766513, + "grad_norm": 5.990005016326904, + "kl": 0.51953125, + "learning_rate": 3.0615384615384617e-07, + "loss": -0.0418, + "reward": 0.7875573337078094, + "reward_std": 0.0322537386091426, + "reward_std_all": 0.029921164736151695, + "rewards-max/forensics_sinkhorn_reward": 0.9873166084289551, + "rewards-max/forensics_strict_edge_reward": 0.5874149799346924, + "rewards-max/forensics_tiou_reward": 0.9948931336402893, + "rewards/forensics_sinkhorn_reward": 0.8055607378482819, + "rewards/forensics_strict_edge_reward": 0.05829743668437004, + "rewards/forensics_tiou_reward": 0.6694565415382385, + "step": 903 + }, + { + "completion_length": 26.796875, + "epoch": 2.774193548387097, + "grad_norm": 7.522788047790527, + "kl": 0.33984375, + "learning_rate": 3.0538461538461533e-07, + "loss": 0.0407, + "reward": 0.7867434322834015, + "reward_std": 0.020179421175271273, + "reward_std_all": 0.02129947580397129, + "rewards-max/forensics_sinkhorn_reward": 0.9602608382701874, + "rewards-max/forensics_strict_edge_reward": 0.5853640735149384, + "rewards-max/forensics_tiou_reward": 0.9261642694473267, + "rewards/forensics_sinkhorn_reward": 0.7731817364692688, + "rewards/forensics_strict_edge_reward": 0.057137180119752884, + "rewards/forensics_tiou_reward": 0.5957903563976288, + "step": 904 + }, + { + "completion_length": 25.109375, + "epoch": 2.7772657450076803, + "grad_norm": 8.108012199401855, + "kl": 0.4755859375, + "learning_rate": 3.046153846153846e-07, + "loss": 0.0107, + "reward": 0.7771784663200378, + "reward_std": 0.037675535306334496, + "reward_std_all": 0.03637154493480921, + "rewards-max/forensics_sinkhorn_reward": 0.9664741158485413, + "rewards-max/forensics_strict_edge_reward": 0.4409151151776314, + "rewards-max/forensics_tiou_reward": 0.940037727355957, + "rewards/forensics_sinkhorn_reward": 0.7153068780899048, + "rewards/forensics_strict_edge_reward": 0.04710871633142233, + "rewards/forensics_tiou_reward": 0.5713710784912109, + "step": 905 + }, + { + "completion_length": 24.5, + "epoch": 2.780337941628264, + "grad_norm": 8.801897048950195, + "kl": 0.4404296875, + "learning_rate": 3.038461538461538e-07, + "loss": 0.0435, + "reward": 0.7836224436759949, + "reward_std": 0.03468533232808113, + "reward_std_all": 0.032629622146487236, + "rewards-max/forensics_sinkhorn_reward": 0.9711574912071228, + "rewards-max/forensics_strict_edge_reward": 0.5318287312984467, + "rewards-max/forensics_tiou_reward": 0.9480286836624146, + "rewards/forensics_sinkhorn_reward": 0.7860738635063171, + "rewards/forensics_strict_edge_reward": 0.055933209136128426, + "rewards/forensics_tiou_reward": 0.6276043355464935, + "step": 906 + }, + { + "completion_length": 24.4375, + "epoch": 2.783410138248848, + "grad_norm": 5.951756954193115, + "kl": 0.5078125, + "learning_rate": 3.030769230769231e-07, + "loss": 0.046, + "reward": 0.7818838953971863, + "reward_std": 0.04016185365617275, + "reward_std_all": 0.03700745664536953, + "rewards-max/forensics_sinkhorn_reward": 0.9752790927886963, + "rewards-max/forensics_strict_edge_reward": 0.5571974515914917, + "rewards-max/forensics_tiou_reward": 0.9665398001670837, + "rewards/forensics_sinkhorn_reward": 0.7252273559570312, + "rewards/forensics_strict_edge_reward": 0.04328002408146858, + "rewards/forensics_tiou_reward": 0.5795449018478394, + "step": 907 + }, + { + "completion_length": 25.234375, + "epoch": 2.7864823348694316, + "grad_norm": 6.9809184074401855, + "kl": 0.40625, + "learning_rate": 3.0230769230769235e-07, + "loss": -0.0767, + "reward": 0.7484119534492493, + "reward_std": 0.031628736294806004, + "reward_std_all": 0.04254252091050148, + "rewards-max/forensics_sinkhorn_reward": 0.9800585508346558, + "rewards-max/forensics_strict_edge_reward": 0.37805598974227905, + "rewards-max/forensics_tiou_reward": 0.9706751108169556, + "rewards/forensics_sinkhorn_reward": 0.730874627828598, + "rewards/forensics_strict_edge_reward": 0.028805828653275967, + "rewards/forensics_tiou_reward": 0.5539413690567017, + "step": 908 + }, + { + "completion_length": 26.53125, + "epoch": 2.7895545314900154, + "grad_norm": 6.270783424377441, + "kl": 0.353515625, + "learning_rate": 3.015384615384615e-07, + "loss": -0.018, + "reward": 0.7768244743347168, + "reward_std": 0.017955382936634123, + "reward_std_all": 0.02304707537405193, + "rewards-max/forensics_sinkhorn_reward": 0.9813632965087891, + "rewards-max/forensics_strict_edge_reward": 0.6293076276779175, + "rewards-max/forensics_tiou_reward": 0.9807457327842712, + "rewards/forensics_sinkhorn_reward": 0.7764657139778137, + "rewards/forensics_strict_edge_reward": 0.06674566864967346, + "rewards/forensics_tiou_reward": 0.6398045718669891, + "step": 909 + }, + { + "completion_length": 26.3125, + "epoch": 2.792626728110599, + "grad_norm": 7.26330041885376, + "kl": 0.60546875, + "learning_rate": 3.0076923076923077e-07, + "loss": 0.0339, + "reward": 0.7577600181102753, + "reward_std": 0.037182146683335304, + "reward_std_all": 0.044283706694841385, + "rewards-max/forensics_sinkhorn_reward": 0.981595367193222, + "rewards-max/forensics_strict_edge_reward": 0.4144160747528076, + "rewards-max/forensics_tiou_reward": 0.9801248610019684, + "rewards/forensics_sinkhorn_reward": 0.7512537240982056, + "rewards/forensics_strict_edge_reward": 0.06264987960457802, + "rewards/forensics_tiou_reward": 0.6466365158557892, + "step": 910 + }, + { + "completion_length": 25.875, + "epoch": 2.795698924731183, + "grad_norm": 8.86171817779541, + "kl": 0.451171875, + "learning_rate": 3e-07, + "loss": 0.1575, + "reward": 0.7655938565731049, + "reward_std": 0.04630686063319445, + "reward_std_all": 0.045005598571151495, + "rewards-max/forensics_sinkhorn_reward": 0.9889476001262665, + "rewards-max/forensics_strict_edge_reward": 0.9836080372333527, + "rewards-max/forensics_tiou_reward": 0.9998351633548737, + "rewards/forensics_sinkhorn_reward": 0.7409592568874359, + "rewards/forensics_strict_edge_reward": 0.10101692751049995, + "rewards/forensics_tiou_reward": 0.5944383144378662, + "step": 911 + }, + { + "completion_length": 27.78125, + "epoch": 2.7987711213517663, + "grad_norm": 6.334739685058594, + "kl": 0.55078125, + "learning_rate": 2.9923076923076925e-07, + "loss": 0.0245, + "reward": 0.7219364047050476, + "reward_std": 0.05915191024541855, + "reward_std_all": 0.07042524591088295, + "rewards-max/forensics_sinkhorn_reward": 0.9731059372425079, + "rewards-max/forensics_strict_edge_reward": 0.5049435645341873, + "rewards-max/forensics_tiou_reward": 0.9630025625228882, + "rewards/forensics_sinkhorn_reward": 0.6946338415145874, + "rewards/forensics_strict_edge_reward": 0.0695800930261612, + "rewards/forensics_tiou_reward": 0.5433146357536316, + "step": 912 + }, + { + "completion_length": 27.203125, + "epoch": 2.80184331797235, + "grad_norm": 6.308077812194824, + "kl": 0.37890625, + "learning_rate": 2.984615384615384e-07, + "loss": -0.0802, + "reward": 0.7749734818935394, + "reward_std": 0.03405106533318758, + "reward_std_all": 0.03414603788405657, + "rewards-max/forensics_sinkhorn_reward": 0.9821788966655731, + "rewards-max/forensics_strict_edge_reward": 0.5232763588428497, + "rewards-max/forensics_tiou_reward": 0.9821935594081879, + "rewards/forensics_sinkhorn_reward": 0.7643224596977234, + "rewards/forensics_strict_edge_reward": 0.07075396738946438, + "rewards/forensics_tiou_reward": 0.5827005505561829, + "step": 913 + }, + { + "completion_length": 24.09375, + "epoch": 2.804915514592934, + "grad_norm": 7.831546783447266, + "kl": 0.46875, + "learning_rate": 2.976923076923077e-07, + "loss": -0.0029, + "reward": 0.7764838933944702, + "reward_std": 0.019794617837760597, + "reward_std_all": 0.02810971613507718, + "rewards-max/forensics_sinkhorn_reward": 0.9680038094520569, + "rewards-max/forensics_strict_edge_reward": 0.4900742173194885, + "rewards-max/forensics_tiou_reward": 0.948487401008606, + "rewards/forensics_sinkhorn_reward": 0.7755619287490845, + "rewards/forensics_strict_edge_reward": 0.0476869847625494, + "rewards/forensics_tiou_reward": 0.6172890961170197, + "step": 914 + }, + { + "completion_length": 24.171875, + "epoch": 2.8079877112135176, + "grad_norm": 8.185823440551758, + "kl": 0.37109375, + "learning_rate": 2.969230769230769e-07, + "loss": 0.047, + "reward": 0.7628994286060333, + "reward_std": 0.02680477499961853, + "reward_std_all": 0.03679967671632767, + "rewards-max/forensics_sinkhorn_reward": 0.9579406976699829, + "rewards-max/forensics_strict_edge_reward": 0.349105641245842, + "rewards-max/forensics_tiou_reward": 0.9200129806995392, + "rewards/forensics_sinkhorn_reward": 0.7187440991401672, + "rewards/forensics_strict_edge_reward": 0.037636226043105125, + "rewards/forensics_tiou_reward": 0.5525698363780975, + "step": 915 + }, + { + "completion_length": 25.0, + "epoch": 2.8110599078341014, + "grad_norm": 6.508098125457764, + "kl": 0.4267578125, + "learning_rate": 2.9615384615384616e-07, + "loss": 0.0136, + "reward": 0.7139875292778015, + "reward_std": 0.04749990347772837, + "reward_std_all": 0.07142500765621662, + "rewards-max/forensics_sinkhorn_reward": 0.9782067239284515, + "rewards-max/forensics_strict_edge_reward": 0.4909786581993103, + "rewards-max/forensics_tiou_reward": 0.9714046716690063, + "rewards/forensics_sinkhorn_reward": 0.7043171226978302, + "rewards/forensics_strict_edge_reward": 0.050448402762413025, + "rewards/forensics_tiou_reward": 0.5555044114589691, + "step": 916 + }, + { + "completion_length": 28.171875, + "epoch": 2.814132104454685, + "grad_norm": 8.330702781677246, + "kl": 0.4677734375, + "learning_rate": 2.953846153846154e-07, + "loss": 0.0907, + "reward": 0.7489922046661377, + "reward_std": 0.059777723625302315, + "reward_std_all": 0.059085406363010406, + "rewards-max/forensics_sinkhorn_reward": 0.9821949601173401, + "rewards-max/forensics_strict_edge_reward": 0.5037480741739273, + "rewards-max/forensics_tiou_reward": 0.9813636243343353, + "rewards/forensics_sinkhorn_reward": 0.7357710003852844, + "rewards/forensics_strict_edge_reward": 0.05778411775827408, + "rewards/forensics_tiou_reward": 0.6027874052524567, + "step": 917 + }, + { + "completion_length": 23.171875, + "epoch": 2.817204301075269, + "grad_norm": 8.25635051727295, + "kl": 0.333984375, + "learning_rate": 2.946153846153846e-07, + "loss": 0.0072, + "reward": 0.774203360080719, + "reward_std": 0.0260115135461092, + "reward_std_all": 0.028916033916175365, + "rewards-max/forensics_sinkhorn_reward": 0.9767159521579742, + "rewards-max/forensics_strict_edge_reward": 0.3632843419909477, + "rewards-max/forensics_tiou_reward": 0.9674401581287384, + "rewards/forensics_sinkhorn_reward": 0.7639370560646057, + "rewards/forensics_strict_edge_reward": 0.03915464039891958, + "rewards/forensics_tiou_reward": 0.6057774722576141, + "step": 918 + }, + { + "completion_length": 24.375, + "epoch": 2.8202764976958523, + "grad_norm": 8.118263244628906, + "kl": 0.4287109375, + "learning_rate": 2.9384615384615385e-07, + "loss": 0.0177, + "reward": 0.7929093241691589, + "reward_std": 0.015362377278506756, + "reward_std_all": 0.016325123608112335, + "rewards-max/forensics_sinkhorn_reward": 0.981517493724823, + "rewards-max/forensics_strict_edge_reward": 0.6718102097511292, + "rewards-max/forensics_tiou_reward": 0.9790587723255157, + "rewards/forensics_sinkhorn_reward": 0.7766887545585632, + "rewards/forensics_strict_edge_reward": 0.06019308790564537, + "rewards/forensics_tiou_reward": 0.567735880613327, + "step": 919 + }, + { + "completion_length": 26.421875, + "epoch": 2.823348694316436, + "grad_norm": 8.021784782409668, + "kl": 0.3740234375, + "learning_rate": 2.9307692307692307e-07, + "loss": 0.0058, + "reward": 0.7800170183181763, + "reward_std": 0.0318293534219265, + "reward_std_all": 0.03143216483294964, + "rewards-max/forensics_sinkhorn_reward": 0.9701344966888428, + "rewards-max/forensics_strict_edge_reward": 0.47726282477378845, + "rewards-max/forensics_tiou_reward": 0.9542284309864044, + "rewards/forensics_sinkhorn_reward": 0.7645418047904968, + "rewards/forensics_strict_edge_reward": 0.037740763276815414, + "rewards/forensics_tiou_reward": 0.6210250854492188, + "step": 920 + }, + { + "completion_length": 28.328125, + "epoch": 2.82642089093702, + "grad_norm": 7.024162769317627, + "kl": 0.3916015625, + "learning_rate": 2.9230769230769234e-07, + "loss": 0.0571, + "reward": 0.7656112015247345, + "reward_std": 0.028992805629968643, + "reward_std_all": 0.034197873435914516, + "rewards-max/forensics_sinkhorn_reward": 0.9726874828338623, + "rewards-max/forensics_strict_edge_reward": 0.3299512267112732, + "rewards-max/forensics_tiou_reward": 0.9601567685604095, + "rewards/forensics_sinkhorn_reward": 0.7459766566753387, + "rewards/forensics_strict_edge_reward": 0.035631727427244186, + "rewards/forensics_tiou_reward": 0.6007190644741058, + "step": 921 + }, + { + "completion_length": 24.546875, + "epoch": 2.8294930875576036, + "grad_norm": 6.744950771331787, + "kl": 0.537109375, + "learning_rate": 2.915384615384615e-07, + "loss": 0.0848, + "reward": 0.7535214126110077, + "reward_std": 0.06946786493062973, + "reward_std_all": 0.06466707587242126, + "rewards-max/forensics_sinkhorn_reward": 0.9746828079223633, + "rewards-max/forensics_strict_edge_reward": 0.44672510027885437, + "rewards-max/forensics_tiou_reward": 0.9603354930877686, + "rewards/forensics_sinkhorn_reward": 0.7282577753067017, + "rewards/forensics_strict_edge_reward": 0.04676198400557041, + "rewards/forensics_tiou_reward": 0.5778793692588806, + "step": 922 + }, + { + "completion_length": 25.96875, + "epoch": 2.8325652841781874, + "grad_norm": 6.888558387756348, + "kl": 0.328125, + "learning_rate": 2.9076923076923076e-07, + "loss": -0.0451, + "reward": 0.758033961057663, + "reward_std": 0.059626493602991104, + "reward_std_all": 0.05660632532089949, + "rewards-max/forensics_sinkhorn_reward": 0.9740896224975586, + "rewards-max/forensics_strict_edge_reward": 0.6526533365249634, + "rewards-max/forensics_tiou_reward": 0.9644397497177124, + "rewards/forensics_sinkhorn_reward": 0.7474022209644318, + "rewards/forensics_strict_edge_reward": 0.061662089079618454, + "rewards/forensics_tiou_reward": 0.6306732296943665, + "step": 923 + }, + { + "completion_length": 26.9375, + "epoch": 2.835637480798771, + "grad_norm": 7.365160942077637, + "kl": 0.41015625, + "learning_rate": 2.9e-07, + "loss": -0.0856, + "reward": 0.7842489778995514, + "reward_std": 0.02117240009829402, + "reward_std_all": 0.022663431242108345, + "rewards-max/forensics_sinkhorn_reward": 0.9817533493041992, + "rewards-max/forensics_strict_edge_reward": 0.6271176636219025, + "rewards-max/forensics_tiou_reward": 0.9816955029964447, + "rewards/forensics_sinkhorn_reward": 0.7662067115306854, + "rewards/forensics_strict_edge_reward": 0.053620846942067146, + "rewards/forensics_tiou_reward": 0.6015049815177917, + "step": 924 + }, + { + "completion_length": 22.59375, + "epoch": 2.838709677419355, + "grad_norm": 5.642494201660156, + "kl": 0.5185546875, + "learning_rate": 2.8923076923076924e-07, + "loss": -0.0466, + "reward": 0.7922323346138, + "reward_std": 0.024000099394470453, + "reward_std_all": 0.02284224273171276, + "rewards-max/forensics_sinkhorn_reward": 0.9852249026298523, + "rewards-max/forensics_strict_edge_reward": 0.835382878780365, + "rewards-max/forensics_tiou_reward": 0.9889653027057648, + "rewards/forensics_sinkhorn_reward": 0.769429475069046, + "rewards/forensics_strict_edge_reward": 0.12318657338619232, + "rewards/forensics_tiou_reward": 0.6267462968826294, + "step": 925 + }, + { + "completion_length": 24.703125, + "epoch": 2.8417818740399383, + "grad_norm": 6.6969122886657715, + "kl": 0.412109375, + "learning_rate": 2.884615384615384e-07, + "loss": -0.0095, + "reward": 0.7766340672969818, + "reward_std": 0.04565940424799919, + "reward_std_all": 0.04204892925918102, + "rewards-max/forensics_sinkhorn_reward": 0.9855242371559143, + "rewards-max/forensics_strict_edge_reward": 0.8704091012477875, + "rewards-max/forensics_tiou_reward": 0.9909568727016449, + "rewards/forensics_sinkhorn_reward": 0.7598891854286194, + "rewards/forensics_strict_edge_reward": 0.09891121461987495, + "rewards/forensics_tiou_reward": 0.5777687132358551, + "step": 926 + }, + { + "completion_length": 27.203125, + "epoch": 2.8448540706605225, + "grad_norm": 6.609552383422852, + "kl": 0.451171875, + "learning_rate": 2.8769230769230767e-07, + "loss": 0.0227, + "reward": 0.7908080816268921, + "reward_std": 0.015530155156739056, + "reward_std_all": 0.017845966620370746, + "rewards-max/forensics_sinkhorn_reward": 0.9847975671291351, + "rewards-max/forensics_strict_edge_reward": 0.6245522797107697, + "rewards-max/forensics_tiou_reward": 0.9895066916942596, + "rewards/forensics_sinkhorn_reward": 0.7006420791149139, + "rewards/forensics_strict_edge_reward": 0.05287803336977959, + "rewards/forensics_tiou_reward": 0.6231949925422668, + "step": 927 + }, + { + "completion_length": 23.328125, + "epoch": 2.847926267281106, + "grad_norm": 6.198256492614746, + "kl": 0.455078125, + "learning_rate": 2.8692307692307694e-07, + "loss": -0.0025, + "reward": 0.7966485917568207, + "reward_std": 0.014330969075672328, + "reward_std_all": 0.01502344710752368, + "rewards-max/forensics_sinkhorn_reward": 0.9688233733177185, + "rewards-max/forensics_strict_edge_reward": 0.4429181069135666, + "rewards-max/forensics_tiou_reward": 0.9462155103683472, + "rewards/forensics_sinkhorn_reward": 0.797956794500351, + "rewards/forensics_strict_edge_reward": 0.036617111414670944, + "rewards/forensics_tiou_reward": 0.5864440202713013, + "step": 928 + }, + { + "completion_length": 24.046875, + "epoch": 2.8509984639016897, + "grad_norm": 8.908499717712402, + "kl": 0.3095703125, + "learning_rate": 2.8615384615384615e-07, + "loss": -0.0692, + "reward": 0.7704367637634277, + "reward_std": 0.0010869739344343543, + "reward_std_all": 0.020777163095772266, + "rewards-max/forensics_sinkhorn_reward": 0.969714492559433, + "rewards-max/forensics_strict_edge_reward": 0.5433452129364014, + "rewards-max/forensics_tiou_reward": 0.9519162476062775, + "rewards/forensics_sinkhorn_reward": 0.7968369424343109, + "rewards/forensics_strict_edge_reward": 0.04038672149181366, + "rewards/forensics_tiou_reward": 0.6034599840641022, + "step": 929 + }, + { + "completion_length": 24.1875, + "epoch": 2.8540706605222734, + "grad_norm": 7.081463813781738, + "kl": 0.5556640625, + "learning_rate": 2.8538461538461536e-07, + "loss": 0.017, + "reward": 0.7799231112003326, + "reward_std": 0.040014040656387806, + "reward_std_all": 0.03702925052493811, + "rewards-max/forensics_sinkhorn_reward": 0.988603949546814, + "rewards-max/forensics_strict_edge_reward": 0.8351600170135498, + "rewards-max/forensics_tiou_reward": 0.998933345079422, + "rewards/forensics_sinkhorn_reward": 0.8030959665775299, + "rewards/forensics_strict_edge_reward": 0.09001892805099487, + "rewards/forensics_tiou_reward": 0.6375750601291656, + "step": 930 + }, + { + "completion_length": 23.265625, + "epoch": 2.857142857142857, + "grad_norm": 7.081519603729248, + "kl": 0.3583984375, + "learning_rate": 2.846153846153846e-07, + "loss": 0.1171, + "reward": 0.7696177661418915, + "reward_std": 0.020617639645934105, + "reward_std_all": 0.02959362044930458, + "rewards-max/forensics_sinkhorn_reward": 0.9798076748847961, + "rewards-max/forensics_strict_edge_reward": 0.449468195438385, + "rewards-max/forensics_tiou_reward": 0.9755732715129852, + "rewards/forensics_sinkhorn_reward": 0.7290118634700775, + "rewards/forensics_strict_edge_reward": 0.038628931157290936, + "rewards/forensics_tiou_reward": 0.48900406062602997, + "step": 931 + }, + { + "completion_length": 27.078125, + "epoch": 2.860215053763441, + "grad_norm": 7.055200099945068, + "kl": 0.3984375, + "learning_rate": 2.8384615384615384e-07, + "loss": 0.0223, + "reward": 0.7867268919944763, + "reward_std": 0.03058099653571844, + "reward_std_all": 0.02901769056916237, + "rewards-max/forensics_sinkhorn_reward": 0.8820379376411438, + "rewards-max/forensics_strict_edge_reward": 0.33027738193050027, + "rewards-max/forensics_tiou_reward": 0.7757726013660431, + "rewards/forensics_sinkhorn_reward": 0.7023569941520691, + "rewards/forensics_strict_edge_reward": 0.0224257618538104, + "rewards/forensics_tiou_reward": 0.5058858245611191, + "step": 932 + }, + { + "completion_length": 23.4375, + "epoch": 2.8632872503840243, + "grad_norm": 7.321588039398193, + "kl": 0.4404296875, + "learning_rate": 2.8307692307692306e-07, + "loss": 0.0103, + "reward": 0.7620472311973572, + "reward_std": 0.017221026122570038, + "reward_std_all": 0.031066534109413624, + "rewards-max/forensics_sinkhorn_reward": 0.9687156081199646, + "rewards-max/forensics_strict_edge_reward": 0.49006423354148865, + "rewards-max/forensics_tiou_reward": 0.9517787098884583, + "rewards/forensics_sinkhorn_reward": 0.7286492884159088, + "rewards/forensics_strict_edge_reward": 0.05978582799434662, + "rewards/forensics_tiou_reward": 0.5429383516311646, + "step": 933 + }, + { + "completion_length": 21.703125, + "epoch": 2.8663594470046085, + "grad_norm": 6.684999942779541, + "kl": 0.390625, + "learning_rate": 2.823076923076923e-07, + "loss": -0.0568, + "reward": 0.8006062209606171, + "reward_std": 0.010737600270658731, + "reward_std_all": 0.01100843376480043, + "rewards-max/forensics_sinkhorn_reward": 0.9719516038894653, + "rewards-max/forensics_strict_edge_reward": 0.4587109684944153, + "rewards-max/forensics_tiou_reward": 0.9578175842761993, + "rewards/forensics_sinkhorn_reward": 0.8587198257446289, + "rewards/forensics_strict_edge_reward": 0.05180475115776062, + "rewards/forensics_tiou_reward": 0.7040501832962036, + "step": 934 + }, + { + "completion_length": 25.46875, + "epoch": 2.869431643625192, + "grad_norm": 7.2362751960754395, + "kl": 0.51953125, + "learning_rate": 2.815384615384615e-07, + "loss": 0.0212, + "reward": 0.7689208984375, + "reward_std": 0.029245007783174515, + "reward_std_all": 0.03261550609022379, + "rewards-max/forensics_sinkhorn_reward": 0.9715719819068909, + "rewards-max/forensics_strict_edge_reward": 0.09770025685429573, + "rewards-max/forensics_tiou_reward": 0.9411921501159668, + "rewards/forensics_sinkhorn_reward": 0.7271925806999207, + "rewards/forensics_strict_edge_reward": 0.00852808472700417, + "rewards/forensics_tiou_reward": 0.501012772321701, + "step": 935 + }, + { + "completion_length": 28.15625, + "epoch": 2.8725038402457757, + "grad_norm": 6.281267166137695, + "kl": 0.47265625, + "learning_rate": 2.8076923076923075e-07, + "loss": 0.1108, + "reward": 0.7765541970729828, + "reward_std": 0.03168294276110828, + "reward_std_all": 0.032137541100382805, + "rewards-max/forensics_sinkhorn_reward": 0.9588975310325623, + "rewards-max/forensics_strict_edge_reward": 0.6232984811067581, + "rewards-max/forensics_tiou_reward": 0.9317988753318787, + "rewards/forensics_sinkhorn_reward": 0.7027584314346313, + "rewards/forensics_strict_edge_reward": 0.05075506493449211, + "rewards/forensics_tiou_reward": 0.5552860498428345, + "step": 936 + }, + { + "completion_length": 25.78125, + "epoch": 2.8755760368663594, + "grad_norm": 7.919331073760986, + "kl": 0.517578125, + "learning_rate": 2.8e-07, + "loss": -0.0101, + "reward": 0.7810827493667603, + "reward_std": 0.03887930396012962, + "reward_std_all": 0.036178025882691145, + "rewards-max/forensics_sinkhorn_reward": 0.9877321124076843, + "rewards-max/forensics_strict_edge_reward": 0.7203241437673569, + "rewards-max/forensics_tiou_reward": 0.9965329766273499, + "rewards/forensics_sinkhorn_reward": 0.7551643252372742, + "rewards/forensics_strict_edge_reward": 0.060534946620464325, + "rewards/forensics_tiou_reward": 0.6510393917560577, + "step": 937 + }, + { + "completion_length": 22.109375, + "epoch": 2.878648233486943, + "grad_norm": 7.324624061584473, + "kl": 0.36328125, + "learning_rate": 2.7923076923076923e-07, + "loss": -0.0404, + "reward": 0.7891996204853058, + "reward_std": 0.029430747032165527, + "reward_std_all": 0.027416241355240345, + "rewards-max/forensics_sinkhorn_reward": 0.9679878652095795, + "rewards-max/forensics_strict_edge_reward": 0.5770299434661865, + "rewards-max/forensics_tiou_reward": 0.9500374495983124, + "rewards/forensics_sinkhorn_reward": 0.7967754304409027, + "rewards/forensics_strict_edge_reward": 0.08052029460668564, + "rewards/forensics_tiou_reward": 0.6116373836994171, + "step": 938 + }, + { + "completion_length": 24.234375, + "epoch": 2.881720430107527, + "grad_norm": 6.754698753356934, + "kl": 0.4931640625, + "learning_rate": 2.7846153846153845e-07, + "loss": -0.0682, + "reward": 0.7882456481456757, + "reward_std": 0.009034478978719562, + "reward_std_all": 0.01626364141702652, + "rewards-max/forensics_sinkhorn_reward": 0.9818589389324188, + "rewards-max/forensics_strict_edge_reward": 0.7367854118347168, + "rewards-max/forensics_tiou_reward": 0.9814803600311279, + "rewards/forensics_sinkhorn_reward": 0.826215922832489, + "rewards/forensics_strict_edge_reward": 0.06254585087299347, + "rewards/forensics_tiou_reward": 0.6514821350574493, + "step": 939 + }, + { + "completion_length": 28.171875, + "epoch": 2.8847926267281108, + "grad_norm": 6.578000545501709, + "kl": 0.3291015625, + "learning_rate": 2.7769230769230766e-07, + "loss": 0.0699, + "reward": 0.7758644223213196, + "reward_std": 0.05176812782883644, + "reward_std_all": 0.046643707901239395, + "rewards-max/forensics_sinkhorn_reward": 0.9736407101154327, + "rewards-max/forensics_strict_edge_reward": 0.5581126511096954, + "rewards-max/forensics_tiou_reward": 0.9627957940101624, + "rewards/forensics_sinkhorn_reward": 0.7462176382541656, + "rewards/forensics_strict_edge_reward": 0.035338809713721275, + "rewards/forensics_tiou_reward": 0.6208951473236084, + "step": 940 + }, + { + "completion_length": 23.546875, + "epoch": 2.8878648233486945, + "grad_norm": 7.257696151733398, + "kl": 0.505859375, + "learning_rate": 2.7692307692307693e-07, + "loss": 0.0194, + "reward": 0.7431051433086395, + "reward_std": 0.02633785706711933, + "reward_std_all": 0.043338613351807, + "rewards-max/forensics_sinkhorn_reward": 0.9577248096466064, + "rewards-max/forensics_strict_edge_reward": 0.35839712619781494, + "rewards-max/forensics_tiou_reward": 0.9220490157604218, + "rewards/forensics_sinkhorn_reward": 0.7166392505168915, + "rewards/forensics_strict_edge_reward": 0.02115405537188053, + "rewards/forensics_tiou_reward": 0.5286171138286591, + "step": 941 + }, + { + "completion_length": 23.453125, + "epoch": 2.890937019969278, + "grad_norm": 10.8037109375, + "kl": 0.90625, + "learning_rate": 2.7615384615384614e-07, + "loss": 0.0613, + "reward": 0.780790388584137, + "reward_std": 0.04429083690047264, + "reward_std_all": 0.040060948580503464, + "rewards-max/forensics_sinkhorn_reward": 0.9610792398452759, + "rewards-max/forensics_strict_edge_reward": 0.3706422299146652, + "rewards-max/forensics_tiou_reward": 0.9322117865085602, + "rewards/forensics_sinkhorn_reward": 0.8069355487823486, + "rewards/forensics_strict_edge_reward": 0.06934577040374279, + "rewards/forensics_tiou_reward": 0.6201151311397552, + "step": 942 + }, + { + "completion_length": 24.84375, + "epoch": 2.8940092165898617, + "grad_norm": 7.848241329193115, + "kl": 0.4228515625, + "learning_rate": 2.753846153846154e-07, + "loss": 0.0068, + "reward": 0.7881429195404053, + "reward_std": 0.024078617803752422, + "reward_std_all": 0.023690467700362206, + "rewards-max/forensics_sinkhorn_reward": 0.9806357622146606, + "rewards-max/forensics_strict_edge_reward": 0.6574748158454895, + "rewards-max/forensics_tiou_reward": 0.9799443781375885, + "rewards/forensics_sinkhorn_reward": 0.7834616005420685, + "rewards/forensics_strict_edge_reward": 0.08449464663863182, + "rewards/forensics_tiou_reward": 0.6465408205986023, + "step": 943 + }, + { + "completion_length": 24.171875, + "epoch": 2.8970814132104454, + "grad_norm": 7.7300519943237305, + "kl": 0.3232421875, + "learning_rate": 2.7461538461538457e-07, + "loss": -0.0445, + "reward": 0.7707008719444275, + "reward_std": 0.017732896376401186, + "reward_std_all": 0.025791064836084843, + "rewards-max/forensics_sinkhorn_reward": 0.9836601912975311, + "rewards-max/forensics_strict_edge_reward": 0.6685703098773956, + "rewards-max/forensics_tiou_reward": 0.9875317513942719, + "rewards/forensics_sinkhorn_reward": 0.7532080411911011, + "rewards/forensics_strict_edge_reward": 0.0801900140941143, + "rewards/forensics_tiou_reward": 0.6334126591682434, + "step": 944 + }, + { + "completion_length": 23.875, + "epoch": 2.900153609831029, + "grad_norm": 9.968488693237305, + "kl": 0.35546875, + "learning_rate": 2.7384615384615383e-07, + "loss": -0.0018, + "reward": 0.728171706199646, + "reward_std": 0.055488064885139465, + "reward_std_all": 0.06893421709537506, + "rewards-max/forensics_sinkhorn_reward": 0.9784240126609802, + "rewards-max/forensics_strict_edge_reward": 0.756708562374115, + "rewards-max/forensics_tiou_reward": 0.9744287729263306, + "rewards/forensics_sinkhorn_reward": 0.6639314293861389, + "rewards/forensics_strict_edge_reward": 0.08126336522400379, + "rewards/forensics_tiou_reward": 0.49765294790267944, + "step": 945 + }, + { + "completion_length": 25.71875, + "epoch": 2.903225806451613, + "grad_norm": 6.587117671966553, + "kl": 0.470703125, + "learning_rate": 2.7307692307692305e-07, + "loss": 0.0805, + "reward": 0.7780935168266296, + "reward_std": 0.026691137260058895, + "reward_std_all": 0.028106097131967545, + "rewards-max/forensics_sinkhorn_reward": 0.9843672811985016, + "rewards-max/forensics_strict_edge_reward": 0.5452546775341034, + "rewards-max/forensics_tiou_reward": 0.9866340756416321, + "rewards/forensics_sinkhorn_reward": 0.7722283899784088, + "rewards/forensics_strict_edge_reward": 0.045196276158094406, + "rewards/forensics_tiou_reward": 0.6203247606754303, + "step": 946 + }, + { + "completion_length": 25.953125, + "epoch": 2.9062980030721968, + "grad_norm": 7.575273036956787, + "kl": 0.3857421875, + "learning_rate": 2.723076923076923e-07, + "loss": 0.0172, + "reward": 0.7632414996623993, + "reward_std": 0.04312732629477978, + "reward_std_all": 0.04364883154630661, + "rewards-max/forensics_sinkhorn_reward": 0.9819283485412598, + "rewards-max/forensics_strict_edge_reward": 0.7450321167707443, + "rewards-max/forensics_tiou_reward": 0.9809988737106323, + "rewards/forensics_sinkhorn_reward": 0.7609653770923615, + "rewards/forensics_strict_edge_reward": 0.1053792592138052, + "rewards/forensics_tiou_reward": 0.5971010029315948, + "step": 947 + }, + { + "completion_length": 23.796875, + "epoch": 2.9093701996927805, + "grad_norm": 5.814935207366943, + "kl": 0.46875, + "learning_rate": 2.7153846153846153e-07, + "loss": 0.0221, + "reward": 0.7768068909645081, + "reward_std": 0.03587312810122967, + "reward_std_all": 0.03497241344302893, + "rewards-max/forensics_sinkhorn_reward": 0.9776073694229126, + "rewards-max/forensics_strict_edge_reward": 0.6863083839416504, + "rewards-max/forensics_tiou_reward": 0.9705581367015839, + "rewards/forensics_sinkhorn_reward": 0.7871916890144348, + "rewards/forensics_strict_edge_reward": 0.09443159401416779, + "rewards/forensics_tiou_reward": 0.6313529312610626, + "step": 948 + }, + { + "completion_length": 25.671875, + "epoch": 2.912442396313364, + "grad_norm": 6.767332553863525, + "kl": 0.3349609375, + "learning_rate": 2.7076923076923074e-07, + "loss": -0.0175, + "reward": 0.8029094934463501, + "reward_std": 0.00455299683380872, + "reward_std_all": 0.0062673966167494655, + "rewards-max/forensics_sinkhorn_reward": 0.9488003551959991, + "rewards-max/forensics_strict_edge_reward": 0.25205671042203903, + "rewards-max/forensics_tiou_reward": 0.8986313939094543, + "rewards/forensics_sinkhorn_reward": 0.7959472835063934, + "rewards/forensics_strict_edge_reward": 0.05882720183581114, + "rewards/forensics_tiou_reward": 0.620488852262497, + "step": 949 + }, + { + "completion_length": 25.578125, + "epoch": 2.9155145929339477, + "grad_norm": 7.209163665771484, + "kl": 0.462890625, + "learning_rate": 2.7e-07, + "loss": 0.0909, + "reward": 0.7450507581233978, + "reward_std": 0.04903627559542656, + "reward_std_all": 0.05454105883836746, + "rewards-max/forensics_sinkhorn_reward": 0.976503998041153, + "rewards-max/forensics_strict_edge_reward": 0.5474467873573303, + "rewards-max/forensics_tiou_reward": 0.9676402807235718, + "rewards/forensics_sinkhorn_reward": 0.6956360936164856, + "rewards/forensics_strict_edge_reward": 0.07165367901325226, + "rewards/forensics_tiou_reward": 0.5411740243434906, + "step": 950 + }, + { + "completion_length": 26.6875, + "epoch": 2.9185867895545314, + "grad_norm": 5.712553977966309, + "kl": 0.400390625, + "learning_rate": 2.692307692307692e-07, + "loss": 0.0684, + "reward": 0.777999997138977, + "reward_std": 0.04757733270525932, + "reward_std_all": 0.04336962662637234, + "rewards-max/forensics_sinkhorn_reward": 0.9718083739280701, + "rewards-max/forensics_strict_edge_reward": 0.34341900050640106, + "rewards-max/forensics_tiou_reward": 0.9539950788021088, + "rewards/forensics_sinkhorn_reward": 0.7537479102611542, + "rewards/forensics_strict_edge_reward": 0.03577696159482002, + "rewards/forensics_tiou_reward": 0.5661676526069641, + "step": 951 + }, + { + "completion_length": 21.71875, + "epoch": 2.921658986175115, + "grad_norm": 8.188546180725098, + "kl": 0.4833984375, + "learning_rate": 2.6846153846153844e-07, + "loss": -0.0402, + "reward": 0.7766425609588623, + "reward_std": 0.02345351391704753, + "reward_std_all": 0.027731850743293762, + "rewards-max/forensics_sinkhorn_reward": 0.9756255149841309, + "rewards-max/forensics_strict_edge_reward": 0.4927365332841873, + "rewards-max/forensics_tiou_reward": 0.9641706645488739, + "rewards/forensics_sinkhorn_reward": 0.761321097612381, + "rewards/forensics_strict_edge_reward": 0.056398067623376846, + "rewards/forensics_tiou_reward": 0.5925051271915436, + "step": 952 + }, + { + "completion_length": 27.765625, + "epoch": 2.924731182795699, + "grad_norm": 6.219192028045654, + "kl": 0.5166015625, + "learning_rate": 2.6769230769230765e-07, + "loss": -0.0265, + "reward": 0.8032417595386505, + "reward_std": 0.005385168129578233, + "reward_std_all": 0.006582313682883978, + "rewards-max/forensics_sinkhorn_reward": 0.983323872089386, + "rewards-max/forensics_strict_edge_reward": 0.4881605803966522, + "rewards-max/forensics_tiou_reward": 0.9846062064170837, + "rewards/forensics_sinkhorn_reward": 0.8031603395938873, + "rewards/forensics_strict_edge_reward": 0.06947860680520535, + "rewards/forensics_tiou_reward": 0.677990049123764, + "step": 953 + }, + { + "completion_length": 25.1875, + "epoch": 2.9278033794162828, + "grad_norm": 7.937047004699707, + "kl": 0.474609375, + "learning_rate": 2.669230769230769e-07, + "loss": 0.0613, + "reward": 0.7171484231948853, + "reward_std": 0.05756609886884689, + "reward_std_all": 0.0728277862071991, + "rewards-max/forensics_sinkhorn_reward": 0.9868464767932892, + "rewards-max/forensics_strict_edge_reward": 0.8351600170135498, + "rewards-max/forensics_tiou_reward": 0.992067962884903, + "rewards/forensics_sinkhorn_reward": 0.6533005237579346, + "rewards/forensics_strict_edge_reward": 0.10305001214146614, + "rewards/forensics_tiou_reward": 0.5275052487850189, + "step": 954 + }, + { + "completion_length": 26.328125, + "epoch": 2.9308755760368665, + "grad_norm": 7.844270706176758, + "kl": 0.4013671875, + "learning_rate": 2.6615384615384613e-07, + "loss": -0.0647, + "reward": 0.7798566520214081, + "reward_std": 0.016598884132690728, + "reward_std_all": 0.022904403507709503, + "rewards-max/forensics_sinkhorn_reward": 0.9826542437076569, + "rewards-max/forensics_strict_edge_reward": 0.6293683648109436, + "rewards-max/forensics_tiou_reward": 0.9834535419940948, + "rewards/forensics_sinkhorn_reward": 0.7647383809089661, + "rewards/forensics_strict_edge_reward": 0.06759935989975929, + "rewards/forensics_tiou_reward": 0.5914831459522247, + "step": 955 + }, + { + "completion_length": 26.671875, + "epoch": 2.93394777265745, + "grad_norm": 6.491158962249756, + "kl": 0.537109375, + "learning_rate": 2.653846153846154e-07, + "loss": -0.0568, + "reward": 0.7876829206943512, + "reward_std": 0.01884190598502755, + "reward_std_all": 0.020907631143927574, + "rewards-max/forensics_sinkhorn_reward": 0.9837282598018646, + "rewards-max/forensics_strict_edge_reward": 0.5731385350227356, + "rewards-max/forensics_tiou_reward": 0.9851179122924805, + "rewards/forensics_sinkhorn_reward": 0.7724824249744415, + "rewards/forensics_strict_edge_reward": 0.08627705089747906, + "rewards/forensics_tiou_reward": 0.6666090190410614, + "step": 956 + }, + { + "completion_length": 24.984375, + "epoch": 2.9370199692780337, + "grad_norm": 6.524048328399658, + "kl": 0.64453125, + "learning_rate": 2.646153846153846e-07, + "loss": 0.1348, + "reward": 0.7642638981342316, + "reward_std": 0.04580376297235489, + "reward_std_all": 0.0449965912848711, + "rewards-max/forensics_sinkhorn_reward": 0.9822123944759369, + "rewards-max/forensics_strict_edge_reward": 0.4741404205560684, + "rewards-max/forensics_tiou_reward": 0.9834523499011993, + "rewards/forensics_sinkhorn_reward": 0.7140891849994659, + "rewards/forensics_strict_edge_reward": 0.05162654258310795, + "rewards/forensics_tiou_reward": 0.5499118268489838, + "step": 957 + }, + { + "completion_length": 24.0, + "epoch": 2.9400921658986174, + "grad_norm": 8.215468406677246, + "kl": 0.396484375, + "learning_rate": 2.638461538461538e-07, + "loss": -0.0008, + "reward": 0.7840223610401154, + "reward_std": 0.0238872398622334, + "reward_std_all": 0.024334818357601762, + "rewards-max/forensics_sinkhorn_reward": 0.9777982532978058, + "rewards-max/forensics_strict_edge_reward": 0.5248452723026276, + "rewards-max/forensics_tiou_reward": 0.972400039434433, + "rewards/forensics_sinkhorn_reward": 0.7818456590175629, + "rewards/forensics_strict_edge_reward": 0.06662937067449093, + "rewards/forensics_tiou_reward": 0.5895299911499023, + "step": 958 + }, + { + "completion_length": 23.640625, + "epoch": 2.943164362519201, + "grad_norm": 7.593491554260254, + "kl": 0.3662109375, + "learning_rate": 2.630769230769231e-07, + "loss": 0.0218, + "reward": 0.8019487261772156, + "reward_std": 0.00562573061324656, + "reward_std_all": 0.007541171042248607, + "rewards-max/forensics_sinkhorn_reward": 0.9639676213264465, + "rewards-max/forensics_strict_edge_reward": 0.49711741507053375, + "rewards-max/forensics_tiou_reward": 0.9398997128009796, + "rewards/forensics_sinkhorn_reward": 0.7809586226940155, + "rewards/forensics_strict_edge_reward": 0.03917744383215904, + "rewards/forensics_tiou_reward": 0.5775384604930878, + "step": 959 + }, + { + "completion_length": 26.015625, + "epoch": 2.946236559139785, + "grad_norm": 7.937549591064453, + "kl": 0.462890625, + "learning_rate": 2.623076923076923e-07, + "loss": -0.0976, + "reward": 0.7358059287071228, + "reward_std": 0.036097897216677666, + "reward_std_all": 0.05537593923509121, + "rewards-max/forensics_sinkhorn_reward": 0.987221896648407, + "rewards-max/forensics_strict_edge_reward": 0.7244336307048798, + "rewards-max/forensics_tiou_reward": 0.994076132774353, + "rewards/forensics_sinkhorn_reward": 0.719146341085434, + "rewards/forensics_strict_edge_reward": 0.07909358665347099, + "rewards/forensics_tiou_reward": 0.5849636793136597, + "step": 960 + }, + { + "completion_length": 24.984375, + "epoch": 2.9493087557603688, + "grad_norm": 6.510005474090576, + "kl": 0.359375, + "learning_rate": 2.615384615384615e-07, + "loss": 0.0749, + "reward": 0.7188578844070435, + "reward_std": 0.05141858942806721, + "reward_std_all": 0.06771676614880562, + "rewards-max/forensics_sinkhorn_reward": 0.9859280586242676, + "rewards-max/forensics_strict_edge_reward": 0.7367854118347168, + "rewards-max/forensics_tiou_reward": 0.992435872554779, + "rewards/forensics_sinkhorn_reward": 0.6369504034519196, + "rewards/forensics_strict_edge_reward": 0.05384108051657677, + "rewards/forensics_tiou_reward": 0.41681016981601715, + "step": 961 + }, + { + "completion_length": 22.015625, + "epoch": 2.9523809523809526, + "grad_norm": 6.9136152267456055, + "kl": 0.361328125, + "learning_rate": 2.6076923076923073e-07, + "loss": 0.0145, + "reward": 0.7669604420661926, + "reward_std": 0.04184460826218128, + "reward_std_all": 0.041565753519535065, + "rewards-max/forensics_sinkhorn_reward": 0.9845168888568878, + "rewards-max/forensics_strict_edge_reward": 0.7367854118347168, + "rewards-max/forensics_tiou_reward": 0.9869495630264282, + "rewards/forensics_sinkhorn_reward": 0.7716591358184814, + "rewards/forensics_strict_edge_reward": 0.09013520926237106, + "rewards/forensics_tiou_reward": 0.6334872841835022, + "step": 962 + }, + { + "completion_length": 26.375, + "epoch": 2.955453149001536, + "grad_norm": 6.878596305847168, + "kl": 0.3779296875, + "learning_rate": 2.6e-07, + "loss": 0.0239, + "reward": 0.7906493246555328, + "reward_std": 0.02650513406842947, + "reward_std_all": 0.024946723133325577, + "rewards-max/forensics_sinkhorn_reward": 0.9794703423976898, + "rewards-max/forensics_strict_edge_reward": 0.4077484905719757, + "rewards-max/forensics_tiou_reward": 0.9701957106590271, + "rewards/forensics_sinkhorn_reward": 0.7829802930355072, + "rewards/forensics_strict_edge_reward": 0.05521607957780361, + "rewards/forensics_tiou_reward": 0.6716274917125702, + "step": 963 + }, + { + "completion_length": 26.5, + "epoch": 2.9585253456221197, + "grad_norm": 6.48870849609375, + "kl": 0.4599609375, + "learning_rate": 2.592307692307692e-07, + "loss": 0.0085, + "reward": 0.800326406955719, + "reward_std": 0.01132068270817399, + "reward_std_all": 0.011308913119137287, + "rewards-max/forensics_sinkhorn_reward": 0.9746620953083038, + "rewards-max/forensics_strict_edge_reward": 0.6621335446834564, + "rewards-max/forensics_tiou_reward": 0.9653209745883942, + "rewards/forensics_sinkhorn_reward": 0.8104224503040314, + "rewards/forensics_strict_edge_reward": 0.09692461788654327, + "rewards/forensics_tiou_reward": 0.6585460901260376, + "step": 964 + }, + { + "completion_length": 26.703125, + "epoch": 2.9615975422427034, + "grad_norm": 6.522236347198486, + "kl": 0.3173828125, + "learning_rate": 2.584615384615385e-07, + "loss": -0.0053, + "reward": 0.788444310426712, + "reward_std": 0.03111748117953539, + "reward_std_all": 0.02896041190251708, + "rewards-max/forensics_sinkhorn_reward": 0.9808126986026764, + "rewards-max/forensics_strict_edge_reward": 0.613072544336319, + "rewards-max/forensics_tiou_reward": 0.9773645401000977, + "rewards/forensics_sinkhorn_reward": 0.8240457773208618, + "rewards/forensics_strict_edge_reward": 0.05936295911669731, + "rewards/forensics_tiou_reward": 0.7079609930515289, + "step": 965 + }, + { + "completion_length": 28.171875, + "epoch": 2.964669738863287, + "grad_norm": 6.3739542961120605, + "kl": 0.375, + "learning_rate": 2.5769230769230764e-07, + "loss": -0.0738, + "reward": 0.7735293507575989, + "reward_std": 0.03398998035117984, + "reward_std_all": 0.03442602418363094, + "rewards-max/forensics_sinkhorn_reward": 0.971478283405304, + "rewards-max/forensics_strict_edge_reward": 0.42805178463459015, + "rewards-max/forensics_tiou_reward": 0.9508334398269653, + "rewards/forensics_sinkhorn_reward": 0.7873841524124146, + "rewards/forensics_strict_edge_reward": 0.05267421528697014, + "rewards/forensics_tiou_reward": 0.6585296094417572, + "step": 966 + }, + { + "completion_length": 24.296875, + "epoch": 2.967741935483871, + "grad_norm": 7.559092044830322, + "kl": 0.43359375, + "learning_rate": 2.569230769230769e-07, + "loss": 0.0959, + "reward": 0.771047443151474, + "reward_std": 0.03355161193758249, + "reward_std_all": 0.037537889555096626, + "rewards-max/forensics_sinkhorn_reward": 0.9878287315368652, + "rewards-max/forensics_strict_edge_reward": 0.6901743710041046, + "rewards-max/forensics_tiou_reward": 0.9970380961894989, + "rewards/forensics_sinkhorn_reward": 0.7086267471313477, + "rewards/forensics_strict_edge_reward": 0.0486602820456028, + "rewards/forensics_tiou_reward": 0.5547888875007629, + "step": 967 + }, + { + "completion_length": 22.375, + "epoch": 2.9708141321044548, + "grad_norm": 9.468278884887695, + "kl": 0.408203125, + "learning_rate": 2.5615384615384617e-07, + "loss": -0.0153, + "reward": 0.7985085844993591, + "reward_std": 0.009915906470268965, + "reward_std_all": 0.011249182280153036, + "rewards-max/forensics_sinkhorn_reward": 0.9797400832176208, + "rewards-max/forensics_strict_edge_reward": 0.6930406093597412, + "rewards-max/forensics_tiou_reward": 0.9782598614692688, + "rewards/forensics_sinkhorn_reward": 0.7996494174003601, + "rewards/forensics_strict_edge_reward": 0.05569305457174778, + "rewards/forensics_tiou_reward": 0.6161001920700073, + "step": 968 + }, + { + "completion_length": 26.921875, + "epoch": 2.9738863287250386, + "grad_norm": 6.331875324249268, + "kl": 0.38671875, + "learning_rate": 2.553846153846154e-07, + "loss": 0.0093, + "reward": 0.7849771082401276, + "reward_std": 0.03604381554760039, + "reward_std_all": 0.03335905121639371, + "rewards-max/forensics_sinkhorn_reward": 0.9596502482891083, + "rewards-max/forensics_strict_edge_reward": 0.36183040402829647, + "rewards-max/forensics_tiou_reward": 0.9339575469493866, + "rewards/forensics_sinkhorn_reward": 0.7552791833877563, + "rewards/forensics_strict_edge_reward": 0.04877378814853728, + "rewards/forensics_tiou_reward": 0.6270082592964172, + "step": 969 + }, + { + "completion_length": 26.0625, + "epoch": 2.976958525345622, + "grad_norm": 8.9434232711792, + "kl": 0.501953125, + "learning_rate": 2.546153846153846e-07, + "loss": 0.0355, + "reward": 0.7870338559150696, + "reward_std": 0.022388468496501446, + "reward_std_all": 0.022807452362030745, + "rewards-max/forensics_sinkhorn_reward": 0.9850262701511383, + "rewards-max/forensics_strict_edge_reward": 0.5475059002637863, + "rewards-max/forensics_tiou_reward": 0.9885158538818359, + "rewards/forensics_sinkhorn_reward": 0.7401432991027832, + "rewards/forensics_strict_edge_reward": 0.0526789054274559, + "rewards/forensics_tiou_reward": 0.5861817300319672, + "step": 970 + }, + { + "completion_length": 27.46875, + "epoch": 2.980030721966206, + "grad_norm": 6.745112419128418, + "kl": 0.3759765625, + "learning_rate": 2.538461538461538e-07, + "loss": 0.0297, + "reward": 0.7347279191017151, + "reward_std": 0.02772781439125538, + "reward_std_all": 0.04748234525322914, + "rewards-max/forensics_sinkhorn_reward": 0.9766468703746796, + "rewards-max/forensics_strict_edge_reward": 0.38485318422317505, + "rewards-max/forensics_tiou_reward": 0.9672223925590515, + "rewards/forensics_sinkhorn_reward": 0.7098147571086884, + "rewards/forensics_strict_edge_reward": 0.05640421062707901, + "rewards/forensics_tiou_reward": 0.5747716426849365, + "step": 971 + }, + { + "completion_length": 28.0, + "epoch": 2.9831029185867894, + "grad_norm": 5.936853408813477, + "kl": 0.4130859375, + "learning_rate": 2.530769230769231e-07, + "loss": 0.021, + "reward": 0.7784466743469238, + "reward_std": 0.030732744839042425, + "reward_std_all": 0.03060782328248024, + "rewards-max/forensics_sinkhorn_reward": 0.9740714728832245, + "rewards-max/forensics_strict_edge_reward": 0.37596484273672104, + "rewards-max/forensics_tiou_reward": 0.9621375501155853, + "rewards/forensics_sinkhorn_reward": 0.7525494694709778, + "rewards/forensics_strict_edge_reward": 0.06636691838502884, + "rewards/forensics_tiou_reward": 0.6143344640731812, + "step": 972 + }, + { + "completion_length": 24.453125, + "epoch": 2.986175115207373, + "grad_norm": 6.968201160430908, + "kl": 0.451171875, + "learning_rate": 2.523076923076923e-07, + "loss": -0.066, + "reward": 0.7897646725177765, + "reward_std": 0.028367752209305763, + "reward_std_all": 0.026498244144022465, + "rewards-max/forensics_sinkhorn_reward": 0.969995528459549, + "rewards-max/forensics_strict_edge_reward": 0.32777850329875946, + "rewards-max/forensics_tiou_reward": 0.9504086971282959, + "rewards/forensics_sinkhorn_reward": 0.8058056831359863, + "rewards/forensics_strict_edge_reward": 0.03248700127005577, + "rewards/forensics_tiou_reward": 0.6650077700614929, + "step": 973 + }, + { + "completion_length": 21.75, + "epoch": 2.989247311827957, + "grad_norm": 7.630611896514893, + "kl": 0.326171875, + "learning_rate": 2.515384615384615e-07, + "loss": -0.0257, + "reward": 0.7445805072784424, + "reward_std": 0.01777314394712448, + "reward_std_all": 0.040354618802666664, + "rewards-max/forensics_sinkhorn_reward": 0.9864535927772522, + "rewards-max/forensics_strict_edge_reward": 0.6839397251605988, + "rewards-max/forensics_tiou_reward": 0.990413635969162, + "rewards/forensics_sinkhorn_reward": 0.7069192230701447, + "rewards/forensics_strict_edge_reward": 0.05551617033779621, + "rewards/forensics_tiou_reward": 0.5631810128688812, + "step": 974 + }, + { + "completion_length": 24.125, + "epoch": 2.9923195084485408, + "grad_norm": 6.748687744140625, + "kl": 0.498046875, + "learning_rate": 2.507692307692307e-07, + "loss": -0.0224, + "reward": 0.7755720913410187, + "reward_std": 0.03703900892287493, + "reward_std_all": 0.036076219752430916, + "rewards-max/forensics_sinkhorn_reward": 0.9814753830432892, + "rewards-max/forensics_strict_edge_reward": 0.5623251795768738, + "rewards-max/forensics_tiou_reward": 0.9809662401676178, + "rewards/forensics_sinkhorn_reward": 0.8051010966300964, + "rewards/forensics_strict_edge_reward": 0.08262880146503448, + "rewards/forensics_tiou_reward": 0.6389864683151245, + "step": 975 + }, + { + "completion_length": 26.75, + "epoch": 2.9953917050691246, + "grad_norm": 6.150122165679932, + "kl": 0.353515625, + "learning_rate": 2.5e-07, + "loss": -0.1146, + "reward": 0.8054490685462952, + "reward_std": 0.0016713774530217052, + "reward_std_all": 0.0035919154761359096, + "rewards-max/forensics_sinkhorn_reward": 0.9852890372276306, + "rewards-max/forensics_strict_edge_reward": 0.7727466821670532, + "rewards-max/forensics_tiou_reward": 0.9896262884140015, + "rewards/forensics_sinkhorn_reward": 0.7784042954444885, + "rewards/forensics_strict_edge_reward": 0.10330202803015709, + "rewards/forensics_tiou_reward": 0.7005855739116669, + "step": 976 + }, + { + "completion_length": 23.921875, + "epoch": 2.998463901689708, + "grad_norm": 7.847219944000244, + "kl": 0.5302734375, + "learning_rate": 2.492307692307692e-07, + "loss": 0.0689, + "reward": 0.7662035822868347, + "reward_std": 0.06925229355692863, + "reward_std_all": 0.061765123158693314, + "rewards-max/forensics_sinkhorn_reward": 0.9688055217266083, + "rewards-max/forensics_strict_edge_reward": 0.4493289589881897, + "rewards-max/forensics_tiou_reward": 0.9448994398117065, + "rewards/forensics_sinkhorn_reward": 0.7840766906738281, + "rewards/forensics_strict_edge_reward": 0.05512079410254955, + "rewards/forensics_tiou_reward": 0.6299729645252228, + "step": 977 + }, + { + "completion_length": 29.399999618530273, + "epoch": 3.0, + "grad_norm": 4.841268539428711, + "kl": 0.75, + "learning_rate": 2.4846153846153847e-07, + "loss": -0.0052, + "reward": 0.728158175945282, + "reward_std": 0.04463762044906616, + "reward_std_all": 0.046968646347522736, + "rewards-max/forensics_sinkhorn_reward": 0.9608937501907349, + "rewards-max/forensics_strict_edge_reward": 0.010539373382925987, + "rewards-max/forensics_tiou_reward": 0.9331791996955872, + "rewards/forensics_sinkhorn_reward": 0.6265844702720642, + "rewards/forensics_strict_edge_reward": 0.002108100801706314, + "rewards/forensics_tiou_reward": 0.5257677435874939, + "step": 978 + }, + { + "completion_length": 23.5625, + "epoch": 3.003072196620584, + "grad_norm": 7.261849403381348, + "kl": 0.365234375, + "learning_rate": 2.476923076923077e-07, + "loss": 0.0234, + "reward": 0.7713419497013092, + "reward_std": 0.024277186952531338, + "reward_std_all": 0.03190556447952986, + "rewards-max/forensics_sinkhorn_reward": 0.970564991235733, + "rewards-max/forensics_strict_edge_reward": 0.4686219245195389, + "rewards-max/forensics_tiou_reward": 0.9546476006507874, + "rewards/forensics_sinkhorn_reward": 0.7605274021625519, + "rewards/forensics_strict_edge_reward": 0.060670286417007446, + "rewards/forensics_tiou_reward": 0.5759556740522385, + "step": 979 + }, + { + "completion_length": 26.609375, + "epoch": 3.0061443932411676, + "grad_norm": 7.363394260406494, + "kl": 0.37890625, + "learning_rate": 2.469230769230769e-07, + "loss": 0.0697, + "reward": 0.7709479033946991, + "reward_std": 0.025737290270626545, + "reward_std_all": 0.030245433561503887, + "rewards-max/forensics_sinkhorn_reward": 0.972229927778244, + "rewards-max/forensics_strict_edge_reward": 0.561369463801384, + "rewards-max/forensics_tiou_reward": 0.9588571786880493, + "rewards/forensics_sinkhorn_reward": 0.7375697493553162, + "rewards/forensics_strict_edge_reward": 0.05014324560761452, + "rewards/forensics_tiou_reward": 0.606518030166626, + "step": 980 + }, + { + "completion_length": 24.484375, + "epoch": 3.0092165898617513, + "grad_norm": 7.260066032409668, + "kl": 0.5576171875, + "learning_rate": 2.4615384615384616e-07, + "loss": 0.0245, + "reward": 0.7593258917331696, + "reward_std": 0.01579164434224367, + "reward_std_all": 0.033771419897675514, + "rewards-max/forensics_sinkhorn_reward": 0.9731742739677429, + "rewards-max/forensics_strict_edge_reward": 0.6424643248319626, + "rewards-max/forensics_tiou_reward": 0.9613246023654938, + "rewards/forensics_sinkhorn_reward": 0.7644561529159546, + "rewards/forensics_strict_edge_reward": 0.062467269599437714, + "rewards/forensics_tiou_reward": 0.6046904623508453, + "step": 981 + }, + { + "completion_length": 26.34375, + "epoch": 3.0122887864823347, + "grad_norm": 6.391881942749023, + "kl": 0.474609375, + "learning_rate": 2.453846153846154e-07, + "loss": 0.0264, + "reward": 0.7831681370735168, + "reward_std": 0.019982667174190283, + "reward_std_all": 0.02209026087075472, + "rewards-max/forensics_sinkhorn_reward": 0.9843954145908356, + "rewards-max/forensics_strict_edge_reward": 0.5593034327030182, + "rewards-max/forensics_tiou_reward": 0.9905149638652802, + "rewards/forensics_sinkhorn_reward": 0.8006176650524139, + "rewards/forensics_strict_edge_reward": 0.040434274822473526, + "rewards/forensics_tiou_reward": 0.6788970828056335, + "step": 982 + }, + { + "completion_length": 24.796875, + "epoch": 3.0153609831029184, + "grad_norm": 7.208608627319336, + "kl": 0.478515625, + "learning_rate": 2.446153846153846e-07, + "loss": -0.0148, + "reward": 0.8004270493984222, + "reward_std": 0.010965320747345686, + "reward_std_all": 0.011150965932756662, + "rewards-max/forensics_sinkhorn_reward": 0.983225017786026, + "rewards-max/forensics_strict_edge_reward": 0.5563141405582428, + "rewards-max/forensics_tiou_reward": 0.9830823540687561, + "rewards/forensics_sinkhorn_reward": 0.8251080811023712, + "rewards/forensics_strict_edge_reward": 0.0769483894109726, + "rewards/forensics_tiou_reward": 0.6950486600399017, + "step": 983 + }, + { + "completion_length": 23.109375, + "epoch": 3.0184331797235022, + "grad_norm": 8.021319389343262, + "kl": 0.453125, + "learning_rate": 2.4384615384615386e-07, + "loss": 0.042, + "reward": 0.7452542185783386, + "reward_std": 0.07911990582942963, + "reward_std_all": 0.0739007256925106, + "rewards-max/forensics_sinkhorn_reward": 0.9731537401676178, + "rewards-max/forensics_strict_edge_reward": 0.5656128525733948, + "rewards-max/forensics_tiou_reward": 0.957232654094696, + "rewards/forensics_sinkhorn_reward": 0.721723884344101, + "rewards/forensics_strict_edge_reward": 0.06664559990167618, + "rewards/forensics_tiou_reward": 0.5316983312368393, + "step": 984 + }, + { + "completion_length": 23.484375, + "epoch": 3.021505376344086, + "grad_norm": 7.210628986358643, + "kl": 0.4951171875, + "learning_rate": 2.4307692307692307e-07, + "loss": -0.0553, + "reward": 0.7661301791667938, + "reward_std": 0.03544760181102902, + "reward_std_all": 0.0374812928494066, + "rewards-max/forensics_sinkhorn_reward": 0.9825516641139984, + "rewards-max/forensics_strict_edge_reward": 0.5453909933567047, + "rewards-max/forensics_tiou_reward": 0.9833973050117493, + "rewards/forensics_sinkhorn_reward": 0.7434142827987671, + "rewards/forensics_strict_edge_reward": 0.05859265476465225, + "rewards/forensics_tiou_reward": 0.5960331559181213, + "step": 985 + }, + { + "completion_length": 22.1875, + "epoch": 3.02457757296467, + "grad_norm": 7.96949577331543, + "kl": 0.5703125, + "learning_rate": 2.423076923076923e-07, + "loss": -0.08, + "reward": 0.7981315851211548, + "reward_std": 0.007625567028298974, + "reward_std_all": 0.009614885319024324, + "rewards-max/forensics_sinkhorn_reward": 0.9854609072208405, + "rewards-max/forensics_strict_edge_reward": 0.6005697548389435, + "rewards-max/forensics_tiou_reward": 0.9900369942188263, + "rewards/forensics_sinkhorn_reward": 0.8204742670059204, + "rewards/forensics_strict_edge_reward": 0.05466918833553791, + "rewards/forensics_tiou_reward": 0.6844899356365204, + "step": 986 + }, + { + "completion_length": 24.453125, + "epoch": 3.0276497695852536, + "grad_norm": 8.273130416870117, + "kl": 0.3564453125, + "learning_rate": 2.415384615384615e-07, + "loss": 0.0292, + "reward": 0.7895986139774323, + "reward_std": 0.02842753939330578, + "reward_std_all": 0.026913052424788475, + "rewards-max/forensics_sinkhorn_reward": 0.9745115339756012, + "rewards-max/forensics_strict_edge_reward": 0.5081937909126282, + "rewards-max/forensics_tiou_reward": 0.9582580327987671, + "rewards/forensics_sinkhorn_reward": 0.8098797500133514, + "rewards/forensics_strict_edge_reward": 0.06388156861066818, + "rewards/forensics_tiou_reward": 0.6232589483261108, + "step": 987 + }, + { + "completion_length": 23.859375, + "epoch": 3.0307219662058373, + "grad_norm": 7.391808032989502, + "kl": 0.4482421875, + "learning_rate": 2.4076923076923076e-07, + "loss": 0.0439, + "reward": 0.7668816149234772, + "reward_std": 0.036795346066355705, + "reward_std_all": 0.03802269324660301, + "rewards-max/forensics_sinkhorn_reward": 0.9653679728507996, + "rewards-max/forensics_strict_edge_reward": 0.24684785306453705, + "rewards-max/forensics_tiou_reward": 0.9420459270477295, + "rewards/forensics_sinkhorn_reward": 0.7445628046989441, + "rewards/forensics_strict_edge_reward": 0.037715787068009377, + "rewards/forensics_tiou_reward": 0.5998536050319672, + "step": 988 + }, + { + "completion_length": 28.359375, + "epoch": 3.033794162826421, + "grad_norm": 6.330667018890381, + "kl": 0.5302734375, + "learning_rate": 2.4e-07, + "loss": 0.01, + "reward": 0.7249728441238403, + "reward_std": 0.05836691986769438, + "reward_std_all": 0.06756890565156937, + "rewards-max/forensics_sinkhorn_reward": 0.9773143529891968, + "rewards-max/forensics_strict_edge_reward": 0.4741351455450058, + "rewards-max/forensics_tiou_reward": 0.9696952104568481, + "rewards/forensics_sinkhorn_reward": 0.6916212737560272, + "rewards/forensics_strict_edge_reward": 0.07512322068214417, + "rewards/forensics_tiou_reward": 0.5229277908802032, + "step": 989 + }, + { + "completion_length": 23.890625, + "epoch": 3.0368663594470044, + "grad_norm": 8.027928352355957, + "kl": 0.4951171875, + "learning_rate": 2.392307692307692e-07, + "loss": -0.0479, + "reward": 0.7899920046329498, + "reward_std": 0.016151905059814453, + "reward_std_all": 0.017740267794579268, + "rewards-max/forensics_sinkhorn_reward": 0.9772569537162781, + "rewards-max/forensics_strict_edge_reward": 0.49426253139972687, + "rewards-max/forensics_tiou_reward": 0.973350465297699, + "rewards/forensics_sinkhorn_reward": 0.789249062538147, + "rewards/forensics_strict_edge_reward": 0.04394502751529217, + "rewards/forensics_tiou_reward": 0.6024438738822937, + "step": 990 + }, + { + "completion_length": 25.09375, + "epoch": 3.0399385560675882, + "grad_norm": 6.629275798797607, + "kl": 0.4658203125, + "learning_rate": 2.3846153846153846e-07, + "loss": 0.0326, + "reward": 0.8007738590240479, + "reward_std": 0.007992994040250778, + "reward_std_all": 0.009035573340952396, + "rewards-max/forensics_sinkhorn_reward": 0.9799564778804779, + "rewards-max/forensics_strict_edge_reward": 0.34060192853212357, + "rewards-max/forensics_tiou_reward": 0.9747912883758545, + "rewards/forensics_sinkhorn_reward": 0.8237503170967102, + "rewards/forensics_strict_edge_reward": 0.05901896953582764, + "rewards/forensics_tiou_reward": 0.7050036787986755, + "step": 991 + }, + { + "completion_length": 27.46875, + "epoch": 3.043010752688172, + "grad_norm": 17.504581451416016, + "kl": 1.69140625, + "learning_rate": 2.376923076923077e-07, + "loss": -0.003, + "reward": 0.8040960431098938, + "reward_std": 0.003883353667333722, + "reward_std_all": 0.005427355645224452, + "rewards-max/forensics_sinkhorn_reward": 0.9833472669124603, + "rewards-max/forensics_strict_edge_reward": 0.541615754365921, + "rewards-max/forensics_tiou_reward": 0.984884113073349, + "rewards/forensics_sinkhorn_reward": 0.8032616972923279, + "rewards/forensics_strict_edge_reward": 0.07959003001451492, + "rewards/forensics_tiou_reward": 0.7023439407348633, + "step": 992 + }, + { + "completion_length": 25.703125, + "epoch": 3.046082949308756, + "grad_norm": 6.7700934410095215, + "kl": 0.4306640625, + "learning_rate": 2.369230769230769e-07, + "loss": -0.0613, + "reward": 0.7641675472259521, + "reward_std": 0.024592523695901036, + "reward_std_all": 0.032345827436074615, + "rewards-max/forensics_sinkhorn_reward": 0.9817631244659424, + "rewards-max/forensics_strict_edge_reward": 0.40607747435569763, + "rewards-max/forensics_tiou_reward": 0.9777814447879791, + "rewards/forensics_sinkhorn_reward": 0.7854200899600983, + "rewards/forensics_strict_edge_reward": 0.07561686635017395, + "rewards/forensics_tiou_reward": 0.6680464446544647, + "step": 993 + }, + { + "completion_length": 28.734375, + "epoch": 3.0491551459293396, + "grad_norm": 7.046574115753174, + "kl": 0.5576171875, + "learning_rate": 2.3615384615384615e-07, + "loss": -0.0409, + "reward": 0.7857539653778076, + "reward_std": 0.0240644421428442, + "reward_std_all": 0.02409274224191904, + "rewards-max/forensics_sinkhorn_reward": 0.975863516330719, + "rewards-max/forensics_strict_edge_reward": 0.46002286672592163, + "rewards-max/forensics_tiou_reward": 0.9685576260089874, + "rewards/forensics_sinkhorn_reward": 0.759839802980423, + "rewards/forensics_strict_edge_reward": 0.08460687845945358, + "rewards/forensics_tiou_reward": 0.5979922711849213, + "step": 994 + }, + { + "completion_length": 24.953125, + "epoch": 3.0522273425499233, + "grad_norm": 12.40194034576416, + "kl": 1.01953125, + "learning_rate": 2.3538461538461537e-07, + "loss": 0.041, + "reward": 0.7934169173240662, + "reward_std": 0.022237897850573063, + "reward_std_all": 0.021328176371753216, + "rewards-max/forensics_sinkhorn_reward": 0.9823260605335236, + "rewards-max/forensics_strict_edge_reward": 0.8418737053871155, + "rewards-max/forensics_tiou_reward": 0.9849389791488647, + "rewards/forensics_sinkhorn_reward": 0.8099074363708496, + "rewards/forensics_strict_edge_reward": 0.09347739070653915, + "rewards/forensics_tiou_reward": 0.6172614395618439, + "step": 995 + }, + { + "completion_length": 24.09375, + "epoch": 3.055299539170507, + "grad_norm": 8.108648300170898, + "kl": 0.458984375, + "learning_rate": 2.346153846153846e-07, + "loss": 0.1014, + "reward": 0.7771255373954773, + "reward_std": 0.018399535678327084, + "reward_std_all": 0.026102358475327492, + "rewards-max/forensics_sinkhorn_reward": 0.9342207312583923, + "rewards-max/forensics_strict_edge_reward": 0.19799603521823883, + "rewards-max/forensics_tiou_reward": 0.8744065761566162, + "rewards/forensics_sinkhorn_reward": 0.7523138225078583, + "rewards/forensics_strict_edge_reward": 0.022365853190422058, + "rewards/forensics_tiou_reward": 0.5541620552539825, + "step": 996 + }, + { + "completion_length": 24.671875, + "epoch": 3.0583717357910905, + "grad_norm": 7.425316333770752, + "kl": 0.4814453125, + "learning_rate": 2.3384615384615382e-07, + "loss": 0.0673, + "reward": 0.7724748253822327, + "reward_std": 0.013040248188190162, + "reward_std_all": 0.026400895323604345, + "rewards-max/forensics_sinkhorn_reward": 0.9817386567592621, + "rewards-max/forensics_strict_edge_reward": 0.5555279552936554, + "rewards-max/forensics_tiou_reward": 0.9797545671463013, + "rewards/forensics_sinkhorn_reward": 0.7363701164722443, + "rewards/forensics_strict_edge_reward": 0.08182285353541374, + "rewards/forensics_tiou_reward": 0.6161706447601318, + "step": 997 + }, + { + "completion_length": 26.203125, + "epoch": 3.0614439324116742, + "grad_norm": 10.400312423706055, + "kl": 0.419921875, + "learning_rate": 2.3307692307692306e-07, + "loss": 0.1159, + "reward": 0.7265462577342987, + "reward_std": 0.0495717516168952, + "reward_std_all": 0.06609014421701431, + "rewards-max/forensics_sinkhorn_reward": 0.9855284094810486, + "rewards-max/forensics_strict_edge_reward": 0.8084049224853516, + "rewards-max/forensics_tiou_reward": 0.9902050197124481, + "rewards/forensics_sinkhorn_reward": 0.709311842918396, + "rewards/forensics_strict_edge_reward": 0.10580169782042503, + "rewards/forensics_tiou_reward": 0.5509473383426666, + "step": 998 + }, + { + "completion_length": 22.609375, + "epoch": 3.064516129032258, + "grad_norm": 8.1467866897583, + "kl": 0.447265625, + "learning_rate": 2.323076923076923e-07, + "loss": -0.0907, + "reward": 0.7957484722137451, + "reward_std": 0.010127892950549722, + "reward_std_all": 0.011932719964534044, + "rewards-max/forensics_sinkhorn_reward": 0.9839544892311096, + "rewards-max/forensics_strict_edge_reward": 0.5416891872882843, + "rewards-max/forensics_tiou_reward": 0.984266459941864, + "rewards/forensics_sinkhorn_reward": 0.8354012072086334, + "rewards/forensics_strict_edge_reward": 0.059518732130527496, + "rewards/forensics_tiou_reward": 0.6868607401847839, + "step": 999 + }, + { + "completion_length": 25.734375, + "epoch": 3.067588325652842, + "grad_norm": 9.670999526977539, + "kl": 0.490234375, + "learning_rate": 2.3153846153846151e-07, + "loss": 0.0904, + "reward": 0.7737699747085571, + "reward_std": 0.04994021728634834, + "reward_std_all": 0.045946208760142326, + "rewards-max/forensics_sinkhorn_reward": 0.9707669615745544, + "rewards-max/forensics_strict_edge_reward": 0.3907497674226761, + "rewards-max/forensics_tiou_reward": 0.9552126526832581, + "rewards/forensics_sinkhorn_reward": 0.7966498136520386, + "rewards/forensics_strict_edge_reward": 0.03949845116585493, + "rewards/forensics_tiou_reward": 0.6548179984092712, + "step": 1000 + }, + { + "completion_length": 24.484375, + "epoch": 3.0706605222734256, + "grad_norm": 8.177851676940918, + "kl": 0.474609375, + "learning_rate": 2.3076923076923078e-07, + "loss": 0.0764, + "reward": 0.7745853960514069, + "reward_std": 0.04571843147277832, + "reward_std_all": 0.04255045764148235, + "rewards-max/forensics_sinkhorn_reward": 0.9764464795589447, + "rewards-max/forensics_strict_edge_reward": 0.6651247441768646, + "rewards-max/forensics_tiou_reward": 0.9694593846797943, + "rewards/forensics_sinkhorn_reward": 0.7150734066963196, + "rewards/forensics_strict_edge_reward": 0.08442704379558563, + "rewards/forensics_tiou_reward": 0.6089848279953003, + "step": 1001 + }, + { + "completion_length": 25.78125, + "epoch": 3.0737327188940093, + "grad_norm": 7.29801082611084, + "kl": 0.3994140625, + "learning_rate": 2.3e-07, + "loss": -0.0214, + "reward": 0.7817808985710144, + "reward_std": 0.02090227254666388, + "reward_std_all": 0.023223411291837692, + "rewards-max/forensics_sinkhorn_reward": 0.9791323840618134, + "rewards-max/forensics_strict_edge_reward": 0.5484330058097839, + "rewards-max/forensics_tiou_reward": 0.974367767572403, + "rewards/forensics_sinkhorn_reward": 0.7811284065246582, + "rewards/forensics_strict_edge_reward": 0.07142572849988937, + "rewards/forensics_tiou_reward": 0.6102317273616791, + "step": 1002 + }, + { + "completion_length": 26.359375, + "epoch": 3.076804915514593, + "grad_norm": 6.9284257888793945, + "kl": 0.4189453125, + "learning_rate": 2.2923076923076924e-07, + "loss": 0.0824, + "reward": 0.7904230356216431, + "reward_std": 0.018777651013806462, + "reward_std_all": 0.019257127307355404, + "rewards-max/forensics_sinkhorn_reward": 0.9664939641952515, + "rewards-max/forensics_strict_edge_reward": 0.4506318047642708, + "rewards-max/forensics_tiou_reward": 0.9435725808143616, + "rewards/forensics_sinkhorn_reward": 0.8298375904560089, + "rewards/forensics_strict_edge_reward": 0.11049296241253614, + "rewards/forensics_tiou_reward": 0.7209715843200684, + "step": 1003 + }, + { + "completion_length": 25.5, + "epoch": 3.0798771121351765, + "grad_norm": 6.842878818511963, + "kl": 0.4091796875, + "learning_rate": 2.2846153846153845e-07, + "loss": 0.0647, + "reward": 0.7843631207942963, + "reward_std": 0.03361785924062133, + "reward_std_all": 0.031592180486768484, + "rewards-max/forensics_sinkhorn_reward": 0.9759887456893921, + "rewards-max/forensics_strict_edge_reward": 0.7224518060684204, + "rewards-max/forensics_tiou_reward": 0.9720679223537445, + "rewards/forensics_sinkhorn_reward": 0.7476185262203217, + "rewards/forensics_strict_edge_reward": 0.07779887225478888, + "rewards/forensics_tiou_reward": 0.5926875472068787, + "step": 1004 + }, + { + "completion_length": 23.265625, + "epoch": 3.0829493087557602, + "grad_norm": 8.614652633666992, + "kl": 0.447265625, + "learning_rate": 2.276923076923077e-07, + "loss": -0.0363, + "reward": 0.7889849543571472, + "reward_std": 0.017108963802456856, + "reward_std_all": 0.018331191036850214, + "rewards-max/forensics_sinkhorn_reward": 0.9815499782562256, + "rewards-max/forensics_strict_edge_reward": 0.6483739018440247, + "rewards-max/forensics_tiou_reward": 0.9789632260799408, + "rewards/forensics_sinkhorn_reward": 0.784428209066391, + "rewards/forensics_strict_edge_reward": 0.09749317914247513, + "rewards/forensics_tiou_reward": 0.6884097158908844, + "step": 1005 + }, + { + "completion_length": 24.359375, + "epoch": 3.086021505376344, + "grad_norm": 7.887603282928467, + "kl": 0.419921875, + "learning_rate": 2.269230769230769e-07, + "loss": -0.0224, + "reward": 0.79995197057724, + "reward_std": 0.010221235454082489, + "reward_std_all": 0.010861736722290516, + "rewards-max/forensics_sinkhorn_reward": 0.9721891582012177, + "rewards-max/forensics_strict_edge_reward": 0.38722455501556396, + "rewards-max/forensics_tiou_reward": 0.9510970413684845, + "rewards/forensics_sinkhorn_reward": 0.80595862865448, + "rewards/forensics_strict_edge_reward": 0.03159327805042267, + "rewards/forensics_tiou_reward": 0.6059384644031525, + "step": 1006 + }, + { + "completion_length": 28.609375, + "epoch": 3.089093701996928, + "grad_norm": 6.935842037200928, + "kl": 0.3515625, + "learning_rate": 2.2615384615384614e-07, + "loss": -0.0302, + "reward": 0.7876834273338318, + "reward_std": 0.03210638742893934, + "reward_std_all": 0.029809920117259026, + "rewards-max/forensics_sinkhorn_reward": 0.9771892130374908, + "rewards-max/forensics_strict_edge_reward": 0.5010702610015869, + "rewards-max/forensics_tiou_reward": 0.9680319130420685, + "rewards/forensics_sinkhorn_reward": 0.8084287941455841, + "rewards/forensics_strict_edge_reward": 0.05368587374687195, + "rewards/forensics_tiou_reward": 0.6664169132709503, + "step": 1007 + }, + { + "completion_length": 23.234375, + "epoch": 3.0921658986175116, + "grad_norm": 6.961524963378906, + "kl": 0.4443359375, + "learning_rate": 2.2538461538461536e-07, + "loss": -0.0859, + "reward": 0.7366972267627716, + "reward_std": 0.012725755921564996, + "reward_std_all": 0.04367813654243946, + "rewards-max/forensics_sinkhorn_reward": 0.9759908020496368, + "rewards-max/forensics_strict_edge_reward": 0.564366340637207, + "rewards-max/forensics_tiou_reward": 0.9628294408321381, + "rewards/forensics_sinkhorn_reward": 0.7684182524681091, + "rewards/forensics_strict_edge_reward": 0.10583210177719593, + "rewards/forensics_tiou_reward": 0.6260092556476593, + "step": 1008 + }, + { + "completion_length": 24.421875, + "epoch": 3.0952380952380953, + "grad_norm": 6.493067741394043, + "kl": 0.400390625, + "learning_rate": 2.246153846153846e-07, + "loss": 0.0531, + "reward": 0.7642146050930023, + "reward_std": 0.0440933839417994, + "reward_std_all": 0.04575183195993304, + "rewards-max/forensics_sinkhorn_reward": 0.9781408309936523, + "rewards-max/forensics_strict_edge_reward": 0.5404154062271118, + "rewards-max/forensics_tiou_reward": 0.9724633097648621, + "rewards/forensics_sinkhorn_reward": 0.7462518215179443, + "rewards/forensics_strict_edge_reward": 0.07376604527235031, + "rewards/forensics_tiou_reward": 0.6260902285575867, + "step": 1009 + }, + { + "completion_length": 26.015625, + "epoch": 3.098310291858679, + "grad_norm": 7.213451385498047, + "kl": 0.3701171875, + "learning_rate": 2.238461538461538e-07, + "loss": -0.0373, + "reward": 0.7967692613601685, + "reward_std": 0.017157122492790222, + "reward_std_all": 0.01674132887274027, + "rewards-max/forensics_sinkhorn_reward": 0.9863815009593964, + "rewards-max/forensics_strict_edge_reward": 0.8469520509243011, + "rewards-max/forensics_tiou_reward": 0.9923279881477356, + "rewards/forensics_sinkhorn_reward": 0.8284235596656799, + "rewards/forensics_strict_edge_reward": 0.1125289797782898, + "rewards/forensics_tiou_reward": 0.6944003105163574, + "step": 1010 + }, + { + "completion_length": 27.125, + "epoch": 3.1013824884792625, + "grad_norm": 6.680727481842041, + "kl": 0.421875, + "learning_rate": 2.2307692307692308e-07, + "loss": 0.033, + "reward": 0.7758775353431702, + "reward_std": 0.04469330422580242, + "reward_std_all": 0.04156235605478287, + "rewards-max/forensics_sinkhorn_reward": 0.9786031246185303, + "rewards-max/forensics_strict_edge_reward": 0.6798536479473114, + "rewards-max/forensics_tiou_reward": 0.9689613580703735, + "rewards/forensics_sinkhorn_reward": 0.8037994503974915, + "rewards/forensics_strict_edge_reward": 0.1053139865398407, + "rewards/forensics_tiou_reward": 0.6961689591407776, + "step": 1011 + }, + { + "completion_length": 21.671875, + "epoch": 3.1044546850998462, + "grad_norm": 10.150468826293945, + "kl": 0.3759765625, + "learning_rate": 2.2230769230769232e-07, + "loss": -0.0878, + "reward": 0.8038762807846069, + "reward_std": 0.0037477646255865693, + "reward_std_all": 0.005338860675692558, + "rewards-max/forensics_sinkhorn_reward": 0.9764149785041809, + "rewards-max/forensics_strict_edge_reward": 0.528624102473259, + "rewards-max/forensics_tiou_reward": 0.9647823870182037, + "rewards/forensics_sinkhorn_reward": 0.8219843506813049, + "rewards/forensics_strict_edge_reward": 0.08689145650714636, + "rewards/forensics_tiou_reward": 0.6293389201164246, + "step": 1012 + }, + { + "completion_length": 24.96875, + "epoch": 3.10752688172043, + "grad_norm": 7.583367824554443, + "kl": 0.419921875, + "learning_rate": 2.2153846153846153e-07, + "loss": -0.0821, + "reward": 0.762684166431427, + "reward_std": 0.0086094222497195, + "reward_std_all": 0.027523303753696382, + "rewards-max/forensics_sinkhorn_reward": 0.9876669943332672, + "rewards-max/forensics_strict_edge_reward": 0.756708562374115, + "rewards-max/forensics_tiou_reward": 0.9965049922466278, + "rewards/forensics_sinkhorn_reward": 0.8027747273445129, + "rewards/forensics_strict_edge_reward": 0.10292623192071915, + "rewards/forensics_tiou_reward": 0.6588704586029053, + "step": 1013 + }, + { + "completion_length": 27.1875, + "epoch": 3.110599078341014, + "grad_norm": 6.656486988067627, + "kl": 0.388671875, + "learning_rate": 2.2076923076923077e-07, + "loss": 0.0135, + "reward": 0.7940137386322021, + "reward_std": 0.016506826505064964, + "reward_std_all": 0.01681004511192441, + "rewards-max/forensics_sinkhorn_reward": 0.9832956194877625, + "rewards-max/forensics_strict_edge_reward": 0.5599738955497742, + "rewards-max/forensics_tiou_reward": 0.9855867624282837, + "rewards/forensics_sinkhorn_reward": 0.8281154930591583, + "rewards/forensics_strict_edge_reward": 0.06505756266415119, + "rewards/forensics_tiou_reward": 0.685526967048645, + "step": 1014 + }, + { + "completion_length": 25.96875, + "epoch": 3.1136712749615976, + "grad_norm": 7.661181449890137, + "kl": 0.583984375, + "learning_rate": 2.1999999999999998e-07, + "loss": 0.0499, + "reward": 0.7897790968418121, + "reward_std": 0.02950905915349722, + "reward_std_all": 0.02739072684198618, + "rewards-max/forensics_sinkhorn_reward": 0.9810923933982849, + "rewards-max/forensics_strict_edge_reward": 0.5919938534498215, + "rewards-max/forensics_tiou_reward": 0.9747809171676636, + "rewards/forensics_sinkhorn_reward": 0.8326658010482788, + "rewards/forensics_strict_edge_reward": 0.0681152418255806, + "rewards/forensics_tiou_reward": 0.7119101583957672, + "step": 1015 + }, + { + "completion_length": 26.5625, + "epoch": 3.1167434715821813, + "grad_norm": 10.335834503173828, + "kl": 0.4697265625, + "learning_rate": 2.1923076923076922e-07, + "loss": 0.0555, + "reward": 0.7926134765148163, + "reward_std": 0.015647117979824543, + "reward_std_all": 0.01643810933455825, + "rewards-max/forensics_sinkhorn_reward": 0.9761539399623871, + "rewards-max/forensics_strict_edge_reward": 0.6617338359355927, + "rewards-max/forensics_tiou_reward": 0.9679350852966309, + "rewards/forensics_sinkhorn_reward": 0.8052689433097839, + "rewards/forensics_strict_edge_reward": 0.06979472190141678, + "rewards/forensics_tiou_reward": 0.706635981798172, + "step": 1016 + }, + { + "completion_length": 26.1875, + "epoch": 3.119815668202765, + "grad_norm": 7.668642997741699, + "kl": 0.462890625, + "learning_rate": 2.1846153846153844e-07, + "loss": 0.0099, + "reward": 0.7934512197971344, + "reward_std": 0.016125629073940217, + "reward_std_all": 0.017405446153134108, + "rewards-max/forensics_sinkhorn_reward": 0.9692217409610748, + "rewards-max/forensics_strict_edge_reward": 0.5243836045265198, + "rewards-max/forensics_tiou_reward": 0.9449222087860107, + "rewards/forensics_sinkhorn_reward": 0.8027500510215759, + "rewards/forensics_strict_edge_reward": 0.042225511744618416, + "rewards/forensics_tiou_reward": 0.6313093900680542, + "step": 1017 + }, + { + "completion_length": 26.453125, + "epoch": 3.122887864823349, + "grad_norm": 6.215120792388916, + "kl": 0.416015625, + "learning_rate": 2.1769230769230768e-07, + "loss": -0.0397, + "reward": 0.7889706492424011, + "reward_std": 0.030748792923986912, + "reward_std_all": 0.028353894129395485, + "rewards-max/forensics_sinkhorn_reward": 0.9789191484451294, + "rewards-max/forensics_strict_edge_reward": 0.5311143696308136, + "rewards-max/forensics_tiou_reward": 0.9760937988758087, + "rewards/forensics_sinkhorn_reward": 0.8140883445739746, + "rewards/forensics_strict_edge_reward": 0.07392418757081032, + "rewards/forensics_tiou_reward": 0.6962143182754517, + "step": 1018 + }, + { + "completion_length": 27.203125, + "epoch": 3.1259600614439322, + "grad_norm": 7.190629005432129, + "kl": 0.56640625, + "learning_rate": 2.169230769230769e-07, + "loss": 0.0407, + "reward": 0.7810890376567841, + "reward_std": 0.014881220646202564, + "reward_std_all": 0.022906995378434658, + "rewards-max/forensics_sinkhorn_reward": 0.9755865931510925, + "rewards-max/forensics_strict_edge_reward": 0.6331115067005157, + "rewards-max/forensics_tiou_reward": 0.9683420360088348, + "rewards/forensics_sinkhorn_reward": 0.7636039555072784, + "rewards/forensics_strict_edge_reward": 0.08767953142523766, + "rewards/forensics_tiou_reward": 0.608975738286972, + "step": 1019 + }, + { + "completion_length": 27.828125, + "epoch": 3.129032258064516, + "grad_norm": 6.990902900695801, + "kl": 0.4912109375, + "learning_rate": 2.1615384615384613e-07, + "loss": 0.0134, + "reward": 0.7792902290821075, + "reward_std": 0.02390267699956894, + "reward_std_all": 0.025998347904533148, + "rewards-max/forensics_sinkhorn_reward": 0.9848239123821259, + "rewards-max/forensics_strict_edge_reward": 0.3762178048491478, + "rewards-max/forensics_tiou_reward": 0.9852479100227356, + "rewards/forensics_sinkhorn_reward": 0.7928368151187897, + "rewards/forensics_strict_edge_reward": 0.04998097848147154, + "rewards/forensics_tiou_reward": 0.6644915044307709, + "step": 1020 + }, + { + "completion_length": 26.875, + "epoch": 3.1321044546851, + "grad_norm": 8.408106803894043, + "kl": 0.55859375, + "learning_rate": 2.153846153846154e-07, + "loss": 0.0767, + "reward": 0.7562015056610107, + "reward_std": 0.054890797240659595, + "reward_std_all": 0.05374937877058983, + "rewards-max/forensics_sinkhorn_reward": 0.9757615923881531, + "rewards-max/forensics_strict_edge_reward": 0.5273320078849792, + "rewards-max/forensics_tiou_reward": 0.9687017500400543, + "rewards/forensics_sinkhorn_reward": 0.7207038998603821, + "rewards/forensics_strict_edge_reward": 0.04529947694391012, + "rewards/forensics_tiou_reward": 0.5858631730079651, + "step": 1021 + }, + { + "completion_length": 24.3125, + "epoch": 3.1351766513056836, + "grad_norm": 8.159573554992676, + "kl": 0.38671875, + "learning_rate": 2.146153846153846e-07, + "loss": -0.1036, + "reward": 0.7718049883842468, + "reward_std": 0.037980444729328156, + "reward_std_all": 0.03800486959517002, + "rewards-max/forensics_sinkhorn_reward": 0.957023948431015, + "rewards-max/forensics_strict_edge_reward": 0.3298901841044426, + "rewards-max/forensics_tiou_reward": 0.9210334718227386, + "rewards/forensics_sinkhorn_reward": 0.7302446961402893, + "rewards/forensics_strict_edge_reward": 0.040679833851754665, + "rewards/forensics_tiou_reward": 0.5918581187725067, + "step": 1022 + }, + { + "completion_length": 26.953125, + "epoch": 3.1382488479262673, + "grad_norm": 8.555658340454102, + "kl": 0.767578125, + "learning_rate": 2.1384615384615385e-07, + "loss": -0.0014, + "reward": 0.7948299050331116, + "reward_std": 0.011712431733030826, + "reward_std_all": 0.01325845846440643, + "rewards-max/forensics_sinkhorn_reward": 0.9795923233032227, + "rewards-max/forensics_strict_edge_reward": 0.5865857601165771, + "rewards-max/forensics_tiou_reward": 0.9743378460407257, + "rewards/forensics_sinkhorn_reward": 0.7986903488636017, + "rewards/forensics_strict_edge_reward": 0.0847470872104168, + "rewards/forensics_tiou_reward": 0.6245011389255524, + "step": 1023 + }, + { + "completion_length": 25.640625, + "epoch": 3.141321044546851, + "grad_norm": 7.868666648864746, + "kl": 0.4248046875, + "learning_rate": 2.1307692307692307e-07, + "loss": -0.0523, + "reward": 0.7906572818756104, + "reward_std": 0.026796068297699094, + "reward_std_all": 0.025135920150205493, + "rewards-max/forensics_sinkhorn_reward": 0.9722594916820526, + "rewards-max/forensics_strict_edge_reward": 0.2759946882724762, + "rewards-max/forensics_tiou_reward": 0.955478698015213, + "rewards/forensics_sinkhorn_reward": 0.8083164989948273, + "rewards/forensics_strict_edge_reward": 0.04861003300175071, + "rewards/forensics_tiou_reward": 0.6561229228973389, + "step": 1024 + }, + { + "completion_length": 21.53125, + "epoch": 3.144393241167435, + "grad_norm": 7.69385290145874, + "kl": 0.45703125, + "learning_rate": 2.123076923076923e-07, + "loss": 0.0116, + "reward": 0.7768125832080841, + "reward_std": 0.026595043018460274, + "reward_std_all": 0.02824302576482296, + "rewards-max/forensics_sinkhorn_reward": 0.9867472648620605, + "rewards-max/forensics_strict_edge_reward": 0.5669038891792297, + "rewards-max/forensics_tiou_reward": 0.992876410484314, + "rewards/forensics_sinkhorn_reward": 0.8141984045505524, + "rewards/forensics_strict_edge_reward": 0.07904010079801083, + "rewards/forensics_tiou_reward": 0.6555732488632202, + "step": 1025 + }, + { + "completion_length": 28.40625, + "epoch": 3.1474654377880182, + "grad_norm": 8.237658500671387, + "kl": 0.626953125, + "learning_rate": 2.1153846153846152e-07, + "loss": 0.0215, + "reward": 0.783410906791687, + "reward_std": 0.03655404644086957, + "reward_std_all": 0.03406071290373802, + "rewards-max/forensics_sinkhorn_reward": 0.9839868247509003, + "rewards-max/forensics_strict_edge_reward": 0.4500138610601425, + "rewards-max/forensics_tiou_reward": 0.9825803339481354, + "rewards/forensics_sinkhorn_reward": 0.7550501227378845, + "rewards/forensics_strict_edge_reward": 0.04518595710396767, + "rewards/forensics_tiou_reward": 0.6096594929695129, + "step": 1026 + }, + { + "completion_length": 25.796875, + "epoch": 3.150537634408602, + "grad_norm": 8.216240882873535, + "kl": 0.654296875, + "learning_rate": 2.1076923076923076e-07, + "loss": -0.0266, + "reward": 0.8044014573097229, + "reward_std": 0.0032162938732653856, + "reward_std_all": 0.00480616232380271, + "rewards-max/forensics_sinkhorn_reward": 0.9788248836994171, + "rewards-max/forensics_strict_edge_reward": 0.4699527621269226, + "rewards-max/forensics_tiou_reward": 0.9735395014286041, + "rewards/forensics_sinkhorn_reward": 0.801484614610672, + "rewards/forensics_strict_edge_reward": 0.06460425816476345, + "rewards/forensics_tiou_reward": 0.6566577553749084, + "step": 1027 + }, + { + "completion_length": 21.8125, + "epoch": 3.153609831029186, + "grad_norm": 10.00446605682373, + "kl": 0.396484375, + "learning_rate": 2.0999999999999997e-07, + "loss": 0.0041, + "reward": 0.7958729863166809, + "reward_std": 0.017613474861718714, + "reward_std_all": 0.017338739475235343, + "rewards-max/forensics_sinkhorn_reward": 0.9659070372581482, + "rewards-max/forensics_strict_edge_reward": 0.402368500828743, + "rewards-max/forensics_tiou_reward": 0.9449180364608765, + "rewards/forensics_sinkhorn_reward": 0.8098824620246887, + "rewards/forensics_strict_edge_reward": 0.04434182867407799, + "rewards/forensics_tiou_reward": 0.6363314390182495, + "step": 1028 + }, + { + "completion_length": 26.796875, + "epoch": 3.1566820276497696, + "grad_norm": 9.103935241699219, + "kl": 0.46484375, + "learning_rate": 2.0923076923076921e-07, + "loss": -0.0196, + "reward": 0.7847104370594025, + "reward_std": 0.028968666680157185, + "reward_std_all": 0.02806604653596878, + "rewards-max/forensics_sinkhorn_reward": 0.9842043519020081, + "rewards-max/forensics_strict_edge_reward": 0.5745002031326294, + "rewards-max/forensics_tiou_reward": 0.9906452298164368, + "rewards/forensics_sinkhorn_reward": 0.7967678904533386, + "rewards/forensics_strict_edge_reward": 0.07794812880456448, + "rewards/forensics_tiou_reward": 0.6519563794136047, + "step": 1029 + }, + { + "completion_length": 24.71875, + "epoch": 3.1597542242703534, + "grad_norm": 8.362608909606934, + "kl": 0.3427734375, + "learning_rate": 2.0846153846153843e-07, + "loss": -0.0878, + "reward": 0.777361661195755, + "reward_std": 0.00283744465559721, + "reward_std_all": 0.01771081890910864, + "rewards-max/forensics_sinkhorn_reward": 0.9873034656047821, + "rewards-max/forensics_strict_edge_reward": 0.6614123284816742, + "rewards-max/forensics_tiou_reward": 0.9950830340385437, + "rewards/forensics_sinkhorn_reward": 0.8154266178607941, + "rewards/forensics_strict_edge_reward": 0.11333327740430832, + "rewards/forensics_tiou_reward": 0.6954226195812225, + "step": 1030 + }, + { + "completion_length": 25.109375, + "epoch": 3.162826420890937, + "grad_norm": 6.851166248321533, + "kl": 0.439453125, + "learning_rate": 2.076923076923077e-07, + "loss": -0.0828, + "reward": 0.7969793975353241, + "reward_std": 0.018040830618701875, + "reward_std_all": 0.017208204371854663, + "rewards-max/forensics_sinkhorn_reward": 0.9782276749610901, + "rewards-max/forensics_strict_edge_reward": 0.6236754059791565, + "rewards-max/forensics_tiou_reward": 0.9760043025016785, + "rewards/forensics_sinkhorn_reward": 0.8541749715805054, + "rewards/forensics_strict_edge_reward": 0.12085682898759842, + "rewards/forensics_tiou_reward": 0.7442790269851685, + "step": 1031 + }, + { + "completion_length": 27.84375, + "epoch": 3.165898617511521, + "grad_norm": 6.975358486175537, + "kl": 0.3544921875, + "learning_rate": 2.0692307692307694e-07, + "loss": 0.0755, + "reward": 0.7793696522712708, + "reward_std": 0.03444003313779831, + "reward_std_all": 0.03292710892856121, + "rewards-max/forensics_sinkhorn_reward": 0.9857302010059357, + "rewards-max/forensics_strict_edge_reward": 0.7630031704902649, + "rewards-max/forensics_tiou_reward": 0.9938024282455444, + "rewards/forensics_sinkhorn_reward": 0.7846494019031525, + "rewards/forensics_strict_edge_reward": 0.10169463232159615, + "rewards/forensics_tiou_reward": 0.6522201299667358, + "step": 1032 + }, + { + "completion_length": 24.40625, + "epoch": 3.1689708141321047, + "grad_norm": 9.818446159362793, + "kl": 0.4296875, + "learning_rate": 2.0615384615384615e-07, + "loss": -0.0039, + "reward": 0.7281290292739868, + "reward_std": 0.04177575744688511, + "reward_std_all": 0.05871212296187878, + "rewards-max/forensics_sinkhorn_reward": 0.9867121577262878, + "rewards-max/forensics_strict_edge_reward": 0.7071828544139862, + "rewards-max/forensics_tiou_reward": 0.9943590760231018, + "rewards/forensics_sinkhorn_reward": 0.7208279967308044, + "rewards/forensics_strict_edge_reward": 0.07134521380066872, + "rewards/forensics_tiou_reward": 0.5536547005176544, + "step": 1033 + }, + { + "completion_length": 25.515625, + "epoch": 3.172043010752688, + "grad_norm": 8.202656745910645, + "kl": 0.55078125, + "learning_rate": 2.053846153846154e-07, + "loss": 0.0402, + "reward": 0.7689925730228424, + "reward_std": 0.04203528445214033, + "reward_std_all": 0.04147730953991413, + "rewards-max/forensics_sinkhorn_reward": 0.981253445148468, + "rewards-max/forensics_strict_edge_reward": 0.3484451398253441, + "rewards-max/forensics_tiou_reward": 0.979374885559082, + "rewards/forensics_sinkhorn_reward": 0.7786934077739716, + "rewards/forensics_strict_edge_reward": 0.04581030365079641, + "rewards/forensics_tiou_reward": 0.6157251596450806, + "step": 1034 + }, + { + "completion_length": 24.109375, + "epoch": 3.175115207373272, + "grad_norm": 7.258979320526123, + "kl": 0.439453125, + "learning_rate": 2.046153846153846e-07, + "loss": 0.012, + "reward": 0.7877682447433472, + "reward_std": 0.023249669116921723, + "reward_std_all": 0.023461457807570696, + "rewards-max/forensics_sinkhorn_reward": 0.9749146699905396, + "rewards-max/forensics_strict_edge_reward": 0.3420091662555933, + "rewards-max/forensics_tiou_reward": 0.9627169668674469, + "rewards/forensics_sinkhorn_reward": 0.7842714488506317, + "rewards/forensics_strict_edge_reward": 0.07512386457528919, + "rewards/forensics_tiou_reward": 0.6199392676353455, + "step": 1035 + }, + { + "completion_length": 25.375, + "epoch": 3.1781874039938556, + "grad_norm": 7.346764087677002, + "kl": 0.369140625, + "learning_rate": 2.0384615384615384e-07, + "loss": -0.0151, + "reward": 0.7197502553462982, + "reward_std": 0.03408351447433233, + "reward_std_all": 0.05847952328622341, + "rewards-max/forensics_sinkhorn_reward": 0.9763510525226593, + "rewards-max/forensics_strict_edge_reward": 0.5860248506069183, + "rewards-max/forensics_tiou_reward": 0.9674238562583923, + "rewards/forensics_sinkhorn_reward": 0.7157497107982635, + "rewards/forensics_strict_edge_reward": 0.06877275928854942, + "rewards/forensics_tiou_reward": 0.5528600811958313, + "step": 1036 + }, + { + "completion_length": 32.484375, + "epoch": 3.1812596006144394, + "grad_norm": 6.068269729614258, + "kl": 0.388671875, + "learning_rate": 2.0307692307692306e-07, + "loss": -0.0363, + "reward": 0.78689044713974, + "reward_std": 0.018316076428163797, + "reward_std_all": 0.020092529244720936, + "rewards-max/forensics_sinkhorn_reward": 0.9788866937160492, + "rewards-max/forensics_strict_edge_reward": 0.41882238537073135, + "rewards-max/forensics_tiou_reward": 0.9780600666999817, + "rewards/forensics_sinkhorn_reward": 0.7735452353954315, + "rewards/forensics_strict_edge_reward": 0.06451116502285004, + "rewards/forensics_tiou_reward": 0.6561781466007233, + "step": 1037 + }, + { + "completion_length": 30.921875, + "epoch": 3.184331797235023, + "grad_norm": 6.493027687072754, + "kl": 0.486328125, + "learning_rate": 2.023076923076923e-07, + "loss": -0.0553, + "reward": 0.7917705476284027, + "reward_std": 0.02458127122372389, + "reward_std_all": 0.023292982950806618, + "rewards-max/forensics_sinkhorn_reward": 0.9783341884613037, + "rewards-max/forensics_strict_edge_reward": 0.5185752790421247, + "rewards-max/forensics_tiou_reward": 0.9696072041988373, + "rewards/forensics_sinkhorn_reward": 0.7736946046352386, + "rewards/forensics_strict_edge_reward": 0.07251899130642414, + "rewards/forensics_tiou_reward": 0.6634118854999542, + "step": 1038 + }, + { + "completion_length": 26.953125, + "epoch": 3.187403993855607, + "grad_norm": 7.988318920135498, + "kl": 0.5625, + "learning_rate": 2.015384615384615e-07, + "loss": -0.0098, + "reward": 0.7798730432987213, + "reward_std": 0.015260086511261761, + "reward_std_all": 0.020557350013405085, + "rewards-max/forensics_sinkhorn_reward": 0.9817184209823608, + "rewards-max/forensics_strict_edge_reward": 0.5263189375400543, + "rewards-max/forensics_tiou_reward": 0.9853558838367462, + "rewards/forensics_sinkhorn_reward": 0.7969596982002258, + "rewards/forensics_strict_edge_reward": 0.05696333944797516, + "rewards/forensics_tiou_reward": 0.6684344708919525, + "step": 1039 + }, + { + "completion_length": 27.15625, + "epoch": 3.1904761904761907, + "grad_norm": 7.932523727416992, + "kl": 0.4072265625, + "learning_rate": 2.0076923076923075e-07, + "loss": 0.0352, + "reward": 0.7679756581783295, + "reward_std": 0.027791614644229412, + "reward_std_all": 0.036415185779333115, + "rewards-max/forensics_sinkhorn_reward": 0.9818820059299469, + "rewards-max/forensics_strict_edge_reward": 0.49686121940612793, + "rewards-max/forensics_tiou_reward": 0.9843567311763763, + "rewards/forensics_sinkhorn_reward": 0.7511827647686005, + "rewards/forensics_strict_edge_reward": 0.07109191827476025, + "rewards/forensics_tiou_reward": 0.5835441052913666, + "step": 1040 + }, + { + "completion_length": 26.546875, + "epoch": 3.193548387096774, + "grad_norm": 7.270925521850586, + "kl": 0.3876953125, + "learning_rate": 2e-07, + "loss": 0.0571, + "reward": 0.780025988817215, + "reward_std": 0.03121372638270259, + "reward_std_all": 0.030728166224434972, + "rewards-max/forensics_sinkhorn_reward": 0.9840046465396881, + "rewards-max/forensics_strict_edge_reward": 0.6070642471313477, + "rewards-max/forensics_tiou_reward": 0.9871630668640137, + "rewards/forensics_sinkhorn_reward": 0.7626671195030212, + "rewards/forensics_strict_edge_reward": 0.09128066711127758, + "rewards/forensics_tiou_reward": 0.628032922744751, + "step": 1041 + }, + { + "completion_length": 27.5625, + "epoch": 3.196620583717358, + "grad_norm": 6.843864917755127, + "kl": 0.4521484375, + "learning_rate": 1.9923076923076923e-07, + "loss": -0.0067, + "reward": 0.7595220804214478, + "reward_std": 0.03927176725119352, + "reward_std_all": 0.04443201422691345, + "rewards-max/forensics_sinkhorn_reward": 0.976573646068573, + "rewards-max/forensics_strict_edge_reward": 0.4478273540735245, + "rewards-max/forensics_tiou_reward": 0.9678532779216766, + "rewards/forensics_sinkhorn_reward": 0.7432199120521545, + "rewards/forensics_strict_edge_reward": 0.09019119944423437, + "rewards/forensics_tiou_reward": 0.6356484293937683, + "step": 1042 + }, + { + "completion_length": 22.640625, + "epoch": 3.1996927803379416, + "grad_norm": 8.24479866027832, + "kl": 0.421875, + "learning_rate": 1.9846153846153844e-07, + "loss": -0.0927, + "reward": 0.7856774926185608, + "reward_std": 0.019343548105098307, + "reward_std_all": 0.021809590398333967, + "rewards-max/forensics_sinkhorn_reward": 0.9831097722053528, + "rewards-max/forensics_strict_edge_reward": 0.797152578830719, + "rewards-max/forensics_tiou_reward": 0.983992874622345, + "rewards/forensics_sinkhorn_reward": 0.8053874373435974, + "rewards/forensics_strict_edge_reward": 0.12855566293001175, + "rewards/forensics_tiou_reward": 0.6160460114479065, + "step": 1043 + }, + { + "completion_length": 29.734375, + "epoch": 3.2027649769585254, + "grad_norm": 7.277317047119141, + "kl": 0.3427734375, + "learning_rate": 1.9769230769230768e-07, + "loss": 0.0235, + "reward": 0.7988989651203156, + "reward_std": 0.013488799566403031, + "reward_std_all": 0.01358279213309288, + "rewards-max/forensics_sinkhorn_reward": 0.9769331812858582, + "rewards-max/forensics_strict_edge_reward": 0.6988634467124939, + "rewards-max/forensics_tiou_reward": 0.9724652171134949, + "rewards/forensics_sinkhorn_reward": 0.8330403566360474, + "rewards/forensics_strict_edge_reward": 0.09583259746432304, + "rewards/forensics_tiou_reward": 0.7023484706878662, + "step": 1044 + }, + { + "completion_length": 22.859375, + "epoch": 3.205837173579109, + "grad_norm": 7.099738121032715, + "kl": 0.515625, + "learning_rate": 1.9692307692307693e-07, + "loss": 0.072, + "reward": 0.7300364375114441, + "reward_std": 0.0443996787071228, + "reward_std_all": 0.06201275251805782, + "rewards-max/forensics_sinkhorn_reward": 0.9863313138484955, + "rewards-max/forensics_strict_edge_reward": 0.6204805672168732, + "rewards-max/forensics_tiou_reward": 0.9914630651473999, + "rewards/forensics_sinkhorn_reward": 0.7023410201072693, + "rewards/forensics_strict_edge_reward": 0.09466425329446793, + "rewards/forensics_tiou_reward": 0.5560455620288849, + "step": 1045 + }, + { + "completion_length": 24.953125, + "epoch": 3.208909370199693, + "grad_norm": 8.117566108703613, + "kl": 0.439453125, + "learning_rate": 1.9615384615384614e-07, + "loss": -0.0155, + "reward": 0.7417314350605011, + "reward_std": 0.036523159593343735, + "reward_std_all": 0.05126197636127472, + "rewards-max/forensics_sinkhorn_reward": 0.9833679795265198, + "rewards-max/forensics_strict_edge_reward": 0.6342070251703262, + "rewards-max/forensics_tiou_reward": 0.9849042296409607, + "rewards/forensics_sinkhorn_reward": 0.7667313516139984, + "rewards/forensics_strict_edge_reward": 0.09418193995952606, + "rewards/forensics_tiou_reward": 0.6483368873596191, + "step": 1046 + }, + { + "completion_length": 23.03125, + "epoch": 3.2119815668202767, + "grad_norm": 8.566519737243652, + "kl": 0.482421875, + "learning_rate": 1.9538461538461538e-07, + "loss": 0.0143, + "reward": 0.795174777507782, + "reward_std": 0.019701626617461443, + "reward_std_all": 0.018990156706422567, + "rewards-max/forensics_sinkhorn_reward": 0.9821696579456329, + "rewards-max/forensics_strict_edge_reward": 0.6628009527921677, + "rewards-max/forensics_tiou_reward": 0.9739755094051361, + "rewards/forensics_sinkhorn_reward": 0.8245727717876434, + "rewards/forensics_strict_edge_reward": 0.11750605516135693, + "rewards/forensics_tiou_reward": 0.6673066020011902, + "step": 1047 + }, + { + "completion_length": 24.140625, + "epoch": 3.21505376344086, + "grad_norm": 7.320967674255371, + "kl": 0.3984375, + "learning_rate": 1.946153846153846e-07, + "loss": -0.0147, + "reward": 0.7752376198768616, + "reward_std": 0.032065792474895716, + "reward_std_all": 0.03357183141633868, + "rewards-max/forensics_sinkhorn_reward": 0.9876157641410828, + "rewards-max/forensics_strict_edge_reward": 0.6839397251605988, + "rewards-max/forensics_tiou_reward": 0.995990663766861, + "rewards/forensics_sinkhorn_reward": 0.7891585826873779, + "rewards/forensics_strict_edge_reward": 0.07713060081005096, + "rewards/forensics_tiou_reward": 0.6417461037635803, + "step": 1048 + }, + { + "completion_length": 26.609375, + "epoch": 3.218125960061444, + "grad_norm": 5.979068279266357, + "kl": 0.33203125, + "learning_rate": 1.9384615384615383e-07, + "loss": 0.0111, + "reward": 0.780613511800766, + "reward_std": 0.027516813715919852, + "reward_std_all": 0.027967411326244473, + "rewards-max/forensics_sinkhorn_reward": 0.970659464597702, + "rewards-max/forensics_strict_edge_reward": 0.5441151857376099, + "rewards-max/forensics_tiou_reward": 0.9574291408061981, + "rewards/forensics_sinkhorn_reward": 0.736333817243576, + "rewards/forensics_strict_edge_reward": 0.08806504681706429, + "rewards/forensics_tiou_reward": 0.5843434035778046, + "step": 1049 + }, + { + "completion_length": 28.296875, + "epoch": 3.2211981566820276, + "grad_norm": 6.248041152954102, + "kl": 0.369140625, + "learning_rate": 1.9307692307692307e-07, + "loss": 0.0398, + "reward": 0.7749359905719757, + "reward_std": 0.03150477726012468, + "reward_std_all": 0.03206595312803984, + "rewards-max/forensics_sinkhorn_reward": 0.9876964390277863, + "rewards-max/forensics_strict_edge_reward": 0.7957578003406525, + "rewards-max/forensics_tiou_reward": 0.9977405071258545, + "rewards/forensics_sinkhorn_reward": 0.8398092687129974, + "rewards/forensics_strict_edge_reward": 0.14444272965192795, + "rewards/forensics_tiou_reward": 0.7442134618759155, + "step": 1050 + }, + { + "completion_length": 25.0625, + "epoch": 3.2242703533026114, + "grad_norm": 6.192936897277832, + "kl": 0.400390625, + "learning_rate": 1.9230769230769231e-07, + "loss": -0.0729, + "reward": 0.7696964144706726, + "reward_std": 0.0217979084700346, + "reward_std_all": 0.028623271267861128, + "rewards-max/forensics_sinkhorn_reward": 0.9820599853992462, + "rewards-max/forensics_strict_edge_reward": 0.6540074050426483, + "rewards-max/forensics_tiou_reward": 0.9825760722160339, + "rewards/forensics_sinkhorn_reward": 0.7407190501689911, + "rewards/forensics_strict_edge_reward": 0.08743800967931747, + "rewards/forensics_tiou_reward": 0.6798616051673889, + "step": 1051 + }, + { + "completion_length": 25.796875, + "epoch": 3.227342549923195, + "grad_norm": 11.02448844909668, + "kl": 1.125, + "learning_rate": 1.9153846153846153e-07, + "loss": 0.0578, + "reward": 0.7961754500865936, + "reward_std": 0.011685681995004416, + "reward_std_all": 0.012785161612555385, + "rewards-max/forensics_sinkhorn_reward": 0.9852216243743896, + "rewards-max/forensics_strict_edge_reward": 0.5221183001995087, + "rewards-max/forensics_tiou_reward": 0.988459050655365, + "rewards/forensics_sinkhorn_reward": 0.7910301387310028, + "rewards/forensics_strict_edge_reward": 0.08047224208712578, + "rewards/forensics_tiou_reward": 0.6239429414272308, + "step": 1052 + }, + { + "completion_length": 26.609375, + "epoch": 3.230414746543779, + "grad_norm": 7.56751012802124, + "kl": 0.4873046875, + "learning_rate": 1.9076923076923077e-07, + "loss": -0.0069, + "reward": 0.7940143942832947, + "reward_std": 0.014707030495628715, + "reward_std_all": 0.015627610497176647, + "rewards-max/forensics_sinkhorn_reward": 0.986666202545166, + "rewards-max/forensics_strict_edge_reward": 0.6637322008609772, + "rewards-max/forensics_tiou_reward": 0.9932380020618439, + "rewards/forensics_sinkhorn_reward": 0.7943113148212433, + "rewards/forensics_strict_edge_reward": 0.12481994926929474, + "rewards/forensics_tiou_reward": 0.6628043055534363, + "step": 1053 + }, + { + "completion_length": 24.375, + "epoch": 3.2334869431643627, + "grad_norm": 6.420722961425781, + "kl": 0.3759765625, + "learning_rate": 1.8999999999999998e-07, + "loss": -0.0279, + "reward": 0.8053496181964874, + "reward_std": 0.0021689400309696794, + "reward_std_all": 0.003791139810346067, + "rewards-max/forensics_sinkhorn_reward": 0.9828665554523468, + "rewards-max/forensics_strict_edge_reward": 0.6657527685165405, + "rewards-max/forensics_tiou_reward": 0.9836193919181824, + "rewards/forensics_sinkhorn_reward": 0.8313648998737335, + "rewards/forensics_strict_edge_reward": 0.09628157690167427, + "rewards/forensics_tiou_reward": 0.6778690814971924, + "step": 1054 + }, + { + "completion_length": 24.96875, + "epoch": 3.236559139784946, + "grad_norm": 6.772209644317627, + "kl": 0.482421875, + "learning_rate": 1.8923076923076922e-07, + "loss": 0.116, + "reward": 0.7563312649726868, + "reward_std": 0.0627376064658165, + "reward_std_all": 0.05922011658549309, + "rewards-max/forensics_sinkhorn_reward": 0.9783526062965393, + "rewards-max/forensics_strict_edge_reward": 0.7246644794940948, + "rewards-max/forensics_tiou_reward": 0.9704680740833282, + "rewards/forensics_sinkhorn_reward": 0.7253948152065277, + "rewards/forensics_strict_edge_reward": 0.05549016036093235, + "rewards/forensics_tiou_reward": 0.5660023093223572, + "step": 1055 + }, + { + "completion_length": 24.78125, + "epoch": 3.23963133640553, + "grad_norm": 7.324968338012695, + "kl": 0.416015625, + "learning_rate": 1.8846153846153846e-07, + "loss": -0.0478, + "reward": 0.7614753544330597, + "reward_std": 0.03449631808325648, + "reward_std_all": 0.039069488644599915, + "rewards-max/forensics_sinkhorn_reward": 0.9681359529495239, + "rewards-max/forensics_strict_edge_reward": 0.4818538427352905, + "rewards-max/forensics_tiou_reward": 0.950599879026413, + "rewards/forensics_sinkhorn_reward": 0.7220392227172852, + "rewards/forensics_strict_edge_reward": 0.033505178056657314, + "rewards/forensics_tiou_reward": 0.5310748517513275, + "step": 1056 + }, + { + "completion_length": 28.328125, + "epoch": 3.2427035330261136, + "grad_norm": 6.842560291290283, + "kl": 0.455078125, + "learning_rate": 1.8769230769230767e-07, + "loss": -0.0767, + "reward": 0.7987012267112732, + "reward_std": 0.012333989841863513, + "reward_std_all": 0.012893812032416463, + "rewards-max/forensics_sinkhorn_reward": 0.9850294888019562, + "rewards-max/forensics_strict_edge_reward": 0.6774116456508636, + "rewards-max/forensics_tiou_reward": 0.9880642592906952, + "rewards/forensics_sinkhorn_reward": 0.7865395247936249, + "rewards/forensics_strict_edge_reward": 0.1105710044503212, + "rewards/forensics_tiou_reward": 0.682274729013443, + "step": 1057 + }, + { + "completion_length": 23.890625, + "epoch": 3.2457757296466974, + "grad_norm": 6.336462020874023, + "kl": 0.4619140625, + "learning_rate": 1.8692307692307692e-07, + "loss": 0.0072, + "reward": 0.7852421998977661, + "reward_std": 0.03594455495476723, + "reward_std_all": 0.03290982358157635, + "rewards-max/forensics_sinkhorn_reward": 0.9777198433876038, + "rewards-max/forensics_strict_edge_reward": 0.6032288670539856, + "rewards-max/forensics_tiou_reward": 0.972002774477005, + "rewards/forensics_sinkhorn_reward": 0.7746879756450653, + "rewards/forensics_strict_edge_reward": 0.0704632680863142, + "rewards/forensics_tiou_reward": 0.6284694671630859, + "step": 1058 + }, + { + "completion_length": 25.78125, + "epoch": 3.248847926267281, + "grad_norm": 8.300104141235352, + "kl": 0.6162109375, + "learning_rate": 1.8615384615384613e-07, + "loss": -0.0195, + "reward": 0.7955531477928162, + "reward_std": 0.018841643817722797, + "reward_std_all": 0.018257263116538525, + "rewards-max/forensics_sinkhorn_reward": 0.9861427247524261, + "rewards-max/forensics_strict_edge_reward": 0.7212935090065002, + "rewards-max/forensics_tiou_reward": 0.9919536709785461, + "rewards/forensics_sinkhorn_reward": 0.810871809720993, + "rewards/forensics_strict_edge_reward": 0.09022495150566101, + "rewards/forensics_tiou_reward": 0.6647710204124451, + "step": 1059 + }, + { + "completion_length": 23.859375, + "epoch": 3.251920122887865, + "grad_norm": 8.511722564697266, + "kl": 0.44921875, + "learning_rate": 1.853846153846154e-07, + "loss": 0.1031, + "reward": 0.7493058443069458, + "reward_std": 0.05396169261075556, + "reward_std_all": 0.057338664308190346, + "rewards-max/forensics_sinkhorn_reward": 0.9817594587802887, + "rewards-max/forensics_strict_edge_reward": 0.5251639038324356, + "rewards-max/forensics_tiou_reward": 0.980440229177475, + "rewards/forensics_sinkhorn_reward": 0.7170992493629456, + "rewards/forensics_strict_edge_reward": 0.04068917781114578, + "rewards/forensics_tiou_reward": 0.6022748947143555, + "step": 1060 + }, + { + "completion_length": 24.734375, + "epoch": 3.2549923195084487, + "grad_norm": 7.520751953125, + "kl": 0.357421875, + "learning_rate": 1.846153846153846e-07, + "loss": 0.0331, + "reward": 0.7938041687011719, + "reward_std": 0.01752511691302061, + "reward_std_all": 0.017383461352437735, + "rewards-max/forensics_sinkhorn_reward": 0.9770488142967224, + "rewards-max/forensics_strict_edge_reward": 0.8135445415973663, + "rewards-max/forensics_tiou_reward": 0.9731793701648712, + "rewards/forensics_sinkhorn_reward": 0.8401472866535187, + "rewards/forensics_strict_edge_reward": 0.09522951021790504, + "rewards/forensics_tiou_reward": 0.6955714225769043, + "step": 1061 + }, + { + "completion_length": 24.4375, + "epoch": 3.258064516129032, + "grad_norm": 7.049139022827148, + "kl": 0.4658203125, + "learning_rate": 1.8384615384615385e-07, + "loss": 0.1235, + "reward": 0.7522420585155487, + "reward_std": 0.0501247588545084, + "reward_std_all": 0.053628094494342804, + "rewards-max/forensics_sinkhorn_reward": 0.9827740788459778, + "rewards-max/forensics_strict_edge_reward": 0.6589746475219727, + "rewards-max/forensics_tiou_reward": 0.9840026795864105, + "rewards/forensics_sinkhorn_reward": 0.7010700106620789, + "rewards/forensics_strict_edge_reward": 0.07878305204212666, + "rewards/forensics_tiou_reward": 0.5604262948036194, + "step": 1062 + }, + { + "completion_length": 23.390625, + "epoch": 3.261136712749616, + "grad_norm": 10.02003002166748, + "kl": 0.4072265625, + "learning_rate": 1.8307692307692306e-07, + "loss": 0.0229, + "reward": 0.7751437425613403, + "reward_std": 0.033762694569304585, + "reward_std_all": 0.03373201424255967, + "rewards-max/forensics_sinkhorn_reward": 0.9792163670063019, + "rewards-max/forensics_strict_edge_reward": 0.4710261672735214, + "rewards-max/forensics_tiou_reward": 0.968395859003067, + "rewards/forensics_sinkhorn_reward": 0.7133447527885437, + "rewards/forensics_strict_edge_reward": 0.08201861381530762, + "rewards/forensics_tiou_reward": 0.6405709087848663, + "step": 1063 + }, + { + "completion_length": 25.28125, + "epoch": 3.2642089093701996, + "grad_norm": 9.721633911132812, + "kl": 0.4814453125, + "learning_rate": 1.823076923076923e-07, + "loss": -0.1188, + "reward": 0.7977970540523529, + "reward_std": 0.011621577577898279, + "reward_std_all": 0.01248110830783844, + "rewards-max/forensics_sinkhorn_reward": 0.9715116024017334, + "rewards-max/forensics_strict_edge_reward": 0.4591035544872284, + "rewards-max/forensics_tiou_reward": 0.9527881741523743, + "rewards/forensics_sinkhorn_reward": 0.8277369737625122, + "rewards/forensics_strict_edge_reward": 0.04735742136836052, + "rewards/forensics_tiou_reward": 0.673224925994873, + "step": 1064 + }, + { + "completion_length": 27.046875, + "epoch": 3.2672811059907834, + "grad_norm": 6.858327388763428, + "kl": 0.4580078125, + "learning_rate": 1.8153846153846152e-07, + "loss": -0.0327, + "reward": 0.7968817055225372, + "reward_std": 0.016863767465110868, + "reward_std_all": 0.016390510369092226, + "rewards-max/forensics_sinkhorn_reward": 0.9728848040103912, + "rewards-max/forensics_strict_edge_reward": 0.8333333432674408, + "rewards-max/forensics_tiou_reward": 0.9513835608959198, + "rewards/forensics_sinkhorn_reward": 0.8197125494480133, + "rewards/forensics_strict_edge_reward": 0.09345515817403793, + "rewards/forensics_tiou_reward": 0.6699803471565247, + "step": 1065 + }, + { + "completion_length": 23.46875, + "epoch": 3.270353302611367, + "grad_norm": 7.748424053192139, + "kl": 0.4150390625, + "learning_rate": 1.8076923076923076e-07, + "loss": -0.0265, + "reward": 0.8039393126964569, + "reward_std": 0.003679388202726841, + "reward_std_all": 0.005318271461874247, + "rewards-max/forensics_sinkhorn_reward": 0.9788723289966583, + "rewards-max/forensics_strict_edge_reward": 0.5067087709903717, + "rewards-max/forensics_tiou_reward": 0.9729407131671906, + "rewards/forensics_sinkhorn_reward": 0.8198860883712769, + "rewards/forensics_strict_edge_reward": 0.050222501158714294, + "rewards/forensics_tiou_reward": 0.6627055704593658, + "step": 1066 + }, + { + "completion_length": 27.46875, + "epoch": 3.273425499231951, + "grad_norm": 7.61962890625, + "kl": 0.388671875, + "learning_rate": 1.8e-07, + "loss": 0.0156, + "reward": 0.7930586338043213, + "reward_std": 0.008164682105416432, + "reward_std_all": 0.01248169259633869, + "rewards-max/forensics_sinkhorn_reward": 0.977414071559906, + "rewards-max/forensics_strict_edge_reward": 0.6499901115894318, + "rewards-max/forensics_tiou_reward": 0.9696642160415649, + "rewards/forensics_sinkhorn_reward": 0.7588163614273071, + "rewards/forensics_strict_edge_reward": 0.06238674744963646, + "rewards/forensics_tiou_reward": 0.6005221009254456, + "step": 1067 + }, + { + "completion_length": 25.5625, + "epoch": 3.2764976958525347, + "grad_norm": 9.246583938598633, + "kl": 0.390625, + "learning_rate": 1.792307692307692e-07, + "loss": -0.0475, + "reward": 0.7940008640289307, + "reward_std": 0.022577401250600815, + "reward_std_all": 0.02129753027111292, + "rewards-max/forensics_sinkhorn_reward": 0.9806503355503082, + "rewards-max/forensics_strict_edge_reward": 0.4697008728981018, + "rewards-max/forensics_tiou_reward": 0.9802069067955017, + "rewards/forensics_sinkhorn_reward": 0.8396132588386536, + "rewards/forensics_strict_edge_reward": 0.08478887937963009, + "rewards/forensics_tiou_reward": 0.7260942459106445, + "step": 1068 + }, + { + "completion_length": 24.671875, + "epoch": 3.279569892473118, + "grad_norm": 8.498069763183594, + "kl": 0.4072265625, + "learning_rate": 1.7846153846153845e-07, + "loss": 0.0181, + "reward": 0.7490221560001373, + "reward_std": 0.019114055205136538, + "reward_std_all": 0.03930172324180603, + "rewards-max/forensics_sinkhorn_reward": 0.9719835817813873, + "rewards-max/forensics_strict_edge_reward": 0.43232981860637665, + "rewards-max/forensics_tiou_reward": 0.9485259354114532, + "rewards/forensics_sinkhorn_reward": 0.7523606717586517, + "rewards/forensics_strict_edge_reward": 0.07226268388330936, + "rewards/forensics_tiou_reward": 0.5964905619621277, + "step": 1069 + }, + { + "completion_length": 28.484375, + "epoch": 3.282642089093702, + "grad_norm": 7.970707893371582, + "kl": 0.435546875, + "learning_rate": 1.776923076923077e-07, + "loss": 0.047, + "reward": 0.7860020101070404, + "reward_std": 0.020487491972744465, + "reward_std_all": 0.021650771610438824, + "rewards-max/forensics_sinkhorn_reward": 0.9785468876361847, + "rewards-max/forensics_strict_edge_reward": 0.4924985021352768, + "rewards-max/forensics_tiou_reward": 0.9731152653694153, + "rewards/forensics_sinkhorn_reward": 0.7832756638526917, + "rewards/forensics_strict_edge_reward": 0.051910294219851494, + "rewards/forensics_tiou_reward": 0.6292745172977448, + "step": 1070 + }, + { + "completion_length": 28.21875, + "epoch": 3.2857142857142856, + "grad_norm": 8.018898010253906, + "kl": 0.4287109375, + "learning_rate": 1.7692307692307693e-07, + "loss": -0.0065, + "reward": 0.7936413884162903, + "reward_std": 0.021188434213399887, + "reward_std_all": 0.020387863740324974, + "rewards-max/forensics_sinkhorn_reward": 0.9809367954730988, + "rewards-max/forensics_strict_edge_reward": 0.6523028314113617, + "rewards-max/forensics_tiou_reward": 0.9797930717468262, + "rewards/forensics_sinkhorn_reward": 0.7289661765098572, + "rewards/forensics_strict_edge_reward": 0.09214142337441444, + "rewards/forensics_tiou_reward": 0.6231985688209534, + "step": 1071 + }, + { + "completion_length": 24.78125, + "epoch": 3.2887864823348694, + "grad_norm": 8.327858924865723, + "kl": 0.5, + "learning_rate": 1.7615384615384615e-07, + "loss": 0.0223, + "reward": 0.7916969060897827, + "reward_std": 0.02531509567052126, + "reward_std_all": 0.023795288987457752, + "rewards-max/forensics_sinkhorn_reward": 0.9836744070053101, + "rewards-max/forensics_strict_edge_reward": 0.8665722012519836, + "rewards-max/forensics_tiou_reward": 0.9874068200588226, + "rewards/forensics_sinkhorn_reward": 0.7852544784545898, + "rewards/forensics_strict_edge_reward": 0.09093344584107399, + "rewards/forensics_tiou_reward": 0.6497898101806641, + "step": 1072 + }, + { + "completion_length": 28.09375, + "epoch": 3.291858678955453, + "grad_norm": 8.339033126831055, + "kl": 0.609375, + "learning_rate": 1.7538461538461539e-07, + "loss": 0.0041, + "reward": 0.7928297221660614, + "reward_std": 0.025191571563482285, + "reward_std_all": 0.023407457396388054, + "rewards-max/forensics_sinkhorn_reward": 0.9865407347679138, + "rewards-max/forensics_strict_edge_reward": 0.6654710471630096, + "rewards-max/forensics_tiou_reward": 0.9929576516151428, + "rewards/forensics_sinkhorn_reward": 0.8730583786964417, + "rewards/forensics_strict_edge_reward": 0.09763576835393906, + "rewards/forensics_tiou_reward": 0.7698908448219299, + "step": 1073 + }, + { + "completion_length": 27.125, + "epoch": 3.294930875576037, + "grad_norm": 10.19405746459961, + "kl": 0.9794921875, + "learning_rate": 1.746153846153846e-07, + "loss": 0.0026, + "reward": 0.8001694083213806, + "reward_std": 0.008744372054934502, + "reward_std_all": 0.010049661621451378, + "rewards-max/forensics_sinkhorn_reward": 0.9715978801250458, + "rewards-max/forensics_strict_edge_reward": 0.42911434173583984, + "rewards-max/forensics_tiou_reward": 0.9583940804004669, + "rewards/forensics_sinkhorn_reward": 0.8232496976852417, + "rewards/forensics_strict_edge_reward": 0.03936839662492275, + "rewards/forensics_tiou_reward": 0.6405554413795471, + "step": 1074 + }, + { + "completion_length": 29.328125, + "epoch": 3.2980030721966207, + "grad_norm": 7.63916015625, + "kl": 0.419921875, + "learning_rate": 1.7384615384615384e-07, + "loss": -0.0389, + "reward": 0.7710055708885193, + "reward_std": 0.028132827952504158, + "reward_std_all": 0.0354633592069149, + "rewards-max/forensics_sinkhorn_reward": 0.97636678814888, + "rewards-max/forensics_strict_edge_reward": 0.27595338225364685, + "rewards-max/forensics_tiou_reward": 0.967473715543747, + "rewards/forensics_sinkhorn_reward": 0.708241879940033, + "rewards/forensics_strict_edge_reward": 0.03754098899662495, + "rewards/forensics_tiou_reward": 0.536400318145752, + "step": 1075 + }, + { + "completion_length": 22.3125, + "epoch": 3.3010752688172045, + "grad_norm": 20.8438777923584, + "kl": 0.8173828125, + "learning_rate": 1.7307692307692305e-07, + "loss": 0.0368, + "reward": 0.7952830195426941, + "reward_std": 0.018111884128302336, + "reward_std_all": 0.018058492802083492, + "rewards-max/forensics_sinkhorn_reward": 0.9766179621219635, + "rewards-max/forensics_strict_edge_reward": 0.5097636878490448, + "rewards-max/forensics_tiou_reward": 0.9673238098621368, + "rewards/forensics_sinkhorn_reward": 0.8056995570659637, + "rewards/forensics_strict_edge_reward": 0.0536415446549654, + "rewards/forensics_tiou_reward": 0.6268265247344971, + "step": 1076 + }, + { + "completion_length": 22.921875, + "epoch": 3.3041474654377883, + "grad_norm": 7.512409687042236, + "kl": 0.41015625, + "learning_rate": 1.723076923076923e-07, + "loss": -0.1402, + "reward": 0.7599611580371857, + "reward_std": 0.008298922330141068, + "reward_std_all": 0.027755027636885643, + "rewards-max/forensics_sinkhorn_reward": 0.9760087728500366, + "rewards-max/forensics_strict_edge_reward": 0.48032720386981964, + "rewards-max/forensics_tiou_reward": 0.9632279872894287, + "rewards/forensics_sinkhorn_reward": 0.8024004697799683, + "rewards/forensics_strict_edge_reward": 0.06612098962068558, + "rewards/forensics_tiou_reward": 0.6480452418327332, + "step": 1077 + }, + { + "completion_length": 25.796875, + "epoch": 3.3072196620583716, + "grad_norm": 9.885226249694824, + "kl": 0.5830078125, + "learning_rate": 1.7153846153846153e-07, + "loss": 0.0359, + "reward": 0.7349560260772705, + "reward_std": 0.03580229543149471, + "reward_std_all": 0.055302172899246216, + "rewards-max/forensics_sinkhorn_reward": 0.9820843935012817, + "rewards-max/forensics_strict_edge_reward": 0.5840189903974533, + "rewards-max/forensics_tiou_reward": 0.9736545979976654, + "rewards/forensics_sinkhorn_reward": 0.7299249172210693, + "rewards/forensics_strict_edge_reward": 0.044810907915234566, + "rewards/forensics_tiou_reward": 0.6024823784828186, + "step": 1078 + }, + { + "completion_length": 29.375, + "epoch": 3.3102918586789554, + "grad_norm": 7.841787815093994, + "kl": 0.44921875, + "learning_rate": 1.7076923076923075e-07, + "loss": -0.0519, + "reward": 0.8004821836948395, + "reward_std": 0.009272331837564707, + "reward_std_all": 0.010349575779400766, + "rewards-max/forensics_sinkhorn_reward": 0.9813873171806335, + "rewards-max/forensics_strict_edge_reward": 0.3255263715982437, + "rewards-max/forensics_tiou_reward": 0.9778317213058472, + "rewards/forensics_sinkhorn_reward": 0.7851531207561493, + "rewards/forensics_strict_edge_reward": 0.049367194063961506, + "rewards/forensics_tiou_reward": 0.6616554260253906, + "step": 1079 + }, + { + "completion_length": 21.609375, + "epoch": 3.313364055299539, + "grad_norm": 6.7061614990234375, + "kl": 0.3525390625, + "learning_rate": 1.7000000000000001e-07, + "loss": -0.0011, + "reward": 0.7676483988761902, + "reward_std": 0.025330952368676662, + "reward_std_all": 0.033455509692430496, + "rewards-max/forensics_sinkhorn_reward": 0.987758070230484, + "rewards-max/forensics_strict_edge_reward": 0.7047859728336334, + "rewards-max/forensics_tiou_reward": 0.9965505599975586, + "rewards/forensics_sinkhorn_reward": 0.7307806611061096, + "rewards/forensics_strict_edge_reward": 0.0577908530831337, + "rewards/forensics_tiou_reward": 0.5294638276100159, + "step": 1080 + }, + { + "completion_length": 25.375, + "epoch": 3.316436251920123, + "grad_norm": 8.141916275024414, + "kl": 0.462890625, + "learning_rate": 1.6923076923076923e-07, + "loss": -0.0097, + "reward": 0.7460213601589203, + "reward_std": 0.027681383304297924, + "reward_std_all": 0.046257177367806435, + "rewards-max/forensics_sinkhorn_reward": 0.9804738163948059, + "rewards-max/forensics_strict_edge_reward": 0.46036943793296814, + "rewards-max/forensics_tiou_reward": 0.9786219298839569, + "rewards/forensics_sinkhorn_reward": 0.7569661736488342, + "rewards/forensics_strict_edge_reward": 0.08122388646006584, + "rewards/forensics_tiou_reward": 0.5532487034797668, + "step": 1081 + }, + { + "completion_length": 24.640625, + "epoch": 3.3195084485407067, + "grad_norm": 7.18326473236084, + "kl": 0.4267578125, + "learning_rate": 1.6846153846153847e-07, + "loss": -0.0295, + "reward": 0.7878042459487915, + "reward_std": 0.01901883678510785, + "reward_std_all": 0.020104294177144766, + "rewards-max/forensics_sinkhorn_reward": 0.9760583639144897, + "rewards-max/forensics_strict_edge_reward": 0.5449539422988892, + "rewards-max/forensics_tiou_reward": 0.9670745134353638, + "rewards/forensics_sinkhorn_reward": 0.8033992648124695, + "rewards/forensics_strict_edge_reward": 0.08929379098117352, + "rewards/forensics_tiou_reward": 0.6480697989463806, + "step": 1082 + }, + { + "completion_length": 22.859375, + "epoch": 3.3225806451612905, + "grad_norm": 8.488763809204102, + "kl": 0.673828125, + "learning_rate": 1.6769230769230768e-07, + "loss": 0.0641, + "reward": 0.7338542938232422, + "reward_std": 0.03822324983775616, + "reward_std_all": 0.05387943051755428, + "rewards-max/forensics_sinkhorn_reward": 0.9791199266910553, + "rewards-max/forensics_strict_edge_reward": 0.5900419056415558, + "rewards-max/forensics_tiou_reward": 0.9757606387138367, + "rewards/forensics_sinkhorn_reward": 0.7293540239334106, + "rewards/forensics_strict_edge_reward": 0.1143568754196167, + "rewards/forensics_tiou_reward": 0.5776996612548828, + "step": 1083 + }, + { + "completion_length": 28.96875, + "epoch": 3.3256528417818743, + "grad_norm": 7.977285385131836, + "kl": 0.67578125, + "learning_rate": 1.6692307692307692e-07, + "loss": 0.0418, + "reward": 0.7696480453014374, + "reward_std": 0.01260491309221834, + "reward_std_all": 0.026043985970318317, + "rewards-max/forensics_sinkhorn_reward": 0.9679787158966064, + "rewards-max/forensics_strict_edge_reward": 0.5211784988641739, + "rewards-max/forensics_tiou_reward": 0.9507893919944763, + "rewards/forensics_sinkhorn_reward": 0.7127570509910583, + "rewards/forensics_strict_edge_reward": 0.044003305956721306, + "rewards/forensics_tiou_reward": 0.5488407015800476, + "step": 1084 + }, + { + "completion_length": 25.03125, + "epoch": 3.3287250384024576, + "grad_norm": 6.570627212524414, + "kl": 0.521484375, + "learning_rate": 1.6615384615384613e-07, + "loss": -0.0361, + "reward": 0.7986023128032684, + "reward_std": 0.011406471021473408, + "reward_std_all": 0.01213645376265049, + "rewards-max/forensics_sinkhorn_reward": 0.9827595949172974, + "rewards-max/forensics_strict_edge_reward": 0.4468829333782196, + "rewards-max/forensics_tiou_reward": 0.980763167142868, + "rewards/forensics_sinkhorn_reward": 0.8190217912197113, + "rewards/forensics_strict_edge_reward": 0.06152987852692604, + "rewards/forensics_tiou_reward": 0.6737707257270813, + "step": 1085 + }, + { + "completion_length": 31.015625, + "epoch": 3.3317972350230414, + "grad_norm": 7.01091194152832, + "kl": 0.4794921875, + "learning_rate": 1.6538461538461538e-07, + "loss": -0.0197, + "reward": 0.778023362159729, + "reward_std": 0.0389126343652606, + "reward_std_all": 0.03673299215734005, + "rewards-max/forensics_sinkhorn_reward": 0.9802844822406769, + "rewards-max/forensics_strict_edge_reward": 0.5911939442157745, + "rewards-max/forensics_tiou_reward": 0.9784796237945557, + "rewards/forensics_sinkhorn_reward": 0.8059894144535065, + "rewards/forensics_strict_edge_reward": 0.06977317109704018, + "rewards/forensics_tiou_reward": 0.7034508585929871, + "step": 1086 + }, + { + "completion_length": 27.75, + "epoch": 3.334869431643625, + "grad_norm": 7.94098424911499, + "kl": 0.4462890625, + "learning_rate": 1.646153846153846e-07, + "loss": 0.0178, + "reward": 0.7872752547264099, + "reward_std": 0.02175724063999951, + "reward_std_all": 0.022688958793878555, + "rewards-max/forensics_sinkhorn_reward": 0.9760027229785919, + "rewards-max/forensics_strict_edge_reward": 0.5900419056415558, + "rewards-max/forensics_tiou_reward": 0.9674841463565826, + "rewards/forensics_sinkhorn_reward": 0.7527889609336853, + "rewards/forensics_strict_edge_reward": 0.05014008842408657, + "rewards/forensics_tiou_reward": 0.5985409021377563, + "step": 1087 + }, + { + "completion_length": 24.359375, + "epoch": 3.337941628264209, + "grad_norm": 10.068053245544434, + "kl": 0.412109375, + "learning_rate": 1.6384615384615383e-07, + "loss": -0.053, + "reward": 0.7866791486740112, + "reward_std": 0.02191470324760303, + "reward_std_all": 0.022489882539957762, + "rewards-max/forensics_sinkhorn_reward": 0.9863704442977905, + "rewards-max/forensics_strict_edge_reward": 0.5841981768608093, + "rewards-max/forensics_tiou_reward": 0.9941505789756775, + "rewards/forensics_sinkhorn_reward": 0.8035455346107483, + "rewards/forensics_strict_edge_reward": 0.09036064147949219, + "rewards/forensics_tiou_reward": 0.6419826447963715, + "step": 1088 + }, + { + "completion_length": 24.03125, + "epoch": 3.3410138248847927, + "grad_norm": 7.441035747528076, + "kl": 0.6484375, + "learning_rate": 1.6307692307692307e-07, + "loss": -0.018, + "reward": 0.8011423051357269, + "reward_std": 0.01040602708235383, + "reward_std_all": 0.010655727703124285, + "rewards-max/forensics_sinkhorn_reward": 0.9853994846343994, + "rewards-max/forensics_strict_edge_reward": 0.756708562374115, + "rewards-max/forensics_tiou_reward": 0.9906215667724609, + "rewards/forensics_sinkhorn_reward": 0.8230184018611908, + "rewards/forensics_strict_edge_reward": 0.12675969675183296, + "rewards/forensics_tiou_reward": 0.7062380313873291, + "step": 1089 + }, + { + "completion_length": 27.78125, + "epoch": 3.3440860215053765, + "grad_norm": 7.4757232666015625, + "kl": 0.5712890625, + "learning_rate": 1.623076923076923e-07, + "loss": -0.0623, + "reward": 0.7992480397224426, + "reward_std": 0.007845346233807504, + "reward_std_all": 0.009216398932039738, + "rewards-max/forensics_sinkhorn_reward": 0.9878757894039154, + "rewards-max/forensics_strict_edge_reward": 0.6938109397888184, + "rewards-max/forensics_tiou_reward": 0.9956388473510742, + "rewards/forensics_sinkhorn_reward": 0.8763741850852966, + "rewards/forensics_strict_edge_reward": 0.12716129049658775, + "rewards/forensics_tiou_reward": 0.7552338540554047, + "step": 1090 + }, + { + "completion_length": 23.015625, + "epoch": 3.3471582181259603, + "grad_norm": 8.63615894317627, + "kl": 0.5126953125, + "learning_rate": 1.6153846153846155e-07, + "loss": -0.013, + "reward": 0.7888453006744385, + "reward_std": 0.021622213476803154, + "reward_std_all": 0.021643076790496707, + "rewards-max/forensics_sinkhorn_reward": 0.9840530157089233, + "rewards-max/forensics_strict_edge_reward": 0.6763789355754852, + "rewards-max/forensics_tiou_reward": 0.9867537915706635, + "rewards/forensics_sinkhorn_reward": 0.8318370878696442, + "rewards/forensics_strict_edge_reward": 0.0792480893433094, + "rewards/forensics_tiou_reward": 0.6941572725772858, + "step": 1091 + }, + { + "completion_length": 23.953125, + "epoch": 3.3502304147465436, + "grad_norm": 8.969182014465332, + "kl": 0.4267578125, + "learning_rate": 1.6076923076923076e-07, + "loss": 0.1356, + "reward": 0.7674819827079773, + "reward_std": 0.04331602342426777, + "reward_std_all": 0.04257740080356598, + "rewards-max/forensics_sinkhorn_reward": 0.9859511256217957, + "rewards-max/forensics_strict_edge_reward": 0.9375866651535034, + "rewards-max/forensics_tiou_reward": 0.9930244386196136, + "rewards/forensics_sinkhorn_reward": 0.7490555047988892, + "rewards/forensics_strict_edge_reward": 0.1272093877196312, + "rewards/forensics_tiou_reward": 0.6329284012317657, + "step": 1092 + }, + { + "completion_length": 25.734375, + "epoch": 3.3533026113671274, + "grad_norm": 7.916918754577637, + "kl": 0.5078125, + "learning_rate": 1.6e-07, + "loss": 0.0862, + "reward": 0.7774771451950073, + "reward_std": 0.03931462252512574, + "reward_std_all": 0.03743591997772455, + "rewards-max/forensics_sinkhorn_reward": 0.9804970920085907, + "rewards-max/forensics_strict_edge_reward": 0.7450321167707443, + "rewards-max/forensics_tiou_reward": 0.980070173740387, + "rewards/forensics_sinkhorn_reward": 0.7873176038265228, + "rewards/forensics_strict_edge_reward": 0.11173094809055328, + "rewards/forensics_tiou_reward": 0.661246657371521, + "step": 1093 + }, + { + "completion_length": 25.984375, + "epoch": 3.356374807987711, + "grad_norm": 7.286405086517334, + "kl": 0.453125, + "learning_rate": 1.5923076923076922e-07, + "loss": -0.0253, + "reward": 0.7918239831924438, + "reward_std": 0.016295563313178718, + "reward_std_all": 0.01752606756053865, + "rewards-max/forensics_sinkhorn_reward": 0.983564019203186, + "rewards-max/forensics_strict_edge_reward": 0.6298359930515289, + "rewards-max/forensics_tiou_reward": 0.9872857928276062, + "rewards/forensics_sinkhorn_reward": 0.7343074381351471, + "rewards/forensics_strict_edge_reward": 0.08244410902261734, + "rewards/forensics_tiou_reward": 0.6021267175674438, + "step": 1094 + }, + { + "completion_length": 23.90625, + "epoch": 3.359447004608295, + "grad_norm": 7.735140323638916, + "kl": 0.359375, + "learning_rate": 1.5846153846153846e-07, + "loss": -0.0848, + "reward": 0.7857523262500763, + "reward_std": 0.01765535620506853, + "reward_std_all": 0.020030075451359153, + "rewards-max/forensics_sinkhorn_reward": 0.9835398197174072, + "rewards-max/forensics_strict_edge_reward": 0.5101649016141891, + "rewards-max/forensics_tiou_reward": 0.9804573953151703, + "rewards/forensics_sinkhorn_reward": 0.8058870732784271, + "rewards/forensics_strict_edge_reward": 0.03755796141922474, + "rewards/forensics_tiou_reward": 0.6589646637439728, + "step": 1095 + }, + { + "completion_length": 26.984375, + "epoch": 3.3625192012288787, + "grad_norm": 8.660903930664062, + "kl": 0.62890625, + "learning_rate": 1.5769230769230767e-07, + "loss": 0.0459, + "reward": 0.760492593050003, + "reward_std": 0.05870943795889616, + "reward_std_all": 0.05551551282405853, + "rewards-max/forensics_sinkhorn_reward": 0.9711434245109558, + "rewards-max/forensics_strict_edge_reward": 0.2914600968360901, + "rewards-max/forensics_tiou_reward": 0.9543257355690002, + "rewards/forensics_sinkhorn_reward": 0.7440517246723175, + "rewards/forensics_strict_edge_reward": 0.055523951537907124, + "rewards/forensics_tiou_reward": 0.6188317835330963, + "step": 1096 + }, + { + "completion_length": 22.828125, + "epoch": 3.3655913978494625, + "grad_norm": 8.331805229187012, + "kl": 0.4267578125, + "learning_rate": 1.569230769230769e-07, + "loss": 0.1458, + "reward": 0.7863022983074188, + "reward_std": 0.02516016736626625, + "reward_std_all": 0.02460848819464445, + "rewards-max/forensics_sinkhorn_reward": 0.98085618019104, + "rewards-max/forensics_strict_edge_reward": 0.6976466774940491, + "rewards-max/forensics_tiou_reward": 0.9794489741325378, + "rewards/forensics_sinkhorn_reward": 0.7636095583438873, + "rewards/forensics_strict_edge_reward": 0.077418252825737, + "rewards/forensics_tiou_reward": 0.6076290905475616, + "step": 1097 + }, + { + "completion_length": 27.78125, + "epoch": 3.3686635944700463, + "grad_norm": 6.5053887367248535, + "kl": 0.53125, + "learning_rate": 1.5615384615384612e-07, + "loss": -0.0162, + "reward": 0.757560133934021, + "reward_std": 0.019463242031633854, + "reward_std_all": 0.035315072163939476, + "rewards-max/forensics_sinkhorn_reward": 0.9842289388179779, + "rewards-max/forensics_strict_edge_reward": 0.7113372087478638, + "rewards-max/forensics_tiou_reward": 0.9903730154037476, + "rewards/forensics_sinkhorn_reward": 0.7722003757953644, + "rewards/forensics_strict_edge_reward": 0.07829805463552475, + "rewards/forensics_tiou_reward": 0.6607472002506256, + "step": 1098 + }, + { + "completion_length": 21.75, + "epoch": 3.3717357910906296, + "grad_norm": 7.769695281982422, + "kl": 0.580078125, + "learning_rate": 1.5538461538461536e-07, + "loss": -0.0591, + "reward": 0.790522038936615, + "reward_std": 0.027737214521039277, + "reward_std_all": 0.026036086259409785, + "rewards-max/forensics_sinkhorn_reward": 0.9814918637275696, + "rewards-max/forensics_strict_edge_reward": 0.5895819067955017, + "rewards-max/forensics_tiou_reward": 0.9800592958927155, + "rewards/forensics_sinkhorn_reward": 0.8069036900997162, + "rewards/forensics_strict_edge_reward": 0.11508440598845482, + "rewards/forensics_tiou_reward": 0.6433230638504028, + "step": 1099 + }, + { + "completion_length": 23.1875, + "epoch": 3.3748079877112134, + "grad_norm": 9.490248680114746, + "kl": 0.4892578125, + "learning_rate": 1.5461538461538463e-07, + "loss": 0.1299, + "reward": 0.7669557929039001, + "reward_std": 0.05235152319073677, + "reward_std_all": 0.04911450110375881, + "rewards-max/forensics_sinkhorn_reward": 0.9796162545681, + "rewards-max/forensics_strict_edge_reward": 0.6149742305278778, + "rewards-max/forensics_tiou_reward": 0.9741256833076477, + "rewards/forensics_sinkhorn_reward": 0.6954975426197052, + "rewards/forensics_strict_edge_reward": 0.04599650762975216, + "rewards/forensics_tiou_reward": 0.5481320321559906, + "step": 1100 + }, + { + "completion_length": 22.25, + "epoch": 3.377880184331797, + "grad_norm": 8.614826202392578, + "kl": 0.4326171875, + "learning_rate": 1.5384615384615385e-07, + "loss": -0.0185, + "reward": 0.7839285433292389, + "reward_std": 0.037414949270896614, + "reward_std_all": 0.03456726507283747, + "rewards-max/forensics_sinkhorn_reward": 0.9846193492412567, + "rewards-max/forensics_strict_edge_reward": 0.6515887677669525, + "rewards-max/forensics_tiou_reward": 0.988861620426178, + "rewards/forensics_sinkhorn_reward": 0.7816025614738464, + "rewards/forensics_strict_edge_reward": 0.060610031709074974, + "rewards/forensics_tiou_reward": 0.6086476445198059, + "step": 1101 + }, + { + "completion_length": 23.859375, + "epoch": 3.380952380952381, + "grad_norm": 9.612211227416992, + "kl": 0.375, + "learning_rate": 1.5307692307692309e-07, + "loss": -0.0021, + "reward": 0.7877423167228699, + "reward_std": 0.024955387692898512, + "reward_std_all": 0.024323811754584312, + "rewards-max/forensics_sinkhorn_reward": 0.9775158762931824, + "rewards-max/forensics_strict_edge_reward": 0.5622940063476562, + "rewards-max/forensics_tiou_reward": 0.9702278971672058, + "rewards/forensics_sinkhorn_reward": 0.7733339667320251, + "rewards/forensics_strict_edge_reward": 0.05979903042316437, + "rewards/forensics_tiou_reward": 0.5909209251403809, + "step": 1102 + }, + { + "completion_length": 26.859375, + "epoch": 3.3840245775729647, + "grad_norm": 6.800344944000244, + "kl": 0.453125, + "learning_rate": 1.523076923076923e-07, + "loss": -0.0694, + "reward": 0.796098381280899, + "reward_std": 0.012452252209186554, + "reward_std_all": 0.013372886925935745, + "rewards-max/forensics_sinkhorn_reward": 0.9799899160861969, + "rewards-max/forensics_strict_edge_reward": 0.6371559798717499, + "rewards-max/forensics_tiou_reward": 0.9764894247055054, + "rewards/forensics_sinkhorn_reward": 0.8310425281524658, + "rewards/forensics_strict_edge_reward": 0.12480730563402176, + "rewards/forensics_tiou_reward": 0.7040852606296539, + "step": 1103 + }, + { + "completion_length": 26.453125, + "epoch": 3.3870967741935485, + "grad_norm": 7.239922523498535, + "kl": 0.462890625, + "learning_rate": 1.5153846153846154e-07, + "loss": 0.0111, + "reward": 0.7851977646350861, + "reward_std": 0.02077874355018139, + "reward_std_all": 0.022000043652951717, + "rewards-max/forensics_sinkhorn_reward": 0.9823158979415894, + "rewards-max/forensics_strict_edge_reward": 0.4402783513069153, + "rewards-max/forensics_tiou_reward": 0.981516033411026, + "rewards/forensics_sinkhorn_reward": 0.8323899209499359, + "rewards/forensics_strict_edge_reward": 0.06543899513781071, + "rewards/forensics_tiou_reward": 0.6763521432876587, + "step": 1104 + }, + { + "completion_length": 27.078125, + "epoch": 3.3901689708141323, + "grad_norm": 6.520407199859619, + "kl": 0.3876953125, + "learning_rate": 1.5076923076923075e-07, + "loss": 0.0147, + "reward": 0.7876777946949005, + "reward_std": 0.022691052872687578, + "reward_std_all": 0.022664510644972324, + "rewards-max/forensics_sinkhorn_reward": 0.9851463437080383, + "rewards-max/forensics_strict_edge_reward": 0.7370076179504395, + "rewards-max/forensics_tiou_reward": 0.9905676543712616, + "rewards/forensics_sinkhorn_reward": 0.7401861846446991, + "rewards/forensics_strict_edge_reward": 0.07252469658851624, + "rewards/forensics_tiou_reward": 0.6328109800815582, + "step": 1105 + }, + { + "completion_length": 26.75, + "epoch": 3.3932411674347156, + "grad_norm": 11.291901588439941, + "kl": 0.5185546875, + "learning_rate": 1.5e-07, + "loss": 0.0212, + "reward": 0.7742251753807068, + "reward_std": 0.03822759259492159, + "reward_std_all": 0.03752988716587424, + "rewards-max/forensics_sinkhorn_reward": 0.9829347431659698, + "rewards-max/forensics_strict_edge_reward": 0.6642511188983917, + "rewards-max/forensics_tiou_reward": 0.9832562208175659, + "rewards/forensics_sinkhorn_reward": 0.7833943963050842, + "rewards/forensics_strict_edge_reward": 0.05777676776051521, + "rewards/forensics_tiou_reward": 0.6272078156471252, + "step": 1106 + }, + { + "completion_length": 22.828125, + "epoch": 3.3963133640552994, + "grad_norm": 8.950060844421387, + "kl": 0.505859375, + "learning_rate": 1.492307692307692e-07, + "loss": -0.0081, + "reward": 0.7894582152366638, + "reward_std": 0.019532935693860054, + "reward_std_all": 0.020165071822702885, + "rewards-max/forensics_sinkhorn_reward": 0.9805954694747925, + "rewards-max/forensics_strict_edge_reward": 0.6657527685165405, + "rewards-max/forensics_tiou_reward": 0.9781520068645477, + "rewards/forensics_sinkhorn_reward": 0.7814664542675018, + "rewards/forensics_strict_edge_reward": 0.0820468794554472, + "rewards/forensics_tiou_reward": 0.6694078743457794, + "step": 1107 + }, + { + "completion_length": 23.484375, + "epoch": 3.399385560675883, + "grad_norm": 7.030198097229004, + "kl": 0.5068359375, + "learning_rate": 1.4846153846153845e-07, + "loss": 0.0293, + "reward": 0.7779347002506256, + "reward_std": 0.049811920151114464, + "reward_std_all": 0.04482240043580532, + "rewards-max/forensics_sinkhorn_reward": 0.9875812828540802, + "rewards-max/forensics_strict_edge_reward": 0.8052808940410614, + "rewards-max/forensics_tiou_reward": 0.9977640807628632, + "rewards/forensics_sinkhorn_reward": 0.8115422129631042, + "rewards/forensics_strict_edge_reward": 0.10157451778650284, + "rewards/forensics_tiou_reward": 0.6929713487625122, + "step": 1108 + }, + { + "completion_length": 27.765625, + "epoch": 3.402457757296467, + "grad_norm": 7.939448833465576, + "kl": 0.4619140625, + "learning_rate": 1.476923076923077e-07, + "loss": -0.0622, + "reward": 0.7949253916740417, + "reward_std": 0.020254835952073336, + "reward_std_all": 0.019327988848090172, + "rewards-max/forensics_sinkhorn_reward": 0.9787561893463135, + "rewards-max/forensics_strict_edge_reward": 0.6942535638809204, + "rewards-max/forensics_tiou_reward": 0.9767161309719086, + "rewards/forensics_sinkhorn_reward": 0.8157637715339661, + "rewards/forensics_strict_edge_reward": 0.07023290731012821, + "rewards/forensics_tiou_reward": 0.6969204545021057, + "step": 1109 + }, + { + "completion_length": 21.5, + "epoch": 3.4055299539170507, + "grad_norm": 7.947158336639404, + "kl": 0.4365234375, + "learning_rate": 1.4692307692307693e-07, + "loss": 0.0941, + "reward": 0.7816511392593384, + "reward_std": 0.033144257962703705, + "reward_std_all": 0.03175677265971899, + "rewards-max/forensics_sinkhorn_reward": 0.9752319753170013, + "rewards-max/forensics_strict_edge_reward": 0.6326714754104614, + "rewards-max/forensics_tiou_reward": 0.9656527042388916, + "rewards/forensics_sinkhorn_reward": 0.7560121119022369, + "rewards/forensics_strict_edge_reward": 0.06322379782795906, + "rewards/forensics_tiou_reward": 0.5468243956565857, + "step": 1110 + }, + { + "completion_length": 24.65625, + "epoch": 3.4086021505376345, + "grad_norm": 7.747176647186279, + "kl": 0.4267578125, + "learning_rate": 1.4615384615384617e-07, + "loss": 0.1195, + "reward": 0.7737548649311066, + "reward_std": 0.022677121218293905, + "reward_std_all": 0.030216429382562637, + "rewards-max/forensics_sinkhorn_reward": 0.9772141873836517, + "rewards-max/forensics_strict_edge_reward": 0.4346180260181427, + "rewards-max/forensics_tiou_reward": 0.9676647782325745, + "rewards/forensics_sinkhorn_reward": 0.7342686057090759, + "rewards/forensics_strict_edge_reward": 0.05025179870426655, + "rewards/forensics_tiou_reward": 0.5704347491264343, + "step": 1111 + }, + { + "completion_length": 24.546875, + "epoch": 3.4116743471582183, + "grad_norm": 8.197150230407715, + "kl": 0.6171875, + "learning_rate": 1.4538461538461538e-07, + "loss": 0.0378, + "reward": 0.7836092412471771, + "reward_std": 0.03914392273873091, + "reward_std_all": 0.03591752704232931, + "rewards-max/forensics_sinkhorn_reward": 0.9781399369239807, + "rewards-max/forensics_strict_edge_reward": 0.6904512643814087, + "rewards-max/forensics_tiou_reward": 0.9730218350887299, + "rewards/forensics_sinkhorn_reward": 0.7784218192100525, + "rewards/forensics_strict_edge_reward": 0.07369746640324593, + "rewards/forensics_tiou_reward": 0.6713809370994568, + "step": 1112 + }, + { + "completion_length": 24.0625, + "epoch": 3.4147465437788016, + "grad_norm": 7.255040168762207, + "kl": 0.4931640625, + "learning_rate": 1.4461538461538462e-07, + "loss": -0.074, + "reward": 0.7751950025558472, + "reward_std": 0.020723227877169847, + "reward_std_all": 0.025798638816922903, + "rewards-max/forensics_sinkhorn_reward": 0.9783475399017334, + "rewards-max/forensics_strict_edge_reward": 0.6934256851673126, + "rewards-max/forensics_tiou_reward": 0.9748938977718353, + "rewards/forensics_sinkhorn_reward": 0.8171599209308624, + "rewards/forensics_strict_edge_reward": 0.10194146633148193, + "rewards/forensics_tiou_reward": 0.6926378607749939, + "step": 1113 + }, + { + "completion_length": 25.390625, + "epoch": 3.4178187403993854, + "grad_norm": 8.072212219238281, + "kl": 0.626953125, + "learning_rate": 1.4384615384615384e-07, + "loss": 0.0846, + "reward": 0.7834421098232269, + "reward_std": 0.02948353160172701, + "reward_std_all": 0.028658585622906685, + "rewards-max/forensics_sinkhorn_reward": 0.9774591028690338, + "rewards-max/forensics_strict_edge_reward": 0.5344093441963196, + "rewards-max/forensics_tiou_reward": 0.9679820537567139, + "rewards/forensics_sinkhorn_reward": 0.7919855117797852, + "rewards/forensics_strict_edge_reward": 0.05593864619731903, + "rewards/forensics_tiou_reward": 0.6312931180000305, + "step": 1114 + }, + { + "completion_length": 28.15625, + "epoch": 3.420890937019969, + "grad_norm": 9.941795349121094, + "kl": 0.875, + "learning_rate": 1.4307692307692308e-07, + "loss": 0.0543, + "reward": 0.7965914011001587, + "reward_std": 0.01726142317056656, + "reward_std_all": 0.016564679332077503, + "rewards-max/forensics_sinkhorn_reward": 0.9856412708759308, + "rewards-max/forensics_strict_edge_reward": 0.8187680542469025, + "rewards-max/forensics_tiou_reward": 0.9902812838554382, + "rewards/forensics_sinkhorn_reward": 0.7970694899559021, + "rewards/forensics_strict_edge_reward": 0.0972760021686554, + "rewards/forensics_tiou_reward": 0.6676794290542603, + "step": 1115 + }, + { + "completion_length": 24.34375, + "epoch": 3.423963133640553, + "grad_norm": 6.500853061676025, + "kl": 0.4453125, + "learning_rate": 1.423076923076923e-07, + "loss": 0.0251, + "reward": 0.7673279345035553, + "reward_std": 0.04018234647810459, + "reward_std_all": 0.04141983576118946, + "rewards-max/forensics_sinkhorn_reward": 0.9748877882957458, + "rewards-max/forensics_strict_edge_reward": 0.5834526717662811, + "rewards-max/forensics_tiou_reward": 0.9607429802417755, + "rewards/forensics_sinkhorn_reward": 0.7387103140354156, + "rewards/forensics_strict_edge_reward": 0.049770537763834, + "rewards/forensics_tiou_reward": 0.5644925832748413, + "step": 1116 + }, + { + "completion_length": 25.109375, + "epoch": 3.4270353302611367, + "grad_norm": 8.648920059204102, + "kl": 0.416015625, + "learning_rate": 1.4153846153846153e-07, + "loss": 0.0599, + "reward": 0.7511454820632935, + "reward_std": 0.06240514665842056, + "reward_std_all": 0.06018255837261677, + "rewards-max/forensics_sinkhorn_reward": 0.982671707868576, + "rewards-max/forensics_strict_edge_reward": 0.7165313363075256, + "rewards-max/forensics_tiou_reward": 0.9845954477787018, + "rewards/forensics_sinkhorn_reward": 0.7309687733650208, + "rewards/forensics_strict_edge_reward": 0.06141928397119045, + "rewards/forensics_tiou_reward": 0.5405827462673187, + "step": 1117 + }, + { + "completion_length": 25.609375, + "epoch": 3.4301075268817205, + "grad_norm": 7.797938346862793, + "kl": 0.595703125, + "learning_rate": 1.4076923076923074e-07, + "loss": 0.0102, + "reward": 0.7936286628246307, + "reward_std": 0.02245534025132656, + "reward_std_all": 0.021092492155730724, + "rewards-max/forensics_sinkhorn_reward": 0.9700445234775543, + "rewards-max/forensics_strict_edge_reward": 0.579744964838028, + "rewards-max/forensics_tiou_reward": 0.9493367373943329, + "rewards/forensics_sinkhorn_reward": 0.808057576417923, + "rewards/forensics_strict_edge_reward": 0.08712108805775642, + "rewards/forensics_tiou_reward": 0.6643903851509094, + "step": 1118 + }, + { + "completion_length": 23.15625, + "epoch": 3.4331797235023043, + "grad_norm": 9.011590957641602, + "kl": 0.39453125, + "learning_rate": 1.4e-07, + "loss": -0.0223, + "reward": 0.7776239514350891, + "reward_std": 0.03499258775264025, + "reward_std_all": 0.0341902282088995, + "rewards-max/forensics_sinkhorn_reward": 0.9828955233097076, + "rewards-max/forensics_strict_edge_reward": 0.639672726392746, + "rewards-max/forensics_tiou_reward": 0.9810968339443207, + "rewards/forensics_sinkhorn_reward": 0.774346798658371, + "rewards/forensics_strict_edge_reward": 0.07603604160249233, + "rewards/forensics_tiou_reward": 0.564677894115448, + "step": 1119 + }, + { + "completion_length": 24.03125, + "epoch": 3.436251920122888, + "grad_norm": 7.796039581298828, + "kl": 0.451171875, + "learning_rate": 1.3923076923076922e-07, + "loss": 0.0151, + "reward": 0.7769953906536102, + "reward_std": 0.032861617393791676, + "reward_std_all": 0.03298649098724127, + "rewards-max/forensics_sinkhorn_reward": 0.9880319535732269, + "rewards-max/forensics_strict_edge_reward": 0.7727466821670532, + "rewards-max/forensics_tiou_reward": 0.9963907599449158, + "rewards/forensics_sinkhorn_reward": 0.7873237729072571, + "rewards/forensics_strict_edge_reward": 0.0833296924829483, + "rewards/forensics_tiou_reward": 0.6196805238723755, + "step": 1120 + }, + { + "completion_length": 24.46875, + "epoch": 3.439324116743472, + "grad_norm": 7.518969535827637, + "kl": 0.4345703125, + "learning_rate": 1.3846153846153846e-07, + "loss": -0.0027, + "reward": 0.7711426317691803, + "reward_std": 0.033117092156317085, + "reward_std_all": 0.035285600228235126, + "rewards-max/forensics_sinkhorn_reward": 0.9767235815525055, + "rewards-max/forensics_strict_edge_reward": 0.47752194851636887, + "rewards-max/forensics_tiou_reward": 0.9615413844585419, + "rewards/forensics_sinkhorn_reward": 0.7803667187690735, + "rewards/forensics_strict_edge_reward": 0.05266437865793705, + "rewards/forensics_tiou_reward": 0.6435553133487701, + "step": 1121 + }, + { + "completion_length": 22.203125, + "epoch": 3.442396313364055, + "grad_norm": 12.465517044067383, + "kl": 0.5859375, + "learning_rate": 1.376923076923077e-07, + "loss": -0.0463, + "reward": 0.7594833672046661, + "reward_std": 0.04844934679567814, + "reward_std_all": 0.04804188385605812, + "rewards-max/forensics_sinkhorn_reward": 0.9849841594696045, + "rewards-max/forensics_strict_edge_reward": 0.6899624168872833, + "rewards-max/forensics_tiou_reward": 0.9885509312152863, + "rewards/forensics_sinkhorn_reward": 0.8078000545501709, + "rewards/forensics_strict_edge_reward": 0.0920306071639061, + "rewards/forensics_tiou_reward": 0.6910782158374786, + "step": 1122 + }, + { + "completion_length": 26.15625, + "epoch": 3.445468509984639, + "grad_norm": 7.395604133605957, + "kl": 0.3642578125, + "learning_rate": 1.3692307692307692e-07, + "loss": 0.0085, + "reward": 0.7949215471744537, + "reward_std": 0.013357376679778099, + "reward_std_all": 0.014623706694692373, + "rewards-max/forensics_sinkhorn_reward": 0.9784988164901733, + "rewards-max/forensics_strict_edge_reward": 0.4008854106068611, + "rewards-max/forensics_tiou_reward": 0.9710573554039001, + "rewards/forensics_sinkhorn_reward": 0.7428095638751984, + "rewards/forensics_strict_edge_reward": 0.058157605584710836, + "rewards/forensics_tiou_reward": 0.5981986224651337, + "step": 1123 + }, + { + "completion_length": 26.75, + "epoch": 3.4485407066052227, + "grad_norm": 9.055049896240234, + "kl": 0.4638671875, + "learning_rate": 1.3615384615384616e-07, + "loss": -0.0476, + "reward": 0.8027592599391937, + "reward_std": 0.005449139513075352, + "reward_std_all": 0.006954599637538195, + "rewards-max/forensics_sinkhorn_reward": 0.975984513759613, + "rewards-max/forensics_strict_edge_reward": 0.3416936919093132, + "rewards-max/forensics_tiou_reward": 0.966715931892395, + "rewards/forensics_sinkhorn_reward": 0.8117506802082062, + "rewards/forensics_strict_edge_reward": 0.045739445835351944, + "rewards/forensics_tiou_reward": 0.6726606786251068, + "step": 1124 + }, + { + "completion_length": 22.125, + "epoch": 3.4516129032258065, + "grad_norm": 8.661704063415527, + "kl": 0.58203125, + "learning_rate": 1.3538461538461537e-07, + "loss": -0.0317, + "reward": 0.8031455278396606, + "reward_std": 0.006496769259683788, + "reward_std_all": 0.00731189118232578, + "rewards-max/forensics_sinkhorn_reward": 0.9741065204143524, + "rewards-max/forensics_strict_edge_reward": 0.582292228937149, + "rewards-max/forensics_tiou_reward": 0.9623362720012665, + "rewards/forensics_sinkhorn_reward": 0.8419440388679504, + "rewards/forensics_strict_edge_reward": 0.10114145651459694, + "rewards/forensics_tiou_reward": 0.6765750646591187, + "step": 1125 + }, + { + "completion_length": 26.578125, + "epoch": 3.4546850998463903, + "grad_norm": 7.374004364013672, + "kl": 0.55078125, + "learning_rate": 1.346153846153846e-07, + "loss": -0.0381, + "reward": 0.7934686541557312, + "reward_std": 0.02369211334735155, + "reward_std_all": 0.022180378437042236, + "rewards-max/forensics_sinkhorn_reward": 0.9824163317680359, + "rewards-max/forensics_strict_edge_reward": 0.8135445415973663, + "rewards-max/forensics_tiou_reward": 0.9829773306846619, + "rewards/forensics_sinkhorn_reward": 0.8528580367565155, + "rewards/forensics_strict_edge_reward": 0.12160136550664902, + "rewards/forensics_tiou_reward": 0.7452985644340515, + "step": 1126 + }, + { + "completion_length": 25.9375, + "epoch": 3.457757296466974, + "grad_norm": 8.422975540161133, + "kl": 0.4638671875, + "learning_rate": 1.3384615384615383e-07, + "loss": 0.0092, + "reward": 0.7762046456336975, + "reward_std": 0.0355311743915081, + "reward_std_all": 0.03474130108952522, + "rewards-max/forensics_sinkhorn_reward": 0.9772053956985474, + "rewards-max/forensics_strict_edge_reward": 0.4685726761817932, + "rewards-max/forensics_tiou_reward": 0.9667543470859528, + "rewards/forensics_sinkhorn_reward": 0.767113596200943, + "rewards/forensics_strict_edge_reward": 0.05881497636437416, + "rewards/forensics_tiou_reward": 0.5894461274147034, + "step": 1127 + }, + { + "completion_length": 25.71875, + "epoch": 3.460829493087558, + "grad_norm": 7.89640998840332, + "kl": 0.4072265625, + "learning_rate": 1.3307692307692307e-07, + "loss": -0.0405, + "reward": 0.7816987633705139, + "reward_std": 0.027604859322309494, + "reward_std_all": 0.02774530928581953, + "rewards-max/forensics_sinkhorn_reward": 0.9755041897296906, + "rewards-max/forensics_strict_edge_reward": 0.42128483951091766, + "rewards-max/forensics_tiou_reward": 0.9656627476215363, + "rewards/forensics_sinkhorn_reward": 0.7839978039264679, + "rewards/forensics_strict_edge_reward": 0.057129207998514175, + "rewards/forensics_tiou_reward": 0.6312914788722992, + "step": 1128 + }, + { + "completion_length": 24.015625, + "epoch": 3.463901689708141, + "grad_norm": 7.003757476806641, + "kl": 0.3271484375, + "learning_rate": 1.323076923076923e-07, + "loss": -0.009, + "reward": 0.8037143051624298, + "reward_std": 0.006023234338499606, + "reward_std_all": 0.006984645035117865, + "rewards-max/forensics_sinkhorn_reward": 0.9873001575469971, + "rewards-max/forensics_strict_edge_reward": 0.9375866651535034, + "rewards-max/forensics_tiou_reward": 0.9939988553524017, + "rewards/forensics_sinkhorn_reward": 0.870946854352951, + "rewards/forensics_strict_edge_reward": 0.09510993212461472, + "rewards/forensics_tiou_reward": 0.7672064900398254, + "step": 1129 + }, + { + "completion_length": 28.59375, + "epoch": 3.466973886328725, + "grad_norm": 7.247668743133545, + "kl": 0.4951171875, + "learning_rate": 1.3153846153846155e-07, + "loss": 0.019, + "reward": 0.7886997163295746, + "reward_std": 0.020424597896635532, + "reward_std_all": 0.020846951752901077, + "rewards-max/forensics_sinkhorn_reward": 0.984940230846405, + "rewards-max/forensics_strict_edge_reward": 0.3127523958683014, + "rewards-max/forensics_tiou_reward": 0.9892416894435883, + "rewards/forensics_sinkhorn_reward": 0.8202354311943054, + "rewards/forensics_strict_edge_reward": 0.03344072541221976, + "rewards/forensics_tiou_reward": 0.6901730298995972, + "step": 1130 + }, + { + "completion_length": 23.875, + "epoch": 3.4700460829493087, + "grad_norm": 9.062738418579102, + "kl": 0.52734375, + "learning_rate": 1.3076923076923076e-07, + "loss": -0.0259, + "reward": 0.7825844883918762, + "reward_std": 0.02676741685718298, + "reward_std_all": 0.02725680870935321, + "rewards-max/forensics_sinkhorn_reward": 0.9872029423713684, + "rewards-max/forensics_strict_edge_reward": 0.7511312067508698, + "rewards-max/forensics_tiou_reward": 0.9946709871292114, + "rewards/forensics_sinkhorn_reward": 0.8264995217323303, + "rewards/forensics_strict_edge_reward": 0.05986126512289047, + "rewards/forensics_tiou_reward": 0.6655621528625488, + "step": 1131 + }, + { + "completion_length": 26.265625, + "epoch": 3.4731182795698925, + "grad_norm": 8.867504119873047, + "kl": 0.462890625, + "learning_rate": 1.3e-07, + "loss": 0.0749, + "reward": 0.774854451417923, + "reward_std": 0.031171662732958794, + "reward_std_all": 0.03404960874468088, + "rewards-max/forensics_sinkhorn_reward": 0.9803714752197266, + "rewards-max/forensics_strict_edge_reward": 0.5324672311544418, + "rewards-max/forensics_tiou_reward": 0.9790855348110199, + "rewards/forensics_sinkhorn_reward": 0.7118871808052063, + "rewards/forensics_strict_edge_reward": 0.06190888024866581, + "rewards/forensics_tiou_reward": 0.5725515484809875, + "step": 1132 + }, + { + "completion_length": 25.0, + "epoch": 3.4761904761904763, + "grad_norm": 7.905651569366455, + "kl": 0.47265625, + "learning_rate": 1.2923076923076924e-07, + "loss": 0.0126, + "reward": 0.8004339337348938, + "reward_std": 0.010881821101065725, + "reward_std_all": 0.011252319323830307, + "rewards-max/forensics_sinkhorn_reward": 0.97637540102005, + "rewards-max/forensics_strict_edge_reward": 0.6355590671300888, + "rewards-max/forensics_tiou_reward": 0.9691869020462036, + "rewards/forensics_sinkhorn_reward": 0.8380181789398193, + "rewards/forensics_strict_edge_reward": 0.1050223670899868, + "rewards/forensics_tiou_reward": 0.683896541595459, + "step": 1133 + }, + { + "completion_length": 24.9375, + "epoch": 3.47926267281106, + "grad_norm": 9.666022300720215, + "kl": 0.72265625, + "learning_rate": 1.2846153846153845e-07, + "loss": 0.0292, + "reward": 0.77999147772789, + "reward_std": 0.04658130370080471, + "reward_std_all": 0.04180821403861046, + "rewards-max/forensics_sinkhorn_reward": 0.9836282730102539, + "rewards-max/forensics_strict_edge_reward": 0.6796516180038452, + "rewards-max/forensics_tiou_reward": 0.9852176904678345, + "rewards/forensics_sinkhorn_reward": 0.8140560984611511, + "rewards/forensics_strict_edge_reward": 0.08734697848558426, + "rewards/forensics_tiou_reward": 0.6704056262969971, + "step": 1134 + }, + { + "completion_length": 28.234375, + "epoch": 3.482334869431644, + "grad_norm": 7.792408466339111, + "kl": 0.333984375, + "learning_rate": 1.276923076923077e-07, + "loss": 0.042, + "reward": 0.7610735297203064, + "reward_std": 0.008493275148794055, + "reward_std_all": 0.02783176116645336, + "rewards-max/forensics_sinkhorn_reward": 0.9851495921611786, + "rewards-max/forensics_strict_edge_reward": 0.5081382244825363, + "rewards-max/forensics_tiou_reward": 0.9890740215778351, + "rewards/forensics_sinkhorn_reward": 0.7540059089660645, + "rewards/forensics_strict_edge_reward": 0.048616185784339905, + "rewards/forensics_tiou_reward": 0.5924609750509262, + "step": 1135 + }, + { + "completion_length": 25.09375, + "epoch": 3.485407066052227, + "grad_norm": 7.6365556716918945, + "kl": 0.4150390625, + "learning_rate": 1.269230769230769e-07, + "loss": 0.0522, + "reward": 0.7767724096775055, + "reward_std": 0.034937139600515366, + "reward_std_all": 0.034286766313016415, + "rewards-max/forensics_sinkhorn_reward": 0.9620071053504944, + "rewards-max/forensics_strict_edge_reward": 0.4256758689880371, + "rewards-max/forensics_tiou_reward": 0.9307790696620941, + "rewards/forensics_sinkhorn_reward": 0.7823577523231506, + "rewards/forensics_strict_edge_reward": 0.04580957256257534, + "rewards/forensics_tiou_reward": 0.6396408975124359, + "step": 1136 + }, + { + "completion_length": 21.359375, + "epoch": 3.488479262672811, + "grad_norm": 9.069877624511719, + "kl": 0.4375, + "learning_rate": 1.2615384615384615e-07, + "loss": -0.108, + "reward": 0.7745413184165955, + "reward_std": 0.00836506299674511, + "reward_std_all": 0.02180576603859663, + "rewards-max/forensics_sinkhorn_reward": 0.9836313724517822, + "rewards-max/forensics_strict_edge_reward": 0.6271176636219025, + "rewards-max/forensics_tiou_reward": 0.9859862327575684, + "rewards/forensics_sinkhorn_reward": 0.7940677404403687, + "rewards/forensics_strict_edge_reward": 0.07589911296963692, + "rewards/forensics_tiou_reward": 0.6352209448814392, + "step": 1137 + }, + { + "completion_length": 26.359375, + "epoch": 3.4915514592933947, + "grad_norm": 8.120361328125, + "kl": 0.4931640625, + "learning_rate": 1.2538461538461536e-07, + "loss": -0.0321, + "reward": 0.7504355609416962, + "reward_std": 0.03917289711534977, + "reward_std_all": 0.04544157348573208, + "rewards-max/forensics_sinkhorn_reward": 0.9779357612133026, + "rewards-max/forensics_strict_edge_reward": 0.5702531039714813, + "rewards-max/forensics_tiou_reward": 0.9700908064842224, + "rewards/forensics_sinkhorn_reward": 0.7552161812782288, + "rewards/forensics_strict_edge_reward": 0.06318199448287487, + "rewards/forensics_tiou_reward": 0.6260680854320526, + "step": 1138 + }, + { + "completion_length": 25.09375, + "epoch": 3.4946236559139785, + "grad_norm": 9.963260650634766, + "kl": 0.4833984375, + "learning_rate": 1.246153846153846e-07, + "loss": -0.0941, + "reward": 0.7950813174247742, + "reward_std": 0.011605334002524614, + "reward_std_all": 0.012949546566233039, + "rewards-max/forensics_sinkhorn_reward": 0.9775309562683105, + "rewards-max/forensics_strict_edge_reward": 0.6882063448429108, + "rewards-max/forensics_tiou_reward": 0.9657483696937561, + "rewards/forensics_sinkhorn_reward": 0.8129036724567413, + "rewards/forensics_strict_edge_reward": 0.0939597338438034, + "rewards/forensics_tiou_reward": 0.6385356187820435, + "step": 1139 + }, + { + "completion_length": 26.421875, + "epoch": 3.4976958525345623, + "grad_norm": 8.33547306060791, + "kl": 0.41015625, + "learning_rate": 1.2384615384615384e-07, + "loss": -0.0036, + "reward": 0.799540251493454, + "reward_std": 0.01094989525154233, + "reward_std_all": 0.011923266109079123, + "rewards-max/forensics_sinkhorn_reward": 0.9812952876091003, + "rewards-max/forensics_strict_edge_reward": 0.4934597313404083, + "rewards-max/forensics_tiou_reward": 0.9796487390995026, + "rewards/forensics_sinkhorn_reward": 0.8219614326953888, + "rewards/forensics_strict_edge_reward": 0.05600232258439064, + "rewards/forensics_tiou_reward": 0.6658825278282166, + "step": 1140 + }, + { + "completion_length": 21.5, + "epoch": 3.500768049155146, + "grad_norm": 7.3758134841918945, + "kl": 0.3291015625, + "learning_rate": 1.2307692307692308e-07, + "loss": -0.0296, + "reward": 0.7941868305206299, + "reward_std": 0.020103746792301536, + "reward_std_all": 0.019352178554981947, + "rewards-max/forensics_sinkhorn_reward": 0.9817889928817749, + "rewards-max/forensics_strict_edge_reward": 0.7165313363075256, + "rewards-max/forensics_tiou_reward": 0.9822448790073395, + "rewards/forensics_sinkhorn_reward": 0.8206977248191833, + "rewards/forensics_strict_edge_reward": 0.07126395776867867, + "rewards/forensics_tiou_reward": 0.6913174986839294, + "step": 1141 + }, + { + "completion_length": 25.890625, + "epoch": 3.50384024577573, + "grad_norm": 8.322604179382324, + "kl": 0.4453125, + "learning_rate": 1.223076923076923e-07, + "loss": -0.0279, + "reward": 0.8025089800357819, + "reward_std": 0.0070045627653598785, + "reward_std_all": 0.007911299588158727, + "rewards-max/forensics_sinkhorn_reward": 0.9819729924201965, + "rewards-max/forensics_strict_edge_reward": 0.6076286435127258, + "rewards-max/forensics_tiou_reward": 0.982892632484436, + "rewards/forensics_sinkhorn_reward": 0.8134530484676361, + "rewards/forensics_strict_edge_reward": 0.0852934941649437, + "rewards/forensics_tiou_reward": 0.7077050507068634, + "step": 1142 + }, + { + "completion_length": 25.296875, + "epoch": 3.506912442396313, + "grad_norm": 8.680805206298828, + "kl": 0.4541015625, + "learning_rate": 1.2153846153846154e-07, + "loss": -0.0563, + "reward": 0.8010023236274719, + "reward_std": 0.009659864590503275, + "reward_std_all": 0.010241533513180912, + "rewards-max/forensics_sinkhorn_reward": 0.9857929944992065, + "rewards-max/forensics_strict_edge_reward": 0.6030216813087463, + "rewards-max/forensics_tiou_reward": 0.9916356801986694, + "rewards/forensics_sinkhorn_reward": 0.8153940737247467, + "rewards/forensics_strict_edge_reward": 0.08929363638162613, + "rewards/forensics_tiou_reward": 0.6787436008453369, + "step": 1143 + }, + { + "completion_length": 23.453125, + "epoch": 3.509984639016897, + "grad_norm": 8.11839771270752, + "kl": 0.4111328125, + "learning_rate": 1.2076923076923075e-07, + "loss": -0.0491, + "reward": 0.7678949236869812, + "reward_std": 0.04781727027148008, + "reward_std_all": 0.0465164128690958, + "rewards-max/forensics_sinkhorn_reward": 0.9825697541236877, + "rewards-max/forensics_strict_edge_reward": 0.5156270563602448, + "rewards-max/forensics_tiou_reward": 0.9819452464580536, + "rewards/forensics_sinkhorn_reward": 0.8170665204524994, + "rewards/forensics_strict_edge_reward": 0.06285303831100464, + "rewards/forensics_tiou_reward": 0.6402927935123444, + "step": 1144 + }, + { + "completion_length": 25.671875, + "epoch": 3.5130568356374807, + "grad_norm": 7.582192420959473, + "kl": 0.4912109375, + "learning_rate": 1.2e-07, + "loss": -0.0152, + "reward": 0.7918777167797089, + "reward_std": 0.0023113138740882277, + "reward_std_all": 0.010333964368328452, + "rewards-max/forensics_sinkhorn_reward": 0.9767112731933594, + "rewards-max/forensics_strict_edge_reward": 0.5860485136508942, + "rewards-max/forensics_tiou_reward": 0.9706358909606934, + "rewards/forensics_sinkhorn_reward": 0.832320362329483, + "rewards/forensics_strict_edge_reward": 0.09319575875997543, + "rewards/forensics_tiou_reward": 0.7073213756084442, + "step": 1145 + }, + { + "completion_length": 24.46875, + "epoch": 3.5161290322580645, + "grad_norm": 6.656006813049316, + "kl": 0.58203125, + "learning_rate": 1.1923076923076923e-07, + "loss": 0.0288, + "reward": 0.7785913348197937, + "reward_std": 0.046808379935100675, + "reward_std_all": 0.04246037779375911, + "rewards-max/forensics_sinkhorn_reward": 0.9824079871177673, + "rewards-max/forensics_strict_edge_reward": 0.7286747694015503, + "rewards-max/forensics_tiou_reward": 0.9825988709926605, + "rewards/forensics_sinkhorn_reward": 0.7525457739830017, + "rewards/forensics_strict_edge_reward": 0.08929156512022018, + "rewards/forensics_tiou_reward": 0.5698924362659454, + "step": 1146 + }, + { + "completion_length": 27.46875, + "epoch": 3.5192012288786483, + "grad_norm": 8.262359619140625, + "kl": 0.4990234375, + "learning_rate": 1.1846153846153846e-07, + "loss": 0.0307, + "reward": 0.7800318896770477, + "reward_std": 0.007560605532489717, + "reward_std_all": 0.018959232605993748, + "rewards-max/forensics_sinkhorn_reward": 0.9834039807319641, + "rewards-max/forensics_strict_edge_reward": 0.34867842495441437, + "rewards-max/forensics_tiou_reward": 0.9827606081962585, + "rewards/forensics_sinkhorn_reward": 0.7630200982093811, + "rewards/forensics_strict_edge_reward": 0.030668647959828377, + "rewards/forensics_tiou_reward": 0.6058062613010406, + "step": 1147 + }, + { + "completion_length": 29.71875, + "epoch": 3.522273425499232, + "grad_norm": 6.927275657653809, + "kl": 0.4921875, + "learning_rate": 1.1769230769230768e-07, + "loss": -0.0756, + "reward": 0.7579282820224762, + "reward_std": 0.036926257191225886, + "reward_std_all": 0.04382215393707156, + "rewards-max/forensics_sinkhorn_reward": 0.986640602350235, + "rewards-max/forensics_strict_edge_reward": 0.5884982198476791, + "rewards-max/forensics_tiou_reward": 0.991420179605484, + "rewards/forensics_sinkhorn_reward": 0.7769087851047516, + "rewards/forensics_strict_edge_reward": 0.07999307662248611, + "rewards/forensics_tiou_reward": 0.6290931105613708, + "step": 1148 + }, + { + "completion_length": 27.125, + "epoch": 3.525345622119816, + "grad_norm": 9.249014854431152, + "kl": 0.3583984375, + "learning_rate": 1.1692307692307691e-07, + "loss": -0.0138, + "reward": 0.7521187961101532, + "reward_std": 0.050214576069265604, + "reward_std_all": 0.0523311304859817, + "rewards-max/forensics_sinkhorn_reward": 0.9824455976486206, + "rewards-max/forensics_strict_edge_reward": 0.756708562374115, + "rewards-max/forensics_tiou_reward": 0.9822539687156677, + "rewards/forensics_sinkhorn_reward": 0.7770882248878479, + "rewards/forensics_strict_edge_reward": 0.06197221949696541, + "rewards/forensics_tiou_reward": 0.616899847984314, + "step": 1149 + }, + { + "completion_length": 25.28125, + "epoch": 3.528417818740399, + "grad_norm": 8.136479377746582, + "kl": 0.4267578125, + "learning_rate": 1.1615384615384615e-07, + "loss": 0.0801, + "reward": 0.7703363597393036, + "reward_std": 0.027742783538997173, + "reward_std_all": 0.03134776093065739, + "rewards-max/forensics_sinkhorn_reward": 0.9740251004695892, + "rewards-max/forensics_strict_edge_reward": 0.34434500336647034, + "rewards-max/forensics_tiou_reward": 0.9658001363277435, + "rewards/forensics_sinkhorn_reward": 0.6913450658321381, + "rewards/forensics_strict_edge_reward": 0.03894652798771858, + "rewards/forensics_tiou_reward": 0.6161559522151947, + "step": 1150 + }, + { + "completion_length": 26.328125, + "epoch": 3.531490015360983, + "grad_norm": 7.636556148529053, + "kl": 0.4580078125, + "learning_rate": 1.1538461538461539e-07, + "loss": -0.0139, + "reward": 0.8030500113964081, + "reward_std": 0.005915298010222614, + "reward_std_all": 0.006984510459005833, + "rewards-max/forensics_sinkhorn_reward": 0.9841597080230713, + "rewards-max/forensics_strict_edge_reward": 0.636265903711319, + "rewards-max/forensics_tiou_reward": 0.988293468952179, + "rewards/forensics_sinkhorn_reward": 0.8059138655662537, + "rewards/forensics_strict_edge_reward": 0.07698140293359756, + "rewards/forensics_tiou_reward": 0.6554754674434662, + "step": 1151 + }, + { + "completion_length": 22.5, + "epoch": 3.5345622119815667, + "grad_norm": 7.390232563018799, + "kl": 0.41015625, + "learning_rate": 1.1461538461538462e-07, + "loss": -0.0406, + "reward": 0.7908572554588318, + "reward_std": 0.02636615838855505, + "reward_std_all": 0.02487333118915558, + "rewards-max/forensics_sinkhorn_reward": 0.9847103655338287, + "rewards-max/forensics_strict_edge_reward": 0.6398434042930603, + "rewards-max/forensics_tiou_reward": 0.9854332506656647, + "rewards/forensics_sinkhorn_reward": 0.8049850761890411, + "rewards/forensics_strict_edge_reward": 0.04646193981170654, + "rewards/forensics_tiou_reward": 0.6314163208007812, + "step": 1152 + }, + { + "completion_length": 25.609375, + "epoch": 3.5376344086021505, + "grad_norm": 8.456701278686523, + "kl": 0.6318359375, + "learning_rate": 1.1384615384615384e-07, + "loss": 0.0169, + "reward": 0.7849153280258179, + "reward_std": 0.02842338476330042, + "reward_std_all": 0.027610059827566147, + "rewards-max/forensics_sinkhorn_reward": 0.9811284840106964, + "rewards-max/forensics_strict_edge_reward": 0.6195405721664429, + "rewards-max/forensics_tiou_reward": 0.9766955375671387, + "rewards/forensics_sinkhorn_reward": 0.7384239733219147, + "rewards/forensics_strict_edge_reward": 0.06360224634408951, + "rewards/forensics_tiou_reward": 0.6164520680904388, + "step": 1153 + }, + { + "completion_length": 23.4375, + "epoch": 3.5407066052227343, + "grad_norm": 8.91901683807373, + "kl": 0.5009765625, + "learning_rate": 1.1307692307692307e-07, + "loss": -0.0345, + "reward": 0.7947123944759369, + "reward_std": 0.016420900239609182, + "reward_std_all": 0.016513120266608894, + "rewards-max/forensics_sinkhorn_reward": 0.9829126298427582, + "rewards-max/forensics_strict_edge_reward": 0.639672726392746, + "rewards-max/forensics_tiou_reward": 0.9830834269523621, + "rewards/forensics_sinkhorn_reward": 0.8342002928256989, + "rewards/forensics_strict_edge_reward": 0.10480845719575882, + "rewards/forensics_tiou_reward": 0.6785324811935425, + "step": 1154 + }, + { + "completion_length": 25.4375, + "epoch": 3.543778801843318, + "grad_norm": 7.659310817718506, + "kl": 0.40625, + "learning_rate": 1.123076923076923e-07, + "loss": 0.0452, + "reward": 0.7892790138721466, + "reward_std": 0.02836701925843954, + "reward_std_all": 0.02657702285796404, + "rewards-max/forensics_sinkhorn_reward": 0.9806062281131744, + "rewards-max/forensics_strict_edge_reward": 0.6439114809036255, + "rewards-max/forensics_tiou_reward": 0.9801383018493652, + "rewards/forensics_sinkhorn_reward": 0.8294795453548431, + "rewards/forensics_strict_edge_reward": 0.10189446434378624, + "rewards/forensics_tiou_reward": 0.6916021108627319, + "step": 1155 + }, + { + "completion_length": 24.921875, + "epoch": 3.546850998463902, + "grad_norm": 6.718679428100586, + "kl": 0.4970703125, + "learning_rate": 1.1153846153846154e-07, + "loss": -0.0001, + "reward": 0.7949748337268829, + "reward_std": 0.01362246461212635, + "reward_std_all": 0.014567413832992315, + "rewards-max/forensics_sinkhorn_reward": 0.969445526599884, + "rewards-max/forensics_strict_edge_reward": 0.49889203906059265, + "rewards-max/forensics_tiou_reward": 0.9450969099998474, + "rewards/forensics_sinkhorn_reward": 0.8255225121974945, + "rewards/forensics_strict_edge_reward": 0.055608995258808136, + "rewards/forensics_tiou_reward": 0.6436762809753418, + "step": 1156 + }, + { + "completion_length": 29.140625, + "epoch": 3.549923195084485, + "grad_norm": 5.993692398071289, + "kl": 0.7119140625, + "learning_rate": 1.1076923076923077e-07, + "loss": -0.0254, + "reward": 0.797989547252655, + "reward_std": 0.016393361147493124, + "reward_std_all": 0.015754800639115274, + "rewards-max/forensics_sinkhorn_reward": 0.9872317016124725, + "rewards-max/forensics_strict_edge_reward": 0.6783875823020935, + "rewards-max/forensics_tiou_reward": 0.9929811060428619, + "rewards/forensics_sinkhorn_reward": 0.8593900203704834, + "rewards/forensics_strict_edge_reward": 0.11914261057972908, + "rewards/forensics_tiou_reward": 0.771595686674118, + "step": 1157 + }, + { + "completion_length": 24.84375, + "epoch": 3.5529953917050694, + "grad_norm": 7.426616668701172, + "kl": 0.501953125, + "learning_rate": 1.0999999999999999e-07, + "loss": -0.0612, + "reward": 0.8062094748020172, + "reward_std": 0.0019940200727432966, + "reward_std_all": 0.0033785091945901513, + "rewards-max/forensics_sinkhorn_reward": 0.9875228703022003, + "rewards-max/forensics_strict_edge_reward": 0.952418714761734, + "rewards-max/forensics_tiou_reward": 0.994718611240387, + "rewards/forensics_sinkhorn_reward": 0.8299743831157684, + "rewards/forensics_strict_edge_reward": 0.18939653038978577, + "rewards/forensics_tiou_reward": 0.7175249457359314, + "step": 1158 + }, + { + "completion_length": 23.71875, + "epoch": 3.5560675883256527, + "grad_norm": 7.696654796600342, + "kl": 0.4912109375, + "learning_rate": 1.0923076923076922e-07, + "loss": -0.0525, + "reward": 0.7934890687465668, + "reward_std": 0.015624068677425385, + "reward_std_all": 0.016293820459395647, + "rewards-max/forensics_sinkhorn_reward": 0.9789295196533203, + "rewards-max/forensics_strict_edge_reward": 0.6661420166492462, + "rewards-max/forensics_tiou_reward": 0.9762344658374786, + "rewards/forensics_sinkhorn_reward": 0.8213768899440765, + "rewards/forensics_strict_edge_reward": 0.11966278403997421, + "rewards/forensics_tiou_reward": 0.6531975865364075, + "step": 1159 + }, + { + "completion_length": 27.3125, + "epoch": 3.5591397849462365, + "grad_norm": 12.861845016479492, + "kl": 0.8671875, + "learning_rate": 1.0846153846153845e-07, + "loss": 0.0054, + "reward": 0.7827656269073486, + "reward_std": 0.02051668521016836, + "reward_std_all": 0.022762387059628963, + "rewards-max/forensics_sinkhorn_reward": 0.9773913621902466, + "rewards-max/forensics_strict_edge_reward": 0.6764304339885712, + "rewards-max/forensics_tiou_reward": 0.9714843332767487, + "rewards/forensics_sinkhorn_reward": 0.7731581032276154, + "rewards/forensics_strict_edge_reward": 0.07116111740469933, + "rewards/forensics_tiou_reward": 0.6296935379505157, + "step": 1160 + }, + { + "completion_length": 21.921875, + "epoch": 3.5622119815668203, + "grad_norm": 9.576428413391113, + "kl": 0.541015625, + "learning_rate": 1.076923076923077e-07, + "loss": 0.0484, + "reward": 0.7887217998504639, + "reward_std": 0.029725823551416397, + "reward_std_all": 0.027835971675813198, + "rewards-max/forensics_sinkhorn_reward": 0.9808903932571411, + "rewards-max/forensics_strict_edge_reward": 0.5829301476478577, + "rewards-max/forensics_tiou_reward": 0.9791216850280762, + "rewards/forensics_sinkhorn_reward": 0.774372398853302, + "rewards/forensics_strict_edge_reward": 0.06098463386297226, + "rewards/forensics_tiou_reward": 0.6200420260429382, + "step": 1161 + }, + { + "completion_length": 24.609375, + "epoch": 3.565284178187404, + "grad_norm": 8.805953979492188, + "kl": 0.4296875, + "learning_rate": 1.0692307692307693e-07, + "loss": -0.1062, + "reward": 0.805833101272583, + "reward_std": 0.0013474615989252925, + "reward_std_all": 0.003336894907988608, + "rewards-max/forensics_sinkhorn_reward": 0.9786726832389832, + "rewards-max/forensics_strict_edge_reward": 0.7047859728336334, + "rewards-max/forensics_tiou_reward": 0.9752848446369171, + "rewards/forensics_sinkhorn_reward": 0.8653741180896759, + "rewards/forensics_strict_edge_reward": 0.08072483167052269, + "rewards/forensics_tiou_reward": 0.7115827798843384, + "step": 1162 + }, + { + "completion_length": 22.640625, + "epoch": 3.568356374807988, + "grad_norm": 8.480830192565918, + "kl": 0.4755859375, + "learning_rate": 1.0615384615384615e-07, + "loss": -0.0128, + "reward": 0.801476776599884, + "reward_std": 0.008447407744824886, + "reward_std_all": 0.009272999479435384, + "rewards-max/forensics_sinkhorn_reward": 0.976366251707077, + "rewards-max/forensics_strict_edge_reward": 0.7815065383911133, + "rewards-max/forensics_tiou_reward": 0.9673227369785309, + "rewards/forensics_sinkhorn_reward": 0.7874986827373505, + "rewards/forensics_strict_edge_reward": 0.10492672771215439, + "rewards/forensics_tiou_reward": 0.6380440294742584, + "step": 1163 + }, + { + "completion_length": 24.421875, + "epoch": 3.571428571428571, + "grad_norm": 8.571022033691406, + "kl": 0.541015625, + "learning_rate": 1.0538461538461538e-07, + "loss": -0.0041, + "reward": 0.7816667854785919, + "reward_std": 0.018333363346755505, + "reward_std_all": 0.024621020071208477, + "rewards-max/forensics_sinkhorn_reward": 0.9784494340419769, + "rewards-max/forensics_strict_edge_reward": 0.740818202495575, + "rewards-max/forensics_tiou_reward": 0.976097047328949, + "rewards/forensics_sinkhorn_reward": 0.7445903718471527, + "rewards/forensics_strict_edge_reward": 0.09498089551925659, + "rewards/forensics_tiou_reward": 0.5814663916826248, + "step": 1164 + }, + { + "completion_length": 26.34375, + "epoch": 3.5745007680491554, + "grad_norm": 9.121907234191895, + "kl": 0.552734375, + "learning_rate": 1.0461538461538461e-07, + "loss": -0.0785, + "reward": 0.7907271683216095, + "reward_std": 0.026453167898580432, + "reward_std_all": 0.024746983777731657, + "rewards-max/forensics_sinkhorn_reward": 0.9830450117588043, + "rewards-max/forensics_strict_edge_reward": 0.7246644794940948, + "rewards-max/forensics_tiou_reward": 0.9819001853466034, + "rewards/forensics_sinkhorn_reward": 0.7995613813400269, + "rewards/forensics_strict_edge_reward": 0.08406978659331799, + "rewards/forensics_tiou_reward": 0.643925666809082, + "step": 1165 + }, + { + "completion_length": 23.75, + "epoch": 3.5775729646697387, + "grad_norm": 8.045052528381348, + "kl": 0.42578125, + "learning_rate": 1.0384615384615385e-07, + "loss": 0.0046, + "reward": 0.7912097871303558, + "reward_std": 0.025901019107550383, + "reward_std_all": 0.024429607670754194, + "rewards-max/forensics_sinkhorn_reward": 0.9742213785648346, + "rewards-max/forensics_strict_edge_reward": 0.625592440366745, + "rewards-max/forensics_tiou_reward": 0.9652663171291351, + "rewards/forensics_sinkhorn_reward": 0.8118573427200317, + "rewards/forensics_strict_edge_reward": 0.0815519168972969, + "rewards/forensics_tiou_reward": 0.6365963518619537, + "step": 1166 + }, + { + "completion_length": 22.71875, + "epoch": 3.5806451612903225, + "grad_norm": 9.00936508178711, + "kl": 0.435546875, + "learning_rate": 1.0307692307692307e-07, + "loss": -0.0146, + "reward": 0.7727473974227905, + "reward_std": 0.037439897656440735, + "reward_std_all": 0.03969844616949558, + "rewards-max/forensics_sinkhorn_reward": 0.9802711009979248, + "rewards-max/forensics_strict_edge_reward": 0.578471302986145, + "rewards-max/forensics_tiou_reward": 0.9788788259029388, + "rewards/forensics_sinkhorn_reward": 0.8008868098258972, + "rewards/forensics_strict_edge_reward": 0.06083697825670242, + "rewards/forensics_tiou_reward": 0.6477875113487244, + "step": 1167 + }, + { + "completion_length": 29.546875, + "epoch": 3.5837173579109063, + "grad_norm": 7.130138397216797, + "kl": 0.4541015625, + "learning_rate": 1.023076923076923e-07, + "loss": 0.039, + "reward": 0.7827910482883453, + "reward_std": 0.041753645054996014, + "reward_std_all": 0.03766472265124321, + "rewards-max/forensics_sinkhorn_reward": 0.9815508723258972, + "rewards-max/forensics_strict_edge_reward": 0.6555203795433044, + "rewards-max/forensics_tiou_reward": 0.9793646931648254, + "rewards/forensics_sinkhorn_reward": 0.7853864431381226, + "rewards/forensics_strict_edge_reward": 0.12073192372918129, + "rewards/forensics_tiou_reward": 0.6883211731910706, + "step": 1168 + }, + { + "completion_length": 30.03125, + "epoch": 3.58678955453149, + "grad_norm": 8.546669006347656, + "kl": 0.6484375, + "learning_rate": 1.0153846153846153e-07, + "loss": -0.0117, + "reward": 0.7642389833927155, + "reward_std": 0.009054760448634624, + "reward_std_all": 0.02639226010069251, + "rewards-max/forensics_sinkhorn_reward": 0.986856997013092, + "rewards-max/forensics_strict_edge_reward": 0.8260191679000854, + "rewards-max/forensics_tiou_reward": 0.9926665425300598, + "rewards/forensics_sinkhorn_reward": 0.8128292858600616, + "rewards/forensics_strict_edge_reward": 0.09658141061663628, + "rewards/forensics_tiou_reward": 0.7381136119365692, + "step": 1169 + }, + { + "completion_length": 24.484375, + "epoch": 3.589861751152074, + "grad_norm": 8.125470161437988, + "kl": 0.3798828125, + "learning_rate": 1.0076923076923076e-07, + "loss": -0.0104, + "reward": 0.7941041886806488, + "reward_std": 0.021390976384282112, + "reward_std_all": 0.020601334050297737, + "rewards-max/forensics_sinkhorn_reward": 0.9842654466629028, + "rewards-max/forensics_strict_edge_reward": 0.6369597017765045, + "rewards-max/forensics_tiou_reward": 0.9879318177700043, + "rewards/forensics_sinkhorn_reward": 0.8344973623752594, + "rewards/forensics_strict_edge_reward": 0.07111601158976555, + "rewards/forensics_tiou_reward": 0.7012315392494202, + "step": 1170 + }, + { + "completion_length": 24.015625, + "epoch": 3.592933947772657, + "grad_norm": 6.587187767028809, + "kl": 0.4404296875, + "learning_rate": 1e-07, + "loss": -0.0843, + "reward": 0.8015237748622894, + "reward_std": 0.005473490338772535, + "reward_std_all": 0.006960235768929124, + "rewards-max/forensics_sinkhorn_reward": 0.988470196723938, + "rewards-max/forensics_strict_edge_reward": 0.8959447741508484, + "rewards-max/forensics_tiou_reward": 0.9985131025314331, + "rewards/forensics_sinkhorn_reward": 0.8243866860866547, + "rewards/forensics_strict_edge_reward": 0.11694291234016418, + "rewards/forensics_tiou_reward": 0.6392359137535095, + "step": 1171 + }, + { + "completion_length": 25.921875, + "epoch": 3.5960061443932414, + "grad_norm": 8.436439514160156, + "kl": 0.4990234375, + "learning_rate": 9.923076923076922e-08, + "loss": -0.101, + "reward": 0.7612906396389008, + "reward_std": 0.038009021896868944, + "reward_std_all": 0.041869066189974546, + "rewards-max/forensics_sinkhorn_reward": 0.9822906851768494, + "rewards-max/forensics_strict_edge_reward": 0.4388405382633209, + "rewards-max/forensics_tiou_reward": 0.9820884168148041, + "rewards/forensics_sinkhorn_reward": 0.7920248806476593, + "rewards/forensics_strict_edge_reward": 0.048172859475016594, + "rewards/forensics_tiou_reward": 0.6439448595046997, + "step": 1172 + }, + { + "completion_length": 23.78125, + "epoch": 3.5990783410138247, + "grad_norm": 8.351880073547363, + "kl": 0.447265625, + "learning_rate": 9.846153846153846e-08, + "loss": -0.0388, + "reward": 0.7820827662944794, + "reward_std": 0.02379594650119543, + "reward_std_all": 0.02528785727918148, + "rewards-max/forensics_sinkhorn_reward": 0.9822970330715179, + "rewards-max/forensics_strict_edge_reward": 0.7837068438529968, + "rewards-max/forensics_tiou_reward": 0.9795866906642914, + "rewards/forensics_sinkhorn_reward": 0.8068662285804749, + "rewards/forensics_strict_edge_reward": 0.12709370255470276, + "rewards/forensics_tiou_reward": 0.6738826632499695, + "step": 1173 + }, + { + "completion_length": 22.625, + "epoch": 3.6021505376344085, + "grad_norm": 9.47186279296875, + "kl": 0.416015625, + "learning_rate": 9.769230769230769e-08, + "loss": -0.0217, + "reward": 0.7970280051231384, + "reward_std": 0.008724348386749625, + "reward_std_all": 0.011056912830099463, + "rewards-max/forensics_sinkhorn_reward": 0.9825315475463867, + "rewards-max/forensics_strict_edge_reward": 0.7654097378253937, + "rewards-max/forensics_tiou_reward": 0.9826673865318298, + "rewards/forensics_sinkhorn_reward": 0.8279592394828796, + "rewards/forensics_strict_edge_reward": 0.08534564636647701, + "rewards/forensics_tiou_reward": 0.6641126275062561, + "step": 1174 + }, + { + "completion_length": 23.515625, + "epoch": 3.6052227342549923, + "grad_norm": 9.155323028564453, + "kl": 0.376953125, + "learning_rate": 9.692307692307692e-08, + "loss": 0.0126, + "reward": 0.8013424277305603, + "reward_std": 0.0077672896441072226, + "reward_std_all": 0.008653172291815281, + "rewards-max/forensics_sinkhorn_reward": 0.9854745268821716, + "rewards-max/forensics_strict_edge_reward": 0.6487045586109161, + "rewards-max/forensics_tiou_reward": 0.987939864397049, + "rewards/forensics_sinkhorn_reward": 0.8080770671367645, + "rewards/forensics_strict_edge_reward": 0.0918637216091156, + "rewards/forensics_tiou_reward": 0.7012138068675995, + "step": 1175 + }, + { + "completion_length": 25.375, + "epoch": 3.608294930875576, + "grad_norm": 8.155909538269043, + "kl": 0.5625, + "learning_rate": 9.615384615384616e-08, + "loss": -0.0073, + "reward": 0.7949530482292175, + "reward_std": 0.01226621842943132, + "reward_std_all": 0.013479723362252116, + "rewards-max/forensics_sinkhorn_reward": 0.9821023941040039, + "rewards-max/forensics_strict_edge_reward": 0.7412298321723938, + "rewards-max/forensics_tiou_reward": 0.9818608164787292, + "rewards/forensics_sinkhorn_reward": 0.7934631109237671, + "rewards/forensics_strict_edge_reward": 0.07857590168714523, + "rewards/forensics_tiou_reward": 0.6236274242401123, + "step": 1176 + }, + { + "completion_length": 25.59375, + "epoch": 3.61136712749616, + "grad_norm": 7.36611795425415, + "kl": 0.466796875, + "learning_rate": 9.538461538461538e-08, + "loss": 0.0107, + "reward": 0.784905880689621, + "reward_std": 0.0371269378811121, + "reward_std_all": 0.03408806584775448, + "rewards-max/forensics_sinkhorn_reward": 0.9798085391521454, + "rewards-max/forensics_strict_edge_reward": 0.6830973327159882, + "rewards-max/forensics_tiou_reward": 0.9780949354171753, + "rewards/forensics_sinkhorn_reward": 0.7993271052837372, + "rewards/forensics_strict_edge_reward": 0.07784418575465679, + "rewards/forensics_tiou_reward": 0.6480108201503754, + "step": 1177 + }, + { + "completion_length": 25.890625, + "epoch": 3.6144393241167436, + "grad_norm": 8.566965103149414, + "kl": 0.677734375, + "learning_rate": 9.461538461538461e-08, + "loss": -0.0212, + "reward": 0.7801421582698822, + "reward_std": 0.023000874556601048, + "reward_std_all": 0.025174112059175968, + "rewards-max/forensics_sinkhorn_reward": 0.9741595089435577, + "rewards-max/forensics_strict_edge_reward": 0.47726282477378845, + "rewards-max/forensics_tiou_reward": 0.9593113660812378, + "rewards/forensics_sinkhorn_reward": 0.7641931772232056, + "rewards/forensics_strict_edge_reward": 0.06253482028841972, + "rewards/forensics_tiou_reward": 0.6181546747684479, + "step": 1178 + }, + { + "completion_length": 25.46875, + "epoch": 3.6175115207373274, + "grad_norm": 8.260711669921875, + "kl": 0.5283203125, + "learning_rate": 9.384615384615384e-08, + "loss": 0.0048, + "reward": 0.7813746333122253, + "reward_std": 0.01442491274792701, + "reward_std_all": 0.02094996755477041, + "rewards-max/forensics_sinkhorn_reward": 0.9833697378635406, + "rewards-max/forensics_strict_edge_reward": 0.6920900046825409, + "rewards-max/forensics_tiou_reward": 0.9868927001953125, + "rewards/forensics_sinkhorn_reward": 0.8367387056350708, + "rewards/forensics_strict_edge_reward": 0.10232799127697945, + "rewards/forensics_tiou_reward": 0.7115432620048523, + "step": 1179 + }, + { + "completion_length": 24.21875, + "epoch": 3.6205837173579107, + "grad_norm": 15.62476634979248, + "kl": 1.00390625, + "learning_rate": 9.307692307692306e-08, + "loss": -0.0188, + "reward": 0.7751221060752869, + "reward_std": 0.03789298760239035, + "reward_std_all": 0.036755723296664655, + "rewards-max/forensics_sinkhorn_reward": 0.9761134088039398, + "rewards-max/forensics_strict_edge_reward": 0.6733093559741974, + "rewards-max/forensics_tiou_reward": 0.9651846587657928, + "rewards/forensics_sinkhorn_reward": 0.804806262254715, + "rewards/forensics_strict_edge_reward": 0.06913802959024906, + "rewards/forensics_tiou_reward": 0.6660867035388947, + "step": 1180 + }, + { + "completion_length": 24.5625, + "epoch": 3.6236559139784945, + "grad_norm": 8.199807167053223, + "kl": 0.4384765625, + "learning_rate": 9.23076923076923e-08, + "loss": -0.0077, + "reward": 0.7876671552658081, + "reward_std": 0.028137062210589647, + "reward_std_all": 0.026585379149764776, + "rewards-max/forensics_sinkhorn_reward": 0.985273003578186, + "rewards-max/forensics_strict_edge_reward": 0.8665722012519836, + "rewards-max/forensics_tiou_reward": 0.9914792776107788, + "rewards/forensics_sinkhorn_reward": 0.8023868501186371, + "rewards/forensics_strict_edge_reward": 0.08625378087162971, + "rewards/forensics_tiou_reward": 0.676178365945816, + "step": 1181 + }, + { + "completion_length": 22.90625, + "epoch": 3.6267281105990783, + "grad_norm": 8.98829460144043, + "kl": 0.4326171875, + "learning_rate": 9.153846153846153e-08, + "loss": 0.0454, + "reward": 0.7932814955711365, + "reward_std": 0.011779309948906302, + "reward_std_all": 0.015138392802327871, + "rewards-max/forensics_sinkhorn_reward": 0.9821628630161285, + "rewards-max/forensics_strict_edge_reward": 0.5484859831631184, + "rewards-max/forensics_tiou_reward": 0.9834513962268829, + "rewards/forensics_sinkhorn_reward": 0.8223625123500824, + "rewards/forensics_strict_edge_reward": 0.08476836793124676, + "rewards/forensics_tiou_reward": 0.6564145982265472, + "step": 1182 + }, + { + "completion_length": 25.015625, + "epoch": 3.629800307219662, + "grad_norm": 7.893299102783203, + "kl": 0.359375, + "learning_rate": 9.076923076923076e-08, + "loss": -0.0707, + "reward": 0.7988071143627167, + "reward_std": 0.013251532800495625, + "reward_std_all": 0.013426168821752071, + "rewards-max/forensics_sinkhorn_reward": 0.9764418005943298, + "rewards-max/forensics_strict_edge_reward": 0.5308194756507874, + "rewards-max/forensics_tiou_reward": 0.9660487771034241, + "rewards/forensics_sinkhorn_reward": 0.8347618579864502, + "rewards/forensics_strict_edge_reward": 0.0957886129617691, + "rewards/forensics_tiou_reward": 0.7073158621788025, + "step": 1183 + }, + { + "completion_length": 27.71875, + "epoch": 3.632872503840246, + "grad_norm": 8.507463455200195, + "kl": 0.4921875, + "learning_rate": 9e-08, + "loss": 0.0136, + "reward": 0.7935150861740112, + "reward_std": 0.012689821305684745, + "reward_std_all": 0.014303077245131135, + "rewards-max/forensics_sinkhorn_reward": 0.9730841219425201, + "rewards-max/forensics_strict_edge_reward": 0.6204091012477875, + "rewards-max/forensics_tiou_reward": 0.9630628824234009, + "rewards/forensics_sinkhorn_reward": 0.8202918171882629, + "rewards/forensics_strict_edge_reward": 0.048126209527254105, + "rewards/forensics_tiou_reward": 0.6948966681957245, + "step": 1184 + }, + { + "completion_length": 27.421875, + "epoch": 3.6359447004608296, + "grad_norm": 8.584249496459961, + "kl": 0.4140625, + "learning_rate": 8.923076923076923e-08, + "loss": -0.0345, + "reward": 0.7506424784660339, + "reward_std": 0.05284855142235756, + "reward_std_all": 0.05478637106716633, + "rewards-max/forensics_sinkhorn_reward": 0.9847301542758942, + "rewards-max/forensics_strict_edge_reward": 0.7868733704090118, + "rewards-max/forensics_tiou_reward": 0.9882618188858032, + "rewards/forensics_sinkhorn_reward": 0.7377924621105194, + "rewards/forensics_strict_edge_reward": 0.09830205515027046, + "rewards/forensics_tiou_reward": 0.6154935657978058, + "step": 1185 + }, + { + "completion_length": 25.4375, + "epoch": 3.6390168970814134, + "grad_norm": 7.418065547943115, + "kl": 0.3720703125, + "learning_rate": 8.846153846153847e-08, + "loss": -0.0104, + "reward": 0.7747021019458771, + "reward_std": 0.022278944496065378, + "reward_std_all": 0.02956120204180479, + "rewards-max/forensics_sinkhorn_reward": 0.9834731817245483, + "rewards-max/forensics_strict_edge_reward": 0.5918685793876648, + "rewards-max/forensics_tiou_reward": 0.9879098236560822, + "rewards/forensics_sinkhorn_reward": 0.7967574596405029, + "rewards/forensics_strict_edge_reward": 0.09828482568264008, + "rewards/forensics_tiou_reward": 0.6545641124248505, + "step": 1186 + }, + { + "completion_length": 25.71875, + "epoch": 3.6420890937019967, + "grad_norm": 8.513436317443848, + "kl": 0.5380859375, + "learning_rate": 8.769230769230769e-08, + "loss": -0.1153, + "reward": 0.8058144450187683, + "reward_std": 0.0016983990208245814, + "reward_std_all": 0.00332081678789109, + "rewards-max/forensics_sinkhorn_reward": 0.9797661900520325, + "rewards-max/forensics_strict_edge_reward": 0.6886788010597229, + "rewards-max/forensics_tiou_reward": 0.9739188551902771, + "rewards/forensics_sinkhorn_reward": 0.8427798748016357, + "rewards/forensics_strict_edge_reward": 0.11691171303391457, + "rewards/forensics_tiou_reward": 0.6923331916332245, + "step": 1187 + }, + { + "completion_length": 27.328125, + "epoch": 3.6451612903225805, + "grad_norm": 8.232488632202148, + "kl": 0.494140625, + "learning_rate": 8.692307692307692e-08, + "loss": -0.0187, + "reward": 0.7414354085922241, + "reward_std": 0.03729023551568389, + "reward_std_all": 0.0540353748947382, + "rewards-max/forensics_sinkhorn_reward": 0.9844042658805847, + "rewards-max/forensics_strict_edge_reward": 0.7511312067508698, + "rewards-max/forensics_tiou_reward": 0.9882753491401672, + "rewards/forensics_sinkhorn_reward": 0.7339815497398376, + "rewards/forensics_strict_edge_reward": 0.07700037583708763, + "rewards/forensics_tiou_reward": 0.6174667477607727, + "step": 1188 + }, + { + "completion_length": 23.078125, + "epoch": 3.6482334869431643, + "grad_norm": 10.097662925720215, + "kl": 0.50390625, + "learning_rate": 8.615384615384615e-08, + "loss": -0.0049, + "reward": 0.7989388108253479, + "reward_std": 0.007528979796916246, + "reward_std_all": 0.009269434493035078, + "rewards-max/forensics_sinkhorn_reward": 0.9785023927688599, + "rewards-max/forensics_strict_edge_reward": 0.6432462930679321, + "rewards-max/forensics_tiou_reward": 0.9768982231616974, + "rewards/forensics_sinkhorn_reward": 0.8306409120559692, + "rewards/forensics_strict_edge_reward": 0.06263920664787292, + "rewards/forensics_tiou_reward": 0.6800414323806763, + "step": 1189 + }, + { + "completion_length": 22.34375, + "epoch": 3.651305683563748, + "grad_norm": 9.145522117614746, + "kl": 0.4306640625, + "learning_rate": 8.538461538461537e-08, + "loss": -0.0436, + "reward": 0.7983104288578033, + "reward_std": 0.01322524668648839, + "reward_std_all": 0.01332898810505867, + "rewards-max/forensics_sinkhorn_reward": 0.9706577062606812, + "rewards-max/forensics_strict_edge_reward": 0.5883472263813019, + "rewards-max/forensics_tiou_reward": 0.9464753866195679, + "rewards/forensics_sinkhorn_reward": 0.7829074561595917, + "rewards/forensics_strict_edge_reward": 0.10248231142759323, + "rewards/forensics_tiou_reward": 0.6078527867794037, + "step": 1190 + }, + { + "completion_length": 22.421875, + "epoch": 3.654377880184332, + "grad_norm": 9.013897895812988, + "kl": 0.7353515625, + "learning_rate": 8.461538461538461e-08, + "loss": -0.0188, + "reward": 0.7914929687976837, + "reward_std": 0.027993314433842897, + "reward_std_all": 0.025719492230564356, + "rewards-max/forensics_sinkhorn_reward": 0.9877099692821503, + "rewards-max/forensics_strict_edge_reward": 0.776590883731842, + "rewards-max/forensics_tiou_reward": 0.9953327476978302, + "rewards/forensics_sinkhorn_reward": 0.8732280731201172, + "rewards/forensics_strict_edge_reward": 0.13763641938567162, + "rewards/forensics_tiou_reward": 0.763390302658081, + "step": 1191 + }, + { + "completion_length": 20.8125, + "epoch": 3.6574500768049156, + "grad_norm": 9.908526420593262, + "kl": 0.5322265625, + "learning_rate": 8.384615384615384e-08, + "loss": -0.0533, + "reward": 0.797665536403656, + "reward_std": 0.014406085480004549, + "reward_std_all": 0.014627964701503515, + "rewards-max/forensics_sinkhorn_reward": 0.985017329454422, + "rewards-max/forensics_strict_edge_reward": 0.7676310539245605, + "rewards-max/forensics_tiou_reward": 0.9896626770496368, + "rewards/forensics_sinkhorn_reward": 0.788218766450882, + "rewards/forensics_strict_edge_reward": 0.1092676967382431, + "rewards/forensics_tiou_reward": 0.6010983288288116, + "step": 1192 + }, + { + "completion_length": 26.59375, + "epoch": 3.6605222734254994, + "grad_norm": 8.04723834991455, + "kl": 0.50390625, + "learning_rate": 8.307692307692307e-08, + "loss": 0.0321, + "reward": 0.7731610536575317, + "reward_std": 0.057989140041172504, + "reward_std_all": 0.05167241673916578, + "rewards-max/forensics_sinkhorn_reward": 0.9843921363353729, + "rewards-max/forensics_strict_edge_reward": 0.596860408782959, + "rewards-max/forensics_tiou_reward": 0.9868273735046387, + "rewards/forensics_sinkhorn_reward": 0.7563682198524475, + "rewards/forensics_strict_edge_reward": 0.10985036939382553, + "rewards/forensics_tiou_reward": 0.6415907144546509, + "step": 1193 + }, + { + "completion_length": 23.546875, + "epoch": 3.6635944700460827, + "grad_norm": 9.280065536499023, + "kl": 0.50390625, + "learning_rate": 8.23076923076923e-08, + "loss": -0.0537, + "reward": 0.8046171069145203, + "reward_std": 0.003490504575893283, + "reward_std_all": 0.004814596846699715, + "rewards-max/forensics_sinkhorn_reward": 0.982215404510498, + "rewards-max/forensics_strict_edge_reward": 0.5785898119211197, + "rewards-max/forensics_tiou_reward": 0.9842009842395782, + "rewards/forensics_sinkhorn_reward": 0.8382136225700378, + "rewards/forensics_strict_edge_reward": 0.07128802314400673, + "rewards/forensics_tiou_reward": 0.6878090798854828, + "step": 1194 + }, + { + "completion_length": 21.40625, + "epoch": 3.6666666666666665, + "grad_norm": 9.712687492370605, + "kl": 0.4296875, + "learning_rate": 8.153846153846153e-08, + "loss": -0.0286, + "reward": 0.7807305157184601, + "reward_std": 0.0296023259870708, + "reward_std_all": 0.029603535309433937, + "rewards-max/forensics_sinkhorn_reward": 0.9806772172451019, + "rewards-max/forensics_strict_edge_reward": 0.6487045586109161, + "rewards-max/forensics_tiou_reward": 0.97382453083992, + "rewards/forensics_sinkhorn_reward": 0.7837324440479279, + "rewards/forensics_strict_edge_reward": 0.07826989889144897, + "rewards/forensics_tiou_reward": 0.5948190093040466, + "step": 1195 + }, + { + "completion_length": 24.859375, + "epoch": 3.6697388632872503, + "grad_norm": 7.182151794433594, + "kl": 0.580078125, + "learning_rate": 8.076923076923077e-08, + "loss": 0.1657, + "reward": 0.754578709602356, + "reward_std": 0.0835016518831253, + "reward_std_all": 0.07467802613973618, + "rewards-max/forensics_sinkhorn_reward": 0.9787014424800873, + "rewards-max/forensics_strict_edge_reward": 0.7265062034130096, + "rewards-max/forensics_tiou_reward": 0.9759823977947235, + "rewards/forensics_sinkhorn_reward": 0.7191344499588013, + "rewards/forensics_strict_edge_reward": 0.08503123559057713, + "rewards/forensics_tiou_reward": 0.5595747828483582, + "step": 1196 + }, + { + "completion_length": 26.40625, + "epoch": 3.672811059907834, + "grad_norm": 7.4905266761779785, + "kl": 0.3544921875, + "learning_rate": 8e-08, + "loss": 0.0477, + "reward": 0.7947190701961517, + "reward_std": 0.010985899716615677, + "reward_std_all": 0.012855868553742766, + "rewards-max/forensics_sinkhorn_reward": 0.9766258895397186, + "rewards-max/forensics_strict_edge_reward": 0.5076818764209747, + "rewards-max/forensics_tiou_reward": 0.9668629169464111, + "rewards/forensics_sinkhorn_reward": 0.7680238783359528, + "rewards/forensics_strict_edge_reward": 0.0630611777305603, + "rewards/forensics_tiou_reward": 0.6360283195972443, + "step": 1197 + }, + { + "completion_length": 24.671875, + "epoch": 3.675883256528418, + "grad_norm": 8.611437797546387, + "kl": 0.421875, + "learning_rate": 7.923076923076923e-08, + "loss": -0.0588, + "reward": 0.7406233251094818, + "reward_std": 0.04369146376848221, + "reward_std_all": 0.058813463896512985, + "rewards-max/forensics_sinkhorn_reward": 0.9808491170406342, + "rewards-max/forensics_strict_edge_reward": 0.7545943260192871, + "rewards-max/forensics_tiou_reward": 0.9791894555091858, + "rewards/forensics_sinkhorn_reward": 0.7396562695503235, + "rewards/forensics_strict_edge_reward": 0.0813903957605362, + "rewards/forensics_tiou_reward": 0.6076274812221527, + "step": 1198 + }, + { + "completion_length": 24.5, + "epoch": 3.6789554531490016, + "grad_norm": 8.66987419128418, + "kl": 0.4345703125, + "learning_rate": 7.846153846153846e-08, + "loss": 0.0185, + "reward": 0.7898063063621521, + "reward_std": 0.027561553986743093, + "reward_std_all": 0.026177075691521168, + "rewards-max/forensics_sinkhorn_reward": 0.9804354310035706, + "rewards-max/forensics_strict_edge_reward": 0.613023579120636, + "rewards-max/forensics_tiou_reward": 0.9760319292545319, + "rewards/forensics_sinkhorn_reward": 0.8418979048728943, + "rewards/forensics_strict_edge_reward": 0.08982580155134201, + "rewards/forensics_tiou_reward": 0.6980895400047302, + "step": 1199 + }, + { + "completion_length": 22.53125, + "epoch": 3.6820276497695854, + "grad_norm": 10.14116096496582, + "kl": 0.658203125, + "learning_rate": 7.769230769230768e-08, + "loss": -0.0941, + "reward": 0.7863123416900635, + "reward_std": 0.012464586179703474, + "reward_std_all": 0.017799081979319453, + "rewards-max/forensics_sinkhorn_reward": 0.9604304134845734, + "rewards-max/forensics_strict_edge_reward": 0.5368446409702301, + "rewards-max/forensics_tiou_reward": 0.9327940344810486, + "rewards/forensics_sinkhorn_reward": 0.7993449568748474, + "rewards/forensics_strict_edge_reward": 0.0968010937795043, + "rewards/forensics_tiou_reward": 0.6548982262611389, + "step": 1200 + }, + { + "completion_length": 23.96875, + "epoch": 3.6850998463901687, + "grad_norm": 9.798667907714844, + "kl": 0.537109375, + "learning_rate": 7.692307692307692e-08, + "loss": -0.0715, + "reward": 0.7771091461181641, + "reward_std": 0.04970119334757328, + "reward_std_all": 0.04481283761560917, + "rewards-max/forensics_sinkhorn_reward": 0.9825781285762787, + "rewards-max/forensics_strict_edge_reward": 0.7286747694015503, + "rewards-max/forensics_tiou_reward": 0.982970118522644, + "rewards/forensics_sinkhorn_reward": 0.7788808345794678, + "rewards/forensics_strict_edge_reward": 0.07047749124467373, + "rewards/forensics_tiou_reward": 0.624398797750473, + "step": 1201 + }, + { + "completion_length": 22.59375, + "epoch": 3.688172043010753, + "grad_norm": 8.063543319702148, + "kl": 0.4052734375, + "learning_rate": 7.615384615384615e-08, + "loss": -0.0198, + "reward": 0.7794051766395569, + "reward_std": 0.045943102799355984, + "reward_std_all": 0.0419336975319311, + "rewards-max/forensics_sinkhorn_reward": 0.9850017428398132, + "rewards-max/forensics_strict_edge_reward": 0.8419698476791382, + "rewards-max/forensics_tiou_reward": 0.9889829456806183, + "rewards/forensics_sinkhorn_reward": 0.799155980348587, + "rewards/forensics_strict_edge_reward": 0.06867439299821854, + "rewards/forensics_tiou_reward": 0.6194160282611847, + "step": 1202 + }, + { + "completion_length": 27.21875, + "epoch": 3.6912442396313363, + "grad_norm": 7.6652445793151855, + "kl": 0.470703125, + "learning_rate": 7.538461538461538e-08, + "loss": 0.0229, + "reward": 0.7924003303050995, + "reward_std": 0.024416229338385165, + "reward_std_all": 0.022987131495028734, + "rewards-max/forensics_sinkhorn_reward": 0.9706950783729553, + "rewards-max/forensics_strict_edge_reward": 0.3885071277618408, + "rewards-max/forensics_tiou_reward": 0.9529468715190887, + "rewards/forensics_sinkhorn_reward": 0.8256758451461792, + "rewards/forensics_strict_edge_reward": 0.06417987123131752, + "rewards/forensics_tiou_reward": 0.6842042803764343, + "step": 1203 + }, + { + "completion_length": 23.0, + "epoch": 3.69431643625192, + "grad_norm": 8.342829704284668, + "kl": 0.70703125, + "learning_rate": 7.46153846153846e-08, + "loss": 0.0511, + "reward": 0.7769549190998077, + "reward_std": 0.03906981088221073, + "reward_std_all": 0.03719642385840416, + "rewards-max/forensics_sinkhorn_reward": 0.9768860936164856, + "rewards-max/forensics_strict_edge_reward": 0.5004266202449799, + "rewards-max/forensics_tiou_reward": 0.9663298726081848, + "rewards/forensics_sinkhorn_reward": 0.7592498660087585, + "rewards/forensics_strict_edge_reward": 0.05616094172000885, + "rewards/forensics_tiou_reward": 0.5559139549732208, + "step": 1204 + }, + { + "completion_length": 21.515625, + "epoch": 3.697388632872504, + "grad_norm": 8.363968849182129, + "kl": 0.42578125, + "learning_rate": 7.384615384615384e-08, + "loss": -0.0181, + "reward": 0.7948737442493439, + "reward_std": 0.021121093071997166, + "reward_std_all": 0.020090505247935653, + "rewards-max/forensics_sinkhorn_reward": 0.9859295189380646, + "rewards-max/forensics_strict_edge_reward": 0.7797744870185852, + "rewards-max/forensics_tiou_reward": 0.9904360473155975, + "rewards/forensics_sinkhorn_reward": 0.8627430498600006, + "rewards/forensics_strict_edge_reward": 0.10207146778702736, + "rewards/forensics_tiou_reward": 0.7137014269828796, + "step": 1205 + }, + { + "completion_length": 25.765625, + "epoch": 3.7004608294930876, + "grad_norm": 7.6721978187561035, + "kl": 0.533203125, + "learning_rate": 7.307692307692308e-08, + "loss": -0.0822, + "reward": 0.8037729561328888, + "reward_std": 0.005020993412472308, + "reward_std_all": 0.006342769949696958, + "rewards-max/forensics_sinkhorn_reward": 0.9801815748214722, + "rewards-max/forensics_strict_edge_reward": 0.4792220741510391, + "rewards-max/forensics_tiou_reward": 0.9703865051269531, + "rewards/forensics_sinkhorn_reward": 0.8540776371955872, + "rewards/forensics_strict_edge_reward": 0.07223687320947647, + "rewards/forensics_tiou_reward": 0.6978365182876587, + "step": 1206 + }, + { + "completion_length": 24.5, + "epoch": 3.7035330261136714, + "grad_norm": 8.961370468139648, + "kl": 0.5146484375, + "learning_rate": 7.230769230769231e-08, + "loss": 0.0211, + "reward": 0.7820338308811188, + "reward_std": 0.04283355642110109, + "reward_std_all": 0.03900595381855965, + "rewards-max/forensics_sinkhorn_reward": 0.9843628704547882, + "rewards-max/forensics_strict_edge_reward": 0.7419231235980988, + "rewards-max/forensics_tiou_reward": 0.9856897592544556, + "rewards/forensics_sinkhorn_reward": 0.7989960312843323, + "rewards/forensics_strict_edge_reward": 0.13832363858819008, + "rewards/forensics_tiou_reward": 0.6548278629779816, + "step": 1207 + }, + { + "completion_length": 26.015625, + "epoch": 3.7066052227342547, + "grad_norm": 7.23323917388916, + "kl": 0.5771484375, + "learning_rate": 7.153846153846154e-08, + "loss": 0.016, + "reward": 0.7652881443500519, + "reward_std": 0.031277835834771395, + "reward_std_all": 0.03592822887003422, + "rewards-max/forensics_sinkhorn_reward": 0.987379789352417, + "rewards-max/forensics_strict_edge_reward": 0.7988193929195404, + "rewards-max/forensics_tiou_reward": 0.997001588344574, + "rewards/forensics_sinkhorn_reward": 0.7887597680091858, + "rewards/forensics_strict_edge_reward": 0.08315027505159378, + "rewards/forensics_tiou_reward": 0.6296443939208984, + "step": 1208 + }, + { + "completion_length": 24.109375, + "epoch": 3.709677419354839, + "grad_norm": 9.575767517089844, + "kl": 0.57421875, + "learning_rate": 7.076923076923076e-08, + "loss": 0.0149, + "reward": 0.783267080783844, + "reward_std": 0.027011362835764885, + "reward_std_all": 0.02684208285063505, + "rewards-max/forensics_sinkhorn_reward": 0.9880090057849884, + "rewards-max/forensics_strict_edge_reward": 0.8135445415973663, + "rewards-max/forensics_tiou_reward": 0.9965444803237915, + "rewards/forensics_sinkhorn_reward": 0.7798941731452942, + "rewards/forensics_strict_edge_reward": 0.10912222415208817, + "rewards/forensics_tiou_reward": 0.63114333152771, + "step": 1209 + }, + { + "completion_length": 25.5625, + "epoch": 3.7127496159754223, + "grad_norm": 7.231744766235352, + "kl": 0.412109375, + "learning_rate": 7e-08, + "loss": 0.1423, + "reward": 0.7847934067249298, + "reward_std": 0.019523418974131346, + "reward_std_all": 0.02256024582311511, + "rewards-max/forensics_sinkhorn_reward": 0.9746264815330505, + "rewards-max/forensics_strict_edge_reward": 0.36655619740486145, + "rewards-max/forensics_tiou_reward": 0.9622183740139008, + "rewards/forensics_sinkhorn_reward": 0.7159139215946198, + "rewards/forensics_strict_edge_reward": 0.03567582368850708, + "rewards/forensics_tiou_reward": 0.5830796957015991, + "step": 1210 + }, + { + "completion_length": 22.4375, + "epoch": 3.715821812596006, + "grad_norm": 7.032373428344727, + "kl": 0.41015625, + "learning_rate": 6.923076923076923e-08, + "loss": -0.0188, + "reward": 0.7918959259986877, + "reward_std": 0.024875984323443845, + "reward_std_all": 0.023734404239803553, + "rewards-max/forensics_sinkhorn_reward": 0.98194220662117, + "rewards-max/forensics_strict_edge_reward": 0.639672726392746, + "rewards-max/forensics_tiou_reward": 0.9834344387054443, + "rewards/forensics_sinkhorn_reward": 0.8285253047943115, + "rewards/forensics_strict_edge_reward": 0.05819141864776611, + "rewards/forensics_tiou_reward": 0.6726105809211731, + "step": 1211 + }, + { + "completion_length": 27.625, + "epoch": 3.71889400921659, + "grad_norm": 6.950202941894531, + "kl": 0.5224609375, + "learning_rate": 6.846153846153846e-08, + "loss": -0.0443, + "reward": 0.7757664322853088, + "reward_std": 0.016604786971583962, + "reward_std_all": 0.026320692151784897, + "rewards-max/forensics_sinkhorn_reward": 0.9796910881996155, + "rewards-max/forensics_strict_edge_reward": 0.625676691532135, + "rewards-max/forensics_tiou_reward": 0.9758522510528564, + "rewards/forensics_sinkhorn_reward": 0.7487581968307495, + "rewards/forensics_strict_edge_reward": 0.03949766978621483, + "rewards/forensics_tiou_reward": 0.5929211378097534, + "step": 1212 + }, + { + "completion_length": 27.046875, + "epoch": 3.7219662058371736, + "grad_norm": 7.052602767944336, + "kl": 0.38671875, + "learning_rate": 6.769230769230769e-08, + "loss": 0.0205, + "reward": 0.7797755599021912, + "reward_std": 0.03177489433437586, + "reward_std_all": 0.031122637912631035, + "rewards-max/forensics_sinkhorn_reward": 0.9870502650737762, + "rewards-max/forensics_strict_edge_reward": 0.6998944580554962, + "rewards-max/forensics_tiou_reward": 0.9964995980262756, + "rewards/forensics_sinkhorn_reward": 0.820328414440155, + "rewards/forensics_strict_edge_reward": 0.08328530192375183, + "rewards/forensics_tiou_reward": 0.6706216037273407, + "step": 1213 + }, + { + "completion_length": 25.203125, + "epoch": 3.7250384024577574, + "grad_norm": 6.500061988830566, + "kl": 0.39453125, + "learning_rate": 6.692307692307691e-08, + "loss": -0.0645, + "reward": 0.7705384194850922, + "reward_std": 0.03245487390086055, + "reward_std_all": 0.034278747625648975, + "rewards-max/forensics_sinkhorn_reward": 0.9768337309360504, + "rewards-max/forensics_strict_edge_reward": 0.4513688236474991, + "rewards-max/forensics_tiou_reward": 0.9697884619235992, + "rewards/forensics_sinkhorn_reward": 0.7609350681304932, + "rewards/forensics_strict_edge_reward": 0.06322244368493557, + "rewards/forensics_tiou_reward": 0.6015645861625671, + "step": 1214 + }, + { + "completion_length": 23.75, + "epoch": 3.7281105990783407, + "grad_norm": 9.928459167480469, + "kl": 0.46484375, + "learning_rate": 6.615384615384615e-08, + "loss": -0.0497, + "reward": 0.8063533902168274, + "reward_std": 0.0012305794516578317, + "reward_std_all": 0.002890750765800476, + "rewards-max/forensics_sinkhorn_reward": 0.9816420674324036, + "rewards-max/forensics_strict_edge_reward": 0.5416891872882843, + "rewards-max/forensics_tiou_reward": 0.9784902036190033, + "rewards/forensics_sinkhorn_reward": 0.8805927634239197, + "rewards/forensics_strict_edge_reward": 0.1011330895125866, + "rewards/forensics_tiou_reward": 0.7484179139137268, + "step": 1215 + }, + { + "completion_length": 23.46875, + "epoch": 3.731182795698925, + "grad_norm": 7.549011707305908, + "kl": 0.4443359375, + "learning_rate": 6.538461538461538e-08, + "loss": -0.001, + "reward": 0.7765059173107147, + "reward_std": 0.028281397186219692, + "reward_std_all": 0.03080481942743063, + "rewards-max/forensics_sinkhorn_reward": 0.9850363731384277, + "rewards-max/forensics_strict_edge_reward": 0.7165313363075256, + "rewards-max/forensics_tiou_reward": 0.9884669184684753, + "rewards/forensics_sinkhorn_reward": 0.7772976160049438, + "rewards/forensics_strict_edge_reward": 0.06777582690119743, + "rewards/forensics_tiou_reward": 0.6533019840717316, + "step": 1216 + }, + { + "completion_length": 27.390625, + "epoch": 3.7342549923195083, + "grad_norm": 7.639670372009277, + "kl": 0.3427734375, + "learning_rate": 6.461538461538462e-08, + "loss": 0.0105, + "reward": 0.7377557754516602, + "reward_std": 0.043158870190382004, + "reward_std_all": 0.058701591566205025, + "rewards-max/forensics_sinkhorn_reward": 0.9815493822097778, + "rewards-max/forensics_strict_edge_reward": 0.594437800347805, + "rewards-max/forensics_tiou_reward": 0.9809350669384003, + "rewards/forensics_sinkhorn_reward": 0.7502382397651672, + "rewards/forensics_strict_edge_reward": 0.07018246781080961, + "rewards/forensics_tiou_reward": 0.623333603143692, + "step": 1217 + }, + { + "completion_length": 23.109375, + "epoch": 3.737327188940092, + "grad_norm": 7.87111234664917, + "kl": 0.537109375, + "learning_rate": 6.384615384615385e-08, + "loss": -0.0356, + "reward": 0.7977283298969269, + "reward_std": 0.017041013576090336, + "reward_std_all": 0.016090456279926002, + "rewards-max/forensics_sinkhorn_reward": 0.9860261976718903, + "rewards-max/forensics_strict_edge_reward": 0.7511312067508698, + "rewards-max/forensics_tiou_reward": 0.9910397827625275, + "rewards/forensics_sinkhorn_reward": 0.836194634437561, + "rewards/forensics_strict_edge_reward": 0.14951223507523537, + "rewards/forensics_tiou_reward": 0.7277235984802246, + "step": 1218 + }, + { + "completion_length": 22.65625, + "epoch": 3.740399385560676, + "grad_norm": 7.821074962615967, + "kl": 0.51953125, + "learning_rate": 6.307692307692307e-08, + "loss": -0.0364, + "reward": 0.7921122610569, + "reward_std": 0.005289523978717625, + "reward_std_all": 0.011300664278678596, + "rewards-max/forensics_sinkhorn_reward": 0.976588785648346, + "rewards-max/forensics_strict_edge_reward": 0.6670474708080292, + "rewards-max/forensics_tiou_reward": 0.9699767529964447, + "rewards/forensics_sinkhorn_reward": 0.7894451022148132, + "rewards/forensics_strict_edge_reward": 0.0785648301243782, + "rewards/forensics_tiou_reward": 0.6103849411010742, + "step": 1219 + }, + { + "completion_length": 21.890625, + "epoch": 3.7434715821812596, + "grad_norm": 7.949988842010498, + "kl": 0.388671875, + "learning_rate": 6.23076923076923e-08, + "loss": 0.0691, + "reward": 0.7970055639743805, + "reward_std": 0.005971861071884632, + "reward_std_all": 0.009539684280753136, + "rewards-max/forensics_sinkhorn_reward": 0.9417774379253387, + "rewards-max/forensics_strict_edge_reward": 0.3467923626303673, + "rewards-max/forensics_tiou_reward": 0.8857291340827942, + "rewards/forensics_sinkhorn_reward": 0.797581285238266, + "rewards/forensics_strict_edge_reward": 0.030426984187215567, + "rewards/forensics_tiou_reward": 0.6037009656429291, + "step": 1220 + }, + { + "completion_length": 26.734375, + "epoch": 3.7465437788018434, + "grad_norm": 8.397916793823242, + "kl": 0.4375, + "learning_rate": 6.153846153846154e-08, + "loss": -0.0353, + "reward": 0.7931728363037109, + "reward_std": 0.015431074250955135, + "reward_std_all": 0.016372217214666307, + "rewards-max/forensics_sinkhorn_reward": 0.9869269728660583, + "rewards-max/forensics_strict_edge_reward": 0.5712012872099876, + "rewards-max/forensics_tiou_reward": 0.9935140907764435, + "rewards/forensics_sinkhorn_reward": 0.8125356137752533, + "rewards/forensics_strict_edge_reward": 0.06334148533642292, + "rewards/forensics_tiou_reward": 0.6835275590419769, + "step": 1221 + }, + { + "completion_length": 22.234375, + "epoch": 3.749615975422427, + "grad_norm": 8.194564819335938, + "kl": 0.4619140625, + "learning_rate": 6.076923076923077e-08, + "loss": -0.0285, + "reward": 0.7735387980937958, + "reward_std": 0.03432257194072008, + "reward_std_all": 0.03500421345233917, + "rewards-max/forensics_sinkhorn_reward": 0.9653784036636353, + "rewards-max/forensics_strict_edge_reward": 0.5593344569206238, + "rewards-max/forensics_tiou_reward": 0.9461203217506409, + "rewards/forensics_sinkhorn_reward": 0.7614540755748749, + "rewards/forensics_strict_edge_reward": 0.06694362312555313, + "rewards/forensics_tiou_reward": 0.6305815279483795, + "step": 1222 + }, + { + "completion_length": 23.6875, + "epoch": 3.752688172043011, + "grad_norm": 10.565230369567871, + "kl": 0.3447265625, + "learning_rate": 6e-08, + "loss": -0.0239, + "reward": 0.7936946451663971, + "reward_std": 0.01821514731273055, + "reward_std_all": 0.018275134963914752, + "rewards-max/forensics_sinkhorn_reward": 0.9804908931255341, + "rewards-max/forensics_strict_edge_reward": 0.5614076256752014, + "rewards-max/forensics_tiou_reward": 0.9794184267520905, + "rewards/forensics_sinkhorn_reward": 0.7975018918514252, + "rewards/forensics_strict_edge_reward": 0.06659932062029839, + "rewards/forensics_tiou_reward": 0.6309545636177063, + "step": 1223 + }, + { + "completion_length": 22.796875, + "epoch": 3.7557603686635943, + "grad_norm": 7.972109317779541, + "kl": 0.3955078125, + "learning_rate": 5.923076923076923e-08, + "loss": 0.0359, + "reward": 0.7925542891025543, + "reward_std": 0.01584379468113184, + "reward_std_all": 0.016575913410633802, + "rewards-max/forensics_sinkhorn_reward": 0.9889904856681824, + "rewards-max/forensics_strict_edge_reward": 1.0, + "rewards-max/forensics_tiou_reward": 1.0, + "rewards/forensics_sinkhorn_reward": 0.8280093967914581, + "rewards/forensics_strict_edge_reward": 0.14963258057832718, + "rewards/forensics_tiou_reward": 0.688983678817749, + "step": 1224 + }, + { + "completion_length": 21.15625, + "epoch": 3.758832565284178, + "grad_norm": 8.515917778015137, + "kl": 0.427734375, + "learning_rate": 5.8461538461538455e-08, + "loss": 0.0389, + "reward": 0.757697582244873, + "reward_std": 0.054807379841804504, + "reward_std_all": 0.05324403755366802, + "rewards-max/forensics_sinkhorn_reward": 0.9836937785148621, + "rewards-max/forensics_strict_edge_reward": 0.5677289068698883, + "rewards-max/forensics_tiou_reward": 0.9823558926582336, + "rewards/forensics_sinkhorn_reward": 0.7655134499073029, + "rewards/forensics_strict_edge_reward": 0.06076149269938469, + "rewards/forensics_tiou_reward": 0.5939923226833344, + "step": 1225 + }, + { + "completion_length": 24.28125, + "epoch": 3.761904761904762, + "grad_norm": 8.529841423034668, + "kl": 0.5546875, + "learning_rate": 5.7692307692307695e-08, + "loss": -0.0548, + "reward": 0.7897942066192627, + "reward_std": 0.01601459551602602, + "reward_std_all": 0.01757347211241722, + "rewards-max/forensics_sinkhorn_reward": 0.9741075932979584, + "rewards-max/forensics_strict_edge_reward": 0.5171007066965103, + "rewards-max/forensics_tiou_reward": 0.9622913002967834, + "rewards/forensics_sinkhorn_reward": 0.8043569922447205, + "rewards/forensics_strict_edge_reward": 0.06724303029477596, + "rewards/forensics_tiou_reward": 0.6161712408065796, + "step": 1226 + }, + { + "completion_length": 25.71875, + "epoch": 3.7649769585253456, + "grad_norm": 8.058137893676758, + "kl": 0.5205078125, + "learning_rate": 5.692307692307692e-08, + "loss": 0.049, + "reward": 0.7745894491672516, + "reward_std": 0.04806426214054227, + "reward_std_all": 0.04410779010504484, + "rewards-max/forensics_sinkhorn_reward": 0.9751825928688049, + "rewards-max/forensics_strict_edge_reward": 0.5253343880176544, + "rewards-max/forensics_tiou_reward": 0.9635870456695557, + "rewards/forensics_sinkhorn_reward": 0.7969470322132111, + "rewards/forensics_strict_edge_reward": 0.098435427993536, + "rewards/forensics_tiou_reward": 0.6294615566730499, + "step": 1227 + }, + { + "completion_length": 25.625, + "epoch": 3.7680491551459294, + "grad_norm": 8.73829174041748, + "kl": 0.3515625, + "learning_rate": 5.615384615384615e-08, + "loss": 0.0508, + "reward": 0.7663436233997345, + "reward_std": 0.038215525448322296, + "reward_std_all": 0.03936487436294556, + "rewards-max/forensics_sinkhorn_reward": 0.9779804348945618, + "rewards-max/forensics_strict_edge_reward": 0.5636615753173828, + "rewards-max/forensics_tiou_reward": 0.9663216173648834, + "rewards/forensics_sinkhorn_reward": 0.7770470380783081, + "rewards/forensics_strict_edge_reward": 0.10275581479072571, + "rewards/forensics_tiou_reward": 0.6555940806865692, + "step": 1228 + }, + { + "completion_length": 24.703125, + "epoch": 3.771121351766513, + "grad_norm": 10.095139503479004, + "kl": 0.40234375, + "learning_rate": 5.538461538461538e-08, + "loss": -0.0338, + "reward": 0.8044894337654114, + "reward_std": 0.0020315832225605845, + "reward_std_all": 0.00415048492141068, + "rewards-max/forensics_sinkhorn_reward": 0.9823513627052307, + "rewards-max/forensics_strict_edge_reward": 0.5905451774597168, + "rewards-max/forensics_tiou_reward": 0.9857993721961975, + "rewards/forensics_sinkhorn_reward": 0.780411422252655, + "rewards/forensics_strict_edge_reward": 0.08325487747788429, + "rewards/forensics_tiou_reward": 0.5752229690551758, + "step": 1229 + }, + { + "completion_length": 23.765625, + "epoch": 3.774193548387097, + "grad_norm": 9.883776664733887, + "kl": 0.455078125, + "learning_rate": 5.461538461538461e-08, + "loss": -0.0047, + "reward": 0.7681702971458435, + "reward_std": 0.017301431391388178, + "reward_std_all": 0.028393413871526718, + "rewards-max/forensics_sinkhorn_reward": 0.9731907546520233, + "rewards-max/forensics_strict_edge_reward": 0.574543759226799, + "rewards-max/forensics_tiou_reward": 0.9602962136268616, + "rewards/forensics_sinkhorn_reward": 0.7215605676174164, + "rewards/forensics_strict_edge_reward": 0.06887075118720531, + "rewards/forensics_tiou_reward": 0.5876310467720032, + "step": 1230 + }, + { + "completion_length": 26.03125, + "epoch": 3.7772657450076803, + "grad_norm": 8.508421897888184, + "kl": 0.5703125, + "learning_rate": 5.384615384615385e-08, + "loss": 0.0177, + "reward": 0.7916978001594543, + "reward_std": 0.02648759074509144, + "reward_std_all": 0.02438976615667343, + "rewards-max/forensics_sinkhorn_reward": 0.9863487184047699, + "rewards-max/forensics_strict_edge_reward": 0.7558856904506683, + "rewards-max/forensics_tiou_reward": 0.993591845035553, + "rewards/forensics_sinkhorn_reward": 0.8346278071403503, + "rewards/forensics_strict_edge_reward": 0.08942266553640366, + "rewards/forensics_tiou_reward": 0.6997207701206207, + "step": 1231 + }, + { + "completion_length": 23.40625, + "epoch": 3.780337941628264, + "grad_norm": 8.1122465133667, + "kl": 0.44140625, + "learning_rate": 5.307692307692308e-08, + "loss": 0.1424, + "reward": 0.7570232450962067, + "reward_std": 0.05148889683187008, + "reward_std_all": 0.055155372247099876, + "rewards-max/forensics_sinkhorn_reward": 0.9751598536968231, + "rewards-max/forensics_strict_edge_reward": 0.6622511446475983, + "rewards-max/forensics_tiou_reward": 0.9650824964046478, + "rewards/forensics_sinkhorn_reward": 0.7325735092163086, + "rewards/forensics_strict_edge_reward": 0.08525865897536278, + "rewards/forensics_tiou_reward": 0.5429864823818207, + "step": 1232 + }, + { + "completion_length": 27.0625, + "epoch": 3.783410138248848, + "grad_norm": 7.015901565551758, + "kl": 0.458984375, + "learning_rate": 5.2307692307692304e-08, + "loss": -0.0129, + "reward": 0.7835459113121033, + "reward_std": 0.03647000715136528, + "reward_std_all": 0.03398693911731243, + "rewards-max/forensics_sinkhorn_reward": 0.976460337638855, + "rewards-max/forensics_strict_edge_reward": 0.4846835881471634, + "rewards-max/forensics_tiou_reward": 0.9702991545200348, + "rewards/forensics_sinkhorn_reward": 0.8233031034469604, + "rewards/forensics_strict_edge_reward": 0.0893109031021595, + "rewards/forensics_tiou_reward": 0.7018048763275146, + "step": 1233 + }, + { + "completion_length": 22.875, + "epoch": 3.7864823348694316, + "grad_norm": 7.729523181915283, + "kl": 0.421875, + "learning_rate": 5.153846153846154e-08, + "loss": 0.024, + "reward": 0.79364413022995, + "reward_std": 0.021020642190705985, + "reward_std_all": 0.020398150896653533, + "rewards-max/forensics_sinkhorn_reward": 0.9813602566719055, + "rewards-max/forensics_strict_edge_reward": 0.4921046942472458, + "rewards-max/forensics_tiou_reward": 0.9750895500183105, + "rewards/forensics_sinkhorn_reward": 0.7848497331142426, + "rewards/forensics_strict_edge_reward": 0.058907363563776016, + "rewards/forensics_tiou_reward": 0.6219275295734406, + "step": 1234 + }, + { + "completion_length": 26.421875, + "epoch": 3.7895545314900154, + "grad_norm": 7.505310535430908, + "kl": 0.3828125, + "learning_rate": 5.0769230769230764e-08, + "loss": 0.0226, + "reward": 0.7489575445652008, + "reward_std": 0.05577012337744236, + "reward_std_all": 0.055921681225299835, + "rewards-max/forensics_sinkhorn_reward": 0.9821240305900574, + "rewards-max/forensics_strict_edge_reward": 0.677129864692688, + "rewards-max/forensics_tiou_reward": 0.9823408424854279, + "rewards/forensics_sinkhorn_reward": 0.7669921517372131, + "rewards/forensics_strict_edge_reward": 0.06074962578713894, + "rewards/forensics_tiou_reward": 0.6308223009109497, + "step": 1235 + }, + { + "completion_length": 25.1875, + "epoch": 3.792626728110599, + "grad_norm": 7.4830780029296875, + "kl": 0.501953125, + "learning_rate": 5e-08, + "loss": -0.0012, + "reward": 0.7902224659919739, + "reward_std": 0.016380510292947292, + "reward_std_all": 0.0175869835074991, + "rewards-max/forensics_sinkhorn_reward": 0.9804275929927826, + "rewards-max/forensics_strict_edge_reward": 0.8351600170135498, + "rewards-max/forensics_tiou_reward": 0.9802471995353699, + "rewards/forensics_sinkhorn_reward": 0.7543410360813141, + "rewards/forensics_strict_edge_reward": 0.08864369988441467, + "rewards/forensics_tiou_reward": 0.6094785332679749, + "step": 1236 + }, + { + "completion_length": 23.015625, + "epoch": 3.795698924731183, + "grad_norm": 9.353550910949707, + "kl": 0.384765625, + "learning_rate": 4.923076923076923e-08, + "loss": -0.0501, + "reward": 0.7897819876670837, + "reward_std": 0.029004722833633423, + "reward_std_all": 0.027064829133450985, + "rewards-max/forensics_sinkhorn_reward": 0.9837838113307953, + "rewards-max/forensics_strict_edge_reward": 0.6065583229064941, + "rewards-max/forensics_tiou_reward": 0.9863202273845673, + "rewards/forensics_sinkhorn_reward": 0.8233681321144104, + "rewards/forensics_strict_edge_reward": 0.07264988403767347, + "rewards/forensics_tiou_reward": 0.6882837414741516, + "step": 1237 + }, + { + "completion_length": 26.609375, + "epoch": 3.7987711213517663, + "grad_norm": 7.322236061096191, + "kl": 0.4453125, + "learning_rate": 4.846153846153846e-08, + "loss": -0.0132, + "reward": 0.7590507864952087, + "reward_std": 0.05879400111734867, + "reward_std_all": 0.056429462507367134, + "rewards-max/forensics_sinkhorn_reward": 0.9865106642246246, + "rewards-max/forensics_strict_edge_reward": 0.5792120099067688, + "rewards-max/forensics_tiou_reward": 0.9931776225566864, + "rewards/forensics_sinkhorn_reward": 0.799542248249054, + "rewards/forensics_strict_edge_reward": 0.12312659248709679, + "rewards/forensics_tiou_reward": 0.6815466284751892, + "step": 1238 + }, + { + "completion_length": 29.03125, + "epoch": 3.80184331797235, + "grad_norm": 6.534937858581543, + "kl": 0.3935546875, + "learning_rate": 4.769230769230769e-08, + "loss": 0.0427, + "reward": 0.7906622290611267, + "reward_std": 0.01605094224214554, + "reward_std_all": 0.017393638845533133, + "rewards-max/forensics_sinkhorn_reward": 0.966504693031311, + "rewards-max/forensics_strict_edge_reward": 0.3856436163187027, + "rewards-max/forensics_tiou_reward": 0.950749933719635, + "rewards/forensics_sinkhorn_reward": 0.7508515417575836, + "rewards/forensics_strict_edge_reward": 0.03156699799001217, + "rewards/forensics_tiou_reward": 0.6070311963558197, + "step": 1239 + }, + { + "completion_length": 21.59375, + "epoch": 3.804915514592934, + "grad_norm": 8.959296226501465, + "kl": 0.42578125, + "learning_rate": 4.692307692307692e-08, + "loss": 0.0564, + "reward": 0.7905314862728119, + "reward_std": 0.01427686889655888, + "reward_std_all": 0.017462240997701883, + "rewards-max/forensics_sinkhorn_reward": 0.9700496196746826, + "rewards-max/forensics_strict_edge_reward": 0.5168293714523315, + "rewards-max/forensics_tiou_reward": 0.9560562670230865, + "rewards/forensics_sinkhorn_reward": 0.7886577546596527, + "rewards/forensics_strict_edge_reward": 0.05107590928673744, + "rewards/forensics_tiou_reward": 0.6193693578243256, + "step": 1240 + }, + { + "completion_length": 24.390625, + "epoch": 3.8079877112135176, + "grad_norm": 8.491194725036621, + "kl": 0.84765625, + "learning_rate": 4.615384615384615e-08, + "loss": -0.0106, + "reward": 0.7497747540473938, + "reward_std": 0.03339130058884621, + "reward_std_all": 0.046615928411483765, + "rewards-max/forensics_sinkhorn_reward": 0.9786632061004639, + "rewards-max/forensics_strict_edge_reward": 0.6176748722791672, + "rewards-max/forensics_tiou_reward": 0.9725542962551117, + "rewards/forensics_sinkhorn_reward": 0.7707240879535675, + "rewards/forensics_strict_edge_reward": 0.09605300799012184, + "rewards/forensics_tiou_reward": 0.6314066648483276, + "step": 1241 + }, + { + "completion_length": 22.84375, + "epoch": 3.8110599078341014, + "grad_norm": 12.915925025939941, + "kl": 0.9296875, + "learning_rate": 4.538461538461538e-08, + "loss": -0.0401, + "reward": 0.7812843918800354, + "reward_std": 0.03211125824600458, + "reward_std_all": 0.031069541350007057, + "rewards-max/forensics_sinkhorn_reward": 0.9788492321968079, + "rewards-max/forensics_strict_edge_reward": 0.6326714754104614, + "rewards-max/forensics_tiou_reward": 0.9738697111606598, + "rewards/forensics_sinkhorn_reward": 0.7920805513858795, + "rewards/forensics_strict_edge_reward": 0.1022845208644867, + "rewards/forensics_tiou_reward": 0.6192362606525421, + "step": 1242 + }, + { + "completion_length": 26.359375, + "epoch": 3.814132104454685, + "grad_norm": 6.68419885635376, + "kl": 0.3779296875, + "learning_rate": 4.461538461538461e-08, + "loss": 0.0115, + "reward": 0.7741156816482544, + "reward_std": 0.04788960516452789, + "reward_std_all": 0.0440907534211874, + "rewards-max/forensics_sinkhorn_reward": 0.9859808385372162, + "rewards-max/forensics_strict_edge_reward": 0.7815065383911133, + "rewards-max/forensics_tiou_reward": 0.9926730990409851, + "rewards/forensics_sinkhorn_reward": 0.8046749234199524, + "rewards/forensics_strict_edge_reward": 0.10734172910451889, + "rewards/forensics_tiou_reward": 0.7023879289627075, + "step": 1243 + }, + { + "completion_length": 24.4375, + "epoch": 3.817204301075269, + "grad_norm": 8.26024341583252, + "kl": 0.50390625, + "learning_rate": 4.3846153846153846e-08, + "loss": -0.03, + "reward": 0.7947873175144196, + "reward_std": 0.011735029169358313, + "reward_std_all": 0.013311262591741979, + "rewards-max/forensics_sinkhorn_reward": 0.9796099364757538, + "rewards-max/forensics_strict_edge_reward": 0.49416518211364746, + "rewards-max/forensics_tiou_reward": 0.9742402136325836, + "rewards/forensics_sinkhorn_reward": 0.803359180688858, + "rewards/forensics_strict_edge_reward": 0.047020234167575836, + "rewards/forensics_tiou_reward": 0.639312207698822, + "step": 1244 + }, + { + "completion_length": 32.546875, + "epoch": 3.8202764976958523, + "grad_norm": 7.196791172027588, + "kl": 0.4189453125, + "learning_rate": 4.307692307692307e-08, + "loss": 0.0601, + "reward": 0.7754043936729431, + "reward_std": 0.035191272385418415, + "reward_std_all": 0.03480153903365135, + "rewards-max/forensics_sinkhorn_reward": 0.9769614338874817, + "rewards-max/forensics_strict_edge_reward": 0.6365986764431, + "rewards-max/forensics_tiou_reward": 0.9743637442588806, + "rewards/forensics_sinkhorn_reward": 0.7232769429683685, + "rewards/forensics_strict_edge_reward": 0.08063616789877415, + "rewards/forensics_tiou_reward": 0.5915792882442474, + "step": 1245 + }, + { + "completion_length": 23.53125, + "epoch": 3.823348694316436, + "grad_norm": 9.428620338439941, + "kl": 0.3427734375, + "learning_rate": 4.230769230769231e-08, + "loss": -0.0752, + "reward": 0.7941131591796875, + "reward_std": 0.0014063488924875855, + "reward_std_all": 0.009081288590095937, + "rewards-max/forensics_sinkhorn_reward": 0.9888097047805786, + "rewards-max/forensics_strict_edge_reward": 0.9836080372333527, + "rewards-max/forensics_tiou_reward": 0.9993940591812134, + "rewards/forensics_sinkhorn_reward": 0.8205189406871796, + "rewards/forensics_strict_edge_reward": 0.09556540101766586, + "rewards/forensics_tiou_reward": 0.6816714406013489, + "step": 1246 + }, + { + "completion_length": 27.5625, + "epoch": 3.82642089093702, + "grad_norm": 8.468940734863281, + "kl": 0.5341796875, + "learning_rate": 4.1538461538461534e-08, + "loss": 0.0623, + "reward": 0.771994560956955, + "reward_std": 0.042217045091092587, + "reward_std_all": 0.040613848716020584, + "rewards-max/forensics_sinkhorn_reward": 0.9729607105255127, + "rewards-max/forensics_strict_edge_reward": 0.5404154062271118, + "rewards-max/forensics_tiou_reward": 0.9594020247459412, + "rewards/forensics_sinkhorn_reward": 0.7605079710483551, + "rewards/forensics_strict_edge_reward": 0.06351949460804462, + "rewards/forensics_tiou_reward": 0.6092095971107483, + "step": 1247 + }, + { + "completion_length": 26.3125, + "epoch": 3.8294930875576036, + "grad_norm": 16.033254623413086, + "kl": 0.962890625, + "learning_rate": 4.076923076923077e-08, + "loss": 0.0812, + "reward": 0.7839089632034302, + "reward_std": 0.014703149383421987, + "reward_std_all": 0.020040371920913458, + "rewards-max/forensics_sinkhorn_reward": 0.9803865849971771, + "rewards-max/forensics_strict_edge_reward": 0.5506429374217987, + "rewards-max/forensics_tiou_reward": 0.979792982339859, + "rewards/forensics_sinkhorn_reward": 0.7586119771003723, + "rewards/forensics_strict_edge_reward": 0.06741518527269363, + "rewards/forensics_tiou_reward": 0.5549422651529312, + "step": 1248 + }, + { + "completion_length": 22.09375, + "epoch": 3.8325652841781874, + "grad_norm": 9.06435775756836, + "kl": 0.408203125, + "learning_rate": 4e-08, + "loss": -0.0961, + "reward": 0.7982104420661926, + "reward_std": 0.0026909704320132732, + "reward_std_all": 0.007328299689106643, + "rewards-max/forensics_sinkhorn_reward": 0.9830424189567566, + "rewards-max/forensics_strict_edge_reward": 0.7457418143749237, + "rewards-max/forensics_tiou_reward": 0.9840767085552216, + "rewards/forensics_sinkhorn_reward": 0.8320919275283813, + "rewards/forensics_strict_edge_reward": 0.15969834104180336, + "rewards/forensics_tiou_reward": 0.680079311132431, + "step": 1249 + }, + { + "completion_length": 25.484375, + "epoch": 3.835637480798771, + "grad_norm": 7.4231157302856445, + "kl": 0.4609375, + "learning_rate": 3.923076923076923e-08, + "loss": -0.0651, + "reward": 0.7978391349315643, + "reward_std": 0.014392931829206645, + "reward_std_all": 0.01463308313395828, + "rewards-max/forensics_sinkhorn_reward": 0.98546302318573, + "rewards-max/forensics_strict_edge_reward": 0.6095658242702484, + "rewards-max/forensics_tiou_reward": 0.9896066188812256, + "rewards/forensics_sinkhorn_reward": 0.8169499933719635, + "rewards/forensics_strict_edge_reward": 0.06151952967047691, + "rewards/forensics_tiou_reward": 0.7013128399848938, + "step": 1250 + }, + { + "completion_length": 23.96875, + "epoch": 3.838709677419355, + "grad_norm": 11.32124137878418, + "kl": 0.388671875, + "learning_rate": 3.846153846153846e-08, + "loss": -0.0152, + "reward": 0.7608439326286316, + "reward_std": 0.048853690386749804, + "reward_std_all": 0.04858078854158521, + "rewards-max/forensics_sinkhorn_reward": 0.9818364083766937, + "rewards-max/forensics_strict_edge_reward": 0.8351600170135498, + "rewards-max/forensics_tiou_reward": 0.9809716045856476, + "rewards/forensics_sinkhorn_reward": 0.7719191908836365, + "rewards/forensics_strict_edge_reward": 0.09174390882253647, + "rewards/forensics_tiou_reward": 0.6245825290679932, + "step": 1251 + }, + { + "completion_length": 24.640625, + "epoch": 3.8417818740399383, + "grad_norm": 7.75275993347168, + "kl": 0.3779296875, + "learning_rate": 3.769230769230769e-08, + "loss": -0.1183, + "reward": 0.7862417995929718, + "reward_std": 0.024556760676205158, + "reward_std_all": 0.02469269186258316, + "rewards-max/forensics_sinkhorn_reward": 0.9818379878997803, + "rewards-max/forensics_strict_edge_reward": 0.5829301476478577, + "rewards-max/forensics_tiou_reward": 0.9788558483123779, + "rewards/forensics_sinkhorn_reward": 0.7982900738716125, + "rewards/forensics_strict_edge_reward": 0.08238793164491653, + "rewards/forensics_tiou_reward": 0.6815212965011597, + "step": 1252 + }, + { + "completion_length": 26.765625, + "epoch": 3.8448540706605225, + "grad_norm": 7.465220928192139, + "kl": 0.4169921875, + "learning_rate": 3.692307692307692e-08, + "loss": -0.0677, + "reward": 0.8022066950798035, + "reward_std": 0.004978966142516583, + "reward_std_all": 0.006723188213072717, + "rewards-max/forensics_sinkhorn_reward": 0.9746313393115997, + "rewards-max/forensics_strict_edge_reward": 0.3577147573232651, + "rewards-max/forensics_tiou_reward": 0.9613098502159119, + "rewards/forensics_sinkhorn_reward": 0.8637992739677429, + "rewards/forensics_strict_edge_reward": 0.06534876301884651, + "rewards/forensics_tiou_reward": 0.7313644886016846, + "step": 1253 + }, + { + "completion_length": 20.5625, + "epoch": 3.847926267281106, + "grad_norm": 9.217443466186523, + "kl": 0.41015625, + "learning_rate": 3.6153846153846155e-08, + "loss": -0.0604, + "reward": 0.789325088262558, + "reward_std": 0.02911152271553874, + "reward_std_all": 0.02739580674096942, + "rewards-max/forensics_sinkhorn_reward": 0.9866142570972443, + "rewards-max/forensics_strict_edge_reward": 0.7789089381694794, + "rewards-max/forensics_tiou_reward": 0.9931600987911224, + "rewards/forensics_sinkhorn_reward": 0.7938006222248077, + "rewards/forensics_strict_edge_reward": 0.12110705301165581, + "rewards/forensics_tiou_reward": 0.6057862043380737, + "step": 1254 + }, + { + "completion_length": 26.171875, + "epoch": 3.8509984639016897, + "grad_norm": 8.71562671661377, + "kl": 0.46875, + "learning_rate": 3.538461538461538e-08, + "loss": -0.0735, + "reward": 0.7746831774711609, + "reward_std": 0.007180980173870921, + "reward_std_all": 0.02162004727870226, + "rewards-max/forensics_sinkhorn_reward": 0.9875763952732086, + "rewards-max/forensics_strict_edge_reward": 0.7501170337200165, + "rewards-max/forensics_tiou_reward": 0.9954599738121033, + "rewards/forensics_sinkhorn_reward": 0.7714409232139587, + "rewards/forensics_strict_edge_reward": 0.0795957874506712, + "rewards/forensics_tiou_reward": 0.6006177365779877, + "step": 1255 + }, + { + "completion_length": 23.234375, + "epoch": 3.8540706605222734, + "grad_norm": 10.492531776428223, + "kl": 0.927734375, + "learning_rate": 3.4615384615384616e-08, + "loss": 0.0127, + "reward": 0.7644394040107727, + "reward_std": 0.04322107322514057, + "reward_std_all": 0.04308139346539974, + "rewards-max/forensics_sinkhorn_reward": 0.9805901050567627, + "rewards-max/forensics_strict_edge_reward": 0.6515887677669525, + "rewards-max/forensics_tiou_reward": 0.9778887033462524, + "rewards/forensics_sinkhorn_reward": 0.7484487295150757, + "rewards/forensics_strict_edge_reward": 0.08128548413515091, + "rewards/forensics_tiou_reward": 0.6132456064224243, + "step": 1256 + }, + { + "completion_length": 24.140625, + "epoch": 3.857142857142857, + "grad_norm": 8.469188690185547, + "kl": 0.560546875, + "learning_rate": 3.384615384615384e-08, + "loss": -0.0739, + "reward": 0.7589064538478851, + "reward_std": 0.04475327581167221, + "reward_std_all": 0.045539405196905136, + "rewards-max/forensics_sinkhorn_reward": 0.9737728536128998, + "rewards-max/forensics_strict_edge_reward": 0.598464846611023, + "rewards-max/forensics_tiou_reward": 0.96109539270401, + "rewards/forensics_sinkhorn_reward": 0.778818279504776, + "rewards/forensics_strict_edge_reward": 0.11804236471652985, + "rewards/forensics_tiou_reward": 0.6187363564968109, + "step": 1257 + }, + { + "completion_length": 28.3125, + "epoch": 3.860215053763441, + "grad_norm": 10.247185707092285, + "kl": 0.45703125, + "learning_rate": 3.3076923076923076e-08, + "loss": -0.0747, + "reward": 0.7825580835342407, + "reward_std": 0.02694813534617424, + "reward_std_all": 0.028937920928001404, + "rewards-max/forensics_sinkhorn_reward": 0.9783482253551483, + "rewards-max/forensics_strict_edge_reward": 0.6914834380149841, + "rewards-max/forensics_tiou_reward": 0.9729137718677521, + "rewards/forensics_sinkhorn_reward": 0.7901219725608826, + "rewards/forensics_strict_edge_reward": 0.11341432854533195, + "rewards/forensics_tiou_reward": 0.6550216972827911, + "step": 1258 + }, + { + "completion_length": 23.203125, + "epoch": 3.8632872503840243, + "grad_norm": 11.161550521850586, + "kl": 0.4150390625, + "learning_rate": 3.230769230769231e-08, + "loss": -0.0106, + "reward": 0.7999512851238251, + "reward_std": 0.006314085912890732, + "reward_std_all": 0.008299278561025858, + "rewards-max/forensics_sinkhorn_reward": 0.9853884279727936, + "rewards-max/forensics_strict_edge_reward": 0.6435305178165436, + "rewards-max/forensics_tiou_reward": 0.9894258677959442, + "rewards/forensics_sinkhorn_reward": 0.8478007614612579, + "rewards/forensics_strict_edge_reward": 0.08430752716958523, + "rewards/forensics_tiou_reward": 0.6810705065727234, + "step": 1259 + }, + { + "completion_length": 22.875, + "epoch": 3.8663594470046085, + "grad_norm": 7.0123724937438965, + "kl": 0.482421875, + "learning_rate": 3.153846153846154e-08, + "loss": -0.0051, + "reward": 0.7808339893817902, + "reward_std": 0.028737173648551106, + "reward_std_all": 0.02968803746625781, + "rewards-max/forensics_sinkhorn_reward": 0.9822718501091003, + "rewards-max/forensics_strict_edge_reward": 0.6945812702178955, + "rewards-max/forensics_tiou_reward": 0.9842558801174164, + "rewards/forensics_sinkhorn_reward": 0.7810414135456085, + "rewards/forensics_strict_edge_reward": 0.09006171301007271, + "rewards/forensics_tiou_reward": 0.5763648748397827, + "step": 1260 + }, + { + "completion_length": 26.875, + "epoch": 3.869431643625192, + "grad_norm": 7.684326171875, + "kl": 0.623046875, + "learning_rate": 3.076923076923077e-08, + "loss": -0.013, + "reward": 0.7581495046615601, + "reward_std": 0.025478500872850418, + "reward_std_all": 0.040795041248202324, + "rewards-max/forensics_sinkhorn_reward": 0.9858158826828003, + "rewards-max/forensics_strict_edge_reward": 0.7412298321723938, + "rewards-max/forensics_tiou_reward": 0.9905295073986053, + "rewards/forensics_sinkhorn_reward": 0.8282010853290558, + "rewards/forensics_strict_edge_reward": 0.12272359430789948, + "rewards/forensics_tiou_reward": 0.7179125547409058, + "step": 1261 + }, + { + "completion_length": 25.734375, + "epoch": 3.8725038402457757, + "grad_norm": 11.634329795837402, + "kl": 0.4970703125, + "learning_rate": 3e-08, + "loss": -0.0022, + "reward": 0.7921886146068573, + "reward_std": 0.025301375542767346, + "reward_std_all": 0.023516435408964753, + "rewards-max/forensics_sinkhorn_reward": 0.9840063452720642, + "rewards-max/forensics_strict_edge_reward": 0.5820295512676239, + "rewards-max/forensics_tiou_reward": 0.9874004423618317, + "rewards/forensics_sinkhorn_reward": 0.8268657028675079, + "rewards/forensics_strict_edge_reward": 0.10618161782622337, + "rewards/forensics_tiou_reward": 0.7154662609100342, + "step": 1262 + }, + { + "completion_length": 22.6875, + "epoch": 3.8755760368663594, + "grad_norm": 9.22838306427002, + "kl": 0.498046875, + "learning_rate": 2.9230769230769228e-08, + "loss": 0.01, + "reward": 0.7987298965454102, + "reward_std": 0.014630768448114395, + "reward_std_all": 0.014071357436478138, + "rewards-max/forensics_sinkhorn_reward": 0.9852750599384308, + "rewards-max/forensics_strict_edge_reward": 0.751229465007782, + "rewards-max/forensics_tiou_reward": 0.987895280122757, + "rewards/forensics_sinkhorn_reward": 0.8412299454212189, + "rewards/forensics_strict_edge_reward": 0.13189087808132172, + "rewards/forensics_tiou_reward": 0.7020100355148315, + "step": 1263 + }, + { + "completion_length": 26.1875, + "epoch": 3.878648233486943, + "grad_norm": 10.140551567077637, + "kl": 0.4423828125, + "learning_rate": 2.846153846153846e-08, + "loss": -0.0215, + "reward": 0.7991338968276978, + "reward_std": 0.007954611093737185, + "reward_std_all": 0.00930839404463768, + "rewards-max/forensics_sinkhorn_reward": 0.9838616251945496, + "rewards-max/forensics_strict_edge_reward": 0.882964164018631, + "rewards-max/forensics_tiou_reward": 0.9879413545131683, + "rewards/forensics_sinkhorn_reward": 0.8236874341964722, + "rewards/forensics_strict_edge_reward": 0.11145494878292084, + "rewards/forensics_tiou_reward": 0.7295026183128357, + "step": 1264 + }, + { + "completion_length": 26.828125, + "epoch": 3.881720430107527, + "grad_norm": 9.594141006469727, + "kl": 0.396484375, + "learning_rate": 2.769230769230769e-08, + "loss": -0.0539, + "reward": 0.8053400218486786, + "reward_std": 0.002727231942117214, + "reward_std_all": 0.004108323249965906, + "rewards-max/forensics_sinkhorn_reward": 0.9839503765106201, + "rewards-max/forensics_strict_edge_reward": 0.8062050342559814, + "rewards-max/forensics_tiou_reward": 0.9859642088413239, + "rewards/forensics_sinkhorn_reward": 0.8450876474380493, + "rewards/forensics_strict_edge_reward": 0.11254137009382248, + "rewards/forensics_tiou_reward": 0.6989279985427856, + "step": 1265 + }, + { + "completion_length": 22.859375, + "epoch": 3.8847926267281108, + "grad_norm": 8.830119132995605, + "kl": 0.515625, + "learning_rate": 2.6923076923076925e-08, + "loss": 0.0004, + "reward": 0.7825167775154114, + "reward_std": 0.028255537152290344, + "reward_std_all": 0.028354118578135967, + "rewards-max/forensics_sinkhorn_reward": 0.975498229265213, + "rewards-max/forensics_strict_edge_reward": 0.5622940063476562, + "rewards-max/forensics_tiou_reward": 0.9633102118968964, + "rewards/forensics_sinkhorn_reward": 0.7721569538116455, + "rewards/forensics_strict_edge_reward": 0.08915766142308712, + "rewards/forensics_tiou_reward": 0.603043794631958, + "step": 1266 + }, + { + "completion_length": 24.859375, + "epoch": 3.8878648233486945, + "grad_norm": 8.111495971679688, + "kl": 0.408203125, + "learning_rate": 2.6153846153846152e-08, + "loss": -0.0705, + "reward": 0.8033985495567322, + "reward_std": 0.005290809087455273, + "reward_std_all": 0.00655162800103426, + "rewards-max/forensics_sinkhorn_reward": 0.9771113693714142, + "rewards-max/forensics_strict_edge_reward": 0.3744247481226921, + "rewards-max/forensics_tiou_reward": 0.9703488051891327, + "rewards/forensics_sinkhorn_reward": 0.8260268270969391, + "rewards/forensics_strict_edge_reward": 0.04658297263085842, + "rewards/forensics_tiou_reward": 0.6908695697784424, + "step": 1267 + }, + { + "completion_length": 27.078125, + "epoch": 3.890937019969278, + "grad_norm": 8.741840362548828, + "kl": 0.408203125, + "learning_rate": 2.5384615384615382e-08, + "loss": -0.1082, + "reward": 0.805390864610672, + "reward_std": 0.0031510288827121258, + "reward_std_all": 0.004315412254072726, + "rewards-max/forensics_sinkhorn_reward": 0.9772938191890717, + "rewards-max/forensics_strict_edge_reward": 0.6195405721664429, + "rewards-max/forensics_tiou_reward": 0.9704245626926422, + "rewards/forensics_sinkhorn_reward": 0.850364476442337, + "rewards/forensics_strict_edge_reward": 0.14010707288980484, + "rewards/forensics_tiou_reward": 0.7097570896148682, + "step": 1268 + }, + { + "completion_length": 24.21875, + "epoch": 3.8940092165898617, + "grad_norm": 8.793659210205078, + "kl": 0.521484375, + "learning_rate": 2.4615384615384616e-08, + "loss": 0.0914, + "reward": 0.7875931262969971, + "reward_std": 0.031852636486291885, + "reward_std_all": 0.0294810738414526, + "rewards-max/forensics_sinkhorn_reward": 0.9855129718780518, + "rewards-max/forensics_strict_edge_reward": 0.7165313363075256, + "rewards-max/forensics_tiou_reward": 0.9899840354919434, + "rewards/forensics_sinkhorn_reward": 0.7767427265644073, + "rewards/forensics_strict_edge_reward": 0.09540760330855846, + "rewards/forensics_tiou_reward": 0.6248531341552734, + "step": 1269 + }, + { + "completion_length": 24.953125, + "epoch": 3.8970814132104454, + "grad_norm": 9.051867485046387, + "kl": 0.6455078125, + "learning_rate": 2.3846153846153846e-08, + "loss": 0.0155, + "reward": 0.7887023091316223, + "reward_std": 0.026788163871970028, + "reward_std_all": 0.02580379880964756, + "rewards-max/forensics_sinkhorn_reward": 0.9839027523994446, + "rewards-max/forensics_strict_edge_reward": 0.756708562374115, + "rewards-max/forensics_tiou_reward": 0.9850912988185883, + "rewards/forensics_sinkhorn_reward": 0.8099124133586884, + "rewards/forensics_strict_edge_reward": 0.07815095409750938, + "rewards/forensics_tiou_reward": 0.6915778517723083, + "step": 1270 + }, + { + "completion_length": 27.609375, + "epoch": 3.900153609831029, + "grad_norm": 6.9307780265808105, + "kl": 0.35546875, + "learning_rate": 2.3076923076923076e-08, + "loss": -0.054, + "reward": 0.746433436870575, + "reward_std": 0.02622990310192108, + "reward_std_all": 0.044730838388204575, + "rewards-max/forensics_sinkhorn_reward": 0.9840092658996582, + "rewards-max/forensics_strict_edge_reward": 0.7361255586147308, + "rewards-max/forensics_tiou_reward": 0.985249251127243, + "rewards/forensics_sinkhorn_reward": 0.7640146911144257, + "rewards/forensics_strict_edge_reward": 0.09442402794957161, + "rewards/forensics_tiou_reward": 0.6647516787052155, + "step": 1271 + }, + { + "completion_length": 23.96875, + "epoch": 3.903225806451613, + "grad_norm": 9.7222900390625, + "kl": 0.5732421875, + "learning_rate": 2.2307692307692306e-08, + "loss": -0.1054, + "reward": 0.7976771891117096, + "reward_std": 0.014520141819957644, + "reward_std_all": 0.014777794014662504, + "rewards-max/forensics_sinkhorn_reward": 0.9690045714378357, + "rewards-max/forensics_strict_edge_reward": 0.44981616735458374, + "rewards-max/forensics_tiou_reward": 0.949805498123169, + "rewards/forensics_sinkhorn_reward": 0.8197221159934998, + "rewards/forensics_strict_edge_reward": 0.057217225432395935, + "rewards/forensics_tiou_reward": 0.6569579541683197, + "step": 1272 + }, + { + "completion_length": 25.15625, + "epoch": 3.9062980030721968, + "grad_norm": 7.635753154754639, + "kl": 0.4443359375, + "learning_rate": 2.1538461538461537e-08, + "loss": -0.0614, + "reward": 0.8031492233276367, + "reward_std": 0.005550396861508489, + "reward_std_all": 0.006579675013199449, + "rewards-max/forensics_sinkhorn_reward": 0.9804466366767883, + "rewards-max/forensics_strict_edge_reward": 0.615243673324585, + "rewards-max/forensics_tiou_reward": 0.980049341917038, + "rewards/forensics_sinkhorn_reward": 0.8330540060997009, + "rewards/forensics_strict_edge_reward": 0.08463668078184128, + "rewards/forensics_tiou_reward": 0.6920128464698792, + "step": 1273 + }, + { + "completion_length": 26.328125, + "epoch": 3.9093701996927805, + "grad_norm": 9.09959602355957, + "kl": 0.662109375, + "learning_rate": 2.0769230769230767e-08, + "loss": 0.0579, + "reward": 0.7795649170875549, + "reward_std": 0.031424276530742645, + "reward_std_all": 0.031181178987026215, + "rewards-max/forensics_sinkhorn_reward": 0.9872377216815948, + "rewards-max/forensics_strict_edge_reward": 0.6345149278640747, + "rewards-max/forensics_tiou_reward": 0.9964613318443298, + "rewards/forensics_sinkhorn_reward": 0.7664192914962769, + "rewards/forensics_strict_edge_reward": 0.09411169961094856, + "rewards/forensics_tiou_reward": 0.639763742685318, + "step": 1274 + }, + { + "completion_length": 23.171875, + "epoch": 3.912442396313364, + "grad_norm": 9.408918380737305, + "kl": 0.404296875, + "learning_rate": 2e-08, + "loss": -0.0772, + "reward": 0.8058548271656036, + "reward_std": 0.0010822229087352753, + "reward_std_all": 0.003097443259321153, + "rewards-max/forensics_sinkhorn_reward": 0.9741529822349548, + "rewards-max/forensics_strict_edge_reward": 0.5599738955497742, + "rewards-max/forensics_tiou_reward": 0.9655754566192627, + "rewards/forensics_sinkhorn_reward": 0.8529114127159119, + "rewards/forensics_strict_edge_reward": 0.07201986387372017, + "rewards/forensics_tiou_reward": 0.7117142975330353, + "step": 1275 + }, + { + "completion_length": 26.859375, + "epoch": 3.9155145929339477, + "grad_norm": 8.411789894104004, + "kl": 0.451171875, + "learning_rate": 1.923076923076923e-08, + "loss": 0.0424, + "reward": 0.7803696990013123, + "reward_std": 0.039335768204182386, + "reward_std_all": 0.03654384089168161, + "rewards-max/forensics_sinkhorn_reward": 0.9855732023715973, + "rewards-max/forensics_strict_edge_reward": 0.7500508725643158, + "rewards-max/forensics_tiou_reward": 0.9902148544788361, + "rewards/forensics_sinkhorn_reward": 0.7881937325000763, + "rewards/forensics_strict_edge_reward": 0.09678582847118378, + "rewards/forensics_tiou_reward": 0.6538958549499512, + "step": 1276 + }, + { + "completion_length": 22.640625, + "epoch": 3.9185867895545314, + "grad_norm": 7.8380303382873535, + "kl": 0.388671875, + "learning_rate": 1.846153846153846e-08, + "loss": -0.011, + "reward": 0.7972604036331177, + "reward_std": 0.01225940277799964, + "reward_std_all": 0.012928823009133339, + "rewards-max/forensics_sinkhorn_reward": 0.9840297698974609, + "rewards-max/forensics_strict_edge_reward": 0.7837068438529968, + "rewards-max/forensics_tiou_reward": 0.9860692918300629, + "rewards/forensics_sinkhorn_reward": 0.8383458852767944, + "rewards/forensics_strict_edge_reward": 0.11716052889823914, + "rewards/forensics_tiou_reward": 0.6908473670482635, + "step": 1277 + }, + { + "completion_length": 25.9375, + "epoch": 3.921658986175115, + "grad_norm": 9.292684555053711, + "kl": 0.412109375, + "learning_rate": 1.769230769230769e-08, + "loss": 0.0423, + "reward": 0.7906694412231445, + "reward_std": 0.013815959449857473, + "reward_std_all": 0.016924167051911354, + "rewards-max/forensics_sinkhorn_reward": 0.9737279415130615, + "rewards-max/forensics_strict_edge_reward": 0.4589976519346237, + "rewards-max/forensics_tiou_reward": 0.962316483259201, + "rewards/forensics_sinkhorn_reward": 0.7786381244659424, + "rewards/forensics_strict_edge_reward": 0.07149197719991207, + "rewards/forensics_tiou_reward": 0.6776512861251831, + "step": 1278 + }, + { + "completion_length": 21.828125, + "epoch": 3.924731182795699, + "grad_norm": 8.822842597961426, + "kl": 0.458984375, + "learning_rate": 1.692307692307692e-08, + "loss": 0.0028, + "reward": 0.7715052962303162, + "reward_std": 0.05400402192026377, + "reward_std_all": 0.04936034791171551, + "rewards-max/forensics_sinkhorn_reward": 0.98675736784935, + "rewards-max/forensics_strict_edge_reward": 0.7467198520898819, + "rewards-max/forensics_tiou_reward": 0.9951932430267334, + "rewards/forensics_sinkhorn_reward": 0.789607435464859, + "rewards/forensics_strict_edge_reward": 0.1179654598236084, + "rewards/forensics_tiou_reward": 0.6422723531723022, + "step": 1279 + }, + { + "completion_length": 27.78125, + "epoch": 3.9278033794162828, + "grad_norm": 7.36871337890625, + "kl": 0.5185546875, + "learning_rate": 1.6153846153846155e-08, + "loss": 0.0515, + "reward": 0.7869957089424133, + "reward_std": 0.03500385023653507, + "reward_std_all": 0.03176579996943474, + "rewards-max/forensics_sinkhorn_reward": 0.9828702509403229, + "rewards-max/forensics_strict_edge_reward": 0.5689603686332703, + "rewards-max/forensics_tiou_reward": 0.9840605556964874, + "rewards/forensics_sinkhorn_reward": 0.8053000271320343, + "rewards/forensics_strict_edge_reward": 0.09108243510127068, + "rewards/forensics_tiou_reward": 0.7227230370044708, + "step": 1280 + }, + { + "completion_length": 29.234375, + "epoch": 3.9308755760368665, + "grad_norm": 6.712173938751221, + "kl": 0.4560546875, + "learning_rate": 1.5384615384615385e-08, + "loss": 0.074, + "reward": 0.7782232761383057, + "reward_std": 0.03701908700168133, + "reward_std_all": 0.03528372943401337, + "rewards-max/forensics_sinkhorn_reward": 0.9763859510421753, + "rewards-max/forensics_strict_edge_reward": 0.45875632762908936, + "rewards-max/forensics_tiou_reward": 0.9700238704681396, + "rewards/forensics_sinkhorn_reward": 0.8022392392158508, + "rewards/forensics_strict_edge_reward": 0.09986292105168104, + "rewards/forensics_tiou_reward": 0.6867009997367859, + "step": 1281 + }, + { + "completion_length": 27.265625, + "epoch": 3.93394777265745, + "grad_norm": 6.579344272613525, + "kl": 0.5546875, + "learning_rate": 1.4615384615384614e-08, + "loss": -0.0218, + "reward": 0.8055247962474823, + "reward_std": 0.0015749036683700979, + "reward_std_all": 0.0034398200223222375, + "rewards-max/forensics_sinkhorn_reward": 0.9860701858997345, + "rewards-max/forensics_strict_edge_reward": 0.7956043183803558, + "rewards-max/forensics_tiou_reward": 0.9939848482608795, + "rewards/forensics_sinkhorn_reward": 0.8088822066783905, + "rewards/forensics_strict_edge_reward": 0.11781138181686401, + "rewards/forensics_tiou_reward": 0.681900829076767, + "step": 1282 + }, + { + "completion_length": 22.734375, + "epoch": 3.9370199692780337, + "grad_norm": 8.785881996154785, + "kl": 0.4951171875, + "learning_rate": 1.3846153846153846e-08, + "loss": 0.0136, + "reward": 0.7957175374031067, + "reward_std": 0.019215767737478018, + "reward_std_all": 0.018479865859262645, + "rewards-max/forensics_sinkhorn_reward": 0.9720202088356018, + "rewards-max/forensics_strict_edge_reward": 0.502072662115097, + "rewards-max/forensics_tiou_reward": 0.9583485722541809, + "rewards/forensics_sinkhorn_reward": 0.8372654318809509, + "rewards/forensics_strict_edge_reward": 0.0676966980099678, + "rewards/forensics_tiou_reward": 0.7069622874259949, + "step": 1283 + }, + { + "completion_length": 24.859375, + "epoch": 3.9400921658986174, + "grad_norm": 7.747325420379639, + "kl": 0.4921875, + "learning_rate": 1.3076923076923076e-08, + "loss": -0.013, + "reward": 0.8050338923931122, + "reward_std": 0.0032150602200999856, + "reward_std_all": 0.004599237581714988, + "rewards-max/forensics_sinkhorn_reward": 0.9852622449398041, + "rewards-max/forensics_strict_edge_reward": 0.633294403553009, + "rewards-max/forensics_tiou_reward": 0.9915623664855957, + "rewards/forensics_sinkhorn_reward": 0.8455919921398163, + "rewards/forensics_strict_edge_reward": 0.10436151921749115, + "rewards/forensics_tiou_reward": 0.7151931524276733, + "step": 1284 + }, + { + "completion_length": 23.3125, + "epoch": 3.943164362519201, + "grad_norm": 9.836764335632324, + "kl": 0.568359375, + "learning_rate": 1.2307692307692308e-08, + "loss": -0.0297, + "reward": 0.7871753573417664, + "reward_std": 0.031037538312375546, + "reward_std_all": 0.02938957791775465, + "rewards-max/forensics_sinkhorn_reward": 0.9709891080856323, + "rewards-max/forensics_strict_edge_reward": 0.6151010096073151, + "rewards-max/forensics_tiou_reward": 0.9620668292045593, + "rewards/forensics_sinkhorn_reward": 0.7502604722976685, + "rewards/forensics_strict_edge_reward": 0.04924185201525688, + "rewards/forensics_tiou_reward": 0.5548003762960434, + "step": 1285 + }, + { + "completion_length": 28.40625, + "epoch": 3.946236559139785, + "grad_norm": 8.357725143432617, + "kl": 0.443359375, + "learning_rate": 1.1538461538461538e-08, + "loss": 0.0691, + "reward": 0.7850872874259949, + "reward_std": 0.010152701754122972, + "reward_std_all": 0.01706210058182478, + "rewards-max/forensics_sinkhorn_reward": 0.9783198237419128, + "rewards-max/forensics_strict_edge_reward": 0.5378695726394653, + "rewards-max/forensics_tiou_reward": 0.9743282794952393, + "rewards/forensics_sinkhorn_reward": 0.7922979891300201, + "rewards/forensics_strict_edge_reward": 0.07705586683005095, + "rewards/forensics_tiou_reward": 0.7116432189941406, + "step": 1286 + }, + { + "completion_length": 23.796875, + "epoch": 3.9493087557603688, + "grad_norm": 6.964393615722656, + "kl": 0.3916015625, + "learning_rate": 1.0769230769230768e-08, + "loss": -0.0011, + "reward": 0.7968840301036835, + "reward_std": 0.01784505066461861, + "reward_std_all": 0.01706145890057087, + "rewards-max/forensics_sinkhorn_reward": 0.9853653013706207, + "rewards-max/forensics_strict_edge_reward": 0.7558856904506683, + "rewards-max/forensics_tiou_reward": 0.9885571300983429, + "rewards/forensics_sinkhorn_reward": 0.8308348655700684, + "rewards/forensics_strict_edge_reward": 0.12467864155769348, + "rewards/forensics_tiou_reward": 0.718787282705307, + "step": 1287 + }, + { + "completion_length": 24.375, + "epoch": 3.9523809523809526, + "grad_norm": 8.039413452148438, + "kl": 0.568359375, + "learning_rate": 1e-08, + "loss": -0.0537, + "reward": 0.7904866337776184, + "reward_std": 0.027249882579781115, + "reward_std_all": 0.02552299899980426, + "rewards-max/forensics_sinkhorn_reward": 0.9796207845211029, + "rewards-max/forensics_strict_edge_reward": 0.48429200053215027, + "rewards-max/forensics_tiou_reward": 0.974711537361145, + "rewards/forensics_sinkhorn_reward": 0.814874529838562, + "rewards/forensics_strict_edge_reward": 0.0997470524162054, + "rewards/forensics_tiou_reward": 0.6400115489959717, + "step": 1288 + }, + { + "completion_length": 23.3125, + "epoch": 3.955453149001536, + "grad_norm": 17.445262908935547, + "kl": 0.7109375, + "learning_rate": 9.23076923076923e-09, + "loss": 0.0677, + "reward": 0.7830430269241333, + "reward_std": 0.029439950827509165, + "reward_std_all": 0.028593186289072037, + "rewards-max/forensics_sinkhorn_reward": 0.9802097082138062, + "rewards-max/forensics_strict_edge_reward": 0.6266644895076752, + "rewards-max/forensics_tiou_reward": 0.9745886027812958, + "rewards/forensics_sinkhorn_reward": 0.7952301800251007, + "rewards/forensics_strict_edge_reward": 0.09000447019934654, + "rewards/forensics_tiou_reward": 0.636448323726654, + "step": 1289 + }, + { + "completion_length": 24.953125, + "epoch": 3.9585253456221197, + "grad_norm": 8.217129707336426, + "kl": 0.4306640625, + "learning_rate": 8.46153846153846e-09, + "loss": -0.0608, + "reward": 0.7616052627563477, + "reward_std": 0.044323699781671166, + "reward_std_all": 0.04449107381515205, + "rewards-max/forensics_sinkhorn_reward": 0.978186845779419, + "rewards-max/forensics_strict_edge_reward": 0.661531001329422, + "rewards-max/forensics_tiou_reward": 0.9704070091247559, + "rewards/forensics_sinkhorn_reward": 0.7687682509422302, + "rewards/forensics_strict_edge_reward": 0.11323430761694908, + "rewards/forensics_tiou_reward": 0.6355542242527008, + "step": 1290 + }, + { + "completion_length": 24.5, + "epoch": 3.9615975422427034, + "grad_norm": 7.848527908325195, + "kl": 0.3818359375, + "learning_rate": 7.692307692307693e-09, + "loss": -0.0726, + "reward": 0.8061857521533966, + "reward_std": 0.0015525214257650077, + "reward_std_all": 0.003142085624858737, + "rewards-max/forensics_sinkhorn_reward": 0.983005940914154, + "rewards-max/forensics_strict_edge_reward": 0.6390409171581268, + "rewards-max/forensics_tiou_reward": 0.9852040708065033, + "rewards/forensics_sinkhorn_reward": 0.8618643581867218, + "rewards/forensics_strict_edge_reward": 0.09598323330283165, + "rewards/forensics_tiou_reward": 0.7417539954185486, + "step": 1291 + }, + { + "completion_length": 24.375, + "epoch": 3.964669738863287, + "grad_norm": 8.605090141296387, + "kl": 0.802734375, + "learning_rate": 6.923076923076923e-09, + "loss": -0.1017, + "reward": 0.8015066385269165, + "reward_std": 0.006582422822248191, + "reward_std_all": 0.007877291413024068, + "rewards-max/forensics_sinkhorn_reward": 0.9849421679973602, + "rewards-max/forensics_strict_edge_reward": 0.5796550810337067, + "rewards-max/forensics_tiou_reward": 0.9879650175571442, + "rewards/forensics_sinkhorn_reward": 0.8193908929824829, + "rewards/forensics_strict_edge_reward": 0.0771968849003315, + "rewards/forensics_tiou_reward": 0.666138231754303, + "step": 1292 + }, + { + "completion_length": 24.8125, + "epoch": 3.967741935483871, + "grad_norm": 7.782159805297852, + "kl": 0.57421875, + "learning_rate": 6.153846153846154e-09, + "loss": 0.1205, + "reward": 0.7462858557701111, + "reward_std": 0.06122463569045067, + "reward_std_all": 0.06078021228313446, + "rewards-max/forensics_sinkhorn_reward": 0.9841336011886597, + "rewards-max/forensics_strict_edge_reward": 0.5125145614147186, + "rewards-max/forensics_tiou_reward": 0.9878640472888947, + "rewards/forensics_sinkhorn_reward": 0.6908300220966339, + "rewards/forensics_strict_edge_reward": 0.05595044791698456, + "rewards/forensics_tiou_reward": 0.5100317299365997, + "step": 1293 + }, + { + "completion_length": 24.4375, + "epoch": 3.9708141321044548, + "grad_norm": 7.848086833953857, + "kl": 0.4267578125, + "learning_rate": 5.384615384615384e-09, + "loss": -0.0707, + "reward": 0.7482654750347137, + "reward_std": 0.016661107016261667, + "reward_std_all": 0.03864003368653357, + "rewards-max/forensics_sinkhorn_reward": 0.9794067740440369, + "rewards-max/forensics_strict_edge_reward": 0.5787998139858246, + "rewards-max/forensics_tiou_reward": 0.9757927656173706, + "rewards/forensics_sinkhorn_reward": 0.7253587543964386, + "rewards/forensics_strict_edge_reward": 0.0433935821056366, + "rewards/forensics_tiou_reward": 0.533345490694046, + "step": 1294 + }, + { + "completion_length": 30.140625, + "epoch": 3.9738863287250386, + "grad_norm": 6.680178165435791, + "kl": 0.4140625, + "learning_rate": 4.615384615384615e-09, + "loss": -0.0747, + "reward": 0.7917675077915192, + "reward_std": 0.02630785177461803, + "reward_std_all": 0.024446799885481596, + "rewards-max/forensics_sinkhorn_reward": 0.9877848923206329, + "rewards-max/forensics_strict_edge_reward": 0.872870922088623, + "rewards-max/forensics_tiou_reward": 0.9972369968891144, + "rewards/forensics_sinkhorn_reward": 0.8092486560344696, + "rewards/forensics_strict_edge_reward": 0.1400170773267746, + "rewards/forensics_tiou_reward": 0.7020966410636902, + "step": 1295 + }, + { + "completion_length": 28.46875, + "epoch": 3.976958525345622, + "grad_norm": 8.515907287597656, + "kl": 0.439453125, + "learning_rate": 3.846153846153846e-09, + "loss": -0.0077, + "reward": 0.8015888631343842, + "reward_std": 0.009887103689834476, + "reward_std_all": 0.010133370175026357, + "rewards-max/forensics_sinkhorn_reward": 0.9823017120361328, + "rewards-max/forensics_strict_edge_reward": 0.7178648710250854, + "rewards-max/forensics_tiou_reward": 0.9828552305698395, + "rewards/forensics_sinkhorn_reward": 0.8729643523693085, + "rewards/forensics_strict_edge_reward": 0.10236528143286705, + "rewards/forensics_tiou_reward": 0.7623666822910309, + "step": 1296 + }, + { + "completion_length": 29.984375, + "epoch": 3.980030721966206, + "grad_norm": 8.227313041687012, + "kl": 0.693359375, + "learning_rate": 3.076923076923077e-09, + "loss": 0.023, + "reward": 0.7678652107715607, + "reward_std": 0.041887396946549416, + "reward_std_all": 0.04138592816889286, + "rewards-max/forensics_sinkhorn_reward": 0.984939306974411, + "rewards-max/forensics_strict_edge_reward": 0.7309097647666931, + "rewards-max/forensics_tiou_reward": 0.9878322780132294, + "rewards/forensics_sinkhorn_reward": 0.7086197733879089, + "rewards/forensics_strict_edge_reward": 0.11894835159182549, + "rewards/forensics_tiou_reward": 0.6023626923561096, + "step": 1297 + }, + { + "completion_length": 22.75, + "epoch": 3.9831029185867894, + "grad_norm": 6.894609451293945, + "kl": 0.396484375, + "learning_rate": 2.3076923076923076e-09, + "loss": 0.0579, + "reward": 0.7695114314556122, + "reward_std": 0.04437157837674022, + "reward_std_all": 0.04281912697479129, + "rewards-max/forensics_sinkhorn_reward": 0.9818938076496124, + "rewards-max/forensics_strict_edge_reward": 0.7789089381694794, + "rewards-max/forensics_tiou_reward": 0.983728438615799, + "rewards/forensics_sinkhorn_reward": 0.8018767833709717, + "rewards/forensics_strict_edge_reward": 0.09315174631774426, + "rewards/forensics_tiou_reward": 0.6774093210697174, + "step": 1298 + }, + { + "completion_length": 24.796875, + "epoch": 3.986175115207373, + "grad_norm": 9.705682754516602, + "kl": 0.44921875, + "learning_rate": 1.5384615384615385e-09, + "loss": -0.0558, + "reward": 0.8033843040466309, + "reward_std": 0.004861816996708512, + "reward_std_all": 0.0063721726182848215, + "rewards-max/forensics_sinkhorn_reward": 0.9773364365100861, + "rewards-max/forensics_strict_edge_reward": 0.6583559811115265, + "rewards-max/forensics_tiou_reward": 0.9740399718284607, + "rewards/forensics_sinkhorn_reward": 0.7909688353538513, + "rewards/forensics_strict_edge_reward": 0.09510331973433495, + "rewards/forensics_tiou_reward": 0.6491391360759735, + "step": 1299 + }, + { + "completion_length": 26.453125, + "epoch": 3.989247311827957, + "grad_norm": 8.327875137329102, + "kl": 0.453125, + "learning_rate": 7.692307692307692e-10, + "loss": -0.0294, + "reward": 0.8053975701332092, + "reward_std": 0.0017077755182981491, + "reward_std_all": 0.003669049357995391, + "rewards-max/forensics_sinkhorn_reward": 0.9855926632881165, + "rewards-max/forensics_strict_edge_reward": 0.6743711233139038, + "rewards-max/forensics_tiou_reward": 0.9916247725486755, + "rewards/forensics_sinkhorn_reward": 0.8254659175872803, + "rewards/forensics_strict_edge_reward": 0.0978970043361187, + "rewards/forensics_tiou_reward": 0.6947337090969086, + "step": 1300 + } + ], + "logging_steps": 1.0, + "max_steps": 1300, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 1300, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +}