| { |
| "best_global_step": 12500, |
| "best_metric": 0.7083570957183838, |
| "best_model_checkpoint": "./sft_dpo_45k/checkpoint-12500", |
| "epoch": 2.8500983382264913, |
| "eval_steps": 2500, |
| "global_step": 12500, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.005700766753128296, |
| "grad_norm": 450.0, |
| "learning_rate": 1.9999940896317337e-05, |
| "logits/chosen": -1.0914154052734375, |
| "logits/rejected": -0.8119749426841736, |
| "logps/chosen": -233.2249755859375, |
| "logps/rejected": -233.35504150390625, |
| "loss": 0.7723, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.3098715543746948, |
| "rewards/margins": 1.4599758386611938, |
| "rewards/rejected": -1.1501044034957886, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.011401533506256592, |
| "grad_norm": 968.0, |
| "learning_rate": 1.9999753632813937e-05, |
| "logits/chosen": -0.9890195727348328, |
| "logits/rejected": -0.7339245080947876, |
| "logps/chosen": -254.09083557128906, |
| "logps/rejected": -276.44415283203125, |
| "loss": 1.1039, |
| "rewards/accuracies": 0.6399999856948853, |
| "rewards/chosen": 0.6527794599533081, |
| "rewards/margins": 1.386103630065918, |
| "rewards/rejected": -0.7333241105079651, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.017102300259384886, |
| "grad_norm": 520.0, |
| "learning_rate": 1.999943810928104e-05, |
| "logits/chosen": -0.9036900401115417, |
| "logits/rejected": -0.6605126857757568, |
| "logps/chosen": -257.6153259277344, |
| "logps/rejected": -272.2424621582031, |
| "loss": 0.9278, |
| "rewards/accuracies": 0.6050000190734863, |
| "rewards/chosen": 0.7515385150909424, |
| "rewards/margins": 1.7310659885406494, |
| "rewards/rejected": -0.9795275330543518, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.022803067012513185, |
| "grad_norm": 207.0, |
| "learning_rate": 1.9998994329765654e-05, |
| "logits/chosen": -0.9496411085128784, |
| "logits/rejected": -0.6262346506118774, |
| "logps/chosen": -231.91595458984375, |
| "logps/rejected": -256.36358642578125, |
| "loss": 1.0349, |
| "rewards/accuracies": 0.6150000095367432, |
| "rewards/chosen": 0.6428021192550659, |
| "rewards/margins": 1.3770060539245605, |
| "rewards/rejected": -0.7342040538787842, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.02850383376564148, |
| "grad_norm": 340.0, |
| "learning_rate": 1.9998422299959834e-05, |
| "logits/chosen": -1.0474913120269775, |
| "logits/rejected": -0.8129882216453552, |
| "logps/chosen": -238.02285766601562, |
| "logps/rejected": -258.8080139160156, |
| "loss": 0.9048, |
| "rewards/accuracies": 0.6800000071525574, |
| "rewards/chosen": 0.6418118476867676, |
| "rewards/margins": 2.0746071338653564, |
| "rewards/rejected": -1.4327950477600098, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.03420460051876977, |
| "grad_norm": 211.0, |
| "learning_rate": 1.999772202720062e-05, |
| "logits/chosen": -0.7310837507247925, |
| "logits/rejected": -0.45146456360816956, |
| "logps/chosen": -243.75872802734375, |
| "logps/rejected": -256.59503173828125, |
| "loss": 0.8691, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.9112188220024109, |
| "rewards/margins": 1.9378777742385864, |
| "rewards/rejected": -1.0266591310501099, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.03990536727189807, |
| "grad_norm": 107.5, |
| "learning_rate": 1.9996893520469934e-05, |
| "logits/chosen": -0.8938100934028625, |
| "logits/rejected": -0.636574923992157, |
| "logps/chosen": -240.33453369140625, |
| "logps/rejected": -251.5081024169922, |
| "loss": 0.764, |
| "rewards/accuracies": 0.6899999976158142, |
| "rewards/chosen": 1.2257814407348633, |
| "rewards/margins": 2.543311834335327, |
| "rewards/rejected": -1.3175301551818848, |
| "step": 175 |
| }, |
| { |
| "epoch": 0.04560613402502637, |
| "grad_norm": 422.0, |
| "learning_rate": 1.9995936790394464e-05, |
| "logits/chosen": -0.7267706990242004, |
| "logits/rejected": -0.5509985685348511, |
| "logps/chosen": -261.2969970703125, |
| "logps/rejected": -293.4281921386719, |
| "loss": 0.7902, |
| "rewards/accuracies": 0.6600000262260437, |
| "rewards/chosen": 1.2394615411758423, |
| "rewards/margins": 2.079812526702881, |
| "rewards/rejected": -0.8403508067131042, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.05130690077815466, |
| "grad_norm": 422.0, |
| "learning_rate": 1.9994851849245536e-05, |
| "logits/chosen": -0.8919548988342285, |
| "logits/rejected": -0.6996061205863953, |
| "logps/chosen": -251.43653869628906, |
| "logps/rejected": -264.9438781738281, |
| "loss": 0.8933, |
| "rewards/accuracies": 0.6700000166893005, |
| "rewards/chosen": 0.6621707081794739, |
| "rewards/margins": 2.149031162261963, |
| "rewards/rejected": -1.4868603944778442, |
| "step": 225 |
| }, |
| { |
| "epoch": 0.05700766753128296, |
| "grad_norm": 280.0, |
| "learning_rate": 1.9993638710938952e-05, |
| "logits/chosen": -0.614574670791626, |
| "logits/rejected": -0.5049194097518921, |
| "logps/chosen": -257.8453369140625, |
| "logps/rejected": -266.7996520996094, |
| "loss": 1.0079, |
| "rewards/accuracies": 0.6299999952316284, |
| "rewards/chosen": 0.2667715549468994, |
| "rewards/margins": 1.8679660558700562, |
| "rewards/rejected": -1.6011946201324463, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.06270843428441125, |
| "grad_norm": 456.0, |
| "learning_rate": 1.9992297391034813e-05, |
| "logits/chosen": -0.6381824612617493, |
| "logits/rejected": -0.3375353515148163, |
| "logps/chosen": -262.174072265625, |
| "logps/rejected": -277.9561462402344, |
| "loss": 1.1025, |
| "rewards/accuracies": 0.6050000190734863, |
| "rewards/chosen": 0.7835137844085693, |
| "rewards/margins": 1.4511244297027588, |
| "rewards/rejected": -0.6676105260848999, |
| "step": 275 |
| }, |
| { |
| "epoch": 0.06840920103753954, |
| "grad_norm": 548.0, |
| "learning_rate": 1.999082790673731e-05, |
| "logits/chosen": -0.5498209595680237, |
| "logits/rejected": -0.2571995258331299, |
| "logps/chosen": -266.0782165527344, |
| "logps/rejected": -271.9496765136719, |
| "loss": 1.1537, |
| "rewards/accuracies": 0.6449999809265137, |
| "rewards/chosen": 0.2032065987586975, |
| "rewards/margins": 1.5133525133132935, |
| "rewards/rejected": -1.3101459741592407, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.07410996779066785, |
| "grad_norm": 484.0, |
| "learning_rate": 1.9989230276894525e-05, |
| "logits/chosen": -0.7042214274406433, |
| "logits/rejected": -0.5629515051841736, |
| "logps/chosen": -251.5504913330078, |
| "logps/rejected": -259.848876953125, |
| "loss": 1.0746, |
| "rewards/accuracies": 0.6050000190734863, |
| "rewards/chosen": 0.2761840522289276, |
| "rewards/margins": 1.531221866607666, |
| "rewards/rejected": -1.2550379037857056, |
| "step": 325 |
| }, |
| { |
| "epoch": 0.07981073454379614, |
| "grad_norm": 145.0, |
| "learning_rate": 1.9987504521998164e-05, |
| "logits/chosen": -0.6372985243797302, |
| "logits/rejected": -0.33795836567878723, |
| "logps/chosen": -246.87806701660156, |
| "logps/rejected": -265.2651672363281, |
| "loss": 1.0687, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.32188349962234497, |
| "rewards/margins": 1.7044808864593506, |
| "rewards/rejected": -1.3825974464416504, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.08551150129692443, |
| "grad_norm": 400.0, |
| "learning_rate": 1.998565066418331e-05, |
| "logits/chosen": -0.6276116371154785, |
| "logits/rejected": -0.46921128034591675, |
| "logps/chosen": -246.4172821044922, |
| "logps/rejected": -262.1149597167969, |
| "loss": 0.9715, |
| "rewards/accuracies": 0.6700000166893005, |
| "rewards/chosen": 0.20023074746131897, |
| "rewards/margins": 1.764892578125, |
| "rewards/rejected": -1.5646618604660034, |
| "step": 375 |
| }, |
| { |
| "epoch": 0.09121226805005274, |
| "grad_norm": 282.0, |
| "learning_rate": 1.9983668727228132e-05, |
| "logits/chosen": -0.7453694343566895, |
| "logits/rejected": -0.4669770896434784, |
| "logps/chosen": -241.35812377929688, |
| "logps/rejected": -263.9813232421875, |
| "loss": 0.9293, |
| "rewards/accuracies": 0.6650000214576721, |
| "rewards/chosen": 0.6146385669708252, |
| "rewards/margins": 1.933253526687622, |
| "rewards/rejected": -1.3186148405075073, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.09691303480318103, |
| "grad_norm": 72.0, |
| "learning_rate": 1.9981558736553586e-05, |
| "logits/chosen": -0.8182552456855774, |
| "logits/rejected": -0.6007075905799866, |
| "logps/chosen": -250.3661651611328, |
| "logps/rejected": -262.82379150390625, |
| "loss": 0.9998, |
| "rewards/accuracies": 0.6449999809265137, |
| "rewards/chosen": 0.9104523658752441, |
| "rewards/margins": 2.1223745346069336, |
| "rewards/rejected": -1.2119220495224, |
| "step": 425 |
| }, |
| { |
| "epoch": 0.10261380155630932, |
| "grad_norm": 290.0, |
| "learning_rate": 1.997932071922309e-05, |
| "logits/chosen": -0.7490326762199402, |
| "logits/rejected": -0.4431981146335602, |
| "logps/chosen": -243.71060180664062, |
| "logps/rejected": -260.09307861328125, |
| "loss": 1.2074, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.9773914217948914, |
| "rewards/margins": 2.117527961730957, |
| "rewards/rejected": -1.1401368379592896, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.10831456830943761, |
| "grad_norm": 438.0, |
| "learning_rate": 1.997695470394217e-05, |
| "logits/chosen": -0.9417197704315186, |
| "logits/rejected": -0.7031550407409668, |
| "logps/chosen": -256.24053955078125, |
| "logps/rejected": -274.4161071777344, |
| "loss": 0.9034, |
| "rewards/accuracies": 0.6549999713897705, |
| "rewards/chosen": 1.2904267311096191, |
| "rewards/margins": 3.114290237426758, |
| "rewards/rejected": -1.8238635063171387, |
| "step": 475 |
| }, |
| { |
| "epoch": 0.11401533506256592, |
| "grad_norm": 520.0, |
| "learning_rate": 1.997446072105808e-05, |
| "logits/chosen": -0.7224742770195007, |
| "logits/rejected": -0.5729588270187378, |
| "logps/chosen": -235.91407775878906, |
| "logps/rejected": -265.5506591796875, |
| "loss": 0.8249, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": 1.3501222133636475, |
| "rewards/margins": 2.9835822582244873, |
| "rewards/rejected": -1.6334600448608398, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.11971610181569421, |
| "grad_norm": 300.0, |
| "learning_rate": 1.9971838802559453e-05, |
| "logits/chosen": -1.0090041160583496, |
| "logits/rejected": -0.6921093463897705, |
| "logps/chosen": -231.53359985351562, |
| "logps/rejected": -259.6770935058594, |
| "loss": 0.9426, |
| "rewards/accuracies": 0.6600000262260437, |
| "rewards/chosen": 0.42308470606803894, |
| "rewards/margins": 2.7364745140075684, |
| "rewards/rejected": -2.313389539718628, |
| "step": 525 |
| }, |
| { |
| "epoch": 0.1254168685688225, |
| "grad_norm": 292.0, |
| "learning_rate": 1.9969088982075842e-05, |
| "logits/chosen": -1.0311638116836548, |
| "logits/rejected": -0.7545648813247681, |
| "logps/chosen": -254.17587280273438, |
| "logps/rejected": -273.646728515625, |
| "loss": 1.2653, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": 0.06095041334629059, |
| "rewards/margins": 2.5934252738952637, |
| "rewards/rejected": -2.532474994659424, |
| "step": 550 |
| }, |
| { |
| "epoch": 0.1311176353219508, |
| "grad_norm": 368.0, |
| "learning_rate": 1.996621129487733e-05, |
| "logits/chosen": -0.882070004940033, |
| "logits/rejected": -0.5608855485916138, |
| "logps/chosen": -261.51141357421875, |
| "logps/rejected": -278.70849609375, |
| "loss": 1.1298, |
| "rewards/accuracies": 0.6650000214576721, |
| "rewards/chosen": 0.3231360614299774, |
| "rewards/margins": 2.614332675933838, |
| "rewards/rejected": -2.291196346282959, |
| "step": 575 |
| }, |
| { |
| "epoch": 0.1368184020750791, |
| "grad_norm": 420.0, |
| "learning_rate": 1.9963205777874048e-05, |
| "logits/chosen": -0.6045601963996887, |
| "logits/rejected": -0.45467254519462585, |
| "logps/chosen": -268.7801818847656, |
| "logps/rejected": -282.9999694824219, |
| "loss": 1.0993, |
| "rewards/accuracies": 0.6349999904632568, |
| "rewards/chosen": 1.0803488492965698, |
| "rewards/margins": 2.486741542816162, |
| "rewards/rejected": -1.4063928127288818, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.1425191688282074, |
| "grad_norm": 624.0, |
| "learning_rate": 1.9960072469615716e-05, |
| "logits/chosen": -0.620104968547821, |
| "logits/rejected": -0.43143904209136963, |
| "logps/chosen": -253.4304656982422, |
| "logps/rejected": -263.4527587890625, |
| "loss": 1.1091, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 1.1819748878479004, |
| "rewards/margins": 2.4705114364624023, |
| "rewards/rejected": -1.288536548614502, |
| "step": 625 |
| }, |
| { |
| "epoch": 0.1482199355813357, |
| "grad_norm": 400.0, |
| "learning_rate": 1.995681141029114e-05, |
| "logits/chosen": -0.7728636264801025, |
| "logits/rejected": -0.5173014402389526, |
| "logps/chosen": -254.5880126953125, |
| "logps/rejected": -262.836669921875, |
| "loss": 1.0705, |
| "rewards/accuracies": 0.6549999713897705, |
| "rewards/chosen": 1.315026044845581, |
| "rewards/margins": 2.9843475818634033, |
| "rewards/rejected": -1.6693212985992432, |
| "step": 650 |
| }, |
| { |
| "epoch": 0.15392070233446398, |
| "grad_norm": 213.0, |
| "learning_rate": 1.9953422641727714e-05, |
| "logits/chosen": -0.8867356181144714, |
| "logits/rejected": -0.7285749912261963, |
| "logps/chosen": -259.9929504394531, |
| "logps/rejected": -280.42449951171875, |
| "loss": 1.0153, |
| "rewards/accuracies": 0.7099999785423279, |
| "rewards/chosen": 1.535692572593689, |
| "rewards/margins": 2.9881153106689453, |
| "rewards/rejected": -1.452423095703125, |
| "step": 675 |
| }, |
| { |
| "epoch": 0.15962146908759228, |
| "grad_norm": 516.0, |
| "learning_rate": 1.9949906207390857e-05, |
| "logits/chosen": -0.8390807509422302, |
| "logits/rejected": -0.5874150991439819, |
| "logps/chosen": -263.6914367675781, |
| "logps/rejected": -268.042236328125, |
| "loss": 0.9813, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": 0.9048083424568176, |
| "rewards/margins": 2.7781078815460205, |
| "rewards/rejected": -1.873299241065979, |
| "step": 700 |
| }, |
| { |
| "epoch": 0.1653222358407206, |
| "grad_norm": 780.0, |
| "learning_rate": 1.9946262152383486e-05, |
| "logits/chosen": -1.0042121410369873, |
| "logits/rejected": -0.8443993926048279, |
| "logps/chosen": -266.72540283203125, |
| "logps/rejected": -272.64874267578125, |
| "loss": 1.4198, |
| "rewards/accuracies": 0.6100000143051147, |
| "rewards/chosen": 0.821239709854126, |
| "rewards/margins": 2.344177007675171, |
| "rewards/rejected": -1.522937297821045, |
| "step": 725 |
| }, |
| { |
| "epoch": 0.17102300259384887, |
| "grad_norm": 260.0, |
| "learning_rate": 1.994249052344541e-05, |
| "logits/chosen": -1.1082932949066162, |
| "logits/rejected": -0.9135249853134155, |
| "logps/chosen": -267.7679138183594, |
| "logps/rejected": -268.8138732910156, |
| "loss": 0.9783, |
| "rewards/accuracies": 0.6449999809265137, |
| "rewards/chosen": 0.5921077132225037, |
| "rewards/margins": 2.613111972808838, |
| "rewards/rejected": -2.0210041999816895, |
| "step": 750 |
| }, |
| { |
| "epoch": 0.17672376934697717, |
| "grad_norm": 436.0, |
| "learning_rate": 1.993859136895274e-05, |
| "logits/chosen": -1.1738299131393433, |
| "logits/rejected": -0.992180585861206, |
| "logps/chosen": -238.45547485351562, |
| "logps/rejected": -261.0966491699219, |
| "loss": 1.0253, |
| "rewards/accuracies": 0.6850000023841858, |
| "rewards/chosen": 1.2687020301818848, |
| "rewards/margins": 3.19614839553833, |
| "rewards/rejected": -1.9274462461471558, |
| "step": 775 |
| }, |
| { |
| "epoch": 0.18242453610010548, |
| "grad_norm": 288.0, |
| "learning_rate": 1.9934564738917278e-05, |
| "logits/chosen": -1.5361509323120117, |
| "logits/rejected": -1.2878414392471313, |
| "logps/chosen": -244.20046997070312, |
| "logps/rejected": -265.6919860839844, |
| "loss": 0.9318, |
| "rewards/accuracies": 0.6549999713897705, |
| "rewards/chosen": 1.1407175064086914, |
| "rewards/margins": 3.996370792388916, |
| "rewards/rejected": -2.8556532859802246, |
| "step": 800 |
| }, |
| { |
| "epoch": 0.18812530285323376, |
| "grad_norm": 224.0, |
| "learning_rate": 1.9930410684985867e-05, |
| "logits/chosen": -1.4972169399261475, |
| "logits/rejected": -1.2626354694366455, |
| "logps/chosen": -232.57504272460938, |
| "logps/rejected": -261.83416748046875, |
| "loss": 1.113, |
| "rewards/accuracies": 0.6650000214576721, |
| "rewards/chosen": 0.435445100069046, |
| "rewards/margins": 3.3964593410491943, |
| "rewards/rejected": -2.96101450920105, |
| "step": 825 |
| }, |
| { |
| "epoch": 0.19382606960636206, |
| "grad_norm": 624.0, |
| "learning_rate": 1.9926129260439726e-05, |
| "logits/chosen": -1.2795894145965576, |
| "logits/rejected": -1.0612705945968628, |
| "logps/chosen": -294.5425109863281, |
| "logps/rejected": -294.6100769042969, |
| "loss": 1.1766, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.5490455031394958, |
| "rewards/margins": 2.3998987674713135, |
| "rewards/rejected": -1.850853443145752, |
| "step": 850 |
| }, |
| { |
| "epoch": 0.19952683635949034, |
| "grad_norm": 568.0, |
| "learning_rate": 1.992172052019377e-05, |
| "logits/chosen": -1.3455495834350586, |
| "logits/rejected": -1.0349416732788086, |
| "logps/chosen": -264.94366455078125, |
| "logps/rejected": -268.7208251953125, |
| "loss": 1.2642, |
| "rewards/accuracies": 0.6200000047683716, |
| "rewards/chosen": 0.19544316828250885, |
| "rewards/margins": 2.6854217052459717, |
| "rewards/rejected": -2.489978551864624, |
| "step": 875 |
| }, |
| { |
| "epoch": 0.20522760311261865, |
| "grad_norm": 548.0, |
| "learning_rate": 1.9917184520795913e-05, |
| "logits/chosen": -1.2598049640655518, |
| "logits/rejected": -1.0997883081436157, |
| "logps/chosen": -276.93463134765625, |
| "logps/rejected": -296.1929016113281, |
| "loss": 1.0854, |
| "rewards/accuracies": 0.6449999809265137, |
| "rewards/chosen": -0.2938272953033447, |
| "rewards/margins": 2.645054817199707, |
| "rewards/rejected": -2.938882350921631, |
| "step": 900 |
| }, |
| { |
| "epoch": 0.21092836986574695, |
| "grad_norm": 320.0, |
| "learning_rate": 1.9912521320426327e-05, |
| "logits/chosen": -1.486849069595337, |
| "logits/rejected": -1.1710004806518555, |
| "logps/chosen": -260.2918395996094, |
| "logps/rejected": -258.1929626464844, |
| "loss": 1.1041, |
| "rewards/accuracies": 0.6600000262260437, |
| "rewards/chosen": 0.2849799692630768, |
| "rewards/margins": 2.6305792331695557, |
| "rewards/rejected": -2.3455991744995117, |
| "step": 925 |
| }, |
| { |
| "epoch": 0.21662913661887523, |
| "grad_norm": 492.0, |
| "learning_rate": 1.9907730978896703e-05, |
| "logits/chosen": -1.3513752222061157, |
| "logits/rejected": -1.0843216180801392, |
| "logps/chosen": -245.48690795898438, |
| "logps/rejected": -233.47265625, |
| "loss": 1.0083, |
| "rewards/accuracies": 0.6299999952316284, |
| "rewards/chosen": 0.20163799822330475, |
| "rewards/margins": 2.6695971488952637, |
| "rewards/rejected": -2.4679596424102783, |
| "step": 950 |
| }, |
| { |
| "epoch": 0.22232990337200353, |
| "grad_norm": 300.0, |
| "learning_rate": 1.9902813557649495e-05, |
| "logits/chosen": -1.0412503480911255, |
| "logits/rejected": -0.8158297538757324, |
| "logps/chosen": -266.1749267578125, |
| "logps/rejected": -273.68670654296875, |
| "loss": 1.3173, |
| "rewards/accuracies": 0.6200000047683716, |
| "rewards/chosen": -0.1282215118408203, |
| "rewards/margins": 1.696293592453003, |
| "rewards/rejected": -1.8245151042938232, |
| "step": 975 |
| }, |
| { |
| "epoch": 0.22803067012513184, |
| "grad_norm": 165.0, |
| "learning_rate": 1.9897769119757116e-05, |
| "logits/chosen": -1.1691317558288574, |
| "logits/rejected": -0.9282821416854858, |
| "logps/chosen": -271.3135070800781, |
| "logps/rejected": -272.3988342285156, |
| "loss": 1.3322, |
| "rewards/accuracies": 0.6899999976158142, |
| "rewards/chosen": -0.064248226583004, |
| "rewards/margins": 2.6389236450195312, |
| "rewards/rejected": -2.703171730041504, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.23373143687826012, |
| "grad_norm": 54.5, |
| "learning_rate": 1.9892597729921134e-05, |
| "logits/chosen": -1.012527346611023, |
| "logits/rejected": -0.8686437010765076, |
| "logps/chosen": -253.73333740234375, |
| "logps/rejected": -274.84503173828125, |
| "loss": 1.1078, |
| "rewards/accuracies": 0.6549999713897705, |
| "rewards/chosen": -0.18542949855327606, |
| "rewards/margins": 2.423771381378174, |
| "rewards/rejected": -2.609200954437256, |
| "step": 1025 |
| }, |
| { |
| "epoch": 0.23943220363138842, |
| "grad_norm": 175.0, |
| "learning_rate": 1.988729945447144e-05, |
| "logits/chosen": -0.9831740856170654, |
| "logits/rejected": -0.863321840763092, |
| "logps/chosen": -250.1609344482422, |
| "logps/rejected": -246.76043701171875, |
| "loss": 1.1116, |
| "rewards/accuracies": 0.6800000071525574, |
| "rewards/chosen": -0.5303030610084534, |
| "rewards/margins": 2.7494423389434814, |
| "rewards/rejected": -3.279745578765869, |
| "step": 1050 |
| }, |
| { |
| "epoch": 0.24513297038451673, |
| "grad_norm": 440.0, |
| "learning_rate": 1.9881874361365413e-05, |
| "logits/chosen": -1.0947002172470093, |
| "logits/rejected": -0.8605647087097168, |
| "logps/chosen": -244.8800811767578, |
| "logps/rejected": -260.9714660644531, |
| "loss": 1.1596, |
| "rewards/accuracies": 0.6150000095367432, |
| "rewards/chosen": 0.16015632450580597, |
| "rewards/margins": 2.340996503829956, |
| "rewards/rejected": -2.180840253829956, |
| "step": 1075 |
| }, |
| { |
| "epoch": 0.250833737137645, |
| "grad_norm": 444.0, |
| "learning_rate": 1.987632252018702e-05, |
| "logits/chosen": -1.1349825859069824, |
| "logits/rejected": -0.8324615955352783, |
| "logps/chosen": -282.9785461425781, |
| "logps/rejected": -299.1729736328125, |
| "loss": 1.0182, |
| "rewards/accuracies": 0.6299999952316284, |
| "rewards/chosen": 0.10454043745994568, |
| "rewards/margins": 3.014533281326294, |
| "rewards/rejected": -2.9099929332733154, |
| "step": 1100 |
| }, |
| { |
| "epoch": 0.2565345038907733, |
| "grad_norm": 432.0, |
| "learning_rate": 1.9870644002145954e-05, |
| "logits/chosen": -1.348595142364502, |
| "logits/rejected": -1.069776177406311, |
| "logps/chosen": -249.79339599609375, |
| "logps/rejected": -264.1799621582031, |
| "loss": 0.859, |
| "rewards/accuracies": 0.6850000023841858, |
| "rewards/chosen": -0.24737724661827087, |
| "rewards/margins": 3.5905261039733887, |
| "rewards/rejected": -3.8379034996032715, |
| "step": 1125 |
| }, |
| { |
| "epoch": 0.2622352706439016, |
| "grad_norm": 568.0, |
| "learning_rate": 1.9864838880076696e-05, |
| "logits/chosen": -1.410559058189392, |
| "logits/rejected": -1.2540091276168823, |
| "logps/chosen": -269.29364013671875, |
| "logps/rejected": -282.6539611816406, |
| "loss": 1.4656, |
| "rewards/accuracies": 0.6150000095367432, |
| "rewards/chosen": -1.0998212099075317, |
| "rewards/margins": 2.224698543548584, |
| "rewards/rejected": -3.324519634246826, |
| "step": 1150 |
| }, |
| { |
| "epoch": 0.2679360373970299, |
| "grad_norm": 416.0, |
| "learning_rate": 1.9858907228437586e-05, |
| "logits/chosen": -1.1306984424591064, |
| "logits/rejected": -0.9032139778137207, |
| "logps/chosen": -261.55841064453125, |
| "logps/rejected": -271.16400146484375, |
| "loss": 1.2053, |
| "rewards/accuracies": 0.6549999713897705, |
| "rewards/chosen": -0.35265934467315674, |
| "rewards/margins": 2.9411590099334717, |
| "rewards/rejected": -3.293818473815918, |
| "step": 1175 |
| }, |
| { |
| "epoch": 0.2736368041501582, |
| "grad_norm": 300.0, |
| "learning_rate": 1.9852849123309892e-05, |
| "logits/chosen": -1.3609739542007446, |
| "logits/rejected": -1.0884772539138794, |
| "logps/chosen": -219.86376953125, |
| "logps/rejected": -244.00181579589844, |
| "loss": 1.1641, |
| "rewards/accuracies": 0.6600000262260437, |
| "rewards/chosen": 0.29006442427635193, |
| "rewards/margins": 2.9711480140686035, |
| "rewards/rejected": -2.6810836791992188, |
| "step": 1200 |
| }, |
| { |
| "epoch": 0.2793375709032865, |
| "grad_norm": 376.0, |
| "learning_rate": 1.9846664642396793e-05, |
| "logits/chosen": -1.394396185874939, |
| "logits/rejected": -1.1222848892211914, |
| "logps/chosen": -253.79254150390625, |
| "logps/rejected": -290.03765869140625, |
| "loss": 1.0715, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.5299239158630371, |
| "rewards/margins": 2.99424409866333, |
| "rewards/rejected": -3.524168014526367, |
| "step": 1225 |
| }, |
| { |
| "epoch": 0.2850383376564148, |
| "grad_norm": 592.0, |
| "learning_rate": 1.9840353865022418e-05, |
| "logits/chosen": -1.3301948308944702, |
| "logits/rejected": -1.1577705144882202, |
| "logps/chosen": -250.49786376953125, |
| "logps/rejected": -258.0343322753906, |
| "loss": 1.1561, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.9897598028182983, |
| "rewards/margins": 2.9111433029174805, |
| "rewards/rejected": -3.9009029865264893, |
| "step": 1250 |
| }, |
| { |
| "epoch": 0.2907391044095431, |
| "grad_norm": 600.0, |
| "learning_rate": 1.983391687213081e-05, |
| "logits/chosen": -1.095660924911499, |
| "logits/rejected": -0.979175865650177, |
| "logps/chosen": -251.69837951660156, |
| "logps/rejected": -253.51651000976562, |
| "loss": 1.056, |
| "rewards/accuracies": 0.6650000214576721, |
| "rewards/chosen": -0.5402212738990784, |
| "rewards/margins": 2.4007418155670166, |
| "rewards/rejected": -2.94096302986145, |
| "step": 1275 |
| }, |
| { |
| "epoch": 0.2964398711626714, |
| "grad_norm": 245.0, |
| "learning_rate": 1.982735374628489e-05, |
| "logits/chosen": -1.0665302276611328, |
| "logits/rejected": -0.9313135743141174, |
| "logps/chosen": -259.1434326171875, |
| "logps/rejected": -276.8780517578125, |
| "loss": 0.7732, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.30881497263908386, |
| "rewards/margins": 3.3376965522766113, |
| "rewards/rejected": -3.028881311416626, |
| "step": 1300 |
| }, |
| { |
| "epoch": 0.30214063791579965, |
| "grad_norm": 338.0, |
| "learning_rate": 1.982066457166541e-05, |
| "logits/chosen": -1.1438727378845215, |
| "logits/rejected": -0.9572059512138367, |
| "logps/chosen": -271.3417053222656, |
| "logps/rejected": -271.4455871582031, |
| "loss": 0.9104, |
| "rewards/accuracies": 0.6800000071525574, |
| "rewards/chosen": 0.3914209306240082, |
| "rewards/margins": 3.1302664279937744, |
| "rewards/rejected": -2.7388455867767334, |
| "step": 1325 |
| }, |
| { |
| "epoch": 0.30784140466892795, |
| "grad_norm": 251.0, |
| "learning_rate": 1.981384943406985e-05, |
| "logits/chosen": -1.2839680910110474, |
| "logits/rejected": -1.0110063552856445, |
| "logps/chosen": -255.1140594482422, |
| "logps/rejected": -278.3137512207031, |
| "loss": 1.0058, |
| "rewards/accuracies": 0.6800000071525574, |
| "rewards/chosen": 0.10983797162771225, |
| "rewards/margins": 3.2528226375579834, |
| "rewards/rejected": -3.142983913421631, |
| "step": 1350 |
| }, |
| { |
| "epoch": 0.31354217142205626, |
| "grad_norm": 462.0, |
| "learning_rate": 1.980690842091134e-05, |
| "logits/chosen": -1.166993498802185, |
| "logits/rejected": -1.0042352676391602, |
| "logps/chosen": -264.400634765625, |
| "logps/rejected": -277.24383544921875, |
| "loss": 1.3169, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.4301116168498993, |
| "rewards/margins": 2.988276720046997, |
| "rewards/rejected": -3.418388605117798, |
| "step": 1375 |
| }, |
| { |
| "epoch": 0.31924293817518457, |
| "grad_norm": 326.0, |
| "learning_rate": 1.9799841621217533e-05, |
| "logits/chosen": -1.0347505807876587, |
| "logits/rejected": -0.8579542636871338, |
| "logps/chosen": -241.3385009765625, |
| "logps/rejected": -270.1276550292969, |
| "loss": 1.085, |
| "rewards/accuracies": 0.6949999928474426, |
| "rewards/chosen": 0.08915968239307404, |
| "rewards/margins": 2.777833938598633, |
| "rewards/rejected": -2.6886746883392334, |
| "step": 1400 |
| }, |
| { |
| "epoch": 0.32494370492831287, |
| "grad_norm": 135.0, |
| "learning_rate": 1.979264912562945e-05, |
| "logits/chosen": -1.2988579273223877, |
| "logits/rejected": -1.0970826148986816, |
| "logps/chosen": -248.28968811035156, |
| "logps/rejected": -266.11370849609375, |
| "loss": 1.1171, |
| "rewards/accuracies": 0.6800000071525574, |
| "rewards/chosen": 0.37792572379112244, |
| "rewards/margins": 3.0713775157928467, |
| "rewards/rejected": -2.6934518814086914, |
| "step": 1425 |
| }, |
| { |
| "epoch": 0.3306444716814412, |
| "grad_norm": 180.0, |
| "learning_rate": 1.978533102640034e-05, |
| "logits/chosen": -1.1900591850280762, |
| "logits/rejected": -1.0248688459396362, |
| "logps/chosen": -266.3290100097656, |
| "logps/rejected": -275.16925048828125, |
| "loss": 0.9664, |
| "rewards/accuracies": 0.6850000023841858, |
| "rewards/chosen": 0.5474690794944763, |
| "rewards/margins": 3.266464948654175, |
| "rewards/rejected": -2.7189955711364746, |
| "step": 1450 |
| }, |
| { |
| "epoch": 0.3363452384345694, |
| "grad_norm": 844.0, |
| "learning_rate": 1.9777887417394472e-05, |
| "logits/chosen": -1.403022289276123, |
| "logits/rejected": -1.0972503423690796, |
| "logps/chosen": -245.90745544433594, |
| "logps/rejected": -277.2258605957031, |
| "loss": 1.0723, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.6207212805747986, |
| "rewards/margins": 3.5378522872924805, |
| "rewards/rejected": -2.917130947113037, |
| "step": 1475 |
| }, |
| { |
| "epoch": 0.34204600518769773, |
| "grad_norm": 724.0, |
| "learning_rate": 1.977031839408595e-05, |
| "logits/chosen": -1.396769404411316, |
| "logits/rejected": -1.1699669361114502, |
| "logps/chosen": -264.754638671875, |
| "logps/rejected": -305.9522399902344, |
| "loss": 1.2735, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": 0.3658964931964874, |
| "rewards/margins": 3.7579073905944824, |
| "rewards/rejected": -3.392010450363159, |
| "step": 1500 |
| }, |
| { |
| "epoch": 0.34774677194082604, |
| "grad_norm": 286.0, |
| "learning_rate": 1.9762624053557485e-05, |
| "logits/chosen": -1.4281407594680786, |
| "logits/rejected": -1.2444676160812378, |
| "logps/chosen": -251.6646728515625, |
| "logps/rejected": -255.15985107421875, |
| "loss": 1.216, |
| "rewards/accuracies": 0.6600000262260437, |
| "rewards/chosen": 0.3557482957839966, |
| "rewards/margins": 3.1491892337799072, |
| "rewards/rejected": -2.793440818786621, |
| "step": 1525 |
| }, |
| { |
| "epoch": 0.35344753869395434, |
| "grad_norm": 251.0, |
| "learning_rate": 1.975480449449914e-05, |
| "logits/chosen": -1.2902607917785645, |
| "logits/rejected": -0.9710902571678162, |
| "logps/chosen": -236.17356872558594, |
| "logps/rejected": -264.3370056152344, |
| "loss": 1.1113, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.46047019958496094, |
| "rewards/margins": 2.801276206970215, |
| "rewards/rejected": -2.340806245803833, |
| "step": 1550 |
| }, |
| { |
| "epoch": 0.35914830544708265, |
| "grad_norm": 438.0, |
| "learning_rate": 1.974685981720707e-05, |
| "logits/chosen": -1.4735171794891357, |
| "logits/rejected": -1.2437952756881714, |
| "logps/chosen": -269.0889587402344, |
| "logps/rejected": -295.01336669921875, |
| "loss": 1.112, |
| "rewards/accuracies": 0.6650000214576721, |
| "rewards/chosen": 0.6523958444595337, |
| "rewards/margins": 3.259882926940918, |
| "rewards/rejected": -2.607487201690674, |
| "step": 1575 |
| }, |
| { |
| "epoch": 0.36484907220021096, |
| "grad_norm": 536.0, |
| "learning_rate": 1.973879012358225e-05, |
| "logits/chosen": -1.0262943506240845, |
| "logits/rejected": -0.780067503452301, |
| "logps/chosen": -267.7236328125, |
| "logps/rejected": -281.8758544921875, |
| "loss": 1.1099, |
| "rewards/accuracies": 0.6200000047683716, |
| "rewards/chosen": 0.009256744757294655, |
| "rewards/margins": 2.672445297241211, |
| "rewards/rejected": -2.6631884574890137, |
| "step": 1600 |
| }, |
| { |
| "epoch": 0.3705498389533392, |
| "grad_norm": 237.0, |
| "learning_rate": 1.9730595517129135e-05, |
| "logits/chosen": -1.6497623920440674, |
| "logits/rejected": -1.2609881162643433, |
| "logps/chosen": -273.4625244140625, |
| "logps/rejected": -288.0296325683594, |
| "loss": 0.9213, |
| "rewards/accuracies": 0.7200000286102295, |
| "rewards/chosen": -0.000785461685154587, |
| "rewards/margins": 3.715505361557007, |
| "rewards/rejected": -3.7162909507751465, |
| "step": 1625 |
| }, |
| { |
| "epoch": 0.3762506057064675, |
| "grad_norm": 308.0, |
| "learning_rate": 1.972227610295436e-05, |
| "logits/chosen": -1.2562499046325684, |
| "logits/rejected": -1.126264214515686, |
| "logps/chosen": -253.4782257080078, |
| "logps/rejected": -272.88702392578125, |
| "loss": 1.2809, |
| "rewards/accuracies": 0.7350000143051147, |
| "rewards/chosen": -0.07920097559690475, |
| "rewards/margins": 3.1613900661468506, |
| "rewards/rejected": -3.240590810775757, |
| "step": 1650 |
| }, |
| { |
| "epoch": 0.3819513724595958, |
| "grad_norm": 416.0, |
| "learning_rate": 1.9713831987765394e-05, |
| "logits/chosen": -1.128271460533142, |
| "logits/rejected": -0.9645025730133057, |
| "logps/chosen": -237.6045379638672, |
| "logps/rejected": -248.9420928955078, |
| "loss": 1.0105, |
| "rewards/accuracies": 0.6349999904632568, |
| "rewards/chosen": -0.2654229998588562, |
| "rewards/margins": 2.6151678562164307, |
| "rewards/rejected": -2.8805909156799316, |
| "step": 1675 |
| }, |
| { |
| "epoch": 0.3876521392127241, |
| "grad_norm": 484.0, |
| "learning_rate": 1.9705263279869143e-05, |
| "logits/chosen": -0.847245454788208, |
| "logits/rejected": -0.6292633414268494, |
| "logps/chosen": -264.4554138183594, |
| "logps/rejected": -300.7296142578125, |
| "loss": 1.2341, |
| "rewards/accuracies": 0.6449999809265137, |
| "rewards/chosen": -0.5760868787765503, |
| "rewards/margins": 2.681614637374878, |
| "rewards/rejected": -3.2577013969421387, |
| "step": 1700 |
| }, |
| { |
| "epoch": 0.39335290596585243, |
| "grad_norm": 346.0, |
| "learning_rate": 1.969657008917059e-05, |
| "logits/chosen": -1.1323637962341309, |
| "logits/rejected": -0.9998621940612793, |
| "logps/chosen": -247.55746459960938, |
| "logps/rejected": -271.728515625, |
| "loss": 1.1804, |
| "rewards/accuracies": 0.6200000047683716, |
| "rewards/chosen": 0.0011176633415743709, |
| "rewards/margins": 2.8530521392822266, |
| "rewards/rejected": -2.8519344329833984, |
| "step": 1725 |
| }, |
| { |
| "epoch": 0.3990536727189807, |
| "grad_norm": 424.0, |
| "learning_rate": 1.9687752527171372e-05, |
| "logits/chosen": -1.273902416229248, |
| "logits/rejected": -1.0144777297973633, |
| "logps/chosen": -256.39080810546875, |
| "logps/rejected": -279.9971923828125, |
| "loss": 0.9201, |
| "rewards/accuracies": 0.6650000214576721, |
| "rewards/chosen": -0.09659557044506073, |
| "rewards/margins": 3.122262477874756, |
| "rewards/rejected": -3.218858003616333, |
| "step": 1750 |
| }, |
| { |
| "epoch": 0.404754439472109, |
| "grad_norm": 420.0, |
| "learning_rate": 1.9678810706968354e-05, |
| "logits/chosen": -1.2052587270736694, |
| "logits/rejected": -0.9955114722251892, |
| "logps/chosen": -276.763427734375, |
| "logps/rejected": -285.3282470703125, |
| "loss": 1.2864, |
| "rewards/accuracies": 0.6850000023841858, |
| "rewards/chosen": -0.28634077310562134, |
| "rewards/margins": 3.197866201400757, |
| "rewards/rejected": -3.4842073917388916, |
| "step": 1775 |
| }, |
| { |
| "epoch": 0.4104552062252373, |
| "grad_norm": 118.0, |
| "learning_rate": 1.966974474325217e-05, |
| "logits/chosen": -1.1649010181427002, |
| "logits/rejected": -0.9521620869636536, |
| "logps/chosen": -259.9009094238281, |
| "logps/rejected": -260.0208740234375, |
| "loss": 1.2442, |
| "rewards/accuracies": 0.6449999809265137, |
| "rewards/chosen": 0.22394870221614838, |
| "rewards/margins": 3.1581387519836426, |
| "rewards/rejected": -2.934190034866333, |
| "step": 1800 |
| }, |
| { |
| "epoch": 0.4161559729783656, |
| "grad_norm": 564.0, |
| "learning_rate": 1.9660554752305763e-05, |
| "logits/chosen": -1.1627532243728638, |
| "logits/rejected": -0.8920064568519592, |
| "logps/chosen": -236.98739624023438, |
| "logps/rejected": -283.9150695800781, |
| "loss": 1.1474, |
| "rewards/accuracies": 0.6850000023841858, |
| "rewards/chosen": 0.3811478316783905, |
| "rewards/margins": 3.8591396808624268, |
| "rewards/rejected": -3.477992057800293, |
| "step": 1825 |
| }, |
| { |
| "epoch": 0.4218567397314939, |
| "grad_norm": 235.0, |
| "learning_rate": 1.965124085200289e-05, |
| "logits/chosen": -1.1206789016723633, |
| "logits/rejected": -0.9556788802146912, |
| "logps/chosen": -257.28582763671875, |
| "logps/rejected": -255.8318328857422, |
| "loss": 1.1252, |
| "rewards/accuracies": 0.6850000023841858, |
| "rewards/chosen": 0.5607142448425293, |
| "rewards/margins": 3.538761854171753, |
| "rewards/rejected": -2.9780478477478027, |
| "step": 1850 |
| }, |
| { |
| "epoch": 0.4275575064846222, |
| "grad_norm": 221.0, |
| "learning_rate": 1.9641803161806607e-05, |
| "logits/chosen": -1.1132560968399048, |
| "logits/rejected": -0.90582275390625, |
| "logps/chosen": -290.8529357910156, |
| "logps/rejected": -316.3980407714844, |
| "loss": 1.1914, |
| "rewards/accuracies": 0.6399999856948853, |
| "rewards/chosen": 0.19392406940460205, |
| "rewards/margins": 3.4192163944244385, |
| "rewards/rejected": -3.225292682647705, |
| "step": 1875 |
| }, |
| { |
| "epoch": 0.43325827323775046, |
| "grad_norm": 302.0, |
| "learning_rate": 1.9632241802767738e-05, |
| "logits/chosen": -1.2468523979187012, |
| "logits/rejected": -1.0206049680709839, |
| "logps/chosen": -251.34429931640625, |
| "logps/rejected": -272.6199951171875, |
| "loss": 1.1632, |
| "rewards/accuracies": 0.6850000023841858, |
| "rewards/chosen": 0.291097491979599, |
| "rewards/margins": 4.113528251647949, |
| "rewards/rejected": -3.822430372238159, |
| "step": 1900 |
| }, |
| { |
| "epoch": 0.43895903999087876, |
| "grad_norm": 656.0, |
| "learning_rate": 1.962255689752332e-05, |
| "logits/chosen": -1.1402331590652466, |
| "logits/rejected": -0.9489359259605408, |
| "logps/chosen": -267.90252685546875, |
| "logps/rejected": -282.5189514160156, |
| "loss": 1.2744, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.49352777004241943, |
| "rewards/margins": 3.0860934257507324, |
| "rewards/rejected": -2.5925652980804443, |
| "step": 1925 |
| }, |
| { |
| "epoch": 0.44465980674400707, |
| "grad_norm": 280.0, |
| "learning_rate": 1.961274857029504e-05, |
| "logits/chosen": -0.9756186008453369, |
| "logits/rejected": -0.7587201595306396, |
| "logps/chosen": -267.59112548828125, |
| "logps/rejected": -291.5782165527344, |
| "loss": 0.99, |
| "rewards/accuracies": 0.6600000262260437, |
| "rewards/chosen": 0.0824054554104805, |
| "rewards/margins": 3.3685171604156494, |
| "rewards/rejected": -3.286111354827881, |
| "step": 1950 |
| }, |
| { |
| "epoch": 0.4503605734971354, |
| "grad_norm": 193.0, |
| "learning_rate": 1.9602816946887634e-05, |
| "logits/chosen": -1.1163030862808228, |
| "logits/rejected": -0.7850387096405029, |
| "logps/chosen": -254.1883544921875, |
| "logps/rejected": -274.1246643066406, |
| "loss": 1.1358, |
| "rewards/accuracies": 0.7200000286102295, |
| "rewards/chosen": 0.3547518253326416, |
| "rewards/margins": 3.67342472076416, |
| "rewards/rejected": -3.3186728954315186, |
| "step": 1975 |
| }, |
| { |
| "epoch": 0.4560613402502637, |
| "grad_norm": 382.0, |
| "learning_rate": 1.9592762154687267e-05, |
| "logits/chosen": -1.2395323514938354, |
| "logits/rejected": -0.9825863838195801, |
| "logps/chosen": -243.5386199951172, |
| "logps/rejected": -255.34542846679688, |
| "loss": 1.1811, |
| "rewards/accuracies": 0.6399999856948853, |
| "rewards/chosen": 0.1102820560336113, |
| "rewards/margins": 3.283404588699341, |
| "rewards/rejected": -3.1731226444244385, |
| "step": 2000 |
| }, |
| { |
| "epoch": 0.46176210700339193, |
| "grad_norm": 255.0, |
| "learning_rate": 1.958258432265991e-05, |
| "logits/chosen": -1.1586804389953613, |
| "logits/rejected": -0.9523459672927856, |
| "logps/chosen": -254.47714233398438, |
| "logps/rejected": -268.3463134765625, |
| "loss": 1.249, |
| "rewards/accuracies": 0.6650000214576721, |
| "rewards/chosen": 0.20043763518333435, |
| "rewards/margins": 3.5270485877990723, |
| "rewards/rejected": -3.3266103267669678, |
| "step": 2025 |
| }, |
| { |
| "epoch": 0.46746287375652024, |
| "grad_norm": 380.0, |
| "learning_rate": 1.9572283581349685e-05, |
| "logits/chosen": -1.164260745048523, |
| "logits/rejected": -0.8869006633758545, |
| "logps/chosen": -235.7589111328125, |
| "logps/rejected": -260.83331298828125, |
| "loss": 1.0798, |
| "rewards/accuracies": 0.6850000023841858, |
| "rewards/chosen": 0.5844032764434814, |
| "rewards/margins": 3.378695011138916, |
| "rewards/rejected": -2.7942919731140137, |
| "step": 2050 |
| }, |
| { |
| "epoch": 0.47316364050964854, |
| "grad_norm": 438.0, |
| "learning_rate": 1.9561860062877187e-05, |
| "logits/chosen": -1.118167519569397, |
| "logits/rejected": -0.8347035050392151, |
| "logps/chosen": -260.2663879394531, |
| "logps/rejected": -291.6225891113281, |
| "loss": 1.0539, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.23616516590118408, |
| "rewards/margins": 3.250753402709961, |
| "rewards/rejected": -3.0145883560180664, |
| "step": 2075 |
| }, |
| { |
| "epoch": 0.47886440726277685, |
| "grad_norm": 360.0, |
| "learning_rate": 1.9551313900937784e-05, |
| "logits/chosen": -0.9437161087989807, |
| "logits/rejected": -0.6581639647483826, |
| "logps/chosen": -268.835205078125, |
| "logps/rejected": -275.67852783203125, |
| "loss": 1.0915, |
| "rewards/accuracies": 0.6700000166893005, |
| "rewards/chosen": -0.1512981802225113, |
| "rewards/margins": 2.9217278957366943, |
| "rewards/rejected": -3.073026180267334, |
| "step": 2100 |
| }, |
| { |
| "epoch": 0.48456517401590515, |
| "grad_norm": 400.0, |
| "learning_rate": 1.954064523079992e-05, |
| "logits/chosen": -1.2977359294891357, |
| "logits/rejected": -1.0284979343414307, |
| "logps/chosen": -226.16331481933594, |
| "logps/rejected": -261.0665283203125, |
| "loss": 1.0324, |
| "rewards/accuracies": 0.6650000214576721, |
| "rewards/chosen": -0.26197201013565063, |
| "rewards/margins": 3.9423534870147705, |
| "rewards/rejected": -4.2043256759643555, |
| "step": 2125 |
| }, |
| { |
| "epoch": 0.49026594076903346, |
| "grad_norm": 203.0, |
| "learning_rate": 1.9529854189303354e-05, |
| "logits/chosen": -0.9994919300079346, |
| "logits/rejected": -0.7495032548904419, |
| "logps/chosen": -248.0049591064453, |
| "logps/rejected": -266.43609619140625, |
| "loss": 1.0177, |
| "rewards/accuracies": 0.6850000023841858, |
| "rewards/chosen": -0.6823403239250183, |
| "rewards/margins": 3.05741810798645, |
| "rewards/rejected": -3.739758253097534, |
| "step": 2150 |
| }, |
| { |
| "epoch": 0.4959667075221617, |
| "grad_norm": 364.0, |
| "learning_rate": 1.951894091485743e-05, |
| "logits/chosen": -1.2298970222473145, |
| "logits/rejected": -1.039477825164795, |
| "logps/chosen": -260.4476623535156, |
| "logps/rejected": -267.5234069824219, |
| "loss": 1.1951, |
| "rewards/accuracies": 0.6200000047683716, |
| "rewards/chosen": -0.2629883885383606, |
| "rewards/margins": 2.8319931030273438, |
| "rewards/rejected": -3.0949814319610596, |
| "step": 2175 |
| }, |
| { |
| "epoch": 0.50166747427529, |
| "grad_norm": 256.0, |
| "learning_rate": 1.9507905547439285e-05, |
| "logits/chosen": -1.3255802392959595, |
| "logits/rejected": -1.0225969552993774, |
| "logps/chosen": -214.21710205078125, |
| "logps/rejected": -230.23548889160156, |
| "loss": 1.1661, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": 0.25912871956825256, |
| "rewards/margins": 3.139230251312256, |
| "rewards/rejected": -2.8801019191741943, |
| "step": 2200 |
| }, |
| { |
| "epoch": 0.5073682410284184, |
| "grad_norm": 408.0, |
| "learning_rate": 1.949674822859206e-05, |
| "logits/chosen": -1.3236197233200073, |
| "logits/rejected": -0.9879875779151917, |
| "logps/chosen": -241.90606689453125, |
| "logps/rejected": -257.3275146484375, |
| "loss": 0.9191, |
| "rewards/accuracies": 0.7049999833106995, |
| "rewards/chosen": 0.44867461919784546, |
| "rewards/margins": 3.632122278213501, |
| "rewards/rejected": -3.1834475994110107, |
| "step": 2225 |
| }, |
| { |
| "epoch": 0.5130690077815466, |
| "grad_norm": 540.0, |
| "learning_rate": 1.948546910142309e-05, |
| "logits/chosen": -1.034936547279358, |
| "logits/rejected": -0.7606247663497925, |
| "logps/chosen": -255.0516357421875, |
| "logps/rejected": -302.05352783203125, |
| "loss": 1.2027, |
| "rewards/accuracies": 0.6349999904632568, |
| "rewards/chosen": 0.18715853989124298, |
| "rewards/margins": 2.9205405712127686, |
| "rewards/rejected": -2.733381748199463, |
| "step": 2250 |
| }, |
| { |
| "epoch": 0.5187697745346749, |
| "grad_norm": 466.0, |
| "learning_rate": 1.9474068310602048e-05, |
| "logits/chosen": -1.0151162147521973, |
| "logits/rejected": -0.7761868238449097, |
| "logps/chosen": -271.3726806640625, |
| "logps/rejected": -273.3763427734375, |
| "loss": 1.0249, |
| "rewards/accuracies": 0.6700000166893005, |
| "rewards/chosen": 0.22041645646095276, |
| "rewards/margins": 2.663581609725952, |
| "rewards/rejected": -2.4431653022766113, |
| "step": 2275 |
| }, |
| { |
| "epoch": 0.5244705412878032, |
| "grad_norm": 187.0, |
| "learning_rate": 1.9462546002359124e-05, |
| "logits/chosen": -1.1769782304763794, |
| "logits/rejected": -1.007545828819275, |
| "logps/chosen": -241.39300537109375, |
| "logps/rejected": -259.6580505371094, |
| "loss": 1.242, |
| "rewards/accuracies": 0.6549999713897705, |
| "rewards/chosen": 0.2778342366218567, |
| "rewards/margins": 3.119676113128662, |
| "rewards/rejected": -2.841841697692871, |
| "step": 2300 |
| }, |
| { |
| "epoch": 0.5301713080409315, |
| "grad_norm": 406.0, |
| "learning_rate": 1.9450902324483112e-05, |
| "logits/chosen": -1.273148775100708, |
| "logits/rejected": -1.0504608154296875, |
| "logps/chosen": -275.8114318847656, |
| "logps/rejected": -268.6146545410156, |
| "loss": 0.9462, |
| "rewards/accuracies": 0.6600000262260437, |
| "rewards/chosen": 0.34187212586402893, |
| "rewards/margins": 3.256736993789673, |
| "rewards/rejected": -2.9148645401000977, |
| "step": 2325 |
| }, |
| { |
| "epoch": 0.5358720747940598, |
| "grad_norm": 118.0, |
| "learning_rate": 1.9439137426319542e-05, |
| "logits/chosen": -1.3830595016479492, |
| "logits/rejected": -1.0776387453079224, |
| "logps/chosen": -253.1000518798828, |
| "logps/rejected": -278.6458740234375, |
| "loss": 0.8868, |
| "rewards/accuracies": 0.7200000286102295, |
| "rewards/chosen": 0.31237900257110596, |
| "rewards/margins": 4.299036979675293, |
| "rewards/rejected": -3.9866580963134766, |
| "step": 2350 |
| }, |
| { |
| "epoch": 0.5415728415471881, |
| "grad_norm": 318.0, |
| "learning_rate": 1.9427251458768747e-05, |
| "logits/chosen": -1.2302486896514893, |
| "logits/rejected": -1.0658971071243286, |
| "logps/chosen": -287.3409423828125, |
| "logps/rejected": -287.0306091308594, |
| "loss": 1.1061, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.22148369252681732, |
| "rewards/margins": 3.535696029663086, |
| "rewards/rejected": -3.7571799755096436, |
| "step": 2375 |
| }, |
| { |
| "epoch": 0.5472736083003163, |
| "grad_norm": 374.0, |
| "learning_rate": 1.941524457428394e-05, |
| "logits/chosen": -1.1590862274169922, |
| "logits/rejected": -0.9304063320159912, |
| "logps/chosen": -255.72137451171875, |
| "logps/rejected": -286.2277526855469, |
| "loss": 1.2341, |
| "rewards/accuracies": 0.6449999809265137, |
| "rewards/chosen": -0.2174849957227707, |
| "rewards/margins": 3.2461729049682617, |
| "rewards/rejected": -3.463657855987549, |
| "step": 2400 |
| }, |
| { |
| "epoch": 0.5529743750534447, |
| "grad_norm": 256.0, |
| "learning_rate": 1.9403116926869253e-05, |
| "logits/chosen": -1.3541803359985352, |
| "logits/rejected": -1.099515438079834, |
| "logps/chosen": -252.45530700683594, |
| "logps/rejected": -265.0677490234375, |
| "loss": 1.1474, |
| "rewards/accuracies": 0.6549999713897705, |
| "rewards/chosen": -0.46254172921180725, |
| "rewards/margins": 3.677323579788208, |
| "rewards/rejected": -4.139864921569824, |
| "step": 2425 |
| }, |
| { |
| "epoch": 0.558675141806573, |
| "grad_norm": 282.0, |
| "learning_rate": 1.9390868672077753e-05, |
| "logits/chosen": -1.1199222803115845, |
| "logits/rejected": -0.8672134280204773, |
| "logps/chosen": -229.22000122070312, |
| "logps/rejected": -265.7611083984375, |
| "loss": 0.8589, |
| "rewards/accuracies": 0.6549999713897705, |
| "rewards/chosen": 0.1352018117904663, |
| "rewards/margins": 3.6319026947021484, |
| "rewards/rejected": -3.4967007637023926, |
| "step": 2450 |
| }, |
| { |
| "epoch": 0.5643759085597013, |
| "grad_norm": 346.0, |
| "learning_rate": 1.9378499967009467e-05, |
| "logits/chosen": -1.0298895835876465, |
| "logits/rejected": -0.8239762783050537, |
| "logps/chosen": -271.59893798828125, |
| "logps/rejected": -280.8359069824219, |
| "loss": 1.3015, |
| "rewards/accuracies": 0.6449999809265137, |
| "rewards/chosen": -0.6309685707092285, |
| "rewards/margins": 2.80938720703125, |
| "rewards/rejected": -3.4403557777404785, |
| "step": 2475 |
| }, |
| { |
| "epoch": 0.5700766753128296, |
| "grad_norm": 556.0, |
| "learning_rate": 1.9366010970309355e-05, |
| "logits/chosen": -1.2273378372192383, |
| "logits/rejected": -0.9872612953186035, |
| "logps/chosen": -230.03445434570312, |
| "logps/rejected": -256.1549987792969, |
| "loss": 1.0579, |
| "rewards/accuracies": 0.6399999856948853, |
| "rewards/chosen": -0.3971507251262665, |
| "rewards/margins": 3.7465555667877197, |
| "rewards/rejected": -4.143706798553467, |
| "step": 2500 |
| }, |
| { |
| "epoch": 0.5700766753128296, |
| "eval_logits/chosen": -1.1941542625427246, |
| "eval_logits/rejected": -0.9727721810340881, |
| "eval_logps/chosen": -259.9610595703125, |
| "eval_logps/rejected": -276.6463317871094, |
| "eval_loss": 1.0681413412094116, |
| "eval_rewards/accuracies": 0.6721012592315674, |
| "eval_rewards/chosen": -0.30271676182746887, |
| "eval_rewards/margins": 3.3968069553375244, |
| "eval_rewards/rejected": -3.69952392578125, |
| "eval_runtime": 905.9231, |
| "eval_samples_per_second": 9.682, |
| "eval_steps_per_second": 9.682, |
| "step": 2500 |
| }, |
| { |
| "epoch": 0.5757774420659578, |
| "grad_norm": 348.0, |
| "learning_rate": 1.9353401842165262e-05, |
| "logits/chosen": -1.0818673372268677, |
| "logits/rejected": -0.9418809413909912, |
| "logps/chosen": -293.3169860839844, |
| "logps/rejected": -285.7959899902344, |
| "loss": 1.4433, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.18778309226036072, |
| "rewards/margins": 2.372335910797119, |
| "rewards/rejected": -2.5601189136505127, |
| "step": 2525 |
| }, |
| { |
| "epoch": 0.5814782088190862, |
| "grad_norm": 131.0, |
| "learning_rate": 1.93406727443059e-05, |
| "logits/chosen": -0.8230610489845276, |
| "logits/rejected": -0.6167992949485779, |
| "logps/chosen": -237.55992126464844, |
| "logps/rejected": -271.36651611328125, |
| "loss": 1.0082, |
| "rewards/accuracies": 0.6850000023841858, |
| "rewards/chosen": 0.017401594668626785, |
| "rewards/margins": 2.67635440826416, |
| "rewards/rejected": -2.6589527130126953, |
| "step": 2550 |
| }, |
| { |
| "epoch": 0.5871789755722144, |
| "grad_norm": 470.0, |
| "learning_rate": 1.9327823839998726e-05, |
| "logits/chosen": -0.891941249370575, |
| "logits/rejected": -0.6851850152015686, |
| "logps/chosen": -292.5101623535156, |
| "logps/rejected": -287.673583984375, |
| "loss": 1.2146, |
| "rewards/accuracies": 0.6449999809265137, |
| "rewards/chosen": 0.26026153564453125, |
| "rewards/margins": 2.6793158054351807, |
| "rewards/rejected": -2.4190540313720703, |
| "step": 2575 |
| }, |
| { |
| "epoch": 0.5928797423253428, |
| "grad_norm": 244.0, |
| "learning_rate": 1.9314855294047888e-05, |
| "logits/chosen": -1.0238795280456543, |
| "logits/rejected": -0.7717757225036621, |
| "logps/chosen": -265.3813171386719, |
| "logps/rejected": -286.2080383300781, |
| "loss": 1.0681, |
| "rewards/accuracies": 0.6700000166893005, |
| "rewards/chosen": 0.261279821395874, |
| "rewards/margins": 2.847034215927124, |
| "rewards/rejected": -2.58575439453125, |
| "step": 2600 |
| }, |
| { |
| "epoch": 0.598580509078471, |
| "grad_norm": 346.0, |
| "learning_rate": 1.93017672727921e-05, |
| "logits/chosen": -1.0754836797714233, |
| "logits/rejected": -0.760750412940979, |
| "logps/chosen": -253.26507568359375, |
| "logps/rejected": -281.3257751464844, |
| "loss": 0.9716, |
| "rewards/accuracies": 0.6449999809265137, |
| "rewards/chosen": -0.11648125946521759, |
| "rewards/margins": 3.338644504547119, |
| "rewards/rejected": -3.4551258087158203, |
| "step": 2625 |
| }, |
| { |
| "epoch": 0.6042812758315993, |
| "grad_norm": 452.0, |
| "learning_rate": 1.928855994410249e-05, |
| "logits/chosen": -1.2952466011047363, |
| "logits/rejected": -1.0639019012451172, |
| "logps/chosen": -269.1417236328125, |
| "logps/rejected": -276.7212219238281, |
| "loss": 1.0408, |
| "rewards/accuracies": 0.6600000262260437, |
| "rewards/chosen": 0.3271361291408539, |
| "rewards/margins": 3.3854610919952393, |
| "rewards/rejected": -3.0583252906799316, |
| "step": 2650 |
| }, |
| { |
| "epoch": 0.6099820425847277, |
| "grad_norm": 57.0, |
| "learning_rate": 1.927523347738047e-05, |
| "logits/chosen": -1.1273165941238403, |
| "logits/rejected": -0.8825761675834656, |
| "logps/chosen": -255.91920471191406, |
| "logps/rejected": -267.05328369140625, |
| "loss": 1.2955, |
| "rewards/accuracies": 0.6650000214576721, |
| "rewards/chosen": -0.17305634915828705, |
| "rewards/margins": 2.728884696960449, |
| "rewards/rejected": -2.9019408226013184, |
| "step": 2675 |
| }, |
| { |
| "epoch": 0.6156828093378559, |
| "grad_norm": 596.0, |
| "learning_rate": 1.926178804355555e-05, |
| "logits/chosen": -1.3970657587051392, |
| "logits/rejected": -1.1048283576965332, |
| "logps/chosen": -236.66233825683594, |
| "logps/rejected": -257.4218444824219, |
| "loss": 1.1782, |
| "rewards/accuracies": 0.6349999904632568, |
| "rewards/chosen": 0.023759135976433754, |
| "rewards/margins": 3.343031883239746, |
| "rewards/rejected": -3.319272518157959, |
| "step": 2700 |
| }, |
| { |
| "epoch": 0.6213835760909843, |
| "grad_norm": 552.0, |
| "learning_rate": 1.9248223815083155e-05, |
| "logits/chosen": -1.178747296333313, |
| "logits/rejected": -0.9164783358573914, |
| "logps/chosen": -245.88218688964844, |
| "logps/rejected": -258.6275634765625, |
| "loss": 0.9636, |
| "rewards/accuracies": 0.6949999928474426, |
| "rewards/chosen": 0.08073363453149796, |
| "rewards/margins": 3.230246067047119, |
| "rewards/rejected": -3.1495118141174316, |
| "step": 2725 |
| }, |
| { |
| "epoch": 0.6270843428441125, |
| "grad_norm": 8.75, |
| "learning_rate": 1.9234540965942407e-05, |
| "logits/chosen": -1.4600555896759033, |
| "logits/rejected": -1.1421494483947754, |
| "logps/chosen": -248.7782440185547, |
| "logps/rejected": -266.22314453125, |
| "loss": 0.8563, |
| "rewards/accuracies": 0.7350000143051147, |
| "rewards/chosen": 0.4743490517139435, |
| "rewards/margins": 4.6551833152771, |
| "rewards/rejected": -4.1808342933654785, |
| "step": 2750 |
| }, |
| { |
| "epoch": 0.6327851095972409, |
| "grad_norm": 332.0, |
| "learning_rate": 1.922073967163389e-05, |
| "logits/chosen": -1.05692458152771, |
| "logits/rejected": -0.8919233679771423, |
| "logps/chosen": -249.50613403320312, |
| "logps/rejected": -284.79388427734375, |
| "loss": 1.1057, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.42172202467918396, |
| "rewards/margins": 3.101043701171875, |
| "rewards/rejected": -2.6793220043182373, |
| "step": 2775 |
| }, |
| { |
| "epoch": 0.6384858763503691, |
| "grad_norm": 376.0, |
| "learning_rate": 1.9206820109177412e-05, |
| "logits/chosen": -1.0244029760360718, |
| "logits/rejected": -0.8773370385169983, |
| "logps/chosen": -231.66925048828125, |
| "logps/rejected": -245.7607421875, |
| "loss": 0.8917, |
| "rewards/accuracies": 0.7099999785423279, |
| "rewards/chosen": 0.35288897156715393, |
| "rewards/margins": 3.7832424640655518, |
| "rewards/rejected": -3.4303534030914307, |
| "step": 2800 |
| }, |
| { |
| "epoch": 0.6441866431034974, |
| "grad_norm": 76.0, |
| "learning_rate": 1.9192782457109717e-05, |
| "logits/chosen": -1.0910537242889404, |
| "logits/rejected": -0.9162377715110779, |
| "logps/chosen": -262.3955383300781, |
| "logps/rejected": -281.980224609375, |
| "loss": 1.0045, |
| "rewards/accuracies": 0.6650000214576721, |
| "rewards/chosen": 0.8997083902359009, |
| "rewards/margins": 3.1187634468078613, |
| "rewards/rejected": -2.21905517578125, |
| "step": 2825 |
| }, |
| { |
| "epoch": 0.6498874098566257, |
| "grad_norm": 358.0, |
| "learning_rate": 1.9178626895482206e-05, |
| "logits/chosen": -1.2077234983444214, |
| "logits/rejected": -1.045710563659668, |
| "logps/chosen": -277.63250732421875, |
| "logps/rejected": -293.08758544921875, |
| "loss": 1.2775, |
| "rewards/accuracies": 0.6349999904632568, |
| "rewards/chosen": 0.7143364548683167, |
| "rewards/margins": 3.029661178588867, |
| "rewards/rejected": -2.3153247833251953, |
| "step": 2850 |
| }, |
| { |
| "epoch": 0.655588176609754, |
| "grad_norm": 396.0, |
| "learning_rate": 1.916435360585863e-05, |
| "logits/chosen": -1.2753803730010986, |
| "logits/rejected": -1.0552774667739868, |
| "logps/chosen": -219.78517150878906, |
| "logps/rejected": -242.41868591308594, |
| "loss": 1.0436, |
| "rewards/accuracies": 0.6299999952316284, |
| "rewards/chosen": 0.09890857338905334, |
| "rewards/margins": 3.661541700363159, |
| "rewards/rejected": -3.562633752822876, |
| "step": 2875 |
| }, |
| { |
| "epoch": 0.6612889433628824, |
| "grad_norm": 488.0, |
| "learning_rate": 1.9149962771312753e-05, |
| "logits/chosen": -1.350898265838623, |
| "logits/rejected": -1.14180588722229, |
| "logps/chosen": -252.77313232421875, |
| "logps/rejected": -269.1424255371094, |
| "loss": 1.2597, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.5017364621162415, |
| "rewards/margins": 3.287431240081787, |
| "rewards/rejected": -2.7856948375701904, |
| "step": 2900 |
| }, |
| { |
| "epoch": 0.6669897101160106, |
| "grad_norm": 336.0, |
| "learning_rate": 1.913545457642601e-05, |
| "logits/chosen": -1.5227373838424683, |
| "logits/rejected": -1.2297714948654175, |
| "logps/chosen": -254.07168579101562, |
| "logps/rejected": -270.7015075683594, |
| "loss": 0.8367, |
| "rewards/accuracies": 0.7350000143051147, |
| "rewards/chosen": 0.4393278956413269, |
| "rewards/margins": 4.0322794914245605, |
| "rewards/rejected": -3.592951774597168, |
| "step": 2925 |
| }, |
| { |
| "epoch": 0.6726904768691389, |
| "grad_norm": 402.0, |
| "learning_rate": 1.9120829207285135e-05, |
| "logits/chosen": -1.1801203489303589, |
| "logits/rejected": -1.0105456113815308, |
| "logps/chosen": -269.9618835449219, |
| "logps/rejected": -288.91339111328125, |
| "loss": 0.8932, |
| "rewards/accuracies": 0.7049999833106995, |
| "rewards/chosen": 0.6307451725006104, |
| "rewards/margins": 3.552612066268921, |
| "rewards/rejected": -2.9218664169311523, |
| "step": 2950 |
| }, |
| { |
| "epoch": 0.6783912436222672, |
| "grad_norm": 179.0, |
| "learning_rate": 1.910608685147978e-05, |
| "logits/chosen": -1.3146777153015137, |
| "logits/rejected": -1.1052991151809692, |
| "logps/chosen": -245.76792907714844, |
| "logps/rejected": -268.48797607421875, |
| "loss": 1.2028, |
| "rewards/accuracies": 0.6800000071525574, |
| "rewards/chosen": 0.3203747272491455, |
| "rewards/margins": 3.3431179523468018, |
| "rewards/rejected": -3.0227434635162354, |
| "step": 2975 |
| }, |
| { |
| "epoch": 0.6840920103753955, |
| "grad_norm": 340.0, |
| "learning_rate": 1.9091227698100098e-05, |
| "logits/chosen": -1.2000571489334106, |
| "logits/rejected": -1.0043823719024658, |
| "logps/chosen": -278.100341796875, |
| "logps/rejected": -286.00701904296875, |
| "loss": 1.0315, |
| "rewards/accuracies": 0.6650000214576721, |
| "rewards/chosen": 0.6071378588676453, |
| "rewards/margins": 3.336888074874878, |
| "rewards/rejected": -2.729750633239746, |
| "step": 3000 |
| }, |
| { |
| "epoch": 0.6897927771285238, |
| "grad_norm": 322.0, |
| "learning_rate": 1.9076251937734328e-05, |
| "logits/chosen": -1.212092638015747, |
| "logits/rejected": -1.018998384475708, |
| "logps/chosen": -223.50465393066406, |
| "logps/rejected": -245.6395263671875, |
| "loss": 0.9914, |
| "rewards/accuracies": 0.6899999976158142, |
| "rewards/chosen": 0.6325468420982361, |
| "rewards/margins": 3.556051731109619, |
| "rewards/rejected": -2.9235053062438965, |
| "step": 3025 |
| }, |
| { |
| "epoch": 0.6954935438816521, |
| "grad_norm": 322.0, |
| "learning_rate": 1.9061159762466358e-05, |
| "logits/chosen": -1.1529450416564941, |
| "logits/rejected": -0.7803061008453369, |
| "logps/chosen": -254.00906372070312, |
| "logps/rejected": -286.4268493652344, |
| "loss": 0.8715, |
| "rewards/accuracies": 0.6800000071525574, |
| "rewards/chosen": 0.17509357631206512, |
| "rewards/margins": 3.737367630004883, |
| "rewards/rejected": -3.5622735023498535, |
| "step": 3050 |
| }, |
| { |
| "epoch": 0.7011943106347803, |
| "grad_norm": 492.0, |
| "learning_rate": 1.9045951365873235e-05, |
| "logits/chosen": -1.4562079906463623, |
| "logits/rejected": -1.1702617406845093, |
| "logps/chosen": -254.8812255859375, |
| "logps/rejected": -269.29852294921875, |
| "loss": 1.0107, |
| "rewards/accuracies": 0.6549999713897705, |
| "rewards/chosen": 0.34646105766296387, |
| "rewards/margins": 3.7237024307250977, |
| "rewards/rejected": -3.377241611480713, |
| "step": 3075 |
| }, |
| { |
| "epoch": 0.7068950773879087, |
| "grad_norm": 436.0, |
| "learning_rate": 1.9030626943022703e-05, |
| "logits/chosen": -1.3802834749221802, |
| "logits/rejected": -1.152989149093628, |
| "logps/chosen": -278.0682678222656, |
| "logps/rejected": -280.2952880859375, |
| "loss": 1.0967, |
| "rewards/accuracies": 0.6449999809265137, |
| "rewards/chosen": -0.046190012246370316, |
| "rewards/margins": 2.8668227195739746, |
| "rewards/rejected": -2.913013219833374, |
| "step": 3100 |
| }, |
| { |
| "epoch": 0.7125958441410369, |
| "grad_norm": 354.0, |
| "learning_rate": 1.9015186690470703e-05, |
| "logits/chosen": -1.2900662422180176, |
| "logits/rejected": -1.045066475868225, |
| "logps/chosen": -256.0972900390625, |
| "logps/rejected": -268.8332824707031, |
| "loss": 0.8773, |
| "rewards/accuracies": 0.6700000166893005, |
| "rewards/chosen": 0.5201454758644104, |
| "rewards/margins": 3.820733070373535, |
| "rewards/rejected": -3.3005881309509277, |
| "step": 3125 |
| }, |
| { |
| "epoch": 0.7182966108941653, |
| "grad_norm": 233.0, |
| "learning_rate": 1.8999630806258837e-05, |
| "logits/chosen": -1.482657790184021, |
| "logits/rejected": -1.2103347778320312, |
| "logps/chosen": -242.50462341308594, |
| "logps/rejected": -269.5572814941406, |
| "loss": 1.2095, |
| "rewards/accuracies": 0.6449999809265137, |
| "rewards/chosen": 0.3224226236343384, |
| "rewards/margins": 3.234161376953125, |
| "rewards/rejected": -2.911738872528076, |
| "step": 3150 |
| }, |
| { |
| "epoch": 0.7239973776472935, |
| "grad_norm": 408.0, |
| "learning_rate": 1.8983959489911833e-05, |
| "logits/chosen": -1.3495805263519287, |
| "logits/rejected": -1.148563027381897, |
| "logps/chosen": -240.69271850585938, |
| "logps/rejected": -249.7307586669922, |
| "loss": 1.1167, |
| "rewards/accuracies": 0.6349999904632568, |
| "rewards/chosen": -0.2572160065174103, |
| "rewards/margins": 3.0647361278533936, |
| "rewards/rejected": -3.3219521045684814, |
| "step": 3175 |
| }, |
| { |
| "epoch": 0.7296981444004219, |
| "grad_norm": 290.0, |
| "learning_rate": 1.8968172942434996e-05, |
| "logits/chosen": -1.4180634021759033, |
| "logits/rejected": -1.0784398317337036, |
| "logps/chosen": -257.40313720703125, |
| "logps/rejected": -291.177978515625, |
| "loss": 0.8833, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.16765031218528748, |
| "rewards/margins": 4.429814338684082, |
| "rewards/rejected": -4.26216459274292, |
| "step": 3200 |
| }, |
| { |
| "epoch": 0.7353989111535502, |
| "grad_norm": 368.0, |
| "learning_rate": 1.8952271366311618e-05, |
| "logits/chosen": -1.3795121908187866, |
| "logits/rejected": -1.1531827449798584, |
| "logps/chosen": -247.22943115234375, |
| "logps/rejected": -277.8642272949219, |
| "loss": 0.8494, |
| "rewards/accuracies": 0.7200000286102295, |
| "rewards/chosen": -0.09695852547883987, |
| "rewards/margins": 3.957188129425049, |
| "rewards/rejected": -4.054146766662598, |
| "step": 3225 |
| }, |
| { |
| "epoch": 0.7410996779066784, |
| "grad_norm": 207.0, |
| "learning_rate": 1.8936254965500393e-05, |
| "logits/chosen": -1.0625925064086914, |
| "logits/rejected": -0.8639459013938904, |
| "logps/chosen": -260.7442321777344, |
| "logps/rejected": -267.7799072265625, |
| "loss": 0.9378, |
| "rewards/accuracies": 0.6349999904632568, |
| "rewards/chosen": -0.3863774240016937, |
| "rewards/margins": 3.5430545806884766, |
| "rewards/rejected": -3.929431676864624, |
| "step": 3250 |
| }, |
| { |
| "epoch": 0.7468004446598068, |
| "grad_norm": 300.0, |
| "learning_rate": 1.892012394543277e-05, |
| "logits/chosen": -1.4860643148422241, |
| "logits/rejected": -1.2890013456344604, |
| "logps/chosen": -248.3817596435547, |
| "logps/rejected": -276.2065124511719, |
| "loss": 0.9918, |
| "rewards/accuracies": 0.6949999928474426, |
| "rewards/chosen": -0.0690900906920433, |
| "rewards/margins": 3.6559197902679443, |
| "rewards/rejected": -3.7250096797943115, |
| "step": 3275 |
| }, |
| { |
| "epoch": 0.752501211412935, |
| "grad_norm": 420.0, |
| "learning_rate": 1.8903878513010377e-05, |
| "logits/chosen": -1.1266851425170898, |
| "logits/rejected": -0.914747953414917, |
| "logps/chosen": -259.7164001464844, |
| "logps/rejected": -279.09271240234375, |
| "loss": 1.141, |
| "rewards/accuracies": 0.6150000095367432, |
| "rewards/chosen": -1.0861494541168213, |
| "rewards/margins": 3.1036548614501953, |
| "rewards/rejected": -4.1898040771484375, |
| "step": 3300 |
| }, |
| { |
| "epoch": 0.7582019781660634, |
| "grad_norm": 352.0, |
| "learning_rate": 1.88875188766023e-05, |
| "logits/chosen": -1.4356380701065063, |
| "logits/rejected": -1.2561640739440918, |
| "logps/chosen": -246.74855041503906, |
| "logps/rejected": -281.4516296386719, |
| "loss": 0.9301, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.33938464522361755, |
| "rewards/margins": 4.261763572692871, |
| "rewards/rejected": -4.601147651672363, |
| "step": 3325 |
| }, |
| { |
| "epoch": 0.7639027449191916, |
| "grad_norm": 744.0, |
| "learning_rate": 1.8871045246042454e-05, |
| "logits/chosen": -1.4839650392532349, |
| "logits/rejected": -1.200853705406189, |
| "logps/chosen": -281.3594970703125, |
| "logps/rejected": -294.0035400390625, |
| "loss": 1.144, |
| "rewards/accuracies": 0.6600000262260437, |
| "rewards/chosen": -0.6125954389572144, |
| "rewards/margins": 3.574725389480591, |
| "rewards/rejected": -4.187320709228516, |
| "step": 3350 |
| }, |
| { |
| "epoch": 0.7696035116723199, |
| "grad_norm": 528.0, |
| "learning_rate": 1.885445783262688e-05, |
| "logits/chosen": -1.337283968925476, |
| "logits/rejected": -1.0559604167938232, |
| "logps/chosen": -239.36705017089844, |
| "logps/rejected": -257.6148986816406, |
| "loss": 1.1615, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.10132899135351181, |
| "rewards/margins": 3.514072895050049, |
| "rewards/rejected": -3.6154019832611084, |
| "step": 3375 |
| }, |
| { |
| "epoch": 0.7753042784254482, |
| "grad_norm": 230.0, |
| "learning_rate": 1.8837756849111035e-05, |
| "logits/chosen": -1.4483850002288818, |
| "logits/rejected": -1.2353034019470215, |
| "logps/chosen": -244.01458740234375, |
| "logps/rejected": -275.1045227050781, |
| "loss": 1.1078, |
| "rewards/accuracies": 0.6399999856948853, |
| "rewards/chosen": 0.0038251972291618586, |
| "rewards/margins": 3.1398205757141113, |
| "rewards/rejected": -3.1359951496124268, |
| "step": 3400 |
| }, |
| { |
| "epoch": 0.7810050451785765, |
| "grad_norm": 302.0, |
| "learning_rate": 1.882094250970706e-05, |
| "logits/chosen": -1.157295823097229, |
| "logits/rejected": -0.8771103620529175, |
| "logps/chosen": -266.73065185546875, |
| "logps/rejected": -271.2137756347656, |
| "loss": 1.1206, |
| "rewards/accuracies": 0.6150000095367432, |
| "rewards/chosen": -0.026825208216905594, |
| "rewards/margins": 2.5302481651306152, |
| "rewards/rejected": -2.5570735931396484, |
| "step": 3425 |
| }, |
| { |
| "epoch": 0.7867058119317049, |
| "grad_norm": 255.0, |
| "learning_rate": 1.8804015030081033e-05, |
| "logits/chosen": -1.2398334741592407, |
| "logits/rejected": -0.9958074688911438, |
| "logps/chosen": -264.827880859375, |
| "logps/rejected": -259.72088623046875, |
| "loss": 0.9405, |
| "rewards/accuracies": 0.7350000143051147, |
| "rewards/chosen": 0.305253267288208, |
| "rewards/margins": 3.720737934112549, |
| "rewards/rejected": -3.415484666824341, |
| "step": 3450 |
| }, |
| { |
| "epoch": 0.7924065786848331, |
| "grad_norm": 350.0, |
| "learning_rate": 1.8786974627350206e-05, |
| "logits/chosen": -1.4714716672897339, |
| "logits/rejected": -1.2138081789016724, |
| "logps/chosen": -224.94627380371094, |
| "logps/rejected": -264.0721435546875, |
| "loss": 0.9107, |
| "rewards/accuracies": 0.7149999737739563, |
| "rewards/chosen": -0.06054474040865898, |
| "rewards/margins": 4.0769243240356445, |
| "rewards/rejected": -4.137468338012695, |
| "step": 3475 |
| }, |
| { |
| "epoch": 0.7981073454379614, |
| "grad_norm": 588.0, |
| "learning_rate": 1.8769821520080224e-05, |
| "logits/chosen": -1.691823124885559, |
| "logits/rejected": -1.4056679010391235, |
| "logps/chosen": -239.66937255859375, |
| "logps/rejected": -272.1314697265625, |
| "loss": 0.9991, |
| "rewards/accuracies": 0.7300000190734863, |
| "rewards/chosen": 0.018595609813928604, |
| "rewards/margins": 3.6974661350250244, |
| "rewards/rejected": -3.678870916366577, |
| "step": 3500 |
| }, |
| { |
| "epoch": 0.8038081121910897, |
| "grad_norm": 306.0, |
| "learning_rate": 1.8752555928282307e-05, |
| "logits/chosen": -1.5157978534698486, |
| "logits/rejected": -1.2500338554382324, |
| "logps/chosen": -266.5981750488281, |
| "logps/rejected": -282.7295227050781, |
| "loss": 1.0762, |
| "rewards/accuracies": 0.6949999928474426, |
| "rewards/chosen": -0.1609947234392166, |
| "rewards/margins": 3.896528720855713, |
| "rewards/rejected": -4.057523727416992, |
| "step": 3525 |
| }, |
| { |
| "epoch": 0.809508878944218, |
| "grad_norm": 476.0, |
| "learning_rate": 1.873517807341044e-05, |
| "logits/chosen": -1.3880764245986938, |
| "logits/rejected": -1.1662323474884033, |
| "logps/chosen": -254.42062377929688, |
| "logps/rejected": -264.4542236328125, |
| "loss": 1.0019, |
| "rewards/accuracies": 0.6949999928474426, |
| "rewards/chosen": 0.03959185630083084, |
| "rewards/margins": 3.8310658931732178, |
| "rewards/rejected": -3.7914741039276123, |
| "step": 3550 |
| }, |
| { |
| "epoch": 0.8152096456973463, |
| "grad_norm": 394.0, |
| "learning_rate": 1.871768817835854e-05, |
| "logits/chosen": -1.7619051933288574, |
| "logits/rejected": -1.4349865913391113, |
| "logps/chosen": -244.71749877929688, |
| "logps/rejected": -279.2682800292969, |
| "loss": 1.1266, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.6931087374687195, |
| "rewards/margins": 3.8597850799560547, |
| "rewards/rejected": -4.552894115447998, |
| "step": 3575 |
| }, |
| { |
| "epoch": 0.8209104124504746, |
| "grad_norm": 432.0, |
| "learning_rate": 1.8700086467457577e-05, |
| "logits/chosen": -1.4022297859191895, |
| "logits/rejected": -1.3012043237686157, |
| "logps/chosen": -267.1990661621094, |
| "logps/rejected": -274.3907165527344, |
| "loss": 0.8536, |
| "rewards/accuracies": 0.7300000190734863, |
| "rewards/chosen": -0.5109031200408936, |
| "rewards/margins": 4.6965837478637695, |
| "rewards/rejected": -5.207487106323242, |
| "step": 3600 |
| }, |
| { |
| "epoch": 0.8266111792036028, |
| "grad_norm": 153.0, |
| "learning_rate": 1.868237316647271e-05, |
| "logits/chosen": -1.4858689308166504, |
| "logits/rejected": -1.2162619829177856, |
| "logps/chosen": -294.3266906738281, |
| "logps/rejected": -317.95849609375, |
| "loss": 0.7804, |
| "rewards/accuracies": 0.7599999904632568, |
| "rewards/chosen": -0.21449381113052368, |
| "rewards/margins": 4.547573566436768, |
| "rewards/rejected": -4.762067794799805, |
| "step": 3625 |
| }, |
| { |
| "epoch": 0.8323119459567312, |
| "grad_norm": 180.0, |
| "learning_rate": 1.866454850260038e-05, |
| "logits/chosen": -1.4929500818252563, |
| "logits/rejected": -1.2532182931900024, |
| "logps/chosen": -274.46209716796875, |
| "logps/rejected": -274.04779052734375, |
| "loss": 1.0809, |
| "rewards/accuracies": 0.6850000023841858, |
| "rewards/chosen": -0.091336190700531, |
| "rewards/margins": 4.116111755371094, |
| "rewards/rejected": -4.2074480056762695, |
| "step": 3650 |
| }, |
| { |
| "epoch": 0.8380127127098594, |
| "grad_norm": 322.0, |
| "learning_rate": 1.8646612704465418e-05, |
| "logits/chosen": -1.4442182779312134, |
| "logits/rejected": -1.1680885553359985, |
| "logps/chosen": -262.33612060546875, |
| "logps/rejected": -270.7034912109375, |
| "loss": 1.1587, |
| "rewards/accuracies": 0.6399999856948853, |
| "rewards/chosen": -0.118131123483181, |
| "rewards/margins": 3.0996768474578857, |
| "rewards/rejected": -3.2178080081939697, |
| "step": 3675 |
| }, |
| { |
| "epoch": 0.8437134794629878, |
| "grad_norm": 372.0, |
| "learning_rate": 1.8628566002118087e-05, |
| "logits/chosen": -1.3202441930770874, |
| "logits/rejected": -0.9656006097793579, |
| "logps/chosen": -272.114501953125, |
| "logps/rejected": -266.1072082519531, |
| "loss": 0.8882, |
| "rewards/accuracies": 0.6899999976158142, |
| "rewards/chosen": 0.3116624355316162, |
| "rewards/margins": 4.167173862457275, |
| "rewards/rejected": -3.855511426925659, |
| "step": 3700 |
| }, |
| { |
| "epoch": 0.849414246216116, |
| "grad_norm": 394.0, |
| "learning_rate": 1.8610408627031155e-05, |
| "logits/chosen": -1.389511227607727, |
| "logits/rejected": -1.0983564853668213, |
| "logps/chosen": -237.1016387939453, |
| "logps/rejected": -250.37835693359375, |
| "loss": 0.8634, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.814165472984314, |
| "rewards/margins": 3.4882616996765137, |
| "rewards/rejected": -2.6740963459014893, |
| "step": 3725 |
| }, |
| { |
| "epoch": 0.8551150129692444, |
| "grad_norm": 175.0, |
| "learning_rate": 1.8592140812096894e-05, |
| "logits/chosen": -1.4114035367965698, |
| "logits/rejected": -1.170053243637085, |
| "logps/chosen": -233.17356872558594, |
| "logps/rejected": -265.0191345214844, |
| "loss": 1.1255, |
| "rewards/accuracies": 0.6850000023841858, |
| "rewards/chosen": 1.0485285520553589, |
| "rewards/margins": 3.371248245239258, |
| "rewards/rejected": -2.3227202892303467, |
| "step": 3750 |
| }, |
| { |
| "epoch": 0.8608157797223727, |
| "grad_norm": 360.0, |
| "learning_rate": 1.8573762791624132e-05, |
| "logits/chosen": -1.3008290529251099, |
| "logits/rejected": -1.0176551342010498, |
| "logps/chosen": -256.6377258300781, |
| "logps/rejected": -292.4953918457031, |
| "loss": 1.2385, |
| "rewards/accuracies": 0.6200000047683716, |
| "rewards/chosen": 0.6940019130706787, |
| "rewards/margins": 2.6343843936920166, |
| "rewards/rejected": -1.940382480621338, |
| "step": 3775 |
| }, |
| { |
| "epoch": 0.8665165464755009, |
| "grad_norm": 390.0, |
| "learning_rate": 1.8555274801335222e-05, |
| "logits/chosen": -1.4741488695144653, |
| "logits/rejected": -1.2394438982009888, |
| "logps/chosen": -264.1485595703125, |
| "logps/rejected": -279.9443664550781, |
| "loss": 1.1799, |
| "rewards/accuracies": 0.6600000262260437, |
| "rewards/chosen": 0.8473094701766968, |
| "rewards/margins": 3.7362658977508545, |
| "rewards/rejected": -2.8889565467834473, |
| "step": 3800 |
| }, |
| { |
| "epoch": 0.8722173132286293, |
| "grad_norm": 414.0, |
| "learning_rate": 1.8536677078363025e-05, |
| "logits/chosen": -1.399438500404358, |
| "logits/rejected": -1.1499524116516113, |
| "logps/chosen": -284.58538818359375, |
| "logps/rejected": -277.4525451660156, |
| "loss": 1.0623, |
| "rewards/accuracies": 0.7049999833106995, |
| "rewards/chosen": 0.6038010716438293, |
| "rewards/margins": 3.7198760509490967, |
| "rewards/rejected": -3.116075038909912, |
| "step": 3825 |
| }, |
| { |
| "epoch": 0.8779180799817575, |
| "grad_norm": 370.0, |
| "learning_rate": 1.851796986124786e-05, |
| "logits/chosen": -1.402856469154358, |
| "logits/rejected": -1.1005898714065552, |
| "logps/chosen": -249.05885314941406, |
| "logps/rejected": -271.9918518066406, |
| "loss": 0.8729, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.8663185238838196, |
| "rewards/margins": 4.553758144378662, |
| "rewards/rejected": -3.6874396800994873, |
| "step": 3850 |
| }, |
| { |
| "epoch": 0.8836188467348859, |
| "grad_norm": 398.0, |
| "learning_rate": 1.849915338993447e-05, |
| "logits/chosen": -1.5430972576141357, |
| "logits/rejected": -1.3272191286087036, |
| "logps/chosen": -263.81463623046875, |
| "logps/rejected": -295.1082763671875, |
| "loss": 1.1168, |
| "rewards/accuracies": 0.6600000262260437, |
| "rewards/chosen": 0.23071323335170746, |
| "rewards/margins": 4.211090564727783, |
| "rewards/rejected": -3.980377435684204, |
| "step": 3875 |
| }, |
| { |
| "epoch": 0.8893196134880141, |
| "grad_norm": 204.0, |
| "learning_rate": 1.8480227905768912e-05, |
| "logits/chosen": -1.5200251340866089, |
| "logits/rejected": -1.3166491985321045, |
| "logps/chosen": -251.4666748046875, |
| "logps/rejected": -266.7015380859375, |
| "loss": 0.9931, |
| "rewards/accuracies": 0.7300000190734863, |
| "rewards/chosen": 0.4184899926185608, |
| "rewards/margins": 4.242293834686279, |
| "rewards/rejected": -3.8238039016723633, |
| "step": 3900 |
| }, |
| { |
| "epoch": 0.8950203802411424, |
| "grad_norm": 238.0, |
| "learning_rate": 1.8461193651495482e-05, |
| "logits/chosen": -1.614418387413025, |
| "logits/rejected": -1.3200567960739136, |
| "logps/chosen": -242.67994689941406, |
| "logps/rejected": -261.38671875, |
| "loss": 1.0201, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.5080154538154602, |
| "rewards/margins": 3.6761648654937744, |
| "rewards/rejected": -3.168149709701538, |
| "step": 3925 |
| }, |
| { |
| "epoch": 0.9007211469942707, |
| "grad_norm": 218.0, |
| "learning_rate": 1.84420508712536e-05, |
| "logits/chosen": -1.4272301197052002, |
| "logits/rejected": -1.2174086570739746, |
| "logps/chosen": -248.9918975830078, |
| "logps/rejected": -262.44158935546875, |
| "loss": 0.8625, |
| "rewards/accuracies": 0.7099999785423279, |
| "rewards/chosen": 0.7839375138282776, |
| "rewards/margins": 4.552494049072266, |
| "rewards/rejected": -3.768556594848633, |
| "step": 3950 |
| }, |
| { |
| "epoch": 0.906421913747399, |
| "grad_norm": 540.0, |
| "learning_rate": 1.8422799810574667e-05, |
| "logits/chosen": -1.201849102973938, |
| "logits/rejected": -0.9873482584953308, |
| "logps/chosen": -278.1529846191406, |
| "logps/rejected": -292.9454650878906, |
| "loss": 0.9011, |
| "rewards/accuracies": 0.6949999928474426, |
| "rewards/chosen": 1.0268640518188477, |
| "rewards/margins": 3.9061453342437744, |
| "rewards/rejected": -2.8792812824249268, |
| "step": 3975 |
| }, |
| { |
| "epoch": 0.9121226805005274, |
| "grad_norm": 324.0, |
| "learning_rate": 1.840344071637893e-05, |
| "logits/chosen": -1.1672656536102295, |
| "logits/rejected": -0.9044977426528931, |
| "logps/chosen": -262.7538146972656, |
| "logps/rejected": -283.47357177734375, |
| "loss": 1.0097, |
| "rewards/accuracies": 0.6549999713897705, |
| "rewards/chosen": 0.6802607774734497, |
| "rewards/margins": 3.1989760398864746, |
| "rewards/rejected": -2.5187151432037354, |
| "step": 4000 |
| }, |
| { |
| "epoch": 0.9178234472536556, |
| "grad_norm": 224.0, |
| "learning_rate": 1.83839738369723e-05, |
| "logits/chosen": -1.4750059843063354, |
| "logits/rejected": -1.2969045639038086, |
| "logps/chosen": -237.5166015625, |
| "logps/rejected": -277.1262512207031, |
| "loss": 0.7546, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": 0.6131055355072021, |
| "rewards/margins": 4.259562969207764, |
| "rewards/rejected": -3.6464571952819824, |
| "step": 4025 |
| }, |
| { |
| "epoch": 0.9235242140067839, |
| "grad_norm": 438.0, |
| "learning_rate": 1.8364399422043192e-05, |
| "logits/chosen": -1.2076224088668823, |
| "logits/rejected": -1.0032211542129517, |
| "logps/chosen": -288.94256591796875, |
| "logps/rejected": -297.09613037109375, |
| "loss": 1.1385, |
| "rewards/accuracies": 0.6399999856948853, |
| "rewards/chosen": 0.3317786455154419, |
| "rewards/margins": 2.5920369625091553, |
| "rewards/rejected": -2.260258436203003, |
| "step": 4050 |
| }, |
| { |
| "epoch": 0.9292249807599122, |
| "grad_norm": 144.0, |
| "learning_rate": 1.8344717722659285e-05, |
| "logits/chosen": -1.3961739540100098, |
| "logits/rejected": -1.1345940828323364, |
| "logps/chosen": -240.56785583496094, |
| "logps/rejected": -270.02606201171875, |
| "loss": 0.9074, |
| "rewards/accuracies": 0.7099999785423279, |
| "rewards/chosen": 0.3061050474643707, |
| "rewards/margins": 3.946634292602539, |
| "rewards/rejected": -3.640529155731201, |
| "step": 4075 |
| }, |
| { |
| "epoch": 0.9349257475130405, |
| "grad_norm": 278.0, |
| "learning_rate": 1.8324928991264336e-05, |
| "logits/chosen": -1.3009185791015625, |
| "logits/rejected": -1.0670627355575562, |
| "logps/chosen": -259.485595703125, |
| "logps/rejected": -267.2091369628906, |
| "loss": 1.1792, |
| "rewards/accuracies": 0.6549999713897705, |
| "rewards/chosen": 0.12024705857038498, |
| "rewards/margins": 3.223980665206909, |
| "rewards/rejected": -3.103733777999878, |
| "step": 4100 |
| }, |
| { |
| "epoch": 0.9406265142661688, |
| "grad_norm": 450.0, |
| "learning_rate": 1.830503348167493e-05, |
| "logits/chosen": -1.3163522481918335, |
| "logits/rejected": -1.0894160270690918, |
| "logps/chosen": -252.56285095214844, |
| "logps/rejected": -258.06280517578125, |
| "loss": 1.0357, |
| "rewards/accuracies": 0.6200000047683716, |
| "rewards/chosen": 0.31108230352401733, |
| "rewards/margins": 3.2639894485473633, |
| "rewards/rejected": -2.952907085418701, |
| "step": 4125 |
| }, |
| { |
| "epoch": 0.9463272810192971, |
| "grad_norm": 99.0, |
| "learning_rate": 1.828503144907721e-05, |
| "logits/chosen": -1.2648926973342896, |
| "logits/rejected": -1.0434695482254028, |
| "logps/chosen": -246.25991821289062, |
| "logps/rejected": -270.9434814453125, |
| "loss": 1.0096, |
| "rewards/accuracies": 0.6800000071525574, |
| "rewards/chosen": 0.5834211111068726, |
| "rewards/margins": 3.1179962158203125, |
| "rewards/rejected": -2.5345752239227295, |
| "step": 4150 |
| }, |
| { |
| "epoch": 0.9520280477724254, |
| "grad_norm": 568.0, |
| "learning_rate": 1.8264923150023634e-05, |
| "logits/chosen": -1.4738966226577759, |
| "logits/rejected": -1.2659544944763184, |
| "logps/chosen": -231.15699768066406, |
| "logps/rejected": -268.5801086425781, |
| "loss": 0.949, |
| "rewards/accuracies": 0.6899999976158142, |
| "rewards/chosen": 0.6611790657043457, |
| "rewards/margins": 3.937901020050049, |
| "rewards/rejected": -3.276721239089966, |
| "step": 4175 |
| }, |
| { |
| "epoch": 0.9577288145255537, |
| "grad_norm": 508.0, |
| "learning_rate": 1.8244708842429655e-05, |
| "logits/chosen": -1.3032176494598389, |
| "logits/rejected": -1.0636210441589355, |
| "logps/chosen": -261.3492431640625, |
| "logps/rejected": -284.6722717285156, |
| "loss": 1.1561, |
| "rewards/accuracies": 0.5950000286102295, |
| "rewards/chosen": 0.33548858761787415, |
| "rewards/margins": 2.910846471786499, |
| "rewards/rejected": -2.575357675552368, |
| "step": 4200 |
| }, |
| { |
| "epoch": 0.963429581278682, |
| "grad_norm": 312.0, |
| "learning_rate": 1.822438878557043e-05, |
| "logits/chosen": -1.4212863445281982, |
| "logits/rejected": -1.2179858684539795, |
| "logps/chosen": -249.14242553710938, |
| "logps/rejected": -273.12738037109375, |
| "loss": 1.1246, |
| "rewards/accuracies": 0.6200000047683716, |
| "rewards/chosen": 0.051385749131441116, |
| "rewards/margins": 3.6193032264709473, |
| "rewards/rejected": -3.567917585372925, |
| "step": 4225 |
| }, |
| { |
| "epoch": 0.9691303480318103, |
| "grad_norm": 402.0, |
| "learning_rate": 1.820396324007749e-05, |
| "logits/chosen": -1.5029231309890747, |
| "logits/rejected": -1.1934057474136353, |
| "logps/chosen": -248.04672241210938, |
| "logps/rejected": -266.57696533203125, |
| "loss": 1.038, |
| "rewards/accuracies": 0.6949999928474426, |
| "rewards/chosen": 0.7331379652023315, |
| "rewards/margins": 4.142246246337891, |
| "rewards/rejected": -3.4091086387634277, |
| "step": 4250 |
| }, |
| { |
| "epoch": 0.9748311147849386, |
| "grad_norm": 314.0, |
| "learning_rate": 1.818343246793539e-05, |
| "logits/chosen": -1.2048074007034302, |
| "logits/rejected": -1.0337302684783936, |
| "logps/chosen": -259.23504638671875, |
| "logps/rejected": -273.459228515625, |
| "loss": 1.0443, |
| "rewards/accuracies": 0.6449999809265137, |
| "rewards/chosen": 0.580686628818512, |
| "rewards/margins": 3.419651508331299, |
| "rewards/rejected": -2.8389649391174316, |
| "step": 4275 |
| }, |
| { |
| "epoch": 0.9805318815380669, |
| "grad_norm": 276.0, |
| "learning_rate": 1.8162796732478372e-05, |
| "logits/chosen": -1.373368740081787, |
| "logits/rejected": -1.0760862827301025, |
| "logps/chosen": -233.82196044921875, |
| "logps/rejected": -251.22056579589844, |
| "loss": 1.1556, |
| "rewards/accuracies": 0.6800000071525574, |
| "rewards/chosen": 0.24921609461307526, |
| "rewards/margins": 3.753535270690918, |
| "rewards/rejected": -3.504318952560425, |
| "step": 4300 |
| }, |
| { |
| "epoch": 0.9862326482911952, |
| "grad_norm": 316.0, |
| "learning_rate": 1.8142056298386953e-05, |
| "logits/chosen": -1.4451507329940796, |
| "logits/rejected": -1.1794260740280151, |
| "logps/chosen": -258.71942138671875, |
| "logps/rejected": -282.0692443847656, |
| "loss": 0.8886, |
| "rewards/accuracies": 0.7400000095367432, |
| "rewards/chosen": 0.2146664410829544, |
| "rewards/margins": 3.599161386489868, |
| "rewards/rejected": -3.384495258331299, |
| "step": 4325 |
| }, |
| { |
| "epoch": 0.9919334150443234, |
| "grad_norm": 215.0, |
| "learning_rate": 1.8121211431684563e-05, |
| "logits/chosen": -1.3905117511749268, |
| "logits/rejected": -1.1370949745178223, |
| "logps/chosen": -250.67391967773438, |
| "logps/rejected": -268.27764892578125, |
| "loss": 0.9061, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.15705075860023499, |
| "rewards/margins": 3.688293695449829, |
| "rewards/rejected": -3.531243085861206, |
| "step": 4350 |
| }, |
| { |
| "epoch": 0.9976341817974518, |
| "grad_norm": 292.0, |
| "learning_rate": 1.8100262399734102e-05, |
| "logits/chosen": -1.395089864730835, |
| "logits/rejected": -1.149121880531311, |
| "logps/chosen": -261.2833557128906, |
| "logps/rejected": -279.41033935546875, |
| "loss": 0.8665, |
| "rewards/accuracies": 0.6800000071525574, |
| "rewards/chosen": 0.12012651562690735, |
| "rewards/margins": 3.659724712371826, |
| "rewards/rejected": -3.5395984649658203, |
| "step": 4375 |
| }, |
| { |
| "epoch": 1.0031924293817518, |
| "grad_norm": 188.0, |
| "learning_rate": 1.807920947123454e-05, |
| "logits/chosen": -1.532718300819397, |
| "logits/rejected": -1.177968144416809, |
| "logps/chosen": -228.10366821289062, |
| "logps/rejected": -261.8833312988281, |
| "loss": 0.2587, |
| "rewards/accuracies": 0.8871794939041138, |
| "rewards/chosen": 1.5760877132415771, |
| "rewards/margins": 8.103155136108398, |
| "rewards/rejected": -6.527067184448242, |
| "step": 4400 |
| }, |
| { |
| "epoch": 1.0088931961348802, |
| "grad_norm": 2.515625, |
| "learning_rate": 1.8058052916217458e-05, |
| "logits/chosen": -1.5367858409881592, |
| "logits/rejected": -1.2684277296066284, |
| "logps/chosen": -235.0211181640625, |
| "logps/rejected": -274.67364501953125, |
| "loss": 0.1363, |
| "rewards/accuracies": 0.9350000023841858, |
| "rewards/chosen": 2.6452138423919678, |
| "rewards/margins": 9.436513900756836, |
| "rewards/rejected": -6.791298866271973, |
| "step": 4425 |
| }, |
| { |
| "epoch": 1.0145939628880085, |
| "grad_norm": 24.375, |
| "learning_rate": 1.803679300604357e-05, |
| "logits/chosen": -1.5540528297424316, |
| "logits/rejected": -1.2927262783050537, |
| "logps/chosen": -224.42495727539062, |
| "logps/rejected": -281.99542236328125, |
| "loss": 0.1586, |
| "rewards/accuracies": 0.9700000286102295, |
| "rewards/chosen": 3.378166437149048, |
| "rewards/margins": 10.712231636047363, |
| "rewards/rejected": -7.334064960479736, |
| "step": 4450 |
| }, |
| { |
| "epoch": 1.0202947296411367, |
| "grad_norm": 302.0, |
| "learning_rate": 1.8015430013399267e-05, |
| "logits/chosen": -1.5382097959518433, |
| "logits/rejected": -1.1546252965927124, |
| "logps/chosen": -266.4432678222656, |
| "logps/rejected": -282.19097900390625, |
| "loss": 0.0667, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 3.3535168170928955, |
| "rewards/margins": 10.091172218322754, |
| "rewards/rejected": -6.737655162811279, |
| "step": 4475 |
| }, |
| { |
| "epoch": 1.025995496394265, |
| "grad_norm": 51.75, |
| "learning_rate": 1.799396421229311e-05, |
| "logits/chosen": -2.1244959831237793, |
| "logits/rejected": -1.7771044969558716, |
| "logps/chosen": -240.56422424316406, |
| "logps/rejected": -283.2074279785156, |
| "loss": 0.0876, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 3.4198172092437744, |
| "rewards/margins": 10.795231819152832, |
| "rewards/rejected": -7.37541389465332, |
| "step": 4500 |
| }, |
| { |
| "epoch": 1.0316962631473934, |
| "grad_norm": 241.0, |
| "learning_rate": 1.7972395878052304e-05, |
| "logits/chosen": -1.6828385591506958, |
| "logits/rejected": -1.4055473804473877, |
| "logps/chosen": -245.07986450195312, |
| "logps/rejected": -298.92742919921875, |
| "loss": 0.1184, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 3.391951322555542, |
| "rewards/margins": 11.697711944580078, |
| "rewards/rejected": -8.305761337280273, |
| "step": 4525 |
| }, |
| { |
| "epoch": 1.0373970299005215, |
| "grad_norm": 334.0, |
| "learning_rate": 1.795072528731919e-05, |
| "logits/chosen": -2.2481319904327393, |
| "logits/rejected": -1.8644616603851318, |
| "logps/chosen": -222.0325164794922, |
| "logps/rejected": -279.9166259765625, |
| "loss": 0.0856, |
| "rewards/accuracies": 0.9800000190734863, |
| "rewards/chosen": 2.8130486011505127, |
| "rewards/margins": 11.919305801391602, |
| "rewards/rejected": -9.106254577636719, |
| "step": 4550 |
| }, |
| { |
| "epoch": 1.0430977966536499, |
| "grad_norm": 113.5, |
| "learning_rate": 1.792895271804767e-05, |
| "logits/chosen": -2.022143840789795, |
| "logits/rejected": -1.8155516386032104, |
| "logps/chosen": -248.69898986816406, |
| "logps/rejected": -283.99884033203125, |
| "loss": 0.1568, |
| "rewards/accuracies": 0.9399999976158142, |
| "rewards/chosen": 2.6135878562927246, |
| "rewards/margins": 10.619499206542969, |
| "rewards/rejected": -8.005910873413086, |
| "step": 4575 |
| }, |
| { |
| "epoch": 1.0487985634067782, |
| "grad_norm": 59.5, |
| "learning_rate": 1.790707844949967e-05, |
| "logits/chosen": -1.682399868965149, |
| "logits/rejected": -1.4033764600753784, |
| "logps/chosen": -273.5552673339844, |
| "logps/rejected": -277.05352783203125, |
| "loss": 0.1166, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 3.114898681640625, |
| "rewards/margins": 10.942302703857422, |
| "rewards/rejected": -7.827403545379639, |
| "step": 4600 |
| }, |
| { |
| "epoch": 1.0544993301599066, |
| "grad_norm": 139.0, |
| "learning_rate": 1.7885102762241525e-05, |
| "logits/chosen": -1.7889196872711182, |
| "logits/rejected": -1.5190974473953247, |
| "logps/chosen": -230.36546325683594, |
| "logps/rejected": -281.7632751464844, |
| "loss": 0.1449, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.4209177494049072, |
| "rewards/margins": 9.982870101928711, |
| "rewards/rejected": -7.561952114105225, |
| "step": 4625 |
| }, |
| { |
| "epoch": 1.0602000969130347, |
| "grad_norm": 112.5, |
| "learning_rate": 1.7863025938140414e-05, |
| "logits/chosen": -1.5002140998840332, |
| "logits/rejected": -1.2264771461486816, |
| "logps/chosen": -257.8990173339844, |
| "logps/rejected": -287.5025329589844, |
| "loss": 0.0767, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 3.278228759765625, |
| "rewards/margins": 10.354986190795898, |
| "rewards/rejected": -7.076757907867432, |
| "step": 4650 |
| }, |
| { |
| "epoch": 1.065900863666163, |
| "grad_norm": 29.75, |
| "learning_rate": 1.7840848260360728e-05, |
| "logits/chosen": -1.699466347694397, |
| "logits/rejected": -1.340370774269104, |
| "logps/chosen": -276.0038757324219, |
| "logps/rejected": -314.2296447753906, |
| "loss": 0.0898, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 3.5046215057373047, |
| "rewards/margins": 11.646230697631836, |
| "rewards/rejected": -8.141609191894531, |
| "step": 4675 |
| }, |
| { |
| "epoch": 1.0716016304192915, |
| "grad_norm": 114.5, |
| "learning_rate": 1.781857001336043e-05, |
| "logits/chosen": -1.5552254915237427, |
| "logits/rejected": -1.3489419221878052, |
| "logps/chosen": -267.1172180175781, |
| "logps/rejected": -297.3063659667969, |
| "loss": 0.1017, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 3.511922597885132, |
| "rewards/margins": 10.736223220825195, |
| "rewards/rejected": -7.224301815032959, |
| "step": 4700 |
| }, |
| { |
| "epoch": 1.0773023971724196, |
| "grad_norm": 10.0625, |
| "learning_rate": 1.7796191482887425e-05, |
| "logits/chosen": -2.004019260406494, |
| "logits/rejected": -1.6522330045700073, |
| "logps/chosen": -240.71893310546875, |
| "logps/rejected": -278.7403259277344, |
| "loss": 0.0666, |
| "rewards/accuracies": 0.9700000286102295, |
| "rewards/chosen": 2.558114528656006, |
| "rewards/margins": 12.260228157043457, |
| "rewards/rejected": -9.702115058898926, |
| "step": 4725 |
| }, |
| { |
| "epoch": 1.083003163925548, |
| "grad_norm": 21.0, |
| "learning_rate": 1.777371295597588e-05, |
| "logits/chosen": -1.9352818727493286, |
| "logits/rejected": -1.7156339883804321, |
| "logps/chosen": -287.0283508300781, |
| "logps/rejected": -306.7490234375, |
| "loss": 0.1109, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 3.5754754543304443, |
| "rewards/margins": 12.191374778747559, |
| "rewards/rejected": -8.615900039672852, |
| "step": 4750 |
| }, |
| { |
| "epoch": 1.0887039306786763, |
| "grad_norm": 264.0, |
| "learning_rate": 1.7751134720942552e-05, |
| "logits/chosen": -2.0144588947296143, |
| "logits/rejected": -1.69350004196167, |
| "logps/chosen": -263.1171569824219, |
| "logps/rejected": -304.5240478515625, |
| "loss": 0.1354, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 2.6395621299743652, |
| "rewards/margins": 11.496726036071777, |
| "rewards/rejected": -8.85716438293457, |
| "step": 4775 |
| }, |
| { |
| "epoch": 1.0944046974318047, |
| "grad_norm": 2.4375, |
| "learning_rate": 1.7728457067383088e-05, |
| "logits/chosen": -1.8113974332809448, |
| "logits/rejected": -1.5742379426956177, |
| "logps/chosen": -255.96173095703125, |
| "logps/rejected": -286.1318664550781, |
| "loss": 0.1086, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 2.890777349472046, |
| "rewards/margins": 11.620555877685547, |
| "rewards/rejected": -8.729777336120605, |
| "step": 4800 |
| }, |
| { |
| "epoch": 1.1001054641849328, |
| "grad_norm": 148.0, |
| "learning_rate": 1.7705680286168297e-05, |
| "logits/chosen": -1.9463430643081665, |
| "logits/rejected": -1.6461032629013062, |
| "logps/chosen": -241.93032836914062, |
| "logps/rejected": -280.9911193847656, |
| "loss": 0.0598, |
| "rewards/accuracies": 0.9800000190734863, |
| "rewards/chosen": 2.962514638900757, |
| "rewards/margins": 12.630315780639648, |
| "rewards/rejected": -9.667800903320312, |
| "step": 4825 |
| }, |
| { |
| "epoch": 1.1058062309380612, |
| "grad_norm": 6.90625, |
| "learning_rate": 1.768280466944045e-05, |
| "logits/chosen": -1.6835054159164429, |
| "logits/rejected": -1.5242514610290527, |
| "logps/chosen": -282.61480712890625, |
| "logps/rejected": -299.6497802734375, |
| "loss": 0.1541, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.208606481552124, |
| "rewards/margins": 11.768745422363281, |
| "rewards/rejected": -9.560137748718262, |
| "step": 4850 |
| }, |
| { |
| "epoch": 1.1115069976911895, |
| "grad_norm": 93.0, |
| "learning_rate": 1.765983051060949e-05, |
| "logits/chosen": -1.8821115493774414, |
| "logits/rejected": -1.6821262836456299, |
| "logps/chosen": -236.24322509765625, |
| "logps/rejected": -278.8183898925781, |
| "loss": 0.1597, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.8070673942565918, |
| "rewards/margins": 11.600632667541504, |
| "rewards/rejected": -9.793564796447754, |
| "step": 4875 |
| }, |
| { |
| "epoch": 1.1172077644443177, |
| "grad_norm": 13.9375, |
| "learning_rate": 1.7636758104349318e-05, |
| "logits/chosen": -1.5488380193710327, |
| "logits/rejected": -1.354139804840088, |
| "logps/chosen": -238.58587646484375, |
| "logps/rejected": -258.1515197753906, |
| "loss": 0.1155, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 2.1066207885742188, |
| "rewards/margins": 10.422945976257324, |
| "rewards/rejected": -8.316324234008789, |
| "step": 4900 |
| }, |
| { |
| "epoch": 1.122908531197446, |
| "grad_norm": 64.5, |
| "learning_rate": 1.7613587746593974e-05, |
| "logits/chosen": -1.8195234537124634, |
| "logits/rejected": -1.5636537075042725, |
| "logps/chosen": -274.9443664550781, |
| "logps/rejected": -296.54595947265625, |
| "loss": 0.0834, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 3.169476270675659, |
| "rewards/margins": 12.970063209533691, |
| "rewards/rejected": -9.800585746765137, |
| "step": 4925 |
| }, |
| { |
| "epoch": 1.1286092979505744, |
| "grad_norm": 51.0, |
| "learning_rate": 1.759031973453385e-05, |
| "logits/chosen": -1.9985613822937012, |
| "logits/rejected": -1.8280360698699951, |
| "logps/chosen": -238.9024200439453, |
| "logps/rejected": -287.2914733886719, |
| "loss": 0.17, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 2.2904579639434814, |
| "rewards/margins": 11.46274185180664, |
| "rewards/rejected": -9.172284126281738, |
| "step": 4950 |
| }, |
| { |
| "epoch": 1.1343100647037025, |
| "grad_norm": 225.0, |
| "learning_rate": 1.7566954366611896e-05, |
| "logits/chosen": -1.9637246131896973, |
| "logits/rejected": -1.6346756219863892, |
| "logps/chosen": -218.3473663330078, |
| "logps/rejected": -279.531005859375, |
| "loss": 0.191, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 1.8557524681091309, |
| "rewards/margins": 11.950901985168457, |
| "rewards/rejected": -10.095149993896484, |
| "step": 4975 |
| }, |
| { |
| "epoch": 1.140010831456831, |
| "grad_norm": 41.25, |
| "learning_rate": 1.7543491942519772e-05, |
| "logits/chosen": -2.0061745643615723, |
| "logits/rejected": -1.6673415899276733, |
| "logps/chosen": -259.9736633300781, |
| "logps/rejected": -298.2081298828125, |
| "loss": 0.0943, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 2.942664384841919, |
| "rewards/margins": 12.829992294311523, |
| "rewards/rejected": -9.887327194213867, |
| "step": 5000 |
| }, |
| { |
| "epoch": 1.140010831456831, |
| "eval_logits/chosen": -1.8859049081802368, |
| "eval_logits/rejected": -1.6332569122314453, |
| "eval_logps/chosen": -261.0445861816406, |
| "eval_logps/rejected": -283.3099670410156, |
| "eval_loss": 1.077931523323059, |
| "eval_rewards/accuracies": 0.699350118637085, |
| "eval_rewards/chosen": -0.7361209988594055, |
| "eval_rewards/margins": 5.628861427307129, |
| "eval_rewards/rejected": -6.364983081817627, |
| "eval_runtime": 908.0899, |
| "eval_samples_per_second": 9.659, |
| "eval_steps_per_second": 9.659, |
| "step": 5000 |
| }, |
| { |
| "epoch": 1.1457115982099593, |
| "grad_norm": 82.5, |
| "learning_rate": 1.751993276319401e-05, |
| "logits/chosen": -2.099783420562744, |
| "logits/rejected": -1.91457200050354, |
| "logps/chosen": -283.5301513671875, |
| "logps/rejected": -319.3835144042969, |
| "loss": 0.1269, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.3956735134124756, |
| "rewards/margins": 12.881036758422852, |
| "rewards/rejected": -10.485363960266113, |
| "step": 5025 |
| }, |
| { |
| "epoch": 1.1514123649630876, |
| "grad_norm": 172.0, |
| "learning_rate": 1.749627713081216e-05, |
| "logits/chosen": -1.6732853651046753, |
| "logits/rejected": -1.4193447828292847, |
| "logps/chosen": -265.24493408203125, |
| "logps/rejected": -289.1585998535156, |
| "loss": 0.096, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 2.2105226516723633, |
| "rewards/margins": 11.193897247314453, |
| "rewards/rejected": -8.98337459564209, |
| "step": 5050 |
| }, |
| { |
| "epoch": 1.1571131317162158, |
| "grad_norm": 6.15625, |
| "learning_rate": 1.747252534878891e-05, |
| "logits/chosen": -2.0101499557495117, |
| "logits/rejected": -1.6965676546096802, |
| "logps/chosen": -233.79566955566406, |
| "logps/rejected": -273.3131408691406, |
| "loss": 0.1142, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 1.646675705909729, |
| "rewards/margins": 11.91004753112793, |
| "rewards/rejected": -10.263372421264648, |
| "step": 5075 |
| }, |
| { |
| "epoch": 1.1628138984693441, |
| "grad_norm": 170.0, |
| "learning_rate": 1.744867772177219e-05, |
| "logits/chosen": -1.7389349937438965, |
| "logits/rejected": -1.485107421875, |
| "logps/chosen": -260.11883544921875, |
| "logps/rejected": -283.2637939453125, |
| "loss": 0.0834, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 1.6286638975143433, |
| "rewards/margins": 10.847929954528809, |
| "rewards/rejected": -9.219265937805176, |
| "step": 5100 |
| }, |
| { |
| "epoch": 1.1685146652224725, |
| "grad_norm": 10.75, |
| "learning_rate": 1.742473455563927e-05, |
| "logits/chosen": -1.7321842908859253, |
| "logits/rejected": -1.5367344617843628, |
| "logps/chosen": -250.57904052734375, |
| "logps/rejected": -267.0683898925781, |
| "loss": 0.0772, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 2.34086537361145, |
| "rewards/margins": 10.831689834594727, |
| "rewards/rejected": -8.490823745727539, |
| "step": 5125 |
| }, |
| { |
| "epoch": 1.1742154319756006, |
| "grad_norm": 212.0, |
| "learning_rate": 1.7400696157492834e-05, |
| "logits/chosen": -1.8007432222366333, |
| "logits/rejected": -1.5525892972946167, |
| "logps/chosen": -282.451904296875, |
| "logps/rejected": -321.7064208984375, |
| "loss": 0.1595, |
| "rewards/accuracies": 0.9399999976158142, |
| "rewards/chosen": 2.9073173999786377, |
| "rewards/margins": 11.49176025390625, |
| "rewards/rejected": -8.584442138671875, |
| "step": 5150 |
| }, |
| { |
| "epoch": 1.179916198728729, |
| "grad_norm": 32.0, |
| "learning_rate": 1.7376562835657046e-05, |
| "logits/chosen": -1.7951010465621948, |
| "logits/rejected": -1.5606071949005127, |
| "logps/chosen": -221.2120361328125, |
| "logps/rejected": -256.00701904296875, |
| "loss": 0.0904, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 1.8102614879608154, |
| "rewards/margins": 11.408254623413086, |
| "rewards/rejected": -9.597993850708008, |
| "step": 5175 |
| }, |
| { |
| "epoch": 1.1856169654818574, |
| "grad_norm": 153.0, |
| "learning_rate": 1.7352334899673588e-05, |
| "logits/chosen": -1.8842110633850098, |
| "logits/rejected": -1.5704303979873657, |
| "logps/chosen": -238.68101501464844, |
| "logps/rejected": -266.1350402832031, |
| "loss": 0.0927, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 3.062990188598633, |
| "rewards/margins": 12.920307159423828, |
| "rewards/rejected": -9.857316017150879, |
| "step": 5200 |
| }, |
| { |
| "epoch": 1.1913177322349857, |
| "grad_norm": 168.0, |
| "learning_rate": 1.7328012660297694e-05, |
| "logits/chosen": -1.963917851448059, |
| "logits/rejected": -1.6490057706832886, |
| "logps/chosen": -246.3984375, |
| "logps/rejected": -287.54559326171875, |
| "loss": 0.1073, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 2.475822687149048, |
| "rewards/margins": 11.970541954040527, |
| "rewards/rejected": -9.494718551635742, |
| "step": 5225 |
| }, |
| { |
| "epoch": 1.1970184989881139, |
| "grad_norm": 414.0, |
| "learning_rate": 1.7303596429494165e-05, |
| "logits/chosen": -1.9723440408706665, |
| "logits/rejected": -1.6691049337387085, |
| "logps/chosen": -225.93609619140625, |
| "logps/rejected": -275.4795227050781, |
| "loss": 0.0697, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 2.0542354583740234, |
| "rewards/margins": 12.170297622680664, |
| "rewards/rejected": -10.116063117980957, |
| "step": 5250 |
| }, |
| { |
| "epoch": 1.2027192657412422, |
| "grad_norm": 0.1279296875, |
| "learning_rate": 1.727908652043336e-05, |
| "logits/chosen": -1.820959210395813, |
| "logits/rejected": -1.5412307977676392, |
| "logps/chosen": -225.3406219482422, |
| "logps/rejected": -267.9480285644531, |
| "loss": 0.0689, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 3.1279006004333496, |
| "rewards/margins": 11.7685546875, |
| "rewards/rejected": -8.640655517578125, |
| "step": 5275 |
| }, |
| { |
| "epoch": 1.2084200324943706, |
| "grad_norm": 66.5, |
| "learning_rate": 1.72544832474872e-05, |
| "logits/chosen": -1.9110075235366821, |
| "logits/rejected": -1.701636791229248, |
| "logps/chosen": -274.36798095703125, |
| "logps/rejected": -300.1329345703125, |
| "loss": 0.1362, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 2.5121817588806152, |
| "rewards/margins": 12.248281478881836, |
| "rewards/rejected": -9.736100196838379, |
| "step": 5300 |
| }, |
| { |
| "epoch": 1.2141207992474987, |
| "grad_norm": 24.625, |
| "learning_rate": 1.7229786926225104e-05, |
| "logits/chosen": -1.7718925476074219, |
| "logits/rejected": -1.5595852136611938, |
| "logps/chosen": -261.7027282714844, |
| "logps/rejected": -283.3608703613281, |
| "loss": 0.1261, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.974677562713623, |
| "rewards/margins": 11.067282676696777, |
| "rewards/rejected": -9.092605590820312, |
| "step": 5325 |
| }, |
| { |
| "epoch": 1.219821566000627, |
| "grad_norm": 87.5, |
| "learning_rate": 1.7204997873409964e-05, |
| "logits/chosen": -1.8636274337768555, |
| "logits/rejected": -1.5572285652160645, |
| "logps/chosen": -277.6109619140625, |
| "logps/rejected": -336.9969482421875, |
| "loss": 0.1438, |
| "rewards/accuracies": 0.9399999976158142, |
| "rewards/chosen": 2.3193981647491455, |
| "rewards/margins": 12.150267601013184, |
| "rewards/rejected": -9.8308687210083, |
| "step": 5350 |
| }, |
| { |
| "epoch": 1.2255223327537554, |
| "grad_norm": 97.5, |
| "learning_rate": 1.7180116406994085e-05, |
| "logits/chosen": -2.0497095584869385, |
| "logits/rejected": -1.6978613138198853, |
| "logps/chosen": -245.3409423828125, |
| "logps/rejected": -300.523681640625, |
| "loss": 0.0596, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 2.5929198265075684, |
| "rewards/margins": 13.670166969299316, |
| "rewards/rejected": -11.077245712280273, |
| "step": 5375 |
| }, |
| { |
| "epoch": 1.2312230995068836, |
| "grad_norm": 49.75, |
| "learning_rate": 1.7155142846115085e-05, |
| "logits/chosen": -1.6178284883499146, |
| "logits/rejected": -1.3981338739395142, |
| "logps/chosen": -255.72250366210938, |
| "logps/rejected": -278.7118835449219, |
| "loss": 0.1745, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.293889045715332, |
| "rewards/margins": 10.833410263061523, |
| "rewards/rejected": -8.539521217346191, |
| "step": 5400 |
| }, |
| { |
| "epoch": 1.236923866260012, |
| "grad_norm": 76.0, |
| "learning_rate": 1.7130077511091817e-05, |
| "logits/chosen": -2.015709638595581, |
| "logits/rejected": -1.6567312479019165, |
| "logps/chosen": -237.02569580078125, |
| "logps/rejected": -279.99285888671875, |
| "loss": 0.1045, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 1.7193617820739746, |
| "rewards/margins": 12.00366497039795, |
| "rewards/rejected": -10.28430461883545, |
| "step": 5425 |
| }, |
| { |
| "epoch": 1.2426246330131403, |
| "grad_norm": 199.0, |
| "learning_rate": 1.7104920723420277e-05, |
| "logits/chosen": -1.827425241470337, |
| "logits/rejected": -1.6032847166061401, |
| "logps/chosen": -266.4546813964844, |
| "logps/rejected": -287.0042724609375, |
| "loss": 0.1565, |
| "rewards/accuracies": 0.9350000023841858, |
| "rewards/chosen": 2.5845344066619873, |
| "rewards/margins": 11.549398422241211, |
| "rewards/rejected": -8.964864730834961, |
| "step": 5450 |
| }, |
| { |
| "epoch": 1.2483253997662684, |
| "grad_norm": 205.0, |
| "learning_rate": 1.707967280576943e-05, |
| "logits/chosen": -1.9455676078796387, |
| "logits/rejected": -1.7057570219039917, |
| "logps/chosen": -251.77206420898438, |
| "logps/rejected": -315.8971862792969, |
| "loss": 0.0956, |
| "rewards/accuracies": 0.9449999928474426, |
| "rewards/chosen": 2.796236515045166, |
| "rewards/margins": 13.64761734008789, |
| "rewards/rejected": -10.851380348205566, |
| "step": 5475 |
| }, |
| { |
| "epoch": 1.2540261665193968, |
| "grad_norm": 3.296875, |
| "learning_rate": 1.7054334081977133e-05, |
| "logits/chosen": -2.1241354942321777, |
| "logits/rejected": -1.817455768585205, |
| "logps/chosen": -244.8742218017578, |
| "logps/rejected": -300.0165100097656, |
| "loss": 0.0876, |
| "rewards/accuracies": 0.9700000286102295, |
| "rewards/chosen": 2.3153233528137207, |
| "rewards/margins": 13.096363067626953, |
| "rewards/rejected": -10.78104019165039, |
| "step": 5500 |
| }, |
| { |
| "epoch": 1.2597269332725252, |
| "grad_norm": 32.0, |
| "learning_rate": 1.7028904877045937e-05, |
| "logits/chosen": -2.052846908569336, |
| "logits/rejected": -1.7946313619613647, |
| "logps/chosen": -261.7705993652344, |
| "logps/rejected": -286.5797119140625, |
| "loss": 0.1245, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 2.320967435836792, |
| "rewards/margins": 11.751914978027344, |
| "rewards/rejected": -9.430947303771973, |
| "step": 5525 |
| }, |
| { |
| "epoch": 1.2654277000256535, |
| "grad_norm": 136.0, |
| "learning_rate": 1.700338551713893e-05, |
| "logits/chosen": -1.986914038658142, |
| "logits/rejected": -1.7056453227996826, |
| "logps/chosen": -252.62904357910156, |
| "logps/rejected": -296.6147766113281, |
| "loss": 0.1378, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.3417396545410156, |
| "rewards/margins": 13.585430145263672, |
| "rewards/rejected": -11.243690490722656, |
| "step": 5550 |
| }, |
| { |
| "epoch": 1.2711284667787819, |
| "grad_norm": 31.0, |
| "learning_rate": 1.6977776329575566e-05, |
| "logits/chosen": -1.7505297660827637, |
| "logits/rejected": -1.5814154148101807, |
| "logps/chosen": -268.5666809082031, |
| "logps/rejected": -291.3974914550781, |
| "loss": 0.1428, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 2.660893440246582, |
| "rewards/margins": 12.998727798461914, |
| "rewards/rejected": -10.337834358215332, |
| "step": 5575 |
| }, |
| { |
| "epoch": 1.27682923353191, |
| "grad_norm": 1.0390625, |
| "learning_rate": 1.6952077642827453e-05, |
| "logits/chosen": -1.8441439867019653, |
| "logits/rejected": -1.6603599786758423, |
| "logps/chosen": -226.70094299316406, |
| "logps/rejected": -281.52972412109375, |
| "loss": 0.1171, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 1.8822277784347534, |
| "rewards/margins": 11.532305717468262, |
| "rewards/rejected": -9.650078773498535, |
| "step": 5600 |
| }, |
| { |
| "epoch": 1.2825300002850384, |
| "grad_norm": 19.875, |
| "learning_rate": 1.6926289786514138e-05, |
| "logits/chosen": -1.8755531311035156, |
| "logits/rejected": -1.596110224723816, |
| "logps/chosen": -239.03533935546875, |
| "logps/rejected": -283.2279357910156, |
| "loss": 0.1142, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 2.0581634044647217, |
| "rewards/margins": 12.515800476074219, |
| "rewards/rejected": -10.457636833190918, |
| "step": 5625 |
| }, |
| { |
| "epoch": 1.2882307670381667, |
| "grad_norm": 31.25, |
| "learning_rate": 1.69004130913989e-05, |
| "logits/chosen": -1.806092381477356, |
| "logits/rejected": -1.552976369857788, |
| "logps/chosen": -251.2419891357422, |
| "logps/rejected": -294.34259033203125, |
| "loss": 0.1197, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 2.089460849761963, |
| "rewards/margins": 13.072174072265625, |
| "rewards/rejected": -10.98271369934082, |
| "step": 5650 |
| }, |
| { |
| "epoch": 1.2939315337912949, |
| "grad_norm": 15.1875, |
| "learning_rate": 1.6874447889384477e-05, |
| "logits/chosen": -1.6773737668991089, |
| "logits/rejected": -1.4510283470153809, |
| "logps/chosen": -245.45562744140625, |
| "logps/rejected": -270.8011474609375, |
| "loss": 0.1359, |
| "rewards/accuracies": 0.9700000286102295, |
| "rewards/chosen": 2.333652973175049, |
| "rewards/margins": 11.476463317871094, |
| "rewards/rejected": -9.142810821533203, |
| "step": 5675 |
| }, |
| { |
| "epoch": 1.2996323005444232, |
| "grad_norm": 496.0, |
| "learning_rate": 1.684839451350883e-05, |
| "logits/chosen": -2.1503350734710693, |
| "logits/rejected": -1.8960974216461182, |
| "logps/chosen": -225.591796875, |
| "logps/rejected": -264.4457702636719, |
| "loss": 0.1037, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 1.8477816581726074, |
| "rewards/margins": 13.581563949584961, |
| "rewards/rejected": -11.733781814575195, |
| "step": 5700 |
| }, |
| { |
| "epoch": 1.3053330672975516, |
| "grad_norm": 3.6875, |
| "learning_rate": 1.6822253297940876e-05, |
| "logits/chosen": -2.0778918266296387, |
| "logits/rejected": -1.77174711227417, |
| "logps/chosen": -238.21728515625, |
| "logps/rejected": -297.647216796875, |
| "loss": 0.1171, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.063938856124878, |
| "rewards/margins": 12.555736541748047, |
| "rewards/rejected": -10.491798400878906, |
| "step": 5725 |
| }, |
| { |
| "epoch": 1.3110338340506797, |
| "grad_norm": 302.0, |
| "learning_rate": 1.679602457797618e-05, |
| "logits/chosen": -1.9157469272613525, |
| "logits/rejected": -1.6855671405792236, |
| "logps/chosen": -245.2192840576172, |
| "logps/rejected": -290.0827331542969, |
| "loss": 0.1322, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 2.03474760055542, |
| "rewards/margins": 12.721822738647461, |
| "rewards/rejected": -10.687076568603516, |
| "step": 5750 |
| }, |
| { |
| "epoch": 1.316734600803808, |
| "grad_norm": 105.5, |
| "learning_rate": 1.6769708690032668e-05, |
| "logits/chosen": -1.9795626401901245, |
| "logits/rejected": -1.7698581218719482, |
| "logps/chosen": -273.8256530761719, |
| "logps/rejected": -305.0401306152344, |
| "loss": 0.1115, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 2.6065475940704346, |
| "rewards/margins": 12.51846694946289, |
| "rewards/rejected": -9.911919593811035, |
| "step": 5775 |
| }, |
| { |
| "epoch": 1.3224353675569365, |
| "grad_norm": 334.0, |
| "learning_rate": 1.674330597164631e-05, |
| "logits/chosen": -2.0389952659606934, |
| "logits/rejected": -1.675459623336792, |
| "logps/chosen": -262.79595947265625, |
| "logps/rejected": -289.3572692871094, |
| "loss": 0.0778, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 2.589376926422119, |
| "rewards/margins": 14.18992805480957, |
| "rewards/rejected": -11.60055160522461, |
| "step": 5800 |
| }, |
| { |
| "epoch": 1.3281361343100646, |
| "grad_norm": 169.0, |
| "learning_rate": 1.67168167614668e-05, |
| "logits/chosen": -1.917983055114746, |
| "logits/rejected": -1.688857913017273, |
| "logps/chosen": -239.42417907714844, |
| "logps/rejected": -276.8812255859375, |
| "loss": 0.1289, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": 2.0276737213134766, |
| "rewards/margins": 11.047568321228027, |
| "rewards/rejected": -9.019893646240234, |
| "step": 5825 |
| }, |
| { |
| "epoch": 1.333836901063193, |
| "grad_norm": 71.0, |
| "learning_rate": 1.6690241399253188e-05, |
| "logits/chosen": -2.1813580989837646, |
| "logits/rejected": -1.884798288345337, |
| "logps/chosen": -247.67604064941406, |
| "logps/rejected": -281.4664306640625, |
| "loss": 0.1104, |
| "rewards/accuracies": 0.9449999928474426, |
| "rewards/chosen": 2.3602864742279053, |
| "rewards/margins": 13.533843994140625, |
| "rewards/rejected": -11.17355728149414, |
| "step": 5850 |
| }, |
| { |
| "epoch": 1.3395376678163213, |
| "grad_norm": 30.0, |
| "learning_rate": 1.6663580225869554e-05, |
| "logits/chosen": -2.04599666595459, |
| "logits/rejected": -1.7283560037612915, |
| "logps/chosen": -237.533203125, |
| "logps/rejected": -313.9803771972656, |
| "loss": 0.0705, |
| "rewards/accuracies": 0.9700000286102295, |
| "rewards/chosen": 2.146848678588867, |
| "rewards/margins": 14.878925323486328, |
| "rewards/rejected": -12.732077598571777, |
| "step": 5875 |
| }, |
| { |
| "epoch": 1.3452384345694495, |
| "grad_norm": 30.75, |
| "learning_rate": 1.6636833583280606e-05, |
| "logits/chosen": -2.0319464206695557, |
| "logits/rejected": -1.7241321802139282, |
| "logps/chosen": -276.8165588378906, |
| "logps/rejected": -290.7685852050781, |
| "loss": 0.1853, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.5556492805480957, |
| "rewards/margins": 13.04057502746582, |
| "rewards/rejected": -11.4849271774292, |
| "step": 5900 |
| }, |
| { |
| "epoch": 1.3509392013225778, |
| "grad_norm": 11.9375, |
| "learning_rate": 1.6610001814547314e-05, |
| "logits/chosen": -1.9040794372558594, |
| "logits/rejected": -1.6385403871536255, |
| "logps/chosen": -233.7841796875, |
| "logps/rejected": -276.02838134765625, |
| "loss": 0.1197, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 2.0698249340057373, |
| "rewards/margins": 12.877204895019531, |
| "rewards/rejected": -10.807378768920898, |
| "step": 5925 |
| }, |
| { |
| "epoch": 1.3566399680757062, |
| "grad_norm": 143.0, |
| "learning_rate": 1.6583085263822502e-05, |
| "logits/chosen": -2.0757803916931152, |
| "logits/rejected": -1.8336753845214844, |
| "logps/chosen": -245.9808807373047, |
| "logps/rejected": -290.99114990234375, |
| "loss": 0.1127, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 1.8838566541671753, |
| "rewards/margins": 12.81384563446045, |
| "rewards/rejected": -10.929988861083984, |
| "step": 5950 |
| }, |
| { |
| "epoch": 1.3623407348288346, |
| "grad_norm": 142.0, |
| "learning_rate": 1.6556084276346436e-05, |
| "logits/chosen": -1.5928791761398315, |
| "logits/rejected": -1.3860570192337036, |
| "logps/chosen": -245.87054443359375, |
| "logps/rejected": -282.54083251953125, |
| "loss": 0.1219, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.5248727798461914, |
| "rewards/margins": 11.859354019165039, |
| "rewards/rejected": -9.334482192993164, |
| "step": 5975 |
| }, |
| { |
| "epoch": 1.3680415015819627, |
| "grad_norm": 19.875, |
| "learning_rate": 1.652899919844239e-05, |
| "logits/chosen": -1.973768949508667, |
| "logits/rejected": -1.6700533628463745, |
| "logps/chosen": -238.50852966308594, |
| "logps/rejected": -292.51934814453125, |
| "loss": 0.0876, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 1.8946725130081177, |
| "rewards/margins": 11.96185302734375, |
| "rewards/rejected": -10.067181587219238, |
| "step": 6000 |
| }, |
| { |
| "epoch": 1.373742268335091, |
| "grad_norm": 326.0, |
| "learning_rate": 1.6501830377512214e-05, |
| "logits/chosen": -1.8057034015655518, |
| "logits/rejected": -1.5293034315109253, |
| "logps/chosen": -265.75372314453125, |
| "logps/rejected": -341.4354248046875, |
| "loss": 0.1308, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.4888696670532227, |
| "rewards/margins": 12.961655616760254, |
| "rewards/rejected": -10.472785949707031, |
| "step": 6025 |
| }, |
| { |
| "epoch": 1.3794430350882194, |
| "grad_norm": 164.0, |
| "learning_rate": 1.6474578162031873e-05, |
| "logits/chosen": -1.9495043754577637, |
| "logits/rejected": -1.7214263677597046, |
| "logps/chosen": -250.1916961669922, |
| "logps/rejected": -271.11370849609375, |
| "loss": 0.109, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.406526565551758, |
| "rewards/margins": 13.257115364074707, |
| "rewards/rejected": -10.85058879852295, |
| "step": 6050 |
| }, |
| { |
| "epoch": 1.3851438018413478, |
| "grad_norm": 233.0, |
| "learning_rate": 1.644724290154697e-05, |
| "logits/chosen": -1.7470669746398926, |
| "logits/rejected": -1.4726073741912842, |
| "logps/chosen": -249.06405639648438, |
| "logps/rejected": -286.47393798828125, |
| "loss": 0.1325, |
| "rewards/accuracies": 0.9449999928474426, |
| "rewards/chosen": 1.664414644241333, |
| "rewards/margins": 12.541833877563477, |
| "rewards/rejected": -10.877416610717773, |
| "step": 6075 |
| }, |
| { |
| "epoch": 1.390844568594476, |
| "grad_norm": 211.0, |
| "learning_rate": 1.641982494666828e-05, |
| "logits/chosen": -1.9918991327285767, |
| "logits/rejected": -1.704099178314209, |
| "logps/chosen": -228.07943725585938, |
| "logps/rejected": -269.5039978027344, |
| "loss": 0.0971, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 1.6977978944778442, |
| "rewards/margins": 12.857917785644531, |
| "rewards/rejected": -11.160120010375977, |
| "step": 6100 |
| }, |
| { |
| "epoch": 1.3965453353476043, |
| "grad_norm": 73.5, |
| "learning_rate": 1.639232464906723e-05, |
| "logits/chosen": -1.9871395826339722, |
| "logits/rejected": -1.6356009244918823, |
| "logps/chosen": -238.62265014648438, |
| "logps/rejected": -282.8518371582031, |
| "loss": 0.1061, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 1.773512363433838, |
| "rewards/margins": 12.500313758850098, |
| "rewards/rejected": -10.726801872253418, |
| "step": 6125 |
| }, |
| { |
| "epoch": 1.4022461021007326, |
| "grad_norm": 0.87109375, |
| "learning_rate": 1.6364742361471416e-05, |
| "logits/chosen": -1.8053956031799316, |
| "logits/rejected": -1.5105464458465576, |
| "logps/chosen": -261.6184387207031, |
| "logps/rejected": -292.1883544921875, |
| "loss": 0.1099, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 2.704190254211426, |
| "rewards/margins": 13.197578430175781, |
| "rewards/rejected": -10.493387222290039, |
| "step": 6150 |
| }, |
| { |
| "epoch": 1.4079468688538608, |
| "grad_norm": 242.0, |
| "learning_rate": 1.633707843766005e-05, |
| "logits/chosen": -1.9799277782440186, |
| "logits/rejected": -1.7369930744171143, |
| "logps/chosen": -264.1947021484375, |
| "logps/rejected": -283.8174743652344, |
| "loss": 0.1294, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 2.5791547298431396, |
| "rewards/margins": 13.647754669189453, |
| "rewards/rejected": -11.06860065460205, |
| "step": 6175 |
| }, |
| { |
| "epoch": 1.4136476356069891, |
| "grad_norm": 159.0, |
| "learning_rate": 1.6309333232459444e-05, |
| "logits/chosen": -1.9193072319030762, |
| "logits/rejected": -1.618723750114441, |
| "logps/chosen": -242.60552978515625, |
| "logps/rejected": -278.3722839355469, |
| "loss": 0.1429, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 2.2602224349975586, |
| "rewards/margins": 12.864294052124023, |
| "rewards/rejected": -10.604072570800781, |
| "step": 6200 |
| }, |
| { |
| "epoch": 1.4193484023601175, |
| "grad_norm": 21.125, |
| "learning_rate": 1.628150710173845e-05, |
| "logits/chosen": -1.9135751724243164, |
| "logits/rejected": -1.6459423303604126, |
| "logps/chosen": -231.89263916015625, |
| "logps/rejected": -282.2898254394531, |
| "loss": 0.1438, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 2.401604652404785, |
| "rewards/margins": 13.20291519165039, |
| "rewards/rejected": -10.801309585571289, |
| "step": 6225 |
| }, |
| { |
| "epoch": 1.4250491691132456, |
| "grad_norm": 123.5, |
| "learning_rate": 1.6253600402403896e-05, |
| "logits/chosen": -1.941986083984375, |
| "logits/rejected": -1.7515454292297363, |
| "logps/chosen": -236.5397491455078, |
| "logps/rejected": -277.414794921875, |
| "loss": 0.0945, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 2.087970495223999, |
| "rewards/margins": 13.167825698852539, |
| "rewards/rejected": -11.079854965209961, |
| "step": 6250 |
| }, |
| { |
| "epoch": 1.430749935866374, |
| "grad_norm": 8.5, |
| "learning_rate": 1.6225613492396005e-05, |
| "logits/chosen": -2.0523149967193604, |
| "logits/rejected": -1.8074952363967896, |
| "logps/chosen": -258.17828369140625, |
| "logps/rejected": -271.7230529785156, |
| "loss": 0.1884, |
| "rewards/accuracies": 0.9300000071525574, |
| "rewards/chosen": 2.5216095447540283, |
| "rewards/margins": 14.455331802368164, |
| "rewards/rejected": -11.933721542358398, |
| "step": 6275 |
| }, |
| { |
| "epoch": 1.4364507026195024, |
| "grad_norm": 10.0625, |
| "learning_rate": 1.619754673068382e-05, |
| "logits/chosen": -1.8217549324035645, |
| "logits/rejected": -1.6858906745910645, |
| "logps/chosen": -264.71258544921875, |
| "logps/rejected": -295.71539306640625, |
| "loss": 0.0815, |
| "rewards/accuracies": 0.9800000190734863, |
| "rewards/chosen": 3.4202585220336914, |
| "rewards/margins": 13.367759704589844, |
| "rewards/rejected": -9.947502136230469, |
| "step": 6300 |
| }, |
| { |
| "epoch": 1.4421514693726305, |
| "grad_norm": 18.5, |
| "learning_rate": 1.6169400477260566e-05, |
| "logits/chosen": -1.8766337633132935, |
| "logits/rejected": -1.629098653793335, |
| "logps/chosen": -235.7779541015625, |
| "logps/rejected": -283.8551025390625, |
| "loss": 0.0856, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 2.6527211666107178, |
| "rewards/margins": 13.295780181884766, |
| "rewards/rejected": -10.643058776855469, |
| "step": 6325 |
| }, |
| { |
| "epoch": 1.4478522361257589, |
| "grad_norm": 9.5, |
| "learning_rate": 1.6141175093139078e-05, |
| "logits/chosen": -2.2030673027038574, |
| "logits/rejected": -1.9514520168304443, |
| "logps/chosen": -226.92355346679688, |
| "logps/rejected": -290.2125549316406, |
| "loss": 0.194, |
| "rewards/accuracies": 0.9449999928474426, |
| "rewards/chosen": 2.3246076107025146, |
| "rewards/margins": 13.925102233886719, |
| "rewards/rejected": -11.600497245788574, |
| "step": 6350 |
| }, |
| { |
| "epoch": 1.4535530028788872, |
| "grad_norm": 185.0, |
| "learning_rate": 1.611287094034713e-05, |
| "logits/chosen": -1.9147053956985474, |
| "logits/rejected": -1.6553024053573608, |
| "logps/chosen": -254.85595703125, |
| "logps/rejected": -303.0509033203125, |
| "loss": 0.0973, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.726706027984619, |
| "rewards/margins": 12.884346008300781, |
| "rewards/rejected": -10.157639503479004, |
| "step": 6375 |
| }, |
| { |
| "epoch": 1.4592537696320156, |
| "grad_norm": 11.375, |
| "learning_rate": 1.6084488381922808e-05, |
| "logits/chosen": -2.172454833984375, |
| "logits/rejected": -1.9440901279449463, |
| "logps/chosen": -258.2084655761719, |
| "logps/rejected": -293.0820007324219, |
| "loss": 0.0775, |
| "rewards/accuracies": 0.9700000286102295, |
| "rewards/chosen": 2.8246049880981445, |
| "rewards/margins": 13.954553604125977, |
| "rewards/rejected": -11.129948616027832, |
| "step": 6400 |
| }, |
| { |
| "epoch": 1.4649545363851437, |
| "grad_norm": 196.0, |
| "learning_rate": 1.6056027781909867e-05, |
| "logits/chosen": -2.1385555267333984, |
| "logits/rejected": -1.8859370946884155, |
| "logps/chosen": -250.26626586914062, |
| "logps/rejected": -307.28875732421875, |
| "loss": 0.116, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 2.6781036853790283, |
| "rewards/margins": 13.736023902893066, |
| "rewards/rejected": -11.057921409606934, |
| "step": 6425 |
| }, |
| { |
| "epoch": 1.470655303138272, |
| "grad_norm": 3.484375, |
| "learning_rate": 1.602748950535304e-05, |
| "logits/chosen": -1.9784660339355469, |
| "logits/rejected": -1.7451972961425781, |
| "logps/chosen": -245.80516052246094, |
| "logps/rejected": -269.9248962402344, |
| "loss": 0.103, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 2.9505350589752197, |
| "rewards/margins": 13.441423416137695, |
| "rewards/rejected": -10.490888595581055, |
| "step": 6450 |
| }, |
| { |
| "epoch": 1.4763560698914004, |
| "grad_norm": 1.7109375, |
| "learning_rate": 1.599887391829336e-05, |
| "logits/chosen": -2.3491859436035156, |
| "logits/rejected": -2.086282253265381, |
| "logps/chosen": -220.6693572998047, |
| "logps/rejected": -277.89141845703125, |
| "loss": 0.138, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 2.61944580078125, |
| "rewards/margins": 13.533830642700195, |
| "rewards/rejected": -10.914384841918945, |
| "step": 6475 |
| }, |
| { |
| "epoch": 1.4820568366445288, |
| "grad_norm": 40.75, |
| "learning_rate": 1.5970181387763477e-05, |
| "logits/chosen": -1.8637809753417969, |
| "logits/rejected": -1.6721075773239136, |
| "logps/chosen": -254.94412231445312, |
| "logps/rejected": -286.3333740234375, |
| "loss": 0.1934, |
| "rewards/accuracies": 0.9399999976158142, |
| "rewards/chosen": 2.972402572631836, |
| "rewards/margins": 11.81149673461914, |
| "rewards/rejected": -8.839094161987305, |
| "step": 6500 |
| }, |
| { |
| "epoch": 1.487757603397657, |
| "grad_norm": 200.0, |
| "learning_rate": 1.5941412281782934e-05, |
| "logits/chosen": -1.8482853174209595, |
| "logits/rejected": -1.6093626022338867, |
| "logps/chosen": -266.2801818847656, |
| "logps/rejected": -315.1908264160156, |
| "loss": 0.1641, |
| "rewards/accuracies": 0.9399999976158142, |
| "rewards/chosen": 3.5010366439819336, |
| "rewards/margins": 14.189444541931152, |
| "rewards/rejected": -10.688407897949219, |
| "step": 6525 |
| }, |
| { |
| "epoch": 1.4934583701507853, |
| "grad_norm": 71.5, |
| "learning_rate": 1.5912566969353464e-05, |
| "logits/chosen": -2.072434902191162, |
| "logits/rejected": -1.7364941835403442, |
| "logps/chosen": -252.1800537109375, |
| "logps/rejected": -308.68914794921875, |
| "loss": 0.1151, |
| "rewards/accuracies": 0.9700000286102295, |
| "rewards/chosen": 2.9971704483032227, |
| "rewards/margins": 15.033012390136719, |
| "rewards/rejected": -12.035841941833496, |
| "step": 6550 |
| }, |
| { |
| "epoch": 1.4991591369039137, |
| "grad_norm": 0.5625, |
| "learning_rate": 1.5883645820454247e-05, |
| "logits/chosen": -1.8191848993301392, |
| "logits/rejected": -1.5475897789001465, |
| "logps/chosen": -236.23558044433594, |
| "logps/rejected": -306.14617919921875, |
| "loss": 0.1127, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 2.5426418781280518, |
| "rewards/margins": 13.720407485961914, |
| "rewards/rejected": -11.177764892578125, |
| "step": 6575 |
| }, |
| { |
| "epoch": 1.5048599036570418, |
| "grad_norm": 4.09375, |
| "learning_rate": 1.585464920603716e-05, |
| "logits/chosen": -1.8807624578475952, |
| "logits/rejected": -1.6400059461593628, |
| "logps/chosen": -258.6998596191406, |
| "logps/rejected": -299.45452880859375, |
| "loss": 0.1335, |
| "rewards/accuracies": 0.9399999976158142, |
| "rewards/chosen": 2.9638543128967285, |
| "rewards/margins": 13.171612739562988, |
| "rewards/rejected": -10.207757949829102, |
| "step": 6600 |
| }, |
| { |
| "epoch": 1.5105606704101702, |
| "grad_norm": 350.0, |
| "learning_rate": 1.5825577498022027e-05, |
| "logits/chosen": -1.8970913887023926, |
| "logits/rejected": -1.7421754598617554, |
| "logps/chosen": -257.2083740234375, |
| "logps/rejected": -295.31597900390625, |
| "loss": 0.1437, |
| "rewards/accuracies": 0.9350000023841858, |
| "rewards/chosen": 2.764373779296875, |
| "rewards/margins": 13.546743392944336, |
| "rewards/rejected": -10.782369613647461, |
| "step": 6625 |
| }, |
| { |
| "epoch": 1.5162614371632985, |
| "grad_norm": 7.34375, |
| "learning_rate": 1.5796431069291847e-05, |
| "logits/chosen": -2.0372872352600098, |
| "logits/rejected": -1.7386982440948486, |
| "logps/chosen": -244.50808715820312, |
| "logps/rejected": -301.50634765625, |
| "loss": 0.0566, |
| "rewards/accuracies": 0.9700000286102295, |
| "rewards/chosen": 3.125155448913574, |
| "rewards/margins": 14.936068534851074, |
| "rewards/rejected": -11.8109130859375, |
| "step": 6650 |
| }, |
| { |
| "epoch": 1.5219622039164267, |
| "grad_norm": 278.0, |
| "learning_rate": 1.5767210293688015e-05, |
| "logits/chosen": -1.6637896299362183, |
| "logits/rejected": -1.417211890220642, |
| "logps/chosen": -249.45724487304688, |
| "logps/rejected": -282.8021545410156, |
| "loss": 0.2326, |
| "rewards/accuracies": 0.9399999976158142, |
| "rewards/chosen": 2.5139217376708984, |
| "rewards/margins": 12.243398666381836, |
| "rewards/rejected": -9.729477882385254, |
| "step": 6675 |
| }, |
| { |
| "epoch": 1.527662970669555, |
| "grad_norm": 16.625, |
| "learning_rate": 1.5737915546005513e-05, |
| "logits/chosen": -1.710518479347229, |
| "logits/rejected": -1.4187105894088745, |
| "logps/chosen": -240.12564086914062, |
| "logps/rejected": -311.4556579589844, |
| "loss": 0.076, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 3.113593101501465, |
| "rewards/margins": 13.576586723327637, |
| "rewards/rejected": -10.462994575500488, |
| "step": 6700 |
| }, |
| { |
| "epoch": 1.5333637374226834, |
| "grad_norm": 13.4375, |
| "learning_rate": 1.570854720198812e-05, |
| "logits/chosen": -1.8623547554016113, |
| "logits/rejected": -1.6710395812988281, |
| "logps/chosen": -271.5815124511719, |
| "logps/rejected": -318.53216552734375, |
| "loss": 0.1677, |
| "rewards/accuracies": 0.9300000071525574, |
| "rewards/chosen": 2.801870346069336, |
| "rewards/margins": 13.289447784423828, |
| "rewards/rejected": -10.487576484680176, |
| "step": 6725 |
| }, |
| { |
| "epoch": 1.5390645041758115, |
| "grad_norm": 306.0, |
| "learning_rate": 1.567910563832359e-05, |
| "logits/chosen": -1.6973358392715454, |
| "logits/rejected": -1.5325666666030884, |
| "logps/chosen": -238.62979125976562, |
| "logps/rejected": -273.6838684082031, |
| "loss": 0.1984, |
| "rewards/accuracies": 0.9300000071525574, |
| "rewards/chosen": 2.9855337142944336, |
| "rewards/margins": 12.147957801818848, |
| "rewards/rejected": -9.162424087524414, |
| "step": 6750 |
| }, |
| { |
| "epoch": 1.54476527092894, |
| "grad_norm": 28.75, |
| "learning_rate": 1.5649591232638804e-05, |
| "logits/chosen": -1.8299217224121094, |
| "logits/rejected": -1.5630756616592407, |
| "logps/chosen": -243.60951232910156, |
| "logps/rejected": -275.2950744628906, |
| "loss": 0.1063, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 3.354177236557007, |
| "rewards/margins": 12.557991981506348, |
| "rewards/rejected": -9.203814506530762, |
| "step": 6775 |
| }, |
| { |
| "epoch": 1.5504660376820683, |
| "grad_norm": 3.15625, |
| "learning_rate": 1.5620004363494937e-05, |
| "logits/chosen": -2.204270124435425, |
| "logits/rejected": -1.9153170585632324, |
| "logps/chosen": -239.57000732421875, |
| "logps/rejected": -290.0873718261719, |
| "loss": 0.1495, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 3.1389520168304443, |
| "rewards/margins": 14.174966812133789, |
| "rewards/rejected": -11.036014556884766, |
| "step": 6800 |
| }, |
| { |
| "epoch": 1.5561668044351964, |
| "grad_norm": 145.0, |
| "learning_rate": 1.559034541038262e-05, |
| "logits/chosen": -2.1027653217315674, |
| "logits/rejected": -1.8087865114212036, |
| "logps/chosen": -242.76805114746094, |
| "logps/rejected": -276.0024108886719, |
| "loss": 0.0852, |
| "rewards/accuracies": 0.9700000286102295, |
| "rewards/chosen": 2.613564968109131, |
| "rewards/margins": 13.638014793395996, |
| "rewards/rejected": -11.024452209472656, |
| "step": 6825 |
| }, |
| { |
| "epoch": 1.561867571188325, |
| "grad_norm": 145.0, |
| "learning_rate": 1.556061475371704e-05, |
| "logits/chosen": -1.8637328147888184, |
| "logits/rejected": -1.6028027534484863, |
| "logps/chosen": -255.0188446044922, |
| "logps/rejected": -290.7564697265625, |
| "loss": 0.1415, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 3.0880093574523926, |
| "rewards/margins": 12.83209228515625, |
| "rewards/rejected": -9.7440824508667, |
| "step": 6850 |
| }, |
| { |
| "epoch": 1.5675683379414531, |
| "grad_norm": 53.5, |
| "learning_rate": 1.553081277483308e-05, |
| "logits/chosen": -2.003204584121704, |
| "logits/rejected": -1.7315222024917603, |
| "logps/chosen": -244.65298461914062, |
| "logps/rejected": -276.32379150390625, |
| "loss": 0.1309, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 3.247680902481079, |
| "rewards/margins": 13.783878326416016, |
| "rewards/rejected": -10.536197662353516, |
| "step": 6875 |
| }, |
| { |
| "epoch": 1.5732691046945813, |
| "grad_norm": 87.5, |
| "learning_rate": 1.5500939855980427e-05, |
| "logits/chosen": -2.216855049133301, |
| "logits/rejected": -1.9786120653152466, |
| "logps/chosen": -282.1592102050781, |
| "logps/rejected": -304.30645751953125, |
| "loss": 0.0936, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 3.9328317642211914, |
| "rewards/margins": 14.03812026977539, |
| "rewards/rejected": -10.1052885055542, |
| "step": 6900 |
| }, |
| { |
| "epoch": 1.5789698714477098, |
| "grad_norm": 0.91015625, |
| "learning_rate": 1.547099638031867e-05, |
| "logits/chosen": -2.1321074962615967, |
| "logits/rejected": -1.775516152381897, |
| "logps/chosen": -237.65081787109375, |
| "logps/rejected": -273.5619812011719, |
| "loss": 0.1433, |
| "rewards/accuracies": 0.9449999928474426, |
| "rewards/chosen": 3.0764822959899902, |
| "rewards/margins": 14.040980339050293, |
| "rewards/rejected": -10.964496612548828, |
| "step": 6925 |
| }, |
| { |
| "epoch": 1.584670638200838, |
| "grad_norm": 88.5, |
| "learning_rate": 1.544098273191238e-05, |
| "logits/chosen": -1.9686628580093384, |
| "logits/rejected": -1.6433407068252563, |
| "logps/chosen": -258.356201171875, |
| "logps/rejected": -275.55804443359375, |
| "loss": 0.2793, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 3.253434181213379, |
| "rewards/margins": 13.760456085205078, |
| "rewards/rejected": -10.507022857666016, |
| "step": 6950 |
| }, |
| { |
| "epoch": 1.5903714049539663, |
| "grad_norm": 294.0, |
| "learning_rate": 1.5410899295726174e-05, |
| "logits/chosen": -1.9594305753707886, |
| "logits/rejected": -1.7397663593292236, |
| "logps/chosen": -260.6534729003906, |
| "logps/rejected": -266.09027099609375, |
| "loss": 0.1524, |
| "rewards/accuracies": 0.9399999976158142, |
| "rewards/chosen": 3.686640739440918, |
| "rewards/margins": 13.448753356933594, |
| "rewards/rejected": -9.762112617492676, |
| "step": 6975 |
| }, |
| { |
| "epoch": 1.5960721717070947, |
| "grad_norm": 198.0, |
| "learning_rate": 1.5380746457619808e-05, |
| "logits/chosen": -1.902154803276062, |
| "logits/rejected": -1.494299292564392, |
| "logps/chosen": -252.38613891601562, |
| "logps/rejected": -290.7088623046875, |
| "loss": 0.1927, |
| "rewards/accuracies": 0.9350000023841858, |
| "rewards/chosen": 3.082364559173584, |
| "rewards/margins": 14.098587989807129, |
| "rewards/rejected": -11.016223907470703, |
| "step": 7000 |
| }, |
| { |
| "epoch": 1.6017729384602228, |
| "grad_norm": 107.5, |
| "learning_rate": 1.53505246043432e-05, |
| "logits/chosen": -1.9686224460601807, |
| "logits/rejected": -1.6917322874069214, |
| "logps/chosen": -224.60507202148438, |
| "logps/rejected": -269.6410827636719, |
| "loss": 0.0923, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 2.6634521484375, |
| "rewards/margins": 12.749421119689941, |
| "rewards/rejected": -10.085968971252441, |
| "step": 7025 |
| }, |
| { |
| "epoch": 1.6074737052133512, |
| "grad_norm": 56.5, |
| "learning_rate": 1.5320234123531474e-05, |
| "logits/chosen": -1.8641825914382935, |
| "logits/rejected": -1.6425690650939941, |
| "logps/chosen": -241.71844482421875, |
| "logps/rejected": -299.41522216796875, |
| "loss": 0.255, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.572680711746216, |
| "rewards/margins": 12.366220474243164, |
| "rewards/rejected": -9.793538093566895, |
| "step": 7050 |
| }, |
| { |
| "epoch": 1.6131744719664796, |
| "grad_norm": 109.0, |
| "learning_rate": 1.5289875403700005e-05, |
| "logits/chosen": -1.9764251708984375, |
| "logits/rejected": -1.681931495666504, |
| "logps/chosen": -255.88461303710938, |
| "logps/rejected": -283.6371765136719, |
| "loss": 0.1952, |
| "rewards/accuracies": 0.9300000071525574, |
| "rewards/chosen": 2.4234437942504883, |
| "rewards/margins": 12.720088005065918, |
| "rewards/rejected": -10.29664421081543, |
| "step": 7075 |
| }, |
| { |
| "epoch": 1.6188752387196077, |
| "grad_norm": 100.0, |
| "learning_rate": 1.5259448834239412e-05, |
| "logits/chosen": -2.153337240219116, |
| "logits/rejected": -1.8483002185821533, |
| "logps/chosen": -251.46009826660156, |
| "logps/rejected": -294.6589050292969, |
| "loss": 0.0853, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 2.9544403553009033, |
| "rewards/margins": 15.079411506652832, |
| "rewards/rejected": -12.124970436096191, |
| "step": 7100 |
| }, |
| { |
| "epoch": 1.624576005472736, |
| "grad_norm": 6.9375, |
| "learning_rate": 1.5228954805410584e-05, |
| "logits/chosen": -1.6335525512695312, |
| "logits/rejected": -1.431130051612854, |
| "logps/chosen": -261.31829833984375, |
| "logps/rejected": -312.9454650878906, |
| "loss": 0.076, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 3.7951419353485107, |
| "rewards/margins": 13.92431926727295, |
| "rewards/rejected": -10.12917709350586, |
| "step": 7125 |
| }, |
| { |
| "epoch": 1.6302767722258644, |
| "grad_norm": 12.3125, |
| "learning_rate": 1.5198393708339651e-05, |
| "logits/chosen": -2.202697992324829, |
| "logits/rejected": -1.8340469598770142, |
| "logps/chosen": -240.88328552246094, |
| "logps/rejected": -293.5688171386719, |
| "loss": 0.1229, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 2.583376169204712, |
| "rewards/margins": 14.009459495544434, |
| "rewards/rejected": -11.4260835647583, |
| "step": 7150 |
| }, |
| { |
| "epoch": 1.6359775389789926, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.5167765935013004e-05, |
| "logits/chosen": -1.7569880485534668, |
| "logits/rejected": -1.4942305088043213, |
| "logps/chosen": -245.9915771484375, |
| "logps/rejected": -297.4135437011719, |
| "loss": 0.2532, |
| "rewards/accuracies": 0.9150000214576721, |
| "rewards/chosen": 3.1261816024780273, |
| "rewards/margins": 13.673910140991211, |
| "rewards/rejected": -10.5477294921875, |
| "step": 7175 |
| }, |
| { |
| "epoch": 1.641678305732121, |
| "grad_norm": 49.0, |
| "learning_rate": 1.513707187827222e-05, |
| "logits/chosen": -2.0404019355773926, |
| "logits/rejected": -1.8195034265518188, |
| "logps/chosen": -261.67315673828125, |
| "logps/rejected": -311.7972412109375, |
| "loss": 0.1072, |
| "rewards/accuracies": 0.9700000286102295, |
| "rewards/chosen": 2.812009811401367, |
| "rewards/margins": 13.783918380737305, |
| "rewards/rejected": -10.97191047668457, |
| "step": 7200 |
| }, |
| { |
| "epoch": 1.6473790724852493, |
| "grad_norm": 20.375, |
| "learning_rate": 1.510631193180907e-05, |
| "logits/chosen": -1.894914150238037, |
| "logits/rejected": -1.6014800071716309, |
| "logps/chosen": -251.0972137451172, |
| "logps/rejected": -284.2265930175781, |
| "loss": 0.1497, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 3.305243730545044, |
| "rewards/margins": 12.571907997131348, |
| "rewards/rejected": -9.266663551330566, |
| "step": 7225 |
| }, |
| { |
| "epoch": 1.6530798392383774, |
| "grad_norm": 15.8125, |
| "learning_rate": 1.5075486490160436e-05, |
| "logits/chosen": -2.156367301940918, |
| "logits/rejected": -2.00156569480896, |
| "logps/chosen": -234.45913696289062, |
| "logps/rejected": -277.3482666015625, |
| "loss": 0.15, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 3.082796335220337, |
| "rewards/margins": 13.953583717346191, |
| "rewards/rejected": -10.87078857421875, |
| "step": 7250 |
| }, |
| { |
| "epoch": 1.658780605991506, |
| "grad_norm": 113.0, |
| "learning_rate": 1.5044595948703262e-05, |
| "logits/chosen": -1.8482290506362915, |
| "logits/rejected": -1.5448541641235352, |
| "logps/chosen": -261.39654541015625, |
| "logps/rejected": -291.55352783203125, |
| "loss": 0.1357, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 3.4639017581939697, |
| "rewards/margins": 11.93445110321045, |
| "rewards/rejected": -8.470547676086426, |
| "step": 7275 |
| }, |
| { |
| "epoch": 1.6644813727446341, |
| "grad_norm": 292.0, |
| "learning_rate": 1.5013640703649486e-05, |
| "logits/chosen": -2.2465906143188477, |
| "logits/rejected": -1.9755215644836426, |
| "logps/chosen": -236.1417236328125, |
| "logps/rejected": -283.5238037109375, |
| "loss": 0.1664, |
| "rewards/accuracies": 0.9350000023841858, |
| "rewards/chosen": 2.8566501140594482, |
| "rewards/margins": 14.108537673950195, |
| "rewards/rejected": -11.251887321472168, |
| "step": 7300 |
| }, |
| { |
| "epoch": 1.6701821394977623, |
| "grad_norm": 141.0, |
| "learning_rate": 1.4982621152040954e-05, |
| "logits/chosen": -1.961510181427002, |
| "logits/rejected": -1.6759686470031738, |
| "logps/chosen": -252.07322692871094, |
| "logps/rejected": -302.82550048828125, |
| "loss": 0.1298, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 3.9384844303131104, |
| "rewards/margins": 14.664482116699219, |
| "rewards/rejected": -10.725996017456055, |
| "step": 7325 |
| }, |
| { |
| "epoch": 1.6758829062508909, |
| "grad_norm": 52.5, |
| "learning_rate": 1.495153769174433e-05, |
| "logits/chosen": -2.0477404594421387, |
| "logits/rejected": -1.771289348602295, |
| "logps/chosen": -214.1411590576172, |
| "logps/rejected": -260.2171936035156, |
| "loss": 0.1807, |
| "rewards/accuracies": 0.9449999928474426, |
| "rewards/chosen": 2.922196626663208, |
| "rewards/margins": 14.55020523071289, |
| "rewards/rejected": -11.628008842468262, |
| "step": 7350 |
| }, |
| { |
| "epoch": 1.681583673004019, |
| "grad_norm": 262.0, |
| "learning_rate": 1.4920390721445993e-05, |
| "logits/chosen": -2.0264077186584473, |
| "logits/rejected": -1.8375439643859863, |
| "logps/chosen": -260.9549560546875, |
| "logps/rejected": -304.2368469238281, |
| "loss": 0.0967, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 3.0752246379852295, |
| "rewards/margins": 13.806523323059082, |
| "rewards/rejected": -10.731298446655273, |
| "step": 7375 |
| }, |
| { |
| "epoch": 1.6872844397571474, |
| "grad_norm": 14.125, |
| "learning_rate": 1.4889180640646907e-05, |
| "logits/chosen": -1.616461157798767, |
| "logits/rejected": -1.416245460510254, |
| "logps/chosen": -266.1830139160156, |
| "logps/rejected": -299.46942138671875, |
| "loss": 0.1297, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 3.4925310611724854, |
| "rewards/margins": 13.018446922302246, |
| "rewards/rejected": -9.525915145874023, |
| "step": 7400 |
| }, |
| { |
| "epoch": 1.6929852065102757, |
| "grad_norm": 3.5, |
| "learning_rate": 1.4857907849657536e-05, |
| "logits/chosen": -2.224869728088379, |
| "logits/rejected": -1.9717463254928589, |
| "logps/chosen": -242.56460571289062, |
| "logps/rejected": -280.54608154296875, |
| "loss": 0.0427, |
| "rewards/accuracies": 0.9800000190734863, |
| "rewards/chosen": 3.455406427383423, |
| "rewards/margins": 15.338541030883789, |
| "rewards/rejected": -11.883133888244629, |
| "step": 7425 |
| }, |
| { |
| "epoch": 1.6986859732634039, |
| "grad_norm": 56.0, |
| "learning_rate": 1.4826572749592662e-05, |
| "logits/chosen": -1.9944206476211548, |
| "logits/rejected": -1.8091230392456055, |
| "logps/chosen": -231.27273559570312, |
| "logps/rejected": -279.7394104003906, |
| "loss": 0.1036, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 2.6775946617126465, |
| "rewards/margins": 12.199337005615234, |
| "rewards/rejected": -9.521740913391113, |
| "step": 7450 |
| }, |
| { |
| "epoch": 1.7043867400165322, |
| "grad_norm": 0.9296875, |
| "learning_rate": 1.4795175742366277e-05, |
| "logits/chosen": -1.8677728176116943, |
| "logits/rejected": -1.6506212949752808, |
| "logps/chosen": -282.4718017578125, |
| "logps/rejected": -326.9453125, |
| "loss": 0.1367, |
| "rewards/accuracies": 0.9449999928474426, |
| "rewards/chosen": 2.454669237136841, |
| "rewards/margins": 13.52832317352295, |
| "rewards/rejected": -11.073653221130371, |
| "step": 7475 |
| }, |
| { |
| "epoch": 1.7100875067696606, |
| "grad_norm": 318.0, |
| "learning_rate": 1.4763717230686408e-05, |
| "logits/chosen": -1.9016128778457642, |
| "logits/rejected": -1.6556119918823242, |
| "logps/chosen": -254.5514678955078, |
| "logps/rejected": -296.094970703125, |
| "loss": 0.0999, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.876845121383667, |
| "rewards/margins": 13.160791397094727, |
| "rewards/rejected": -10.283946990966797, |
| "step": 7500 |
| }, |
| { |
| "epoch": 1.7100875067696606, |
| "eval_logits/chosen": -1.9768283367156982, |
| "eval_logits/rejected": -1.7413971424102783, |
| "eval_logps/chosen": -259.7589416503906, |
| "eval_logps/rejected": -285.774658203125, |
| "eval_loss": 1.116422414779663, |
| "eval_rewards/accuracies": 0.7050507068634033, |
| "eval_rewards/chosen": -0.2218780815601349, |
| "eval_rewards/margins": 7.128981590270996, |
| "eval_rewards/rejected": -7.35085916519165, |
| "eval_runtime": 907.2818, |
| "eval_samples_per_second": 9.667, |
| "eval_steps_per_second": 9.667, |
| "step": 7500 |
| }, |
| { |
| "epoch": 1.7157882735227887, |
| "grad_norm": 132.0, |
| "learning_rate": 1.473219761804996e-05, |
| "logits/chosen": -2.070918083190918, |
| "logits/rejected": -1.80425226688385, |
| "logps/chosen": -241.0839080810547, |
| "logps/rejected": -301.5720520019531, |
| "loss": 0.0928, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.161179304122925, |
| "rewards/margins": 14.651226043701172, |
| "rewards/rejected": -12.490045547485352, |
| "step": 7525 |
| }, |
| { |
| "epoch": 1.721489040275917, |
| "grad_norm": 106.0, |
| "learning_rate": 1.4700617308737542e-05, |
| "logits/chosen": -1.990986943244934, |
| "logits/rejected": -1.6481989622116089, |
| "logps/chosen": -249.19142150878906, |
| "logps/rejected": -297.8671569824219, |
| "loss": 0.1269, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 2.4406728744506836, |
| "rewards/margins": 13.752496719360352, |
| "rewards/rejected": -11.311823844909668, |
| "step": 7550 |
| }, |
| { |
| "epoch": 1.7271898070290455, |
| "grad_norm": 154.0, |
| "learning_rate": 1.4668976707808267e-05, |
| "logits/chosen": -2.0242061614990234, |
| "logits/rejected": -1.8689017295837402, |
| "logps/chosen": -236.33058166503906, |
| "logps/rejected": -281.4001159667969, |
| "loss": 0.1128, |
| "rewards/accuracies": 0.9399999976158142, |
| "rewards/chosen": 2.692863702774048, |
| "rewards/margins": 13.501513481140137, |
| "rewards/rejected": -10.808650970458984, |
| "step": 7575 |
| }, |
| { |
| "epoch": 1.7328905737821736, |
| "grad_norm": 166.0, |
| "learning_rate": 1.4637276221094577e-05, |
| "logits/chosen": -1.9251701831817627, |
| "logits/rejected": -1.6317967176437378, |
| "logps/chosen": -246.7865447998047, |
| "logps/rejected": -307.6643371582031, |
| "loss": 0.122, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.5770959854125977, |
| "rewards/margins": 13.34862995147705, |
| "rewards/rejected": -10.771533966064453, |
| "step": 7600 |
| }, |
| { |
| "epoch": 1.738591340535302, |
| "grad_norm": 60.25, |
| "learning_rate": 1.4605516255197025e-05, |
| "logits/chosen": -1.8339169025421143, |
| "logits/rejected": -1.5677050352096558, |
| "logps/chosen": -248.37278747558594, |
| "logps/rejected": -320.5432434082031, |
| "loss": 0.1285, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 2.610668897628784, |
| "rewards/margins": 13.691341400146484, |
| "rewards/rejected": -11.080672264099121, |
| "step": 7625 |
| }, |
| { |
| "epoch": 1.7442921072884303, |
| "grad_norm": 322.0, |
| "learning_rate": 1.4573697217479068e-05, |
| "logits/chosen": -2.1090891361236572, |
| "logits/rejected": -1.779563546180725, |
| "logps/chosen": -265.631591796875, |
| "logps/rejected": -289.65899658203125, |
| "loss": 0.1939, |
| "rewards/accuracies": 0.9300000071525574, |
| "rewards/chosen": 2.4606215953826904, |
| "rewards/margins": 12.846070289611816, |
| "rewards/rejected": -10.385449409484863, |
| "step": 7650 |
| }, |
| { |
| "epoch": 1.7499928740415585, |
| "grad_norm": 82.5, |
| "learning_rate": 1.4541819516061824e-05, |
| "logits/chosen": -1.8630539178848267, |
| "logits/rejected": -1.7309105396270752, |
| "logps/chosen": -277.66473388671875, |
| "logps/rejected": -303.040771484375, |
| "loss": 0.1596, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 2.398030996322632, |
| "rewards/margins": 12.045849800109863, |
| "rewards/rejected": -9.647817611694336, |
| "step": 7675 |
| }, |
| { |
| "epoch": 1.755693640794687, |
| "grad_norm": 32.25, |
| "learning_rate": 1.4509883559818862e-05, |
| "logits/chosen": -1.76078462600708, |
| "logits/rejected": -1.519452691078186, |
| "logps/chosen": -251.12339782714844, |
| "logps/rejected": -289.4566345214844, |
| "loss": 0.2053, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": 2.9382872581481934, |
| "rewards/margins": 12.285896301269531, |
| "rewards/rejected": -9.347609519958496, |
| "step": 7700 |
| }, |
| { |
| "epoch": 1.7613944075478152, |
| "grad_norm": 43.5, |
| "learning_rate": 1.4477889758370945e-05, |
| "logits/chosen": -1.961332082748413, |
| "logits/rejected": -1.695451021194458, |
| "logps/chosen": -256.0299072265625, |
| "logps/rejected": -309.4319763183594, |
| "loss": 0.1388, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 2.8903799057006836, |
| "rewards/margins": 13.495387077331543, |
| "rewards/rejected": -10.605008125305176, |
| "step": 7725 |
| }, |
| { |
| "epoch": 1.7670951743009433, |
| "grad_norm": 104.0, |
| "learning_rate": 1.4445838522080775e-05, |
| "logits/chosen": -2.014139413833618, |
| "logits/rejected": -1.8211010694503784, |
| "logps/chosen": -238.07003784179688, |
| "logps/rejected": -272.3392028808594, |
| "loss": 0.097, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 2.3981170654296875, |
| "rewards/margins": 13.369488716125488, |
| "rewards/rejected": -10.9713716506958, |
| "step": 7750 |
| }, |
| { |
| "epoch": 1.772795941054072, |
| "grad_norm": 6.625, |
| "learning_rate": 1.4413730262047729e-05, |
| "logits/chosen": -2.0013039112091064, |
| "logits/rejected": -1.7016913890838623, |
| "logps/chosen": -229.35426330566406, |
| "logps/rejected": -312.97967529296875, |
| "loss": 0.0585, |
| "rewards/accuracies": 0.9800000190734863, |
| "rewards/chosen": 3.62532377243042, |
| "rewards/margins": 13.768503189086914, |
| "rewards/rejected": -10.143178939819336, |
| "step": 7775 |
| }, |
| { |
| "epoch": 1.7784967078072, |
| "grad_norm": 6.4375, |
| "learning_rate": 1.4381565390102591e-05, |
| "logits/chosen": -2.1853156089782715, |
| "logits/rejected": -1.9105966091156006, |
| "logps/chosen": -231.92718505859375, |
| "logps/rejected": -302.2670593261719, |
| "loss": 0.0984, |
| "rewards/accuracies": 0.9700000286102295, |
| "rewards/chosen": 2.807570457458496, |
| "rewards/margins": 14.134294509887695, |
| "rewards/rejected": -11.3267240524292, |
| "step": 7800 |
| }, |
| { |
| "epoch": 1.7841974745603284, |
| "grad_norm": 13.9375, |
| "learning_rate": 1.434934431880227e-05, |
| "logits/chosen": -2.0825164318084717, |
| "logits/rejected": -1.672012209892273, |
| "logps/chosen": -254.80860900878906, |
| "logps/rejected": -295.489990234375, |
| "loss": 0.0659, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 3.493819236755371, |
| "rewards/margins": 15.302542686462402, |
| "rewards/rejected": -11.808724403381348, |
| "step": 7825 |
| }, |
| { |
| "epoch": 1.7898982413134568, |
| "grad_norm": 0.0025482177734375, |
| "learning_rate": 1.4317067461424498e-05, |
| "logits/chosen": -2.1154448986053467, |
| "logits/rejected": -1.842889428138733, |
| "logps/chosen": -267.07061767578125, |
| "logps/rejected": -313.9446105957031, |
| "loss": 0.215, |
| "rewards/accuracies": 0.9350000023841858, |
| "rewards/chosen": 2.498810291290283, |
| "rewards/margins": 13.930930137634277, |
| "rewards/rejected": -11.432120323181152, |
| "step": 7850 |
| }, |
| { |
| "epoch": 1.795599008066585, |
| "grad_norm": 5.0, |
| "learning_rate": 1.4284735231962547e-05, |
| "logits/chosen": -2.0149598121643066, |
| "logits/rejected": -1.740877389907837, |
| "logps/chosen": -262.04559326171875, |
| "logps/rejected": -306.4689025878906, |
| "loss": 0.0854, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 2.8888344764709473, |
| "rewards/margins": 14.674062728881836, |
| "rewards/rejected": -11.785226821899414, |
| "step": 7875 |
| }, |
| { |
| "epoch": 1.8012997748197133, |
| "grad_norm": 41.25, |
| "learning_rate": 1.4252348045119907e-05, |
| "logits/chosen": -1.9099676609039307, |
| "logits/rejected": -1.739209771156311, |
| "logps/chosen": -267.03961181640625, |
| "logps/rejected": -304.4351806640625, |
| "loss": 0.0925, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 2.5929791927337646, |
| "rewards/margins": 13.497885704040527, |
| "rewards/rejected": -10.904906272888184, |
| "step": 7900 |
| }, |
| { |
| "epoch": 1.8070005415728416, |
| "grad_norm": 49.0, |
| "learning_rate": 1.421990631630496e-05, |
| "logits/chosen": -2.1238083839416504, |
| "logits/rejected": -1.8365452289581299, |
| "logps/chosen": -231.23265075683594, |
| "logps/rejected": -273.32373046875, |
| "loss": 0.0958, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.3405230045318604, |
| "rewards/margins": 13.894143104553223, |
| "rewards/rejected": -11.553619384765625, |
| "step": 7925 |
| }, |
| { |
| "epoch": 1.8127013083259698, |
| "grad_norm": 52.75, |
| "learning_rate": 1.418741046162567e-05, |
| "logits/chosen": -1.9506292343139648, |
| "logits/rejected": -1.7106003761291504, |
| "logps/chosen": -280.2424011230469, |
| "logps/rejected": -327.38311767578125, |
| "loss": 0.1375, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 2.943302869796753, |
| "rewards/margins": 14.869828224182129, |
| "rewards/rejected": -11.926523208618164, |
| "step": 7950 |
| }, |
| { |
| "epoch": 1.8184020750790981, |
| "grad_norm": 129.0, |
| "learning_rate": 1.4154860897884234e-05, |
| "logits/chosen": -1.9906669855117798, |
| "logits/rejected": -1.7677677869796753, |
| "logps/chosen": -239.0503692626953, |
| "logps/rejected": -275.10791015625, |
| "loss": 0.0844, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 2.4054694175720215, |
| "rewards/margins": 13.978160858154297, |
| "rewards/rejected": -11.5726900100708, |
| "step": 7975 |
| }, |
| { |
| "epoch": 1.8241028418322265, |
| "grad_norm": 282.0, |
| "learning_rate": 1.4122258042571737e-05, |
| "logits/chosen": -1.910211205482483, |
| "logits/rejected": -1.7320126295089722, |
| "logps/chosen": -270.5806579589844, |
| "logps/rejected": -301.3614807128906, |
| "loss": 0.0855, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 3.0857489109039307, |
| "rewards/margins": 14.666860580444336, |
| "rewards/rejected": -11.581114768981934, |
| "step": 8000 |
| }, |
| { |
| "epoch": 1.8298036085853546, |
| "grad_norm": 27.5, |
| "learning_rate": 1.40896023138628e-05, |
| "logits/chosen": -2.0649936199188232, |
| "logits/rejected": -1.8236372470855713, |
| "logps/chosen": -246.25279235839844, |
| "logps/rejected": -297.18585205078125, |
| "loss": 0.0984, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 2.6773693561553955, |
| "rewards/margins": 13.745685577392578, |
| "rewards/rejected": -11.068316459655762, |
| "step": 8025 |
| }, |
| { |
| "epoch": 1.835504375338483, |
| "grad_norm": 52.0, |
| "learning_rate": 1.4056894130610213e-05, |
| "logits/chosen": -2.3353066444396973, |
| "logits/rejected": -2.0363428592681885, |
| "logps/chosen": -230.9992218017578, |
| "logps/rejected": -287.4957275390625, |
| "loss": 0.1055, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 2.523937940597534, |
| "rewards/margins": 15.438687324523926, |
| "rewards/rejected": -12.914750099182129, |
| "step": 8050 |
| }, |
| { |
| "epoch": 1.8412051420916113, |
| "grad_norm": 118.0, |
| "learning_rate": 1.402413391233957e-05, |
| "logits/chosen": -1.9912445545196533, |
| "logits/rejected": -1.6964432001113892, |
| "logps/chosen": -235.52078247070312, |
| "logps/rejected": -293.48602294921875, |
| "loss": 0.1481, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 2.204087495803833, |
| "rewards/margins": 14.86922836303711, |
| "rewards/rejected": -12.665140151977539, |
| "step": 8075 |
| }, |
| { |
| "epoch": 1.8469059088447395, |
| "grad_norm": 246.0, |
| "learning_rate": 1.3991322079243879e-05, |
| "logits/chosen": -2.086920738220215, |
| "logits/rejected": -1.8497196435928345, |
| "logps/chosen": -226.25999450683594, |
| "logps/rejected": -273.4587097167969, |
| "loss": 0.2345, |
| "rewards/accuracies": 0.9300000071525574, |
| "rewards/chosen": 2.000837802886963, |
| "rewards/margins": 12.694421768188477, |
| "rewards/rejected": -10.693582534790039, |
| "step": 8100 |
| }, |
| { |
| "epoch": 1.852606675597868, |
| "grad_norm": 384.0, |
| "learning_rate": 1.3958459052178175e-05, |
| "logits/chosen": -2.2162396907806396, |
| "logits/rejected": -1.9216876029968262, |
| "logps/chosen": -230.62911987304688, |
| "logps/rejected": -287.9597473144531, |
| "loss": 0.1276, |
| "rewards/accuracies": 0.9300000071525574, |
| "rewards/chosen": 2.6894161701202393, |
| "rewards/margins": 14.399084091186523, |
| "rewards/rejected": -11.70966911315918, |
| "step": 8125 |
| }, |
| { |
| "epoch": 1.8583074423509962, |
| "grad_norm": 0.392578125, |
| "learning_rate": 1.3925545252654126e-05, |
| "logits/chosen": -2.0470008850097656, |
| "logits/rejected": -1.7049438953399658, |
| "logps/chosen": -259.9233093261719, |
| "logps/rejected": -320.90234375, |
| "loss": 0.0548, |
| "rewards/accuracies": 0.9800000190734863, |
| "rewards/chosen": 2.336545467376709, |
| "rewards/margins": 14.675707817077637, |
| "rewards/rejected": -12.339163780212402, |
| "step": 8150 |
| }, |
| { |
| "epoch": 1.8640082091041243, |
| "grad_norm": 3.640625, |
| "learning_rate": 1.389258110283462e-05, |
| "logits/chosen": -2.1936633586883545, |
| "logits/rejected": -1.8129730224609375, |
| "logps/chosen": -246.93276977539062, |
| "logps/rejected": -284.6800231933594, |
| "loss": 0.1008, |
| "rewards/accuracies": 0.9700000286102295, |
| "rewards/chosen": 2.605966091156006, |
| "rewards/margins": 13.6336669921875, |
| "rewards/rejected": -11.027701377868652, |
| "step": 8175 |
| }, |
| { |
| "epoch": 1.869708975857253, |
| "grad_norm": 135.0, |
| "learning_rate": 1.3859567025528362e-05, |
| "logits/chosen": -2.356243848800659, |
| "logits/rejected": -2.024355411529541, |
| "logps/chosen": -226.45521545410156, |
| "logps/rejected": -276.6752624511719, |
| "loss": 0.1864, |
| "rewards/accuracies": 0.9449999928474426, |
| "rewards/chosen": 1.5739303827285767, |
| "rewards/margins": 12.86932373046875, |
| "rewards/rejected": -11.295392990112305, |
| "step": 8200 |
| }, |
| { |
| "epoch": 1.875409742610381, |
| "grad_norm": 9.5, |
| "learning_rate": 1.3826503444184429e-05, |
| "logits/chosen": -2.0198090076446533, |
| "logits/rejected": -1.7975252866744995, |
| "logps/chosen": -223.4656982421875, |
| "logps/rejected": -288.279052734375, |
| "loss": 0.0985, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.11321759223938, |
| "rewards/margins": 14.056639671325684, |
| "rewards/rejected": -11.943421363830566, |
| "step": 8225 |
| }, |
| { |
| "epoch": 1.8811105093635094, |
| "grad_norm": 384.0, |
| "learning_rate": 1.3793390782886868e-05, |
| "logits/chosen": -1.7256178855895996, |
| "logits/rejected": -1.3665509223937988, |
| "logps/chosen": -271.695556640625, |
| "logps/rejected": -313.2015686035156, |
| "loss": 0.1011, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 2.7012927532196045, |
| "rewards/margins": 13.311800956726074, |
| "rewards/rejected": -10.61050796508789, |
| "step": 8250 |
| }, |
| { |
| "epoch": 1.8868112761166378, |
| "grad_norm": 2.484375, |
| "learning_rate": 1.376022946634923e-05, |
| "logits/chosen": -2.2418694496154785, |
| "logits/rejected": -1.917104959487915, |
| "logps/chosen": -225.27407836914062, |
| "logps/rejected": -290.65203857421875, |
| "loss": 0.11, |
| "rewards/accuracies": 0.9449999928474426, |
| "rewards/chosen": 2.867067337036133, |
| "rewards/margins": 13.970172882080078, |
| "rewards/rejected": -11.103105545043945, |
| "step": 8275 |
| }, |
| { |
| "epoch": 1.892512042869766, |
| "grad_norm": 0.0296630859375, |
| "learning_rate": 1.372701991990914e-05, |
| "logits/chosen": -2.0289347171783447, |
| "logits/rejected": -1.815872311592102, |
| "logps/chosen": -271.4940185546875, |
| "logps/rejected": -315.0787048339844, |
| "loss": 0.1032, |
| "rewards/accuracies": 0.9599999785423279, |
| "rewards/chosen": 2.9256272315979004, |
| "rewards/margins": 14.312517166137695, |
| "rewards/rejected": -11.386889457702637, |
| "step": 8300 |
| }, |
| { |
| "epoch": 1.8982128096228943, |
| "grad_norm": 73.5, |
| "learning_rate": 1.3693762569522833e-05, |
| "logits/chosen": -2.1043567657470703, |
| "logits/rejected": -1.8467495441436768, |
| "logps/chosen": -207.53750610351562, |
| "logps/rejected": -260.2503967285156, |
| "loss": 0.1383, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.223466396331787, |
| "rewards/margins": 14.125040054321289, |
| "rewards/rejected": -11.901573181152344, |
| "step": 8325 |
| }, |
| { |
| "epoch": 1.9039135763760227, |
| "grad_norm": 346.0, |
| "learning_rate": 1.3660457841759691e-05, |
| "logits/chosen": -1.9220653772354126, |
| "logits/rejected": -1.6327801942825317, |
| "logps/chosen": -250.41867065429688, |
| "logps/rejected": -301.1881408691406, |
| "loss": 0.1374, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.2271950244903564, |
| "rewards/margins": 13.532753944396973, |
| "rewards/rejected": -11.305559158325195, |
| "step": 8350 |
| }, |
| { |
| "epoch": 1.9096143431291508, |
| "grad_norm": 36.5, |
| "learning_rate": 1.3627106163796775e-05, |
| "logits/chosen": -2.018688917160034, |
| "logits/rejected": -1.878637433052063, |
| "logps/chosen": -220.7308807373047, |
| "logps/rejected": -265.2090148925781, |
| "loss": 0.1122, |
| "rewards/accuracies": 0.9449999928474426, |
| "rewards/chosen": 1.9046249389648438, |
| "rewards/margins": 12.441152572631836, |
| "rewards/rejected": -10.53652572631836, |
| "step": 8375 |
| }, |
| { |
| "epoch": 1.9153151098822792, |
| "grad_norm": 432.0, |
| "learning_rate": 1.3593707963413342e-05, |
| "logits/chosen": -1.998526930809021, |
| "logits/rejected": -1.6480460166931152, |
| "logps/chosen": -224.95663452148438, |
| "logps/rejected": -284.6125183105469, |
| "loss": 0.1299, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.88581120967865, |
| "rewards/margins": 13.219887733459473, |
| "rewards/rejected": -11.334074974060059, |
| "step": 8400 |
| }, |
| { |
| "epoch": 1.9210158766354075, |
| "grad_norm": 4.15625, |
| "learning_rate": 1.3560263668985366e-05, |
| "logits/chosen": -2.1739501953125, |
| "logits/rejected": -1.9797008037567139, |
| "logps/chosen": -252.2518310546875, |
| "logps/rejected": -306.0657958984375, |
| "loss": 0.121, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 3.656425714492798, |
| "rewards/margins": 15.10020637512207, |
| "rewards/rejected": -11.443780899047852, |
| "step": 8425 |
| }, |
| { |
| "epoch": 1.9267166433885357, |
| "grad_norm": 183.0, |
| "learning_rate": 1.3526773709480035e-05, |
| "logits/chosen": -1.8325295448303223, |
| "logits/rejected": -1.5882664918899536, |
| "logps/chosen": -260.21319580078125, |
| "logps/rejected": -294.474365234375, |
| "loss": 0.1327, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 2.3948607444763184, |
| "rewards/margins": 12.935813903808594, |
| "rewards/rejected": -10.540953636169434, |
| "step": 8450 |
| }, |
| { |
| "epoch": 1.932417410141664, |
| "grad_norm": 10.75, |
| "learning_rate": 1.3493238514450245e-05, |
| "logits/chosen": -2.0013108253479004, |
| "logits/rejected": -1.6929172277450562, |
| "logps/chosen": -238.84693908691406, |
| "logps/rejected": -284.284912109375, |
| "loss": 0.1217, |
| "rewards/accuracies": 0.9649999737739563, |
| "rewards/chosen": 2.1962409019470215, |
| "rewards/margins": 12.911901473999023, |
| "rewards/rejected": -10.715659141540527, |
| "step": 8475 |
| }, |
| { |
| "epoch": 1.9381181768947924, |
| "grad_norm": 68.0, |
| "learning_rate": 1.3459658514029111e-05, |
| "logits/chosen": -1.9809402227401733, |
| "logits/rejected": -1.6711370944976807, |
| "logps/chosen": -261.8890686035156, |
| "logps/rejected": -300.2699890136719, |
| "loss": 0.1227, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 3.1035964488983154, |
| "rewards/margins": 13.04238510131836, |
| "rewards/rejected": -9.938789367675781, |
| "step": 8500 |
| }, |
| { |
| "epoch": 1.9438189436479205, |
| "grad_norm": 262.0, |
| "learning_rate": 1.3426034138924423e-05, |
| "logits/chosen": -2.015519380569458, |
| "logits/rejected": -1.6916810274124146, |
| "logps/chosen": -251.7118377685547, |
| "logps/rejected": -295.4588623046875, |
| "loss": 0.0827, |
| "rewards/accuracies": 0.9700000286102295, |
| "rewards/chosen": 2.805845022201538, |
| "rewards/margins": 13.242785453796387, |
| "rewards/rejected": -10.436941146850586, |
| "step": 8525 |
| }, |
| { |
| "epoch": 1.949519710401049, |
| "grad_norm": 96.5, |
| "learning_rate": 1.3392365820413143e-05, |
| "logits/chosen": -1.6579842567443848, |
| "logits/rejected": -1.5407274961471558, |
| "logps/chosen": -248.76759338378906, |
| "logps/rejected": -280.9204406738281, |
| "loss": 0.0909, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 2.7339792251586914, |
| "rewards/margins": 11.091508865356445, |
| "rewards/rejected": -8.35753059387207, |
| "step": 8550 |
| }, |
| { |
| "epoch": 1.9552204771541772, |
| "grad_norm": 122.5, |
| "learning_rate": 1.3358653990335863e-05, |
| "logits/chosen": -1.9517282247543335, |
| "logits/rejected": -1.7671226263046265, |
| "logps/chosen": -215.39657592773438, |
| "logps/rejected": -267.6592102050781, |
| "loss": 0.0797, |
| "rewards/accuracies": 0.9800000190734863, |
| "rewards/chosen": 2.8591017723083496, |
| "rewards/margins": 13.642084121704102, |
| "rewards/rejected": -10.782980918884277, |
| "step": 8575 |
| }, |
| { |
| "epoch": 1.9609212439073054, |
| "grad_norm": 7.4375, |
| "learning_rate": 1.3324899081091264e-05, |
| "logits/chosen": -1.9911832809448242, |
| "logits/rejected": -1.691896677017212, |
| "logps/chosen": -246.5900421142578, |
| "logps/rejected": -282.4193420410156, |
| "loss": 0.1266, |
| "rewards/accuracies": 0.9449999928474426, |
| "rewards/chosen": 2.503645658493042, |
| "rewards/margins": 12.673896789550781, |
| "rewards/rejected": -10.170249938964844, |
| "step": 8600 |
| }, |
| { |
| "epoch": 1.966622010660434, |
| "grad_norm": 0.00072479248046875, |
| "learning_rate": 1.3291101525630583e-05, |
| "logits/chosen": -2.085864305496216, |
| "logits/rejected": -1.8053228855133057, |
| "logps/chosen": -256.64776611328125, |
| "logps/rejected": -303.3675537109375, |
| "loss": 0.1999, |
| "rewards/accuracies": 0.9200000166893005, |
| "rewards/chosen": 2.869741201400757, |
| "rewards/margins": 13.292259216308594, |
| "rewards/rejected": -10.422518730163574, |
| "step": 8625 |
| }, |
| { |
| "epoch": 1.972322777413562, |
| "grad_norm": 260.0, |
| "learning_rate": 1.3257261757452046e-05, |
| "logits/chosen": -2.0077784061431885, |
| "logits/rejected": -1.747369408607483, |
| "logps/chosen": -250.4134521484375, |
| "logps/rejected": -286.2821350097656, |
| "loss": 0.1477, |
| "rewards/accuracies": 0.9449999928474426, |
| "rewards/chosen": 2.5728821754455566, |
| "rewards/margins": 13.076778411865234, |
| "rewards/rejected": -10.503894805908203, |
| "step": 8650 |
| }, |
| { |
| "epoch": 1.9780235441666905, |
| "grad_norm": 217.0, |
| "learning_rate": 1.322338021059531e-05, |
| "logits/chosen": -1.976699709892273, |
| "logits/rejected": -1.7259711027145386, |
| "logps/chosen": -246.50457763671875, |
| "logps/rejected": -300.0041198730469, |
| "loss": 0.1487, |
| "rewards/accuracies": 0.9399999976158142, |
| "rewards/chosen": 2.4680352210998535, |
| "rewards/margins": 12.91933822631836, |
| "rewards/rejected": -10.451303482055664, |
| "step": 8675 |
| }, |
| { |
| "epoch": 1.9837243109198188, |
| "grad_norm": 288.0, |
| "learning_rate": 1.3189457319635901e-05, |
| "logits/chosen": -2.0450527667999268, |
| "logits/rejected": -1.6978213787078857, |
| "logps/chosen": -226.27236938476562, |
| "logps/rejected": -275.918212890625, |
| "loss": 0.1089, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.395735740661621, |
| "rewards/margins": 12.083142280578613, |
| "rewards/rejected": -9.687406539916992, |
| "step": 8700 |
| }, |
| { |
| "epoch": 1.989425077672947, |
| "grad_norm": 72.0, |
| "learning_rate": 1.315549351967964e-05, |
| "logits/chosen": -1.6967864036560059, |
| "logits/rejected": -1.4557839632034302, |
| "logps/chosen": -270.1540222167969, |
| "logps/rejected": -282.2798156738281, |
| "loss": 0.059, |
| "rewards/accuracies": 0.9850000143051147, |
| "rewards/chosen": 3.5456576347351074, |
| "rewards/margins": 13.195769309997559, |
| "rewards/rejected": -9.650110244750977, |
| "step": 8725 |
| }, |
| { |
| "epoch": 1.9951258444260753, |
| "grad_norm": 1.09375, |
| "learning_rate": 1.312148924635706e-05, |
| "logits/chosen": -2.0327582359313965, |
| "logits/rejected": -1.7850172519683838, |
| "logps/chosen": -236.4394989013672, |
| "logps/rejected": -279.5561828613281, |
| "loss": 0.0915, |
| "rewards/accuracies": 0.9549999833106995, |
| "rewards/chosen": 3.0190200805664062, |
| "rewards/margins": 13.890503883361816, |
| "rewards/rejected": -10.871484756469727, |
| "step": 8750 |
| }, |
| { |
| "epoch": 2.0006840920103754, |
| "grad_norm": 0.103515625, |
| "learning_rate": 1.3087444935817807e-05, |
| "logits/chosen": -1.6433484554290771, |
| "logits/rejected": -1.441351294517517, |
| "logps/chosen": -273.1808166503906, |
| "logps/rejected": -296.0864562988281, |
| "loss": 0.0968, |
| "rewards/accuracies": 0.9743589758872986, |
| "rewards/chosen": 3.6085171699523926, |
| "rewards/margins": 13.520092010498047, |
| "rewards/rejected": -9.91157341003418, |
| "step": 8775 |
| }, |
| { |
| "epoch": 2.0063848587635036, |
| "grad_norm": 0.01116943359375, |
| "learning_rate": 1.3053361024725077e-05, |
| "logits/chosen": -1.8738739490509033, |
| "logits/rejected": -1.580491304397583, |
| "logps/chosen": -244.27285766601562, |
| "logps/rejected": -290.1165466308594, |
| "loss": 0.0083, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 4.263757228851318, |
| "rewards/margins": 15.660774230957031, |
| "rewards/rejected": -11.397017478942871, |
| "step": 8800 |
| }, |
| { |
| "epoch": 2.012085625516632, |
| "grad_norm": 2.53125, |
| "learning_rate": 1.3019237950249984e-05, |
| "logits/chosen": -1.988223910331726, |
| "logits/rejected": -1.7000367641448975, |
| "logps/chosen": -247.82696533203125, |
| "logps/rejected": -305.9439697265625, |
| "loss": 0.0065, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.689967155456543, |
| "rewards/margins": 15.384774208068848, |
| "rewards/rejected": -11.694807052612305, |
| "step": 8825 |
| }, |
| { |
| "epoch": 2.0177863922697603, |
| "grad_norm": 11.3125, |
| "learning_rate": 1.2985076150065959e-05, |
| "logits/chosen": -1.863277792930603, |
| "logits/rejected": -1.6423251628875732, |
| "logps/chosen": -261.5575866699219, |
| "logps/rejected": -310.3473205566406, |
| "loss": 0.0085, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 4.1325201988220215, |
| "rewards/margins": 15.035168647766113, |
| "rewards/rejected": -10.902647972106934, |
| "step": 8850 |
| }, |
| { |
| "epoch": 2.0234871590228884, |
| "grad_norm": 2.1875, |
| "learning_rate": 1.2950876062343147e-05, |
| "logits/chosen": -1.7462306022644043, |
| "logits/rejected": -1.6066529750823975, |
| "logps/chosen": -254.12863159179688, |
| "logps/rejected": -295.3841552734375, |
| "loss": 0.0051, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 4.026098728179932, |
| "rewards/margins": 14.60362434387207, |
| "rewards/rejected": -10.577526092529297, |
| "step": 8875 |
| }, |
| { |
| "epoch": 2.029187925776017, |
| "grad_norm": 1.5234375, |
| "learning_rate": 1.2916638125742786e-05, |
| "logits/chosen": -1.9403685331344604, |
| "logits/rejected": -1.6443599462509155, |
| "logps/chosen": -244.88809204101562, |
| "logps/rejected": -306.5077209472656, |
| "loss": 0.0048, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 4.403689384460449, |
| "rewards/margins": 16.318897247314453, |
| "rewards/rejected": -11.915207862854004, |
| "step": 8900 |
| }, |
| { |
| "epoch": 2.034888692529145, |
| "grad_norm": 1.828125, |
| "learning_rate": 1.2882362779411568e-05, |
| "logits/chosen": -2.0906968116760254, |
| "logits/rejected": -1.9488120079040527, |
| "logps/chosen": -235.38775634765625, |
| "logps/rejected": -273.1040344238281, |
| "loss": 0.0153, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 3.671297073364258, |
| "rewards/margins": 15.871660232543945, |
| "rewards/rejected": -12.200364112854004, |
| "step": 8925 |
| }, |
| { |
| "epoch": 2.0405894592822733, |
| "grad_norm": 2.5, |
| "learning_rate": 1.2848050462976018e-05, |
| "logits/chosen": -2.311669111251831, |
| "logits/rejected": -2.038748025894165, |
| "logps/chosen": -246.0143585205078, |
| "logps/rejected": -294.2733154296875, |
| "loss": 0.006, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.995401620864868, |
| "rewards/margins": 16.378503799438477, |
| "rewards/rejected": -12.38310432434082, |
| "step": 8950 |
| }, |
| { |
| "epoch": 2.046290226035402, |
| "grad_norm": 5.28125, |
| "learning_rate": 1.2813701616536846e-05, |
| "logits/chosen": -2.0229098796844482, |
| "logits/rejected": -1.8306366205215454, |
| "logps/chosen": -257.13311767578125, |
| "logps/rejected": -289.5687561035156, |
| "loss": 0.0084, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.134369134902954, |
| "rewards/margins": 14.71181869506836, |
| "rewards/rejected": -11.577448844909668, |
| "step": 8975 |
| }, |
| { |
| "epoch": 2.05199099278853, |
| "grad_norm": 11.125, |
| "learning_rate": 1.277931668066332e-05, |
| "logits/chosen": -2.010993003845215, |
| "logits/rejected": -1.7618767023086548, |
| "logps/chosen": -220.96163940429688, |
| "logps/rejected": -282.500732421875, |
| "loss": 0.0111, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.3241493701934814, |
| "rewards/margins": 14.812151908874512, |
| "rewards/rejected": -11.488003730773926, |
| "step": 9000 |
| }, |
| { |
| "epoch": 2.057691759541658, |
| "grad_norm": 0.43359375, |
| "learning_rate": 1.274489609638759e-05, |
| "logits/chosen": -2.3063085079193115, |
| "logits/rejected": -2.0254251956939697, |
| "logps/chosen": -233.05796813964844, |
| "logps/rejected": -280.7294921875, |
| "loss": 0.0114, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 3.0705111026763916, |
| "rewards/margins": 16.323986053466797, |
| "rewards/rejected": -13.2534761428833, |
| "step": 9025 |
| }, |
| { |
| "epoch": 2.0633925262947868, |
| "grad_norm": 3.75, |
| "learning_rate": 1.271044030519905e-05, |
| "logits/chosen": -2.113292932510376, |
| "logits/rejected": -1.7858095169067383, |
| "logps/chosen": -235.46705627441406, |
| "logps/rejected": -271.095703125, |
| "loss": 0.0051, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.4772655963897705, |
| "rewards/margins": 15.449165344238281, |
| "rewards/rejected": -11.971900939941406, |
| "step": 9050 |
| }, |
| { |
| "epoch": 2.069093293047915, |
| "grad_norm": 2.65625, |
| "learning_rate": 1.2675949749038663e-05, |
| "logits/chosen": -2.4397528171539307, |
| "logits/rejected": -2.14162278175354, |
| "logps/chosen": -247.75503540039062, |
| "logps/rejected": -300.3695068359375, |
| "loss": 0.0258, |
| "rewards/accuracies": 0.9900000095367432, |
| "rewards/chosen": 3.116487979888916, |
| "rewards/margins": 15.763169288635254, |
| "rewards/rejected": -12.646681785583496, |
| "step": 9075 |
| }, |
| { |
| "epoch": 2.074794059801043, |
| "grad_norm": 2.203125, |
| "learning_rate": 1.2641424870293308e-05, |
| "logits/chosen": -2.467498540878296, |
| "logits/rejected": -2.139953374862671, |
| "logps/chosen": -263.7476806640625, |
| "logps/rejected": -301.3026123046875, |
| "loss": 0.0046, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.196593999862671, |
| "rewards/margins": 17.082901000976562, |
| "rewards/rejected": -13.886308670043945, |
| "step": 9100 |
| }, |
| { |
| "epoch": 2.0804948265541716, |
| "grad_norm": 128.0, |
| "learning_rate": 1.260686611179009e-05, |
| "logits/chosen": -2.336014986038208, |
| "logits/rejected": -2.0574989318847656, |
| "logps/chosen": -241.77401733398438, |
| "logps/rejected": -298.9541320800781, |
| "loss": 0.0254, |
| "rewards/accuracies": 0.9900000095367432, |
| "rewards/chosen": 3.3185479640960693, |
| "rewards/margins": 16.75035858154297, |
| "rewards/rejected": -13.431809425354004, |
| "step": 9125 |
| }, |
| { |
| "epoch": 2.0861955933072998, |
| "grad_norm": 47.5, |
| "learning_rate": 1.2572273916790665e-05, |
| "logits/chosen": -2.207322359085083, |
| "logits/rejected": -1.9621540307998657, |
| "logps/chosen": -232.20187377929688, |
| "logps/rejected": -254.96359252929688, |
| "loss": 0.0119, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 3.587965726852417, |
| "rewards/margins": 15.707365989685059, |
| "rewards/rejected": -12.119399070739746, |
| "step": 9150 |
| }, |
| { |
| "epoch": 2.0918963600604283, |
| "grad_norm": 1.171875, |
| "learning_rate": 1.2537648728985565e-05, |
| "logits/chosen": -2.4663329124450684, |
| "logits/rejected": -2.183616876602173, |
| "logps/chosen": -246.96670532226562, |
| "logps/rejected": -313.9317932128906, |
| "loss": 0.0074, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.4554989337921143, |
| "rewards/margins": 16.177335739135742, |
| "rewards/rejected": -13.721837043762207, |
| "step": 9175 |
| }, |
| { |
| "epoch": 2.0975971268135565, |
| "grad_norm": 16.25, |
| "learning_rate": 1.2502990992488486e-05, |
| "logits/chosen": -2.3707082271575928, |
| "logits/rejected": -2.080127239227295, |
| "logps/chosen": -237.3252716064453, |
| "logps/rejected": -301.4833679199219, |
| "loss": 0.0089, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.764784574508667, |
| "rewards/margins": 18.60264015197754, |
| "rewards/rejected": -14.837857246398926, |
| "step": 9200 |
| }, |
| { |
| "epoch": 2.1032978935666846, |
| "grad_norm": 5.65625, |
| "learning_rate": 1.2468301151830617e-05, |
| "logits/chosen": -2.441214084625244, |
| "logits/rejected": -2.0738654136657715, |
| "logps/chosen": -225.4950408935547, |
| "logps/rejected": -302.107666015625, |
| "loss": 0.0038, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.54949688911438, |
| "rewards/margins": 17.444049835205078, |
| "rewards/rejected": -14.894554138183594, |
| "step": 9225 |
| }, |
| { |
| "epoch": 2.108998660319813, |
| "grad_norm": 16.875, |
| "learning_rate": 1.2433579651954915e-05, |
| "logits/chosen": -1.9151206016540527, |
| "logits/rejected": -1.7221667766571045, |
| "logps/chosen": -265.61865234375, |
| "logps/rejected": -320.0135192871094, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 3.5291004180908203, |
| "rewards/margins": 15.784294128417969, |
| "rewards/rejected": -12.255193710327148, |
| "step": 9250 |
| }, |
| { |
| "epoch": 2.1146994270729413, |
| "grad_norm": 4.34375, |
| "learning_rate": 1.2398826938210408e-05, |
| "logits/chosen": -2.2539315223693848, |
| "logits/rejected": -2.0502426624298096, |
| "logps/chosen": -242.580078125, |
| "logps/rejected": -327.8447570800781, |
| "loss": 0.0155, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8473329544067383, |
| "rewards/margins": 16.492361068725586, |
| "rewards/rejected": -13.645027160644531, |
| "step": 9275 |
| }, |
| { |
| "epoch": 2.1204001938260695, |
| "grad_norm": 1.8671875, |
| "learning_rate": 1.2364043456346493e-05, |
| "logits/chosen": -2.5342278480529785, |
| "logits/rejected": -2.3017780780792236, |
| "logps/chosen": -255.255859375, |
| "logps/rejected": -300.1855773925781, |
| "loss": 0.003, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.531118154525757, |
| "rewards/margins": 17.513072967529297, |
| "rewards/rejected": -13.981954574584961, |
| "step": 9300 |
| }, |
| { |
| "epoch": 2.126100960579198, |
| "grad_norm": 5.364418029785156e-05, |
| "learning_rate": 1.2329229652507199e-05, |
| "logits/chosen": -2.4392752647399902, |
| "logits/rejected": -2.082689046859741, |
| "logps/chosen": -231.39601135253906, |
| "logps/rejected": -305.51141357421875, |
| "loss": 0.0072, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8143885135650635, |
| "rewards/margins": 17.79964828491211, |
| "rewards/rejected": -14.985260009765625, |
| "step": 9325 |
| }, |
| { |
| "epoch": 2.131801727332326, |
| "grad_norm": 0.94140625, |
| "learning_rate": 1.229438597322548e-05, |
| "logits/chosen": -2.3230245113372803, |
| "logits/rejected": -2.0473361015319824, |
| "logps/chosen": -245.57974243164062, |
| "logps/rejected": -281.0194396972656, |
| "loss": 0.0092, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 3.5029819011688232, |
| "rewards/margins": 17.410547256469727, |
| "rewards/rejected": -13.90756607055664, |
| "step": 9350 |
| }, |
| { |
| "epoch": 2.1375024940854543, |
| "grad_norm": 90.5, |
| "learning_rate": 1.2259512865417478e-05, |
| "logits/chosen": -2.735585927963257, |
| "logits/rejected": -2.486250400543213, |
| "logps/chosen": -207.12779235839844, |
| "logps/rejected": -277.7867431640625, |
| "loss": 0.0057, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.455670118331909, |
| "rewards/margins": 17.534330368041992, |
| "rewards/rejected": -14.07866096496582, |
| "step": 9375 |
| }, |
| { |
| "epoch": 2.143203260838583, |
| "grad_norm": 6.71875, |
| "learning_rate": 1.2224610776376797e-05, |
| "logits/chosen": -2.5794248580932617, |
| "logits/rejected": -2.2541115283966064, |
| "logps/chosen": -243.10426330566406, |
| "logps/rejected": -317.0326232910156, |
| "loss": 0.0031, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.939870595932007, |
| "rewards/margins": 18.44550132751465, |
| "rewards/rejected": -15.505631446838379, |
| "step": 9400 |
| }, |
| { |
| "epoch": 2.148904027591711, |
| "grad_norm": 0.036376953125, |
| "learning_rate": 1.2189680153768764e-05, |
| "logits/chosen": -2.411877155303955, |
| "logits/rejected": -2.111809730529785, |
| "logps/chosen": -237.30917358398438, |
| "logps/rejected": -297.800048828125, |
| "loss": 0.0167, |
| "rewards/accuracies": 0.9900000095367432, |
| "rewards/chosen": 2.9920904636383057, |
| "rewards/margins": 16.064294815063477, |
| "rewards/rejected": -13.072205543518066, |
| "step": 9425 |
| }, |
| { |
| "epoch": 2.154604794344839, |
| "grad_norm": 1.1328125, |
| "learning_rate": 1.2154721445624687e-05, |
| "logits/chosen": -2.60996675491333, |
| "logits/rejected": -2.3409676551818848, |
| "logps/chosen": -255.41358947753906, |
| "logps/rejected": -314.8902587890625, |
| "loss": 0.011, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.9025919437408447, |
| "rewards/margins": 18.217742919921875, |
| "rewards/rejected": -15.315152168273926, |
| "step": 9450 |
| }, |
| { |
| "epoch": 2.160305561097968, |
| "grad_norm": 49.75, |
| "learning_rate": 1.21197351003361e-05, |
| "logits/chosen": -2.64809513092041, |
| "logits/rejected": -2.417316198348999, |
| "logps/chosen": -258.0915222167969, |
| "logps/rejected": -300.19573974609375, |
| "loss": 0.0162, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 3.1944146156311035, |
| "rewards/margins": 17.184829711914062, |
| "rewards/rejected": -13.990413665771484, |
| "step": 9475 |
| }, |
| { |
| "epoch": 2.166006327851096, |
| "grad_norm": 8.0625, |
| "learning_rate": 1.2084721566649033e-05, |
| "logits/chosen": -2.1573989391326904, |
| "logits/rejected": -1.8981646299362183, |
| "logps/chosen": -280.8773193359375, |
| "logps/rejected": -303.4893798828125, |
| "loss": 0.0062, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.1069250106811523, |
| "rewards/margins": 15.502786636352539, |
| "rewards/rejected": -12.395862579345703, |
| "step": 9500 |
| }, |
| { |
| "epoch": 2.171707094604224, |
| "grad_norm": 0.05810546875, |
| "learning_rate": 1.2049681293658233e-05, |
| "logits/chosen": -2.5380330085754395, |
| "logits/rejected": -2.2112014293670654, |
| "logps/chosen": -234.0776824951172, |
| "logps/rejected": -300.87548828125, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.9195189476013184, |
| "rewards/margins": 16.195554733276367, |
| "rewards/rejected": -13.276036262512207, |
| "step": 9525 |
| }, |
| { |
| "epoch": 2.1774078613573526, |
| "grad_norm": 0.1416015625, |
| "learning_rate": 1.201461473080142e-05, |
| "logits/chosen": -2.4986467361450195, |
| "logits/rejected": -2.262061357498169, |
| "logps/chosen": -246.8690185546875, |
| "logps/rejected": -284.34735107421875, |
| "loss": 0.0105, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.721853733062744, |
| "rewards/margins": 17.29242515563965, |
| "rewards/rejected": -14.570570945739746, |
| "step": 9550 |
| }, |
| { |
| "epoch": 2.183108628110481, |
| "grad_norm": 4.0, |
| "learning_rate": 1.1979522327853514e-05, |
| "logits/chosen": -2.453469753265381, |
| "logits/rejected": -2.1603918075561523, |
| "logps/chosen": -262.1788330078125, |
| "logps/rejected": -285.7619323730469, |
| "loss": 0.0054, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.3857316970825195, |
| "rewards/margins": 15.468609809875488, |
| "rewards/rejected": -12.082880020141602, |
| "step": 9575 |
| }, |
| { |
| "epoch": 2.1888093948636094, |
| "grad_norm": 0.16015625, |
| "learning_rate": 1.1944404534920864e-05, |
| "logits/chosen": -2.1470394134521484, |
| "logits/rejected": -1.9332739114761353, |
| "logps/chosen": -248.39537048339844, |
| "logps/rejected": -292.4302978515625, |
| "loss": 0.0086, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.763734817504883, |
| "rewards/margins": 15.512297630310059, |
| "rewards/rejected": -12.74856185913086, |
| "step": 9600 |
| }, |
| { |
| "epoch": 2.1945101616167375, |
| "grad_norm": 18.75, |
| "learning_rate": 1.1909261802435485e-05, |
| "logits/chosen": -2.7159311771392822, |
| "logits/rejected": -2.4957785606384277, |
| "logps/chosen": -243.65638732910156, |
| "logps/rejected": -298.7859802246094, |
| "loss": 0.0089, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.9943935871124268, |
| "rewards/margins": 16.801027297973633, |
| "rewards/rejected": -13.806632995605469, |
| "step": 9625 |
| }, |
| { |
| "epoch": 2.2002109283698656, |
| "grad_norm": 0.85546875, |
| "learning_rate": 1.1874094581149275e-05, |
| "logits/chosen": -2.338468074798584, |
| "logits/rejected": -2.020089626312256, |
| "logps/chosen": -256.72265625, |
| "logps/rejected": -292.9359436035156, |
| "loss": 0.0042, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.846083641052246, |
| "rewards/margins": 16.0994930267334, |
| "rewards/rejected": -13.253408432006836, |
| "step": 9650 |
| }, |
| { |
| "epoch": 2.2059116951229942, |
| "grad_norm": 0.64453125, |
| "learning_rate": 1.1838903322128228e-05, |
| "logits/chosen": -2.5586507320404053, |
| "logits/rejected": -2.2193734645843506, |
| "logps/chosen": -242.12037658691406, |
| "logps/rejected": -299.1712341308594, |
| "loss": 0.0245, |
| "rewards/accuracies": 0.9900000095367432, |
| "rewards/chosen": 2.9592761993408203, |
| "rewards/margins": 18.15392303466797, |
| "rewards/rejected": -15.194648742675781, |
| "step": 9675 |
| }, |
| { |
| "epoch": 2.2116124618761224, |
| "grad_norm": 0.734375, |
| "learning_rate": 1.1803688476746656e-05, |
| "logits/chosen": -2.588124990463257, |
| "logits/rejected": -2.3092784881591797, |
| "logps/chosen": -245.51832580566406, |
| "logps/rejected": -295.56695556640625, |
| "loss": 0.0025, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.923919677734375, |
| "rewards/margins": 18.11002540588379, |
| "rewards/rejected": -15.186105728149414, |
| "step": 9700 |
| }, |
| { |
| "epoch": 2.2173132286292505, |
| "grad_norm": 4.125, |
| "learning_rate": 1.1768450496681398e-05, |
| "logits/chosen": -2.5620388984680176, |
| "logits/rejected": -2.246670722961426, |
| "logps/chosen": -250.02664184570312, |
| "logps/rejected": -317.2994079589844, |
| "loss": 0.0061, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.4509410858154297, |
| "rewards/margins": 17.20501708984375, |
| "rewards/rejected": -14.754073143005371, |
| "step": 9725 |
| }, |
| { |
| "epoch": 2.223013995382379, |
| "grad_norm": 2.421875, |
| "learning_rate": 1.1733189833906027e-05, |
| "logits/chosen": -2.357501268386841, |
| "logits/rejected": -2.1167967319488525, |
| "logps/chosen": -253.17510986328125, |
| "logps/rejected": -305.1148376464844, |
| "loss": 0.0057, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.1050448417663574, |
| "rewards/margins": 15.851456642150879, |
| "rewards/rejected": -12.746411323547363, |
| "step": 9750 |
| }, |
| { |
| "epoch": 2.2287147621355072, |
| "grad_norm": 2.328125, |
| "learning_rate": 1.169790694068505e-05, |
| "logits/chosen": -2.4859251976013184, |
| "logits/rejected": -2.2429864406585693, |
| "logps/chosen": -253.77841186523438, |
| "logps/rejected": -303.6786804199219, |
| "loss": 0.0103, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.409817695617676, |
| "rewards/margins": 17.642030715942383, |
| "rewards/rejected": -15.232213973999023, |
| "step": 9775 |
| }, |
| { |
| "epoch": 2.2344155288886354, |
| "grad_norm": 0.08935546875, |
| "learning_rate": 1.1662602269568106e-05, |
| "logits/chosen": -2.463902473449707, |
| "logits/rejected": -2.2587900161743164, |
| "logps/chosen": -234.72341918945312, |
| "logps/rejected": -277.3719787597656, |
| "loss": 0.005, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.3745715618133545, |
| "rewards/margins": 16.92755126953125, |
| "rewards/rejected": -14.552979469299316, |
| "step": 9800 |
| }, |
| { |
| "epoch": 2.240116295641764, |
| "grad_norm": 14.5625, |
| "learning_rate": 1.1627276273384172e-05, |
| "logits/chosen": -2.382319927215576, |
| "logits/rejected": -2.2043912410736084, |
| "logps/chosen": -241.7247314453125, |
| "logps/rejected": -300.5317687988281, |
| "loss": 0.0035, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.0352718830108643, |
| "rewards/margins": 17.514387130737305, |
| "rewards/rejected": -14.479113578796387, |
| "step": 9825 |
| }, |
| { |
| "epoch": 2.245817062394892, |
| "grad_norm": 16.625, |
| "learning_rate": 1.1591929405235738e-05, |
| "logits/chosen": -2.452101230621338, |
| "logits/rejected": -2.068936347961426, |
| "logps/chosen": -248.2640380859375, |
| "logps/rejected": -304.6524658203125, |
| "loss": 0.0028, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.455113410949707, |
| "rewards/margins": 18.65233612060547, |
| "rewards/rejected": -15.197220802307129, |
| "step": 9850 |
| }, |
| { |
| "epoch": 2.2515178291480202, |
| "grad_norm": 0.263671875, |
| "learning_rate": 1.1556562118493006e-05, |
| "logits/chosen": -2.188223361968994, |
| "logits/rejected": -1.9161121845245361, |
| "logps/chosen": -263.90740966796875, |
| "logps/rejected": -302.1184387207031, |
| "loss": 0.0096, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.804036855697632, |
| "rewards/margins": 16.26076889038086, |
| "rewards/rejected": -13.456730842590332, |
| "step": 9875 |
| }, |
| { |
| "epoch": 2.257218595901149, |
| "grad_norm": 1.578125, |
| "learning_rate": 1.1521174866788083e-05, |
| "logits/chosen": -2.4245800971984863, |
| "logits/rejected": -2.159546136856079, |
| "logps/chosen": -262.8552551269531, |
| "logps/rejected": -299.6190490722656, |
| "loss": 0.0094, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 3.0432627201080322, |
| "rewards/margins": 16.80018424987793, |
| "rewards/rejected": -13.756921768188477, |
| "step": 9900 |
| }, |
| { |
| "epoch": 2.262919362654277, |
| "grad_norm": 14.0625, |
| "learning_rate": 1.1485768104009141e-05, |
| "logits/chosen": -2.5439891815185547, |
| "logits/rejected": -2.2792105674743652, |
| "logps/chosen": -225.77540588378906, |
| "logps/rejected": -277.60430908203125, |
| "loss": 0.0268, |
| "rewards/accuracies": 0.9900000095367432, |
| "rewards/chosen": 2.9427313804626465, |
| "rewards/margins": 17.067224502563477, |
| "rewards/rejected": -14.124492645263672, |
| "step": 9925 |
| }, |
| { |
| "epoch": 2.268620129407405, |
| "grad_norm": 4.5625, |
| "learning_rate": 1.1450342284294616e-05, |
| "logits/chosen": -2.5703444480895996, |
| "logits/rejected": -2.2223219871520996, |
| "logps/chosen": -235.23886108398438, |
| "logps/rejected": -300.3282470703125, |
| "loss": 0.0061, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.413525104522705, |
| "rewards/margins": 17.385786056518555, |
| "rewards/rejected": -14.972260475158691, |
| "step": 9950 |
| }, |
| { |
| "epoch": 2.2743208961605337, |
| "grad_norm": 0.65234375, |
| "learning_rate": 1.141489786202737e-05, |
| "logits/chosen": -2.4278616905212402, |
| "logits/rejected": -2.180201530456543, |
| "logps/chosen": -240.21783447265625, |
| "logps/rejected": -308.00787353515625, |
| "loss": 0.0063, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.694826126098633, |
| "rewards/margins": 17.01125717163086, |
| "rewards/rejected": -14.31643295288086, |
| "step": 9975 |
| }, |
| { |
| "epoch": 2.280021662913662, |
| "grad_norm": 1.125, |
| "learning_rate": 1.1379435291828865e-05, |
| "logits/chosen": -2.5520613193511963, |
| "logits/rejected": -2.3080861568450928, |
| "logps/chosen": -236.7652130126953, |
| "logps/rejected": -293.0811462402344, |
| "loss": 0.0161, |
| "rewards/accuracies": 0.9900000095367432, |
| "rewards/chosen": 2.2947661876678467, |
| "rewards/margins": 17.031295776367188, |
| "rewards/rejected": -14.736533164978027, |
| "step": 10000 |
| }, |
| { |
| "epoch": 2.280021662913662, |
| "eval_logits/chosen": -2.480435609817505, |
| "eval_logits/rejected": -2.2375354766845703, |
| "eval_logps/chosen": -263.1004943847656, |
| "eval_logps/rejected": -292.2024841308594, |
| "eval_loss": 1.2503352165222168, |
| "eval_rewards/accuracies": 0.7057347893714905, |
| "eval_rewards/chosen": -1.5584917068481445, |
| "eval_rewards/margins": 8.363493919372559, |
| "eval_rewards/rejected": -9.921985626220703, |
| "eval_runtime": 907.385, |
| "eval_samples_per_second": 9.666, |
| "eval_steps_per_second": 9.666, |
| "step": 10000 |
| }, |
| { |
| "epoch": 2.28572242966679, |
| "grad_norm": 0.4453125, |
| "learning_rate": 1.1343955028553339e-05, |
| "logits/chosen": -2.519899606704712, |
| "logits/rejected": -2.259140729904175, |
| "logps/chosen": -258.9615173339844, |
| "logps/rejected": -302.80877685546875, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 3.075371742248535, |
| "rewards/margins": 17.64069366455078, |
| "rewards/rejected": -14.565321922302246, |
| "step": 10025 |
| }, |
| { |
| "epoch": 2.2914231964199185, |
| "grad_norm": 0.04443359375, |
| "learning_rate": 1.1308457527281966e-05, |
| "logits/chosen": -2.7147040367126465, |
| "logits/rejected": -2.484670877456665, |
| "logps/chosen": -252.8388214111328, |
| "logps/rejected": -318.7659912109375, |
| "loss": 0.0036, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.749878168106079, |
| "rewards/margins": 19.654701232910156, |
| "rewards/rejected": -16.904821395874023, |
| "step": 10050 |
| }, |
| { |
| "epoch": 2.2971239631730467, |
| "grad_norm": 31.25, |
| "learning_rate": 1.1272943243317017e-05, |
| "logits/chosen": -2.407388687133789, |
| "logits/rejected": -2.188960552215576, |
| "logps/chosen": -216.96588134765625, |
| "logps/rejected": -264.1772766113281, |
| "loss": 0.0127, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.1731090545654297, |
| "rewards/margins": 15.625144958496094, |
| "rewards/rejected": -13.452037811279297, |
| "step": 10075 |
| }, |
| { |
| "epoch": 2.3028247299261753, |
| "grad_norm": 1.0703125, |
| "learning_rate": 1.1237412632176028e-05, |
| "logits/chosen": -2.3387563228607178, |
| "logits/rejected": -2.0859382152557373, |
| "logps/chosen": -257.5191650390625, |
| "logps/rejected": -307.0444641113281, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.659071683883667, |
| "rewards/margins": 17.472618103027344, |
| "rewards/rejected": -14.813547134399414, |
| "step": 10100 |
| }, |
| { |
| "epoch": 2.3085254966793034, |
| "grad_norm": 0.59375, |
| "learning_rate": 1.1201866149585948e-05, |
| "logits/chosen": -2.519253730773926, |
| "logits/rejected": -2.180464506149292, |
| "logps/chosen": -254.89266967773438, |
| "logps/rejected": -318.1060791015625, |
| "loss": 0.0186, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.828110694885254, |
| "rewards/margins": 17.266719818115234, |
| "rewards/rejected": -14.438608169555664, |
| "step": 10125 |
| }, |
| { |
| "epoch": 2.3142262634324315, |
| "grad_norm": 0.087890625, |
| "learning_rate": 1.1166304251477302e-05, |
| "logits/chosen": -2.7290658950805664, |
| "logits/rejected": -2.414004325866699, |
| "logps/chosen": -228.138671875, |
| "logps/rejected": -295.9573059082031, |
| "loss": 0.0204, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.7012221813201904, |
| "rewards/margins": 18.300796508789062, |
| "rewards/rejected": -15.59957504272461, |
| "step": 10150 |
| }, |
| { |
| "epoch": 2.31992703018556, |
| "grad_norm": 17.625, |
| "learning_rate": 1.1130727393978336e-05, |
| "logits/chosen": -2.5366790294647217, |
| "logits/rejected": -2.1500518321990967, |
| "logps/chosen": -264.43798828125, |
| "logps/rejected": -319.3265380859375, |
| "loss": 0.0204, |
| "rewards/accuracies": 0.9850000143051147, |
| "rewards/chosen": 2.7021288871765137, |
| "rewards/margins": 16.3283634185791, |
| "rewards/rejected": -13.626235008239746, |
| "step": 10175 |
| }, |
| { |
| "epoch": 2.3256277969386883, |
| "grad_norm": 0.138671875, |
| "learning_rate": 1.1095136033409168e-05, |
| "logits/chosen": -2.6784157752990723, |
| "logits/rejected": -2.2757108211517334, |
| "logps/chosen": -249.7666473388672, |
| "logps/rejected": -333.30108642578125, |
| "loss": 0.003, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.3563756942749023, |
| "rewards/margins": 19.736051559448242, |
| "rewards/rejected": -16.379676818847656, |
| "step": 10200 |
| }, |
| { |
| "epoch": 2.3313285636918164, |
| "grad_norm": 0.28515625, |
| "learning_rate": 1.1059530626275948e-05, |
| "logits/chosen": -2.491715908050537, |
| "logits/rejected": -2.3565056324005127, |
| "logps/chosen": -282.6903991699219, |
| "logps/rejected": -299.4869079589844, |
| "loss": 0.0087, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.1658356189727783, |
| "rewards/margins": 17.258220672607422, |
| "rewards/rejected": -14.092382431030273, |
| "step": 10225 |
| }, |
| { |
| "epoch": 2.337029330444945, |
| "grad_norm": 0.1513671875, |
| "learning_rate": 1.102391162926498e-05, |
| "logits/chosen": -2.73898983001709, |
| "logits/rejected": -2.4041645526885986, |
| "logps/chosen": -242.20736694335938, |
| "logps/rejected": -303.24005126953125, |
| "loss": 0.017, |
| "rewards/accuracies": 0.9900000095367432, |
| "rewards/chosen": 3.1999564170837402, |
| "rewards/margins": 17.591045379638672, |
| "rewards/rejected": -14.391088485717773, |
| "step": 10250 |
| }, |
| { |
| "epoch": 2.342730097198073, |
| "grad_norm": 2.890625, |
| "learning_rate": 1.0988279499236876e-05, |
| "logits/chosen": -2.4035749435424805, |
| "logits/rejected": -2.0998432636260986, |
| "logps/chosen": -240.24057006835938, |
| "logps/rejected": -292.30316162109375, |
| "loss": 0.0079, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.4504904747009277, |
| "rewards/margins": 16.707691192626953, |
| "rewards/rejected": -14.257201194763184, |
| "step": 10275 |
| }, |
| { |
| "epoch": 2.3484308639512013, |
| "grad_norm": 13.75, |
| "learning_rate": 1.095263469322071e-05, |
| "logits/chosen": -2.3402915000915527, |
| "logits/rejected": -2.126401424407959, |
| "logps/chosen": -280.3195495605469, |
| "logps/rejected": -309.2961120605469, |
| "loss": 0.0102, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.934191942214966, |
| "rewards/margins": 17.180307388305664, |
| "rewards/rejected": -14.246115684509277, |
| "step": 10300 |
| }, |
| { |
| "epoch": 2.35413163070433, |
| "grad_norm": 2.359375, |
| "learning_rate": 1.0916977668408125e-05, |
| "logits/chosen": -2.2307350635528564, |
| "logits/rejected": -1.9862459897994995, |
| "logps/chosen": -249.7293701171875, |
| "logps/rejected": -300.0370178222656, |
| "loss": 0.0197, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 3.001857280731201, |
| "rewards/margins": 17.074264526367188, |
| "rewards/rejected": -14.072406768798828, |
| "step": 10325 |
| }, |
| { |
| "epoch": 2.359832397457458, |
| "grad_norm": 3.484375, |
| "learning_rate": 1.0881308882147498e-05, |
| "logits/chosen": -2.704667091369629, |
| "logits/rejected": -2.2885918617248535, |
| "logps/chosen": -229.0606231689453, |
| "logps/rejected": -302.312744140625, |
| "loss": 0.0086, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.002713680267334, |
| "rewards/margins": 17.723676681518555, |
| "rewards/rejected": -14.720961570739746, |
| "step": 10350 |
| }, |
| { |
| "epoch": 2.3655331642105866, |
| "grad_norm": 1.6640625, |
| "learning_rate": 1.0845628791938058e-05, |
| "logits/chosen": -2.7726354598999023, |
| "logits/rejected": -2.4777493476867676, |
| "logps/chosen": -239.9902801513672, |
| "logps/rejected": -311.38006591796875, |
| "loss": 0.0052, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.608236074447632, |
| "rewards/margins": 17.5296573638916, |
| "rewards/rejected": -14.92142105102539, |
| "step": 10375 |
| }, |
| { |
| "epoch": 2.3712339309637147, |
| "grad_norm": 1.4296875, |
| "learning_rate": 1.0809937855424019e-05, |
| "logits/chosen": -2.6110775470733643, |
| "logits/rejected": -2.341466188430786, |
| "logps/chosen": -247.22305297851562, |
| "logps/rejected": -280.120361328125, |
| "loss": 0.0286, |
| "rewards/accuracies": 0.9850000143051147, |
| "rewards/chosen": 2.208012819290161, |
| "rewards/margins": 17.511402130126953, |
| "rewards/rejected": -15.303389549255371, |
| "step": 10400 |
| }, |
| { |
| "epoch": 2.376934697716843, |
| "grad_norm": 6.375, |
| "learning_rate": 1.0774236530388722e-05, |
| "logits/chosen": -2.44248104095459, |
| "logits/rejected": -2.1488513946533203, |
| "logps/chosen": -259.2685852050781, |
| "logps/rejected": -302.98260498046875, |
| "loss": 0.0073, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.7573134899139404, |
| "rewards/margins": 17.479970932006836, |
| "rewards/rejected": -14.722657203674316, |
| "step": 10425 |
| }, |
| { |
| "epoch": 2.3826354644699714, |
| "grad_norm": 6.1875, |
| "learning_rate": 1.073852527474874e-05, |
| "logits/chosen": -2.2181177139282227, |
| "logits/rejected": -1.915704369544983, |
| "logps/chosen": -271.42266845703125, |
| "logps/rejected": -304.30438232421875, |
| "loss": 0.0054, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.4309749603271484, |
| "rewards/margins": 15.70462417602539, |
| "rewards/rejected": -13.273650169372559, |
| "step": 10450 |
| }, |
| { |
| "epoch": 2.3883362312230996, |
| "grad_norm": 0.0908203125, |
| "learning_rate": 1.0702804546548036e-05, |
| "logits/chosen": -2.3788654804229736, |
| "logits/rejected": -2.1179566383361816, |
| "logps/chosen": -256.93389892578125, |
| "logps/rejected": -312.0606994628906, |
| "loss": 0.0082, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.9383556842803955, |
| "rewards/margins": 18.157489776611328, |
| "rewards/rejected": -15.219134330749512, |
| "step": 10475 |
| }, |
| { |
| "epoch": 2.3940369979762277, |
| "grad_norm": 0.99609375, |
| "learning_rate": 1.066707480395206e-05, |
| "logits/chosen": -2.35880970954895, |
| "logits/rejected": -2.146643877029419, |
| "logps/chosen": -259.59588623046875, |
| "logps/rejected": -315.8457336425781, |
| "loss": 0.0026, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.976827621459961, |
| "rewards/margins": 17.67559814453125, |
| "rewards/rejected": -14.698770523071289, |
| "step": 10500 |
| }, |
| { |
| "epoch": 2.3997377647293563, |
| "grad_norm": 254.0, |
| "learning_rate": 1.0631336505241885e-05, |
| "logits/chosen": -2.361175537109375, |
| "logits/rejected": -2.1759908199310303, |
| "logps/chosen": -240.54464721679688, |
| "logps/rejected": -294.257080078125, |
| "loss": 0.0153, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.554966688156128, |
| "rewards/margins": 16.587541580200195, |
| "rewards/rejected": -14.032571792602539, |
| "step": 10525 |
| }, |
| { |
| "epoch": 2.4054385314824844, |
| "grad_norm": 3.140625, |
| "learning_rate": 1.0595590108808331e-05, |
| "logits/chosen": -2.0374953746795654, |
| "logits/rejected": -1.7955071926116943, |
| "logps/chosen": -265.0710754394531, |
| "logps/rejected": -308.02789306640625, |
| "loss": 0.0127, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.2079503536224365, |
| "rewards/margins": 15.704275131225586, |
| "rewards/rejected": -13.496323585510254, |
| "step": 10550 |
| }, |
| { |
| "epoch": 2.4111392982356126, |
| "grad_norm": 8.0, |
| "learning_rate": 1.0559836073146081e-05, |
| "logits/chosen": -2.6259799003601074, |
| "logits/rejected": -2.2645726203918457, |
| "logps/chosen": -243.36489868164062, |
| "logps/rejected": -324.1029052734375, |
| "loss": 0.0115, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.4107019901275635, |
| "rewards/margins": 18.143095016479492, |
| "rewards/rejected": -15.732394218444824, |
| "step": 10575 |
| }, |
| { |
| "epoch": 2.416840064988741, |
| "grad_norm": 7.75, |
| "learning_rate": 1.0524074856847794e-05, |
| "logits/chosen": -2.668849229812622, |
| "logits/rejected": -2.3984618186950684, |
| "logps/chosen": -230.97300720214844, |
| "logps/rejected": -292.5263977050781, |
| "loss": 0.0103, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.7089719772338867, |
| "rewards/margins": 18.038372039794922, |
| "rewards/rejected": -15.329397201538086, |
| "step": 10600 |
| }, |
| { |
| "epoch": 2.4225408317418693, |
| "grad_norm": 0.28125, |
| "learning_rate": 1.0488306918598243e-05, |
| "logits/chosen": -2.727012872695923, |
| "logits/rejected": -2.37370228767395, |
| "logps/chosen": -191.98606872558594, |
| "logps/rejected": -252.22752380371094, |
| "loss": 0.0111, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.401088833808899, |
| "rewards/margins": 16.691862106323242, |
| "rewards/rejected": -15.29077434539795, |
| "step": 10625 |
| }, |
| { |
| "epoch": 2.4282415984949974, |
| "grad_norm": 4.1875, |
| "learning_rate": 1.0452532717168412e-05, |
| "logits/chosen": -2.5329430103302, |
| "logits/rejected": -2.319120168685913, |
| "logps/chosen": -238.58970642089844, |
| "logps/rejected": -292.25750732421875, |
| "loss": 0.0135, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.1987924575805664, |
| "rewards/margins": 16.739208221435547, |
| "rewards/rejected": -14.540417671203613, |
| "step": 10650 |
| }, |
| { |
| "epoch": 2.433942365248126, |
| "grad_norm": 0.453125, |
| "learning_rate": 1.0416752711409612e-05, |
| "logits/chosen": -2.2521848678588867, |
| "logits/rejected": -2.0748984813690186, |
| "logps/chosen": -262.38702392578125, |
| "logps/rejected": -311.2496032714844, |
| "loss": 0.0165, |
| "rewards/accuracies": 0.9900000095367432, |
| "rewards/chosen": 2.979421377182007, |
| "rewards/margins": 17.918113708496094, |
| "rewards/rejected": -14.938693046569824, |
| "step": 10675 |
| }, |
| { |
| "epoch": 2.439643132001254, |
| "grad_norm": 5.09375, |
| "learning_rate": 1.0380967360247616e-05, |
| "logits/chosen": -2.5377230644226074, |
| "logits/rejected": -2.209150552749634, |
| "logps/chosen": -283.0435791015625, |
| "logps/rejected": -341.9014892578125, |
| "loss": 0.0144, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.2588086128234863, |
| "rewards/margins": 18.77469253540039, |
| "rewards/rejected": -16.515886306762695, |
| "step": 10700 |
| }, |
| { |
| "epoch": 2.4453438987543823, |
| "grad_norm": 7.3125, |
| "learning_rate": 1.0345177122676747e-05, |
| "logits/chosen": -2.2560508251190186, |
| "logits/rejected": -1.9481168985366821, |
| "logps/chosen": -244.34722900390625, |
| "logps/rejected": -290.7549743652344, |
| "loss": 0.0167, |
| "rewards/accuracies": 0.9900000095367432, |
| "rewards/chosen": 2.0377843379974365, |
| "rewards/margins": 17.224489212036133, |
| "rewards/rejected": -15.186705589294434, |
| "step": 10725 |
| }, |
| { |
| "epoch": 2.451044665507511, |
| "grad_norm": 8.5, |
| "learning_rate": 1.0309382457754009e-05, |
| "logits/chosen": -2.5246341228485107, |
| "logits/rejected": -2.2586400508880615, |
| "logps/chosen": -242.64271545410156, |
| "logps/rejected": -316.3126220703125, |
| "loss": 0.0124, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 1.990309238433838, |
| "rewards/margins": 17.830862045288086, |
| "rewards/rejected": -15.840553283691406, |
| "step": 10750 |
| }, |
| { |
| "epoch": 2.456745432260639, |
| "grad_norm": 35.5, |
| "learning_rate": 1.0273583824593192e-05, |
| "logits/chosen": -2.531233549118042, |
| "logits/rejected": -2.253929853439331, |
| "logps/chosen": -249.7599639892578, |
| "logps/rejected": -302.4462890625, |
| "loss": 0.0254, |
| "rewards/accuracies": 0.9900000095367432, |
| "rewards/chosen": 2.6132864952087402, |
| "rewards/margins": 17.113136291503906, |
| "rewards/rejected": -14.499850273132324, |
| "step": 10775 |
| }, |
| { |
| "epoch": 2.462446199013767, |
| "grad_norm": 12.8125, |
| "learning_rate": 1.023778168235898e-05, |
| "logits/chosen": -2.4625229835510254, |
| "logits/rejected": -2.2500836849212646, |
| "logps/chosen": -233.9299774169922, |
| "logps/rejected": -296.8111267089844, |
| "loss": 0.003, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.886265516281128, |
| "rewards/margins": 17.21172523498535, |
| "rewards/rejected": -15.325458526611328, |
| "step": 10800 |
| }, |
| { |
| "epoch": 2.4681469657668957, |
| "grad_norm": 0.1494140625, |
| "learning_rate": 1.020197649026107e-05, |
| "logits/chosen": -2.5211057662963867, |
| "logits/rejected": -2.213142156600952, |
| "logps/chosen": -234.17660522460938, |
| "logps/rejected": -293.1681823730469, |
| "loss": 0.0066, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.2980597019195557, |
| "rewards/margins": 16.807985305786133, |
| "rewards/rejected": -14.509925842285156, |
| "step": 10825 |
| }, |
| { |
| "epoch": 2.473847732520024, |
| "grad_norm": 0.0791015625, |
| "learning_rate": 1.0166168707548278e-05, |
| "logits/chosen": -2.439523696899414, |
| "logits/rejected": -2.191675901412964, |
| "logps/chosen": -232.5564422607422, |
| "logps/rejected": -301.06610107421875, |
| "loss": 0.0326, |
| "rewards/accuracies": 0.9900000095367432, |
| "rewards/chosen": 2.4902689456939697, |
| "rewards/margins": 16.55835723876953, |
| "rewards/rejected": -14.068087577819824, |
| "step": 10850 |
| }, |
| { |
| "epoch": 2.479548499273152, |
| "grad_norm": 0.2099609375, |
| "learning_rate": 1.0130358793502644e-05, |
| "logits/chosen": -2.590106248855591, |
| "logits/rejected": -2.282782793045044, |
| "logps/chosen": -229.00221252441406, |
| "logps/rejected": -313.0149841308594, |
| "loss": 0.0158, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4736016988754272, |
| "rewards/margins": 16.571056365966797, |
| "rewards/rejected": -15.097452163696289, |
| "step": 10875 |
| }, |
| { |
| "epoch": 2.4852492660262806, |
| "grad_norm": 4.09375, |
| "learning_rate": 1.0094547207433546e-05, |
| "logits/chosen": -2.468318223953247, |
| "logits/rejected": -2.243657112121582, |
| "logps/chosen": -250.49874877929688, |
| "logps/rejected": -313.0626525878906, |
| "loss": 0.0148, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.111060380935669, |
| "rewards/margins": 18.188505172729492, |
| "rewards/rejected": -16.07744598388672, |
| "step": 10900 |
| }, |
| { |
| "epoch": 2.4909500327794087, |
| "grad_norm": 1.1484375, |
| "learning_rate": 1.0058734408671808e-05, |
| "logits/chosen": -2.413649797439575, |
| "logits/rejected": -2.1180853843688965, |
| "logps/chosen": -263.0333557128906, |
| "logps/rejected": -302.119384765625, |
| "loss": 0.0523, |
| "rewards/accuracies": 0.9900000095367432, |
| "rewards/chosen": 2.739366054534912, |
| "rewards/margins": 16.856290817260742, |
| "rewards/rejected": -14.116926193237305, |
| "step": 10925 |
| }, |
| { |
| "epoch": 2.496650799532537, |
| "grad_norm": 0.48046875, |
| "learning_rate": 1.0022920856563814e-05, |
| "logits/chosen": -2.4533162117004395, |
| "logits/rejected": -2.26503324508667, |
| "logps/chosen": -257.395751953125, |
| "logps/rejected": -303.6195983886719, |
| "loss": 0.051, |
| "rewards/accuracies": 0.9900000095367432, |
| "rewards/chosen": 2.8283374309539795, |
| "rewards/margins": 18.25380516052246, |
| "rewards/rejected": -15.425467491149902, |
| "step": 10950 |
| }, |
| { |
| "epoch": 2.5023515662856655, |
| "grad_norm": 4.4375, |
| "learning_rate": 9.9871070104656e-06, |
| "logits/chosen": -2.6997687816619873, |
| "logits/rejected": -2.3915600776672363, |
| "logps/chosen": -228.5899200439453, |
| "logps/rejected": -290.9750061035156, |
| "loss": 0.0042, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.567615032196045, |
| "rewards/margins": 18.70610809326172, |
| "rewards/rejected": -17.13849449157715, |
| "step": 10975 |
| }, |
| { |
| "epoch": 2.5080523330387936, |
| "grad_norm": 7.8125, |
| "learning_rate": 9.951293329736987e-06, |
| "logits/chosen": -2.5102100372314453, |
| "logits/rejected": -2.226412296295166, |
| "logps/chosen": -251.5011749267578, |
| "logps/rejected": -318.5889587402344, |
| "loss": 0.0163, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 1.6067277193069458, |
| "rewards/margins": 17.951580047607422, |
| "rewards/rejected": -16.3448543548584, |
| "step": 11000 |
| }, |
| { |
| "epoch": 2.5137530997919217, |
| "grad_norm": 0.076171875, |
| "learning_rate": 9.915480273735657e-06, |
| "logits/chosen": -2.6389758586883545, |
| "logits/rejected": -2.2492685317993164, |
| "logps/chosen": -287.0631103515625, |
| "logps/rejected": -323.08746337890625, |
| "loss": 0.0055, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.746593952178955, |
| "rewards/margins": 18.032590866088867, |
| "rewards/rejected": -15.28599739074707, |
| "step": 11025 |
| }, |
| { |
| "epoch": 2.5194538665450503, |
| "grad_norm": 0.2294921875, |
| "learning_rate": 9.879668301811294e-06, |
| "logits/chosen": -2.188718318939209, |
| "logits/rejected": -2.042511463165283, |
| "logps/chosen": -233.71607971191406, |
| "logps/rejected": -278.5687561035156, |
| "loss": 0.0076, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.755091667175293, |
| "rewards/margins": 16.62385368347168, |
| "rewards/rejected": -13.868762969970703, |
| "step": 11050 |
| }, |
| { |
| "epoch": 2.5251546332981785, |
| "grad_norm": 1.8125, |
| "learning_rate": 9.843857873299676e-06, |
| "logits/chosen": -2.537353754043579, |
| "logits/rejected": -2.117387294769287, |
| "logps/chosen": -258.01153564453125, |
| "logps/rejected": -310.85223388671875, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.581132173538208, |
| "rewards/margins": 17.749958038330078, |
| "rewards/rejected": -15.168827056884766, |
| "step": 11075 |
| }, |
| { |
| "epoch": 2.530855400051307, |
| "grad_norm": 2.609375, |
| "learning_rate": 9.808049447516772e-06, |
| "logits/chosen": -2.721762180328369, |
| "logits/rejected": -2.454221725463867, |
| "logps/chosen": -227.43165588378906, |
| "logps/rejected": -264.95184326171875, |
| "loss": 0.0121, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.1185920238494873, |
| "rewards/margins": 17.987041473388672, |
| "rewards/rejected": -15.868450164794922, |
| "step": 11100 |
| }, |
| { |
| "epoch": 2.536556166804435, |
| "grad_norm": 9.8125, |
| "learning_rate": 9.772243483752876e-06, |
| "logits/chosen": -2.725499153137207, |
| "logits/rejected": -2.4557807445526123, |
| "logps/chosen": -251.7043914794922, |
| "logps/rejected": -306.6485595703125, |
| "loss": 0.0177, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.301705837249756, |
| "rewards/margins": 17.31427574157715, |
| "rewards/rejected": -15.012572288513184, |
| "step": 11125 |
| }, |
| { |
| "epoch": 2.5422569335575638, |
| "grad_norm": 0.0169677734375, |
| "learning_rate": 9.7364404412667e-06, |
| "logits/chosen": -2.443347454071045, |
| "logits/rejected": -2.1931350231170654, |
| "logps/chosen": -225.1500244140625, |
| "logps/rejected": -264.4849548339844, |
| "loss": 0.0076, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7691653966903687, |
| "rewards/margins": 15.759430885314941, |
| "rewards/rejected": -13.990264892578125, |
| "step": 11150 |
| }, |
| { |
| "epoch": 2.547957700310692, |
| "grad_norm": 0.396484375, |
| "learning_rate": 9.700640779279488e-06, |
| "logits/chosen": -2.507894515991211, |
| "logits/rejected": -2.285839557647705, |
| "logps/chosen": -232.48631286621094, |
| "logps/rejected": -293.859619140625, |
| "loss": 0.0556, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.121201515197754, |
| "rewards/margins": 17.456039428710938, |
| "rewards/rejected": -15.33484172821045, |
| "step": 11175 |
| }, |
| { |
| "epoch": 2.55365846706382, |
| "grad_norm": 3.09375, |
| "learning_rate": 9.664844956969118e-06, |
| "logits/chosen": -2.4913032054901123, |
| "logits/rejected": -2.221055030822754, |
| "logps/chosen": -244.36968994140625, |
| "logps/rejected": -319.10107421875, |
| "loss": 0.0131, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 1.8303358554840088, |
| "rewards/margins": 17.145048141479492, |
| "rewards/rejected": -15.314714431762695, |
| "step": 11200 |
| }, |
| { |
| "epoch": 2.5593592338169486, |
| "grad_norm": 0.045166015625, |
| "learning_rate": 9.629053433464229e-06, |
| "logits/chosen": -2.535670518875122, |
| "logits/rejected": -2.2105841636657715, |
| "logps/chosen": -260.5967102050781, |
| "logps/rejected": -308.6719055175781, |
| "loss": 0.0104, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.174988031387329, |
| "rewards/margins": 16.259153366088867, |
| "rewards/rejected": -14.0841646194458, |
| "step": 11225 |
| }, |
| { |
| "epoch": 2.5650600005700768, |
| "grad_norm": 4.28125, |
| "learning_rate": 9.593266667838318e-06, |
| "logits/chosen": -2.2556569576263428, |
| "logits/rejected": -1.988387107849121, |
| "logps/chosen": -242.94854736328125, |
| "logps/rejected": -287.181640625, |
| "loss": 0.0262, |
| "rewards/accuracies": 0.9850000143051147, |
| "rewards/chosen": 2.321810722351074, |
| "rewards/margins": 17.308103561401367, |
| "rewards/rejected": -14.986291885375977, |
| "step": 11250 |
| }, |
| { |
| "epoch": 2.570760767323205, |
| "grad_norm": 2.015625, |
| "learning_rate": 9.557485119103849e-06, |
| "logits/chosen": -2.2604024410247803, |
| "logits/rejected": -2.009965419769287, |
| "logps/chosen": -264.4687805175781, |
| "logps/rejected": -321.8617248535156, |
| "loss": 0.0053, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.02841854095459, |
| "rewards/margins": 17.836463928222656, |
| "rewards/rejected": -14.80804443359375, |
| "step": 11275 |
| }, |
| { |
| "epoch": 2.5764615340763335, |
| "grad_norm": 0.0302734375, |
| "learning_rate": 9.52170924620638e-06, |
| "logits/chosen": -2.4766149520874023, |
| "logits/rejected": -2.222557306289673, |
| "logps/chosen": -265.8365478515625, |
| "logps/rejected": -336.5409851074219, |
| "loss": 0.0048, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8788745403289795, |
| "rewards/margins": 19.467832565307617, |
| "rewards/rejected": -16.588958740234375, |
| "step": 11300 |
| }, |
| { |
| "epoch": 2.5821623008294616, |
| "grad_norm": 0.60546875, |
| "learning_rate": 9.485939508018674e-06, |
| "logits/chosen": -2.461235284805298, |
| "logits/rejected": -2.1685104370117188, |
| "logps/chosen": -247.51661682128906, |
| "logps/rejected": -298.4482421875, |
| "loss": 0.0032, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.357917308807373, |
| "rewards/margins": 16.800758361816406, |
| "rewards/rejected": -14.44284439086914, |
| "step": 11325 |
| }, |
| { |
| "epoch": 2.5878630675825898, |
| "grad_norm": 2.46875, |
| "learning_rate": 9.450176363334794e-06, |
| "logits/chosen": -2.477795124053955, |
| "logits/rejected": -2.2618613243103027, |
| "logps/chosen": -248.1282196044922, |
| "logps/rejected": -281.45367431640625, |
| "loss": 0.0086, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.2790262699127197, |
| "rewards/margins": 16.411928176879883, |
| "rewards/rejected": -14.13290023803711, |
| "step": 11350 |
| }, |
| { |
| "epoch": 2.5935638343357184, |
| "grad_norm": 22.25, |
| "learning_rate": 9.414420270864239e-06, |
| "logits/chosen": -2.421799421310425, |
| "logits/rejected": -2.1550545692443848, |
| "logps/chosen": -274.53973388671875, |
| "logps/rejected": -301.0137634277344, |
| "loss": 0.0048, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.7840795516967773, |
| "rewards/margins": 17.463281631469727, |
| "rewards/rejected": -14.679200172424316, |
| "step": 11375 |
| }, |
| { |
| "epoch": 2.5992646010888465, |
| "grad_norm": 1.90625, |
| "learning_rate": 9.378671689226065e-06, |
| "logits/chosen": -2.3667001724243164, |
| "logits/rejected": -1.979435920715332, |
| "logps/chosen": -251.78663635253906, |
| "logps/rejected": -288.2210998535156, |
| "loss": 0.0087, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.1289076805114746, |
| "rewards/margins": 17.049039840698242, |
| "rewards/rejected": -14.920132637023926, |
| "step": 11400 |
| }, |
| { |
| "epoch": 2.6049653678419746, |
| "grad_norm": 0.173828125, |
| "learning_rate": 9.342931076942973e-06, |
| "logits/chosen": -2.674487590789795, |
| "logits/rejected": -2.4038474559783936, |
| "logps/chosen": -239.80392456054688, |
| "logps/rejected": -289.2531433105469, |
| "loss": 0.0124, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.359168291091919, |
| "rewards/margins": 17.957656860351562, |
| "rewards/rejected": -15.598488807678223, |
| "step": 11425 |
| }, |
| { |
| "epoch": 2.610666134595103, |
| "grad_norm": 0.7265625, |
| "learning_rate": 9.307198892435459e-06, |
| "logits/chosen": -2.2927722930908203, |
| "logits/rejected": -2.134584426879883, |
| "logps/chosen": -253.97935485839844, |
| "logps/rejected": -307.4478759765625, |
| "loss": 0.012, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 1.3781896829605103, |
| "rewards/margins": 16.367218017578125, |
| "rewards/rejected": -14.989027976989746, |
| "step": 11450 |
| }, |
| { |
| "epoch": 2.6163669013482314, |
| "grad_norm": 1.6640625, |
| "learning_rate": 9.271475594015923e-06, |
| "logits/chosen": -2.4251346588134766, |
| "logits/rejected": -2.186738967895508, |
| "logps/chosen": -254.7713623046875, |
| "logps/rejected": -283.75531005859375, |
| "loss": 0.0137, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.103003740310669, |
| "rewards/margins": 17.4500675201416, |
| "rewards/rejected": -15.347063064575195, |
| "step": 11475 |
| }, |
| { |
| "epoch": 2.6220676681013595, |
| "grad_norm": 28.5, |
| "learning_rate": 9.235761639882774e-06, |
| "logits/chosen": -2.4982593059539795, |
| "logits/rejected": -2.226536512374878, |
| "logps/chosen": -248.42782592773438, |
| "logps/rejected": -299.2533874511719, |
| "loss": 0.0078, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9232771396636963, |
| "rewards/margins": 17.905176162719727, |
| "rewards/rejected": -15.981898307800293, |
| "step": 11500 |
| }, |
| { |
| "epoch": 2.627768434854488, |
| "grad_norm": 37.5, |
| "learning_rate": 9.200057488114585e-06, |
| "logits/chosen": -2.51665997505188, |
| "logits/rejected": -2.321688413619995, |
| "logps/chosen": -270.8257751464844, |
| "logps/rejected": -330.7559509277344, |
| "loss": 0.0109, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.5012738704681396, |
| "rewards/margins": 17.973756790161133, |
| "rewards/rejected": -15.472482681274414, |
| "step": 11525 |
| }, |
| { |
| "epoch": 2.633469201607616, |
| "grad_norm": 33.25, |
| "learning_rate": 9.164363596664193e-06, |
| "logits/chosen": -2.491921901702881, |
| "logits/rejected": -2.172680377960205, |
| "logps/chosen": -241.7287139892578, |
| "logps/rejected": -278.69696044921875, |
| "loss": 0.0136, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.044093370437622, |
| "rewards/margins": 16.389976501464844, |
| "rewards/rejected": -14.345881462097168, |
| "step": 11550 |
| }, |
| { |
| "epoch": 2.6391699683607444, |
| "grad_norm": 25.875, |
| "learning_rate": 9.128680423352839e-06, |
| "logits/chosen": -2.304872751235962, |
| "logits/rejected": -2.1448278427124023, |
| "logps/chosen": -281.0390319824219, |
| "logps/rejected": -316.8353271484375, |
| "loss": 0.0067, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.3318545818328857, |
| "rewards/margins": 17.570289611816406, |
| "rewards/rejected": -15.238434791564941, |
| "step": 11575 |
| }, |
| { |
| "epoch": 2.644870735113873, |
| "grad_norm": 51.25, |
| "learning_rate": 9.093008425864276e-06, |
| "logits/chosen": -2.3745956420898438, |
| "logits/rejected": -2.0462872982025146, |
| "logps/chosen": -236.0822296142578, |
| "logps/rejected": -299.39349365234375, |
| "loss": 0.0078, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.71913480758667, |
| "rewards/margins": 18.365318298339844, |
| "rewards/rejected": -16.646181106567383, |
| "step": 11600 |
| }, |
| { |
| "epoch": 2.650571501867001, |
| "grad_norm": 0.96875, |
| "learning_rate": 9.057348061738926e-06, |
| "logits/chosen": -2.4560763835906982, |
| "logits/rejected": -2.1491658687591553, |
| "logps/chosen": -261.7008361816406, |
| "logps/rejected": -335.5806884765625, |
| "loss": 0.0116, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 1.6623625755310059, |
| "rewards/margins": 17.477306365966797, |
| "rewards/rejected": -15.814942359924316, |
| "step": 11625 |
| }, |
| { |
| "epoch": 2.656272268620129, |
| "grad_norm": 8.875, |
| "learning_rate": 9.021699788367999e-06, |
| "logits/chosen": -2.5862979888916016, |
| "logits/rejected": -2.238278388977051, |
| "logps/chosen": -272.9748840332031, |
| "logps/rejected": -346.0086364746094, |
| "loss": 0.0048, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.011551022529602, |
| "rewards/margins": 17.461071014404297, |
| "rewards/rejected": -16.449522018432617, |
| "step": 11650 |
| }, |
| { |
| "epoch": 2.661973035373258, |
| "grad_norm": 2.9375, |
| "learning_rate": 8.986064062987608e-06, |
| "logits/chosen": -2.406765937805176, |
| "logits/rejected": -2.178860664367676, |
| "logps/chosen": -252.98814392089844, |
| "logps/rejected": -309.38568115234375, |
| "loss": 0.013, |
| "rewards/accuracies": 0.9900000095367432, |
| "rewards/chosen": 1.522336483001709, |
| "rewards/margins": 18.24814224243164, |
| "rewards/rejected": -16.725805282592773, |
| "step": 11675 |
| }, |
| { |
| "epoch": 2.667673802126386, |
| "grad_norm": 42.25, |
| "learning_rate": 8.950441342672936e-06, |
| "logits/chosen": -2.476860761642456, |
| "logits/rejected": -2.292044162750244, |
| "logps/chosen": -272.4510803222656, |
| "logps/rejected": -291.6954040527344, |
| "loss": 0.0441, |
| "rewards/accuracies": 0.9850000143051147, |
| "rewards/chosen": 1.8895237445831299, |
| "rewards/margins": 16.44178009033203, |
| "rewards/rejected": -14.552257537841797, |
| "step": 11700 |
| }, |
| { |
| "epoch": 2.673374568879514, |
| "grad_norm": 0.006378173828125, |
| "learning_rate": 8.914832084332363e-06, |
| "logits/chosen": -2.4683072566986084, |
| "logits/rejected": -2.141472578048706, |
| "logps/chosen": -244.68954467773438, |
| "logps/rejected": -306.8516540527344, |
| "loss": 0.0203, |
| "rewards/accuracies": 0.9900000095367432, |
| "rewards/chosen": 2.8921725749969482, |
| "rewards/margins": 19.332353591918945, |
| "rewards/rejected": -16.4401798248291, |
| "step": 11725 |
| }, |
| { |
| "epoch": 2.6790753356326427, |
| "grad_norm": 0.38671875, |
| "learning_rate": 8.879236744701584e-06, |
| "logits/chosen": -2.492255687713623, |
| "logits/rejected": -2.217851400375366, |
| "logps/chosen": -236.57611083984375, |
| "logps/rejected": -290.78204345703125, |
| "loss": 0.0087, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9523673057556152, |
| "rewards/margins": 17.2410945892334, |
| "rewards/rejected": -15.288726806640625, |
| "step": 11750 |
| }, |
| { |
| "epoch": 2.684776102385771, |
| "grad_norm": 71.0, |
| "learning_rate": 8.843655780337783e-06, |
| "logits/chosen": -2.495654821395874, |
| "logits/rejected": -2.3087100982666016, |
| "logps/chosen": -243.6807098388672, |
| "logps/rejected": -279.55841064453125, |
| "loss": 0.0115, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.480747938156128, |
| "rewards/margins": 17.961509704589844, |
| "rewards/rejected": -15.480761528015137, |
| "step": 11775 |
| }, |
| { |
| "epoch": 2.690476869138899, |
| "grad_norm": 3.453125, |
| "learning_rate": 8.80808964761376e-06, |
| "logits/chosen": -2.3188583850860596, |
| "logits/rejected": -2.1460444927215576, |
| "logps/chosen": -249.7250518798828, |
| "logps/rejected": -292.0519104003906, |
| "loss": 0.0064, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9855836629867554, |
| "rewards/margins": 16.125455856323242, |
| "rewards/rejected": -15.139871597290039, |
| "step": 11800 |
| }, |
| { |
| "epoch": 2.6961776358920275, |
| "grad_norm": 1.6171875, |
| "learning_rate": 8.772538802712072e-06, |
| "logits/chosen": -2.244267225265503, |
| "logits/rejected": -1.9371627569198608, |
| "logps/chosen": -250.5675048828125, |
| "logps/rejected": -312.7926025390625, |
| "loss": 0.0181, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 3.0079262256622314, |
| "rewards/margins": 17.85134506225586, |
| "rewards/rejected": -14.84341812133789, |
| "step": 11825 |
| }, |
| { |
| "epoch": 2.7018784026451557, |
| "grad_norm": 0.26953125, |
| "learning_rate": 8.737003701619197e-06, |
| "logits/chosen": -2.560298204421997, |
| "logits/rejected": -2.2798218727111816, |
| "logps/chosen": -252.13999938964844, |
| "logps/rejected": -304.7752380371094, |
| "loss": 0.0038, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.520753860473633, |
| "rewards/margins": 20.120046615600586, |
| "rewards/rejected": -17.599292755126953, |
| "step": 11850 |
| }, |
| { |
| "epoch": 2.707579169398284, |
| "grad_norm": 2.203125, |
| "learning_rate": 8.701484800119678e-06, |
| "logits/chosen": -2.295139789581299, |
| "logits/rejected": -2.0687553882598877, |
| "logps/chosen": -256.41571044921875, |
| "logps/rejected": -317.96319580078125, |
| "loss": 0.0079, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.6115550994873047, |
| "rewards/margins": 17.304157257080078, |
| "rewards/rejected": -14.692602157592773, |
| "step": 11875 |
| }, |
| { |
| "epoch": 2.7132799361514124, |
| "grad_norm": 70.5, |
| "learning_rate": 8.665982553790268e-06, |
| "logits/chosen": -2.497831344604492, |
| "logits/rejected": -2.229539394378662, |
| "logps/chosen": -279.81292724609375, |
| "logps/rejected": -344.561279296875, |
| "loss": 0.009, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.309954881668091, |
| "rewards/margins": 18.895309448242188, |
| "rewards/rejected": -16.58535385131836, |
| "step": 11900 |
| }, |
| { |
| "epoch": 2.7189807029045405, |
| "grad_norm": 0.09619140625, |
| "learning_rate": 8.630497417994105e-06, |
| "logits/chosen": -2.4627225399017334, |
| "logits/rejected": -2.1932315826416016, |
| "logps/chosen": -237.32298278808594, |
| "logps/rejected": -312.8143615722656, |
| "loss": 0.0088, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 1.5031342506408691, |
| "rewards/margins": 19.145456314086914, |
| "rewards/rejected": -17.642322540283203, |
| "step": 11925 |
| }, |
| { |
| "epoch": 2.724681469657669, |
| "grad_norm": 4.4375, |
| "learning_rate": 8.595029847874858e-06, |
| "logits/chosen": -2.4345059394836426, |
| "logits/rejected": -2.1848456859588623, |
| "logps/chosen": -261.8165283203125, |
| "logps/rejected": -293.5591125488281, |
| "loss": 0.0145, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 1.936252474784851, |
| "rewards/margins": 16.934310913085938, |
| "rewards/rejected": -14.998059272766113, |
| "step": 11950 |
| }, |
| { |
| "epoch": 2.7303822364107972, |
| "grad_norm": 0.337890625, |
| "learning_rate": 8.559580298350898e-06, |
| "logits/chosen": -2.3152289390563965, |
| "logits/rejected": -2.004532814025879, |
| "logps/chosen": -228.34576416015625, |
| "logps/rejected": -297.91845703125, |
| "loss": 0.0207, |
| "rewards/accuracies": 0.9900000095367432, |
| "rewards/chosen": 1.1755906343460083, |
| "rewards/margins": 15.654560089111328, |
| "rewards/rejected": -14.478970527648926, |
| "step": 11975 |
| }, |
| { |
| "epoch": 2.7360830031639254, |
| "grad_norm": 27.25, |
| "learning_rate": 8.524149224109447e-06, |
| "logits/chosen": -2.3487377166748047, |
| "logits/rejected": -2.0978074073791504, |
| "logps/chosen": -235.12484741210938, |
| "logps/rejected": -288.0135498046875, |
| "loss": 0.0085, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5338854789733887, |
| "rewards/margins": 17.318761825561523, |
| "rewards/rejected": -15.784875869750977, |
| "step": 12000 |
| }, |
| { |
| "epoch": 2.741783769917054, |
| "grad_norm": 136.0, |
| "learning_rate": 8.488737079600767e-06, |
| "logits/chosen": -2.4606521129608154, |
| "logits/rejected": -2.2077949047088623, |
| "logps/chosen": -243.62310791015625, |
| "logps/rejected": -301.163330078125, |
| "loss": 0.0063, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 1.9555913209915161, |
| "rewards/margins": 17.216915130615234, |
| "rewards/rejected": -15.26132583618164, |
| "step": 12025 |
| }, |
| { |
| "epoch": 2.747484536670182, |
| "grad_norm": 0.11181640625, |
| "learning_rate": 8.453344319032323e-06, |
| "logits/chosen": -2.530169725418091, |
| "logits/rejected": -2.222634792327881, |
| "logps/chosen": -246.8153076171875, |
| "logps/rejected": -340.2912902832031, |
| "loss": 0.0171, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.0146796703338623, |
| "rewards/margins": 18.922849655151367, |
| "rewards/rejected": -16.908170700073242, |
| "step": 12050 |
| }, |
| { |
| "epoch": 2.7531853034233107, |
| "grad_norm": 6.03125, |
| "learning_rate": 8.417971396362942e-06, |
| "logits/chosen": -2.5295791625976562, |
| "logits/rejected": -2.270059823989868, |
| "logps/chosen": -256.3033142089844, |
| "logps/rejected": -309.3623352050781, |
| "loss": 0.0052, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.3389968872070312, |
| "rewards/margins": 18.08895492553711, |
| "rewards/rejected": -15.749957084655762, |
| "step": 12075 |
| }, |
| { |
| "epoch": 2.758886070176439, |
| "grad_norm": 0.01275634765625, |
| "learning_rate": 8.382618765297017e-06, |
| "logits/chosen": -2.407400608062744, |
| "logits/rejected": -2.1474361419677734, |
| "logps/chosen": -254.24029541015625, |
| "logps/rejected": -314.04620361328125, |
| "loss": 0.0034, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.3599538803100586, |
| "rewards/margins": 18.286041259765625, |
| "rewards/rejected": -15.926085472106934, |
| "step": 12100 |
| }, |
| { |
| "epoch": 2.764586836929567, |
| "grad_norm": 1.9765625, |
| "learning_rate": 8.347286879278669e-06, |
| "logits/chosen": -2.549743890762329, |
| "logits/rejected": -2.146315813064575, |
| "logps/chosen": -245.70220947265625, |
| "logps/rejected": -298.2206726074219, |
| "loss": 0.0042, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9440699815750122, |
| "rewards/margins": 20.051233291625977, |
| "rewards/rejected": -18.107162475585938, |
| "step": 12125 |
| }, |
| { |
| "epoch": 2.7702876036826956, |
| "grad_norm": 0.028564453125, |
| "learning_rate": 8.311976191485935e-06, |
| "logits/chosen": -2.404059410095215, |
| "logits/rejected": -2.1484217643737793, |
| "logps/chosen": -253.27713012695312, |
| "logps/rejected": -304.2370300292969, |
| "loss": 0.0419, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 1.656130313873291, |
| "rewards/margins": 18.277952194213867, |
| "rewards/rejected": -16.621822357177734, |
| "step": 12150 |
| }, |
| { |
| "epoch": 2.7759883704358237, |
| "grad_norm": 1.9609375, |
| "learning_rate": 8.27668715482496e-06, |
| "logits/chosen": -2.5107831954956055, |
| "logits/rejected": -2.2360105514526367, |
| "logps/chosen": -245.90602111816406, |
| "logps/rejected": -299.7825622558594, |
| "loss": 0.0086, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 1.3645967245101929, |
| "rewards/margins": 17.212987899780273, |
| "rewards/rejected": -15.848393440246582, |
| "step": 12175 |
| }, |
| { |
| "epoch": 2.781689137188952, |
| "grad_norm": 41.5, |
| "learning_rate": 8.24142022192419e-06, |
| "logits/chosen": -2.465029001235962, |
| "logits/rejected": -2.1922547817230225, |
| "logps/chosen": -230.12535095214844, |
| "logps/rejected": -303.5553283691406, |
| "loss": 0.0168, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 1.5119483470916748, |
| "rewards/margins": 18.21250343322754, |
| "rewards/rejected": -16.7005558013916, |
| "step": 12200 |
| }, |
| { |
| "epoch": 2.7873899039420804, |
| "grad_norm": 0.1875, |
| "learning_rate": 8.206175845128544e-06, |
| "logits/chosen": -2.261239528656006, |
| "logits/rejected": -1.8719124794006348, |
| "logps/chosen": -246.2677001953125, |
| "logps/rejected": -306.4869384765625, |
| "loss": 0.0047, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.666265606880188, |
| "rewards/margins": 16.640233993530273, |
| "rewards/rejected": -14.973969459533691, |
| "step": 12225 |
| }, |
| { |
| "epoch": 2.7930906706952086, |
| "grad_norm": 4.96875, |
| "learning_rate": 8.170954476493646e-06, |
| "logits/chosen": -2.2322580814361572, |
| "logits/rejected": -1.929784893989563, |
| "logps/chosen": -255.87771606445312, |
| "logps/rejected": -287.69952392578125, |
| "loss": 0.0044, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.898901343345642, |
| "rewards/margins": 16.230009078979492, |
| "rewards/rejected": -14.331107139587402, |
| "step": 12250 |
| }, |
| { |
| "epoch": 2.7987914374483367, |
| "grad_norm": 0.390625, |
| "learning_rate": 8.135756567780004e-06, |
| "logits/chosen": -2.1802163124084473, |
| "logits/rejected": -1.9820066690444946, |
| "logps/chosen": -276.86590576171875, |
| "logps/rejected": -307.0183410644531, |
| "loss": 0.0128, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.4910402297973633, |
| "rewards/margins": 17.84758186340332, |
| "rewards/rejected": -15.356539726257324, |
| "step": 12275 |
| }, |
| { |
| "epoch": 2.8044922042014653, |
| "grad_norm": 0.10302734375, |
| "learning_rate": 8.100582570447217e-06, |
| "logits/chosen": -2.689002275466919, |
| "logits/rejected": -2.2965428829193115, |
| "logps/chosen": -228.2522430419922, |
| "logps/rejected": -296.1205139160156, |
| "loss": 0.0153, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.0592339038848877, |
| "rewards/margins": 19.616744995117188, |
| "rewards/rejected": -17.557512283325195, |
| "step": 12300 |
| }, |
| { |
| "epoch": 2.8101929709545934, |
| "grad_norm": 0.52734375, |
| "learning_rate": 8.0654329356482e-06, |
| "logits/chosen": -2.3836965560913086, |
| "logits/rejected": -2.2295284271240234, |
| "logps/chosen": -263.2698974609375, |
| "logps/rejected": -314.9093322753906, |
| "loss": 0.0039, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.192784547805786, |
| "rewards/margins": 18.350887298583984, |
| "rewards/rejected": -16.158103942871094, |
| "step": 12325 |
| }, |
| { |
| "epoch": 2.8158937377077216, |
| "grad_norm": 1.171875, |
| "learning_rate": 8.030308114223372e-06, |
| "logits/chosen": -2.5672190189361572, |
| "logits/rejected": -2.249885320663452, |
| "logps/chosen": -238.60321044921875, |
| "logps/rejected": -283.582275390625, |
| "loss": 0.0202, |
| "rewards/accuracies": 0.9850000143051147, |
| "rewards/chosen": 1.1639879941940308, |
| "rewards/margins": 17.713640213012695, |
| "rewards/rejected": -16.549654006958008, |
| "step": 12350 |
| }, |
| { |
| "epoch": 2.82159450446085, |
| "grad_norm": 7.84375, |
| "learning_rate": 7.99520855669491e-06, |
| "logits/chosen": -2.377781391143799, |
| "logits/rejected": -2.085705280303955, |
| "logps/chosen": -280.130615234375, |
| "logps/rejected": -318.92071533203125, |
| "loss": 0.005, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.937410831451416, |
| "rewards/margins": 18.928157806396484, |
| "rewards/rejected": -15.990747451782227, |
| "step": 12375 |
| }, |
| { |
| "epoch": 2.8272952712139783, |
| "grad_norm": 30.25, |
| "learning_rate": 7.960134713260915e-06, |
| "logits/chosen": -2.3684072494506836, |
| "logits/rejected": -2.115257501602173, |
| "logps/chosen": -256.7491149902344, |
| "logps/rejected": -311.6601257324219, |
| "loss": 0.0245, |
| "rewards/accuracies": 0.9900000095367432, |
| "rewards/chosen": 0.8965556621551514, |
| "rewards/margins": 17.164628982543945, |
| "rewards/rejected": -16.2680721282959, |
| "step": 12400 |
| }, |
| { |
| "epoch": 2.8329960379671064, |
| "grad_norm": 0.162109375, |
| "learning_rate": 7.9250870337897e-06, |
| "logits/chosen": -2.1900858879089355, |
| "logits/rejected": -1.9305768013000488, |
| "logps/chosen": -252.32525634765625, |
| "logps/rejected": -304.79901123046875, |
| "loss": 0.0105, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 1.9972655773162842, |
| "rewards/margins": 16.608570098876953, |
| "rewards/rejected": -14.611305236816406, |
| "step": 12425 |
| }, |
| { |
| "epoch": 2.838696804720235, |
| "grad_norm": 11.6875, |
| "learning_rate": 7.89006596781398e-06, |
| "logits/chosen": -2.5183777809143066, |
| "logits/rejected": -2.2107715606689453, |
| "logps/chosen": -234.61769104003906, |
| "logps/rejected": -287.01312255859375, |
| "loss": 0.0113, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 2.3843605518341064, |
| "rewards/margins": 18.706073760986328, |
| "rewards/rejected": -16.321714401245117, |
| "step": 12450 |
| }, |
| { |
| "epoch": 2.844397571473363, |
| "grad_norm": 0.2001953125, |
| "learning_rate": 7.855071964525115e-06, |
| "logits/chosen": -2.0112006664276123, |
| "logits/rejected": -1.7871942520141602, |
| "logps/chosen": -258.3428649902344, |
| "logps/rejected": -305.0438537597656, |
| "loss": 0.0123, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 1.7488614320755005, |
| "rewards/margins": 16.58063507080078, |
| "rewards/rejected": -14.831772804260254, |
| "step": 12475 |
| }, |
| { |
| "epoch": 2.8500983382264913, |
| "grad_norm": 0.11865234375, |
| "learning_rate": 7.820105472767355e-06, |
| "logits/chosen": -2.047637939453125, |
| "logits/rejected": -1.784401297569275, |
| "logps/chosen": -285.30426025390625, |
| "logps/rejected": -312.27801513671875, |
| "loss": 0.016, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": 1.8956531286239624, |
| "rewards/margins": 15.76321792602539, |
| "rewards/rejected": -13.867566108703613, |
| "step": 12500 |
| }, |
| { |
| "epoch": 2.8500983382264913, |
| "eval_logits/chosen": -2.3994665145874023, |
| "eval_logits/rejected": -2.156980037689209, |
| "eval_logps/chosen": -265.0719299316406, |
| "eval_logps/rejected": -295.0705871582031, |
| "eval_loss": 1.2708503007888794, |
| "eval_rewards/accuracies": 0.7083570957183838, |
| "eval_rewards/chosen": -2.3470637798309326, |
| "eval_rewards/margins": 8.72215461730957, |
| "eval_rewards/rejected": -11.069217681884766, |
| "eval_runtime": 914.9534, |
| "eval_samples_per_second": 9.586, |
| "eval_steps_per_second": 9.586, |
| "step": 12500 |
| } |
| ], |
| "logging_steps": 25, |
| "max_steps": 21930, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 5, |
| "save_steps": 2500, |
| "stateful_callbacks": { |
| "EarlyStoppingCallback": { |
| "args": { |
| "early_stopping_patience": 4, |
| "early_stopping_threshold": 0.0 |
| }, |
| "attributes": { |
| "early_stopping_patience_counter": 0 |
| } |
| }, |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
