{
  "best_metric": 3.39335244348149e-08,
  "best_model_checkpoint": "/home/yunli/PycharmProjects/lmdrive/pythonProject/LMDrive_backup_20241129_230741/checkpoints/dpo-llama-7b-0119/checkpoint-7350",
  "epoch": 2.978119935170178,
  "eval_steps": 50,
  "global_step": 7350,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.004051863857374392,
      "grad_norm": 5.274203777313232,
      "learning_rate": 1.3495276653171392e-07,
      "logits/chosen": -1.389871597290039,
      "logits/rejected": -1.7107175588607788,
      "logps/chosen": -140.73594665527344,
      "logps/rejected": -174.1074981689453,
      "loss": 0.6924,
      "rewards/accuracies": 0.4037036895751953,
      "rewards/chosen": 0.0005688929231837392,
      "rewards/margins": 0.0017482050461694598,
      "rewards/rejected": -0.0011793121229857206,
      "step": 10
    },
    {
      "epoch": 0.008103727714748784,
      "grad_norm": 5.223576545715332,
      "learning_rate": 2.6990553306342785e-07,
      "logits/chosen": -1.389092206954956,
      "logits/rejected": -1.7161346673965454,
      "logps/chosen": -140.68429565429688,
      "logps/rejected": -173.65988159179688,
      "loss": 0.692,
      "rewards/accuracies": 0.5148148536682129,
      "rewards/chosen": 0.0010909124976024032,
      "rewards/margins": 0.0025206839200109243,
      "rewards/rejected": -0.0014297709567472339,
      "step": 20
    },
    {
      "epoch": 0.012155591572123177,
      "grad_norm": 4.924193859100342,
      "learning_rate": 4.048582995951417e-07,
      "logits/chosen": -1.3841500282287598,
      "logits/rejected": -1.7277183532714844,
      "logps/chosen": -141.29672241210938,
      "logps/rejected": -173.36868286132812,
      "loss": 0.6956,
      "rewards/accuracies": 0.45555558800697327,
      "rewards/chosen": -0.0018164401408284903,
      "rewards/margins": -0.004576291423290968,
      "rewards/rejected": 0.0027598512824624777,
      "step": 30
    },
    {
      "epoch": 0.01620745542949757,
      "grad_norm": 4.993196964263916,
      "learning_rate": 5.398110661268557e-07,
      "logits/chosen": -1.3897249698638916,
      "logits/rejected": -1.7075787782669067,
      "logps/chosen": -140.5751495361328,
      "logps/rejected": -173.57781982421875,
      "loss": 0.6935,
      "rewards/accuracies": 0.5333333611488342,
      "rewards/chosen": -8.312358113471419e-05,
      "rewards/margins": -0.0004725111648440361,
      "rewards/rejected": 0.0003893872199114412,
      "step": 40
    },
    {
      "epoch": 0.02025931928687196,
      "grad_norm": 5.012182712554932,
      "learning_rate": 6.747638326585696e-07,
      "logits/chosen": -1.3940094709396362,
      "logits/rejected": -1.6996283531188965,
      "logps/chosen": -140.99449157714844,
      "logps/rejected": -173.8045196533203,
      "loss": 0.6916,
      "rewards/accuracies": 0.5851852297782898,
      "rewards/chosen": 0.0015224766684696078,
      "rewards/margins": 0.003430356504395604,
      "rewards/rejected": -0.0019078799523413181,
      "step": 50
    },
    {
      "epoch": 0.02025931928687196,
      "eval_logits/chosen": -1.4222594499588013,
      "eval_logits/rejected": -1.712165355682373,
      "eval_logps/chosen": -140.99501037597656,
      "eval_logps/rejected": -173.00729370117188,
      "eval_loss": 0.6892081499099731,
      "eval_rewards/accuracies": 0.5962783098220825,
      "eval_rewards/chosen": 0.0016927955439314246,
      "eval_rewards/margins": 0.008146832697093487,
      "eval_rewards/rejected": -0.006454036571085453,
      "eval_runtime": 658.3826,
      "eval_samples_per_second": 11.246,
      "eval_steps_per_second": 0.626,
      "step": 50
    },
    {
      "epoch": 0.024311183144246355,
      "grad_norm": 5.27742338180542,
      "learning_rate": 8.097165991902834e-07,
      "logits/chosen": -1.397972583770752,
      "logits/rejected": -1.7362706661224365,
      "logps/chosen": -141.21109008789062,
      "logps/rejected": -173.7831573486328,
      "loss": 0.6885,
      "rewards/accuracies": 0.614814817905426,
      "rewards/chosen": 0.0034686957951635122,
      "rewards/margins": 0.009599005803465843,
      "rewards/rejected": -0.006130308844149113,
      "step": 60
    },
    {
      "epoch": 0.028363047001620744,
      "grad_norm": 5.308858394622803,
      "learning_rate": 9.446693657219974e-07,
      "logits/chosen": -1.4025647640228271,
      "logits/rejected": -1.7040235996246338,
      "logps/chosen": -140.78334045410156,
      "logps/rejected": -173.06678771972656,
      "loss": 0.6825,
      "rewards/accuracies": 0.7444444894790649,
      "rewards/chosen": 0.008767657913267612,
      "rewards/margins": 0.02172079123556614,
      "rewards/rejected": -0.012953130528330803,
      "step": 70
    },
    {
      "epoch": 0.03241491085899514,
      "grad_norm": 5.703453540802002,
      "learning_rate": 1.0796221322537114e-06,
      "logits/chosen": -1.3840010166168213,
      "logits/rejected": -1.6990751028060913,
      "logps/chosen": -140.64222717285156,
      "logps/rejected": -173.1254119873047,
      "loss": 0.6762,
      "rewards/accuracies": 0.8370370864868164,
      "rewards/chosen": 0.014597690664231777,
      "rewards/margins": 0.034553270787000656,
      "rewards/rejected": -0.019955581054091454,
      "step": 80
    },
    {
      "epoch": 0.03646677471636953,
      "grad_norm": 5.481204986572266,
      "learning_rate": 1.214574898785425e-06,
      "logits/chosen": -1.3807578086853027,
      "logits/rejected": -1.6951088905334473,
      "logps/chosen": -141.15838623046875,
      "logps/rejected": -173.69468688964844,
      "loss": 0.6647,
      "rewards/accuracies": 0.9481481909751892,
      "rewards/chosen": 0.021328728646039963,
      "rewards/margins": 0.05808370187878609,
      "rewards/rejected": -0.036754969507455826,
      "step": 90
    },
    {
      "epoch": 0.04051863857374392,
      "grad_norm": 6.070982456207275,
      "learning_rate": 1.3495276653171391e-06,
      "logits/chosen": -1.385872483253479,
      "logits/rejected": -1.6873222589492798,
      "logps/chosen": -139.72789001464844,
      "logps/rejected": -175.5059356689453,
      "loss": 0.6509,
      "rewards/accuracies": 0.9851852059364319,
      "rewards/chosen": 0.03459228575229645,
      "rewards/margins": 0.08671749383211136,
      "rewards/rejected": -0.052125200629234314,
      "step": 100
    },
    {
      "epoch": 0.04051863857374392,
      "eval_logits/chosen": -1.407914400100708,
      "eval_logits/rejected": -1.7055085897445679,
      "eval_logps/chosen": -140.61990356445312,
      "eval_logps/rejected": -173.56077575683594,
      "eval_loss": 0.6440849900245667,
      "eval_rewards/accuracies": 0.9978424906730652,
      "eval_rewards/chosen": 0.039202068001031876,
      "eval_rewards/margins": 0.10100384056568146,
      "eval_rewards/rejected": -0.06180177628993988,
      "eval_runtime": 658.1987,
      "eval_samples_per_second": 11.249,
      "eval_steps_per_second": 0.626,
      "step": 100
    },
    {
      "epoch": 0.04457050243111831,
      "grad_norm": 6.017471790313721,
      "learning_rate": 1.4844804318488532e-06,
      "logits/chosen": -1.3702425956726074,
      "logits/rejected": -1.7028979063034058,
      "logps/chosen": -140.50172424316406,
      "logps/rejected": -174.5872039794922,
      "loss": 0.6319,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.046755120158195496,
      "rewards/margins": 0.12688639760017395,
      "rewards/rejected": -0.08013126999139786,
      "step": 110
    },
    {
      "epoch": 0.04862236628849271,
      "grad_norm": 6.183459758758545,
      "learning_rate": 1.6194331983805669e-06,
      "logits/chosen": -1.3835982084274292,
      "logits/rejected": -1.6761491298675537,
      "logps/chosen": -141.2493133544922,
      "logps/rejected": -174.39820861816406,
      "loss": 0.6077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06035376712679863,
      "rewards/margins": 0.17960818111896515,
      "rewards/rejected": -0.11925441771745682,
      "step": 120
    },
    {
      "epoch": 0.0526742301458671,
      "grad_norm": 6.137373447418213,
      "learning_rate": 1.7543859649122807e-06,
      "logits/chosen": -1.3722988367080688,
      "logits/rejected": -1.683157205581665,
      "logps/chosen": -140.2217254638672,
      "logps/rejected": -174.78143310546875,
      "loss": 0.5756,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09007852524518967,
      "rewards/margins": 0.2515007555484772,
      "rewards/rejected": -0.1614222228527069,
      "step": 130
    },
    {
      "epoch": 0.05672609400324149,
      "grad_norm": 6.078924655914307,
      "learning_rate": 1.8893387314439948e-06,
      "logits/chosen": -1.3697142601013184,
      "logits/rejected": -1.7017598152160645,
      "logps/chosen": -140.21607971191406,
      "logps/rejected": -175.9543914794922,
      "loss": 0.5324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12757784128189087,
      "rewards/margins": 0.3537607789039612,
      "rewards/rejected": -0.2261829674243927,
      "step": 140
    },
    {
      "epoch": 0.060777957860615885,
      "grad_norm": 6.127475738525391,
      "learning_rate": 2.0242914979757085e-06,
      "logits/chosen": -1.3413002490997314,
      "logits/rejected": -1.6914191246032715,
      "logps/chosen": -139.25411987304688,
      "logps/rejected": -176.62246704101562,
      "loss": 0.4718,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17613837122917175,
      "rewards/margins": 0.5081630349159241,
      "rewards/rejected": -0.3320246636867523,
      "step": 150
    },
    {
      "epoch": 0.060777957860615885,
      "eval_logits/chosen": -1.3611022233963013,
      "eval_logits/rejected": -1.6823441982269287,
      "eval_logps/chosen": -138.89381408691406,
      "eval_logps/rejected": -176.76927185058594,
      "eval_loss": 0.4402141571044922,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 0.21181075274944305,
      "eval_rewards/margins": 0.5944603085517883,
      "eval_rewards/rejected": -0.38264963030815125,
      "eval_runtime": 658.4077,
      "eval_samples_per_second": 11.245,
      "eval_steps_per_second": 0.626,
      "step": 150
    },
    {
      "epoch": 0.06482982171799027,
      "grad_norm": 5.659084320068359,
      "learning_rate": 2.1592442645074228e-06,
      "logits/chosen": -1.3335126638412476,
      "logits/rejected": -1.6974385976791382,
      "logps/chosen": -138.15963745117188,
      "logps/rejected": -178.5204620361328,
      "loss": 0.3991,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23877687752246857,
      "rewards/margins": 0.7170161008834839,
      "rewards/rejected": -0.4782392084598541,
      "step": 160
    },
    {
      "epoch": 0.06888168557536467,
      "grad_norm": 5.250364303588867,
      "learning_rate": 2.2941970310391366e-06,
      "logits/chosen": -1.3537461757659912,
      "logits/rejected": -1.6897337436676025,
      "logps/chosen": -138.07557678222656,
      "logps/rejected": -179.66754150390625,
      "loss": 0.3186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31979554891586304,
      "rewards/margins": 0.9889208078384399,
      "rewards/rejected": -0.6691252589225769,
      "step": 170
    },
    {
      "epoch": 0.07293354943273905,
      "grad_norm": 4.185105323791504,
      "learning_rate": 2.42914979757085e-06,
      "logits/chosen": -1.3450257778167725,
      "logits/rejected": -1.6798862218856812,
      "logps/chosen": -136.57334899902344,
      "logps/rejected": -182.96791076660156,
      "loss": 0.233,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41608288884162903,
      "rewards/margins": 1.3557884693145752,
      "rewards/rejected": -0.939705491065979,
      "step": 180
    },
    {
      "epoch": 0.07698541329011345,
      "grad_norm": 3.020256757736206,
      "learning_rate": 2.564102564102564e-06,
      "logits/chosen": -1.3098284006118774,
      "logits/rejected": -1.6822631359100342,
      "logps/chosen": -135.35049438476562,
      "logps/rejected": -186.5647430419922,
      "loss": 0.147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.542268693447113,
      "rewards/margins": 1.874348521232605,
      "rewards/rejected": -1.3320801258087158,
      "step": 190
    },
    {
      "epoch": 0.08103727714748785,
      "grad_norm": 1.9457216262817383,
      "learning_rate": 2.6990553306342783e-06,
      "logits/chosen": -1.299726963043213,
      "logits/rejected": -1.669050693511963,
      "logps/chosen": -134.02984619140625,
      "logps/rejected": -189.47650146484375,
      "loss": 0.0919,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6734461188316345,
      "rewards/margins": 2.4008948802948,
      "rewards/rejected": -1.72744882106781,
      "step": 200
    },
    {
      "epoch": 0.08103727714748785,
      "eval_logits/chosen": -1.3029390573501587,
      "eval_logits/rejected": -1.6757805347442627,
      "eval_logps/chosen": -133.63436889648438,
      "eval_logps/rejected": -193.24513244628906,
      "eval_loss": 0.06510648131370544,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 0.737756073474884,
      "eval_rewards/margins": 2.767993927001953,
      "eval_rewards/rejected": -2.030237913131714,
      "eval_runtime": 658.3066,
      "eval_samples_per_second": 11.247,
      "eval_steps_per_second": 0.626,
      "step": 200
    },
    {
      "epoch": 0.08508914100486224,
      "grad_norm": 0.9668936729431152,
      "learning_rate": 2.834008097165992e-06,
      "logits/chosen": -1.265693187713623,
      "logits/rejected": -1.6668410301208496,
      "logps/chosen": -132.6597137451172,
      "logps/rejected": -195.42698669433594,
      "loss": 0.0489,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8187379837036133,
      "rewards/margins": 3.1014816761016846,
      "rewards/rejected": -2.2827439308166504,
      "step": 210
    },
    {
      "epoch": 0.08914100486223663,
      "grad_norm": 0.540846586227417,
      "learning_rate": 2.9689608636977064e-06,
      "logits/chosen": -1.2424174547195435,
      "logits/rejected": -1.6324236392974854,
      "logps/chosen": -130.07528686523438,
      "logps/rejected": -203.93658447265625,
      "loss": 0.0213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9762877225875854,
      "rewards/margins": 4.0026092529296875,
      "rewards/rejected": -3.0263214111328125,
      "step": 220
    },
    {
      "epoch": 0.09319286871961102,
      "grad_norm": 0.24707287549972534,
      "learning_rate": 3.10391363022942e-06,
      "logits/chosen": -1.196012020111084,
      "logits/rejected": -1.600135326385498,
      "logps/chosen": -128.6001434326172,
      "logps/rejected": -211.2908172607422,
      "loss": 0.0094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1740548610687256,
      "rewards/margins": 4.888430595397949,
      "rewards/rejected": -3.7143759727478027,
      "step": 230
    },
    {
      "epoch": 0.09724473257698542,
      "grad_norm": 0.13641344010829926,
      "learning_rate": 3.2388663967611337e-06,
      "logits/chosen": -1.1732877492904663,
      "logits/rejected": -1.5239564180374146,
      "logps/chosen": -127.9998550415039,
      "logps/rejected": -216.40830993652344,
      "loss": 0.0041,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3555636405944824,
      "rewards/margins": 5.810600757598877,
      "rewards/rejected": -4.455036640167236,
      "step": 240
    },
    {
      "epoch": 0.1012965964343598,
      "grad_norm": 0.08147130161523819,
      "learning_rate": 3.373819163292848e-06,
      "logits/chosen": -1.123272180557251,
      "logits/rejected": -1.5325947999954224,
      "logps/chosen": -124.99156188964844,
      "logps/rejected": -224.6184844970703,
      "loss": 0.002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5564848184585571,
      "rewards/margins": 6.633443832397461,
      "rewards/rejected": -5.076958656311035,
      "step": 250
    },
    {
      "epoch": 0.1012965964343598,
      "eval_logits/chosen": -1.1526036262512207,
      "eval_logits/rejected": -1.5364301204681396,
      "eval_logps/chosen": -125.28230285644531,
      "eval_logps/rejected": -226.2448272705078,
      "eval_loss": 0.0015159068861976266,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.572962760925293,
      "eval_rewards/margins": 6.90316915512085,
      "eval_rewards/rejected": -5.330205917358398,
      "eval_runtime": 658.1337,
      "eval_samples_per_second": 11.25,
      "eval_steps_per_second": 0.626,
      "step": 250
    },
    {
      "epoch": 0.1053484602917342,
      "grad_norm": 0.06831583380699158,
      "learning_rate": 3.5087719298245615e-06,
      "logits/chosen": -1.11264169216156,
      "logits/rejected": -1.521393060684204,
      "logps/chosen": -124.75511169433594,
      "logps/rejected": -229.17233276367188,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.65181565284729,
      "rewards/margins": 7.160605430603027,
      "rewards/rejected": -5.508789539337158,
      "step": 260
    },
    {
      "epoch": 0.1094003241491086,
      "grad_norm": 0.021368836984038353,
      "learning_rate": 3.6437246963562758e-06,
      "logits/chosen": -1.126573920249939,
      "logits/rejected": -1.5180697441101074,
      "logps/chosen": -124.58089447021484,
      "logps/rejected": -231.1531219482422,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7233182191848755,
      "rewards/margins": 7.557948112487793,
      "rewards/rejected": -5.834630489349365,
      "step": 270
    },
    {
      "epoch": 0.11345218800648298,
      "grad_norm": 0.022432563826441765,
      "learning_rate": 3.7786774628879896e-06,
      "logits/chosen": -1.1377226114273071,
      "logits/rejected": -1.5355565547943115,
      "logps/chosen": -122.95037078857422,
      "logps/rejected": -234.68670654296875,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7897114753723145,
      "rewards/margins": 7.988018989562988,
      "rewards/rejected": -6.198306083679199,
      "step": 280
    },
    {
      "epoch": 0.11750405186385737,
      "grad_norm": 0.019663700833916664,
      "learning_rate": 3.9136302294197035e-06,
      "logits/chosen": -1.131860613822937,
      "logits/rejected": -1.4960014820098877,
      "logps/chosen": -122.1822738647461,
      "logps/rejected": -235.7807159423828,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.825466513633728,
      "rewards/margins": 8.117959022521973,
      "rewards/rejected": -6.292492389678955,
      "step": 290
    },
    {
      "epoch": 0.12155591572123177,
      "grad_norm": 0.017695888876914978,
      "learning_rate": 4.048582995951417e-06,
      "logits/chosen": -1.13109290599823,
      "logits/rejected": -1.53248929977417,
      "logps/chosen": -122.07437133789062,
      "logps/rejected": -237.8642120361328,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.871293306350708,
      "rewards/margins": 8.233868598937988,
      "rewards/rejected": -6.362576484680176,
      "step": 300
    },
    {
      "epoch": 0.12155591572123177,
      "eval_logits/chosen": -1.134786605834961,
      "eval_logits/rejected": -1.5121405124664307,
      "eval_logps/chosen": -122.26962280273438,
      "eval_logps/rejected": -237.97372436523438,
      "eval_loss": 0.00040686846477910876,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 1.8742293119430542,
      "eval_rewards/margins": 8.377324104309082,
      "eval_rewards/rejected": -6.50309419631958,
      "eval_runtime": 658.1422,
      "eval_samples_per_second": 11.25,
      "eval_steps_per_second": 0.626,
      "step": 300
    },
    {
      "epoch": 0.12560777957860617,
      "grad_norm": 0.012135895900428295,
      "learning_rate": 4.183535762483131e-06,
      "logits/chosen": -1.109740972518921,
      "logits/rejected": -1.5193902254104614,
      "logps/chosen": -121.0916519165039,
      "logps/rejected": -240.30992126464844,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9484295845031738,
      "rewards/margins": 8.592082023620605,
      "rewards/rejected": -6.643653392791748,
      "step": 310
    },
    {
      "epoch": 0.12965964343598055,
      "grad_norm": 0.010144763626158237,
      "learning_rate": 4.3184885290148456e-06,
      "logits/chosen": -1.1031049489974976,
      "logits/rejected": -1.533254861831665,
      "logps/chosen": -120.3249282836914,
      "logps/rejected": -242.37258911132812,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9627697467803955,
      "rewards/margins": 8.811460494995117,
      "rewards/rejected": -6.848690986633301,
      "step": 320
    },
    {
      "epoch": 0.13371150729335493,
      "grad_norm": 0.010636514984071255,
      "learning_rate": 4.453441295546559e-06,
      "logits/chosen": -1.1099517345428467,
      "logits/rejected": -1.4838870763778687,
      "logps/chosen": -120.37928771972656,
      "logps/rejected": -243.31271362304688,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9846471548080444,
      "rewards/margins": 8.979297637939453,
      "rewards/rejected": -6.994650363922119,
      "step": 330
    },
    {
      "epoch": 0.13776337115072934,
      "grad_norm": 0.011891559697687626,
      "learning_rate": 4.588394062078273e-06,
      "logits/chosen": -1.1261647939682007,
      "logits/rejected": -1.5122199058532715,
      "logps/chosen": -120.46336364746094,
      "logps/rejected": -244.4058380126953,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.024125337600708,
      "rewards/margins": 9.096206665039062,
      "rewards/rejected": -7.072080135345459,
      "step": 340
    },
    {
      "epoch": 0.14181523500810372,
      "grad_norm": 0.008843648247420788,
      "learning_rate": 4.723346828609987e-06,
      "logits/chosen": -1.117754340171814,
      "logits/rejected": -1.5109072923660278,
      "logps/chosen": -120.10042572021484,
      "logps/rejected": -245.29542541503906,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0470709800720215,
      "rewards/margins": 9.163729667663574,
      "rewards/rejected": -7.116659164428711,
      "step": 350
    },
    {
      "epoch": 0.14181523500810372,
      "eval_logits/chosen": -1.1409276723861694,
      "eval_logits/rejected": -1.519826889038086,
      "eval_logps/chosen": -120.77721405029297,
      "eval_logps/rejected": -244.39198303222656,
      "eval_loss": 0.0002016112848650664,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.023472547531128,
      "eval_rewards/margins": 9.168397903442383,
      "eval_rewards/rejected": -7.144925117492676,
      "eval_runtime": 658.0871,
      "eval_samples_per_second": 11.251,
      "eval_steps_per_second": 0.626,
      "step": 350
    },
    {
      "epoch": 0.1458670988654781,
      "grad_norm": 0.01200284343212843,
      "learning_rate": 4.8582995951417e-06,
      "logits/chosen": -1.1370290517807007,
      "logits/rejected": -1.5248901844024658,
      "logps/chosen": -120.55464172363281,
      "logps/rejected": -247.7692413330078,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.083092212677002,
      "rewards/margins": 9.412875175476074,
      "rewards/rejected": -7.3297834396362305,
      "step": 360
    },
    {
      "epoch": 0.14991896272285252,
      "grad_norm": 0.005621300544589758,
      "learning_rate": 4.9932523616734145e-06,
      "logits/chosen": -1.1131725311279297,
      "logits/rejected": -1.519146203994751,
      "logps/chosen": -120.15304565429688,
      "logps/rejected": -246.8217315673828,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.124485731124878,
      "rewards/margins": 9.502544403076172,
      "rewards/rejected": -7.378057956695557,
      "step": 370
    },
    {
      "epoch": 0.1539708265802269,
      "grad_norm": 0.004424644634127617,
      "learning_rate": 5.128205128205128e-06,
      "logits/chosen": -1.1332614421844482,
      "logits/rejected": -1.5120524168014526,
      "logps/chosen": -120.06082916259766,
      "logps/rejected": -248.7280731201172,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0993828773498535,
      "rewards/margins": 9.636995315551758,
      "rewards/rejected": -7.5376129150390625,
      "step": 380
    },
    {
      "epoch": 0.1580226904376013,
      "grad_norm": 0.005630790255963802,
      "learning_rate": 5.263157894736842e-06,
      "logits/chosen": -1.116791844367981,
      "logits/rejected": -1.5200690031051636,
      "logps/chosen": -118.8879165649414,
      "logps/rejected": -248.0203399658203,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.148820400238037,
      "rewards/margins": 9.649928092956543,
      "rewards/rejected": -7.501107215881348,
      "step": 390
    },
    {
      "epoch": 0.1620745542949757,
      "grad_norm": 0.007051733788102865,
      "learning_rate": 5.3981106612685565e-06,
      "logits/chosen": -1.099058747291565,
      "logits/rejected": -1.481469988822937,
      "logps/chosen": -119.5155258178711,
      "logps/rejected": -250.01971435546875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.180222749710083,
      "rewards/margins": 9.90124797821045,
      "rewards/rejected": -7.721026420593262,
      "step": 400
    },
    {
      "epoch": 0.1620745542949757,
      "eval_logits/chosen": -1.1320279836654663,
      "eval_logits/rejected": -1.522124171257019,
      "eval_logps/chosen": -119.53314971923828,
      "eval_logps/rejected": -249.70321655273438,
      "eval_loss": 0.00011243653716519475,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.1478781700134277,
      "eval_rewards/margins": 9.82392692565918,
      "eval_rewards/rejected": -7.6760478019714355,
      "eval_runtime": 658.0443,
      "eval_samples_per_second": 11.252,
      "eval_steps_per_second": 0.626,
      "step": 400
    },
    {
      "epoch": 0.16612641815235007,
      "grad_norm": 0.007869623601436615,
      "learning_rate": 5.533063427800271e-06,
      "logits/chosen": -1.1087368726730347,
      "logits/rejected": -1.4966099262237549,
      "logps/chosen": -118.79527282714844,
      "logps/rejected": -250.2746124267578,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1891911029815674,
      "rewards/margins": 9.879663467407227,
      "rewards/rejected": -7.6904730796813965,
      "step": 410
    },
    {
      "epoch": 0.17017828200972449,
      "grad_norm": 0.0037466222420334816,
      "learning_rate": 5.668016194331984e-06,
      "logits/chosen": -1.1184862852096558,
      "logits/rejected": -1.503065586090088,
      "logps/chosen": -118.98182678222656,
      "logps/rejected": -252.21949768066406,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2015702724456787,
      "rewards/margins": 10.133565902709961,
      "rewards/rejected": -7.931995868682861,
      "step": 420
    },
    {
      "epoch": 0.17423014586709887,
      "grad_norm": 0.002610385650768876,
      "learning_rate": 5.8029689608636986e-06,
      "logits/chosen": -1.0745251178741455,
      "logits/rejected": -1.5020885467529297,
      "logps/chosen": -118.70890045166016,
      "logps/rejected": -254.2556915283203,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.255699396133423,
      "rewards/margins": 10.259760856628418,
      "rewards/rejected": -8.004060745239258,
      "step": 430
    },
    {
      "epoch": 0.17828200972447325,
      "grad_norm": 0.0037527934182435274,
      "learning_rate": 5.937921727395413e-06,
      "logits/chosen": -1.0715187788009644,
      "logits/rejected": -1.4950731992721558,
      "logps/chosen": -117.52845001220703,
      "logps/rejected": -253.7173614501953,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.288992404937744,
      "rewards/margins": 10.322542190551758,
      "rewards/rejected": -8.033550262451172,
      "step": 440
    },
    {
      "epoch": 0.18233387358184766,
      "grad_norm": 0.004331698641180992,
      "learning_rate": 6.0728744939271254e-06,
      "logits/chosen": -1.0888125896453857,
      "logits/rejected": -1.5274492502212524,
      "logps/chosen": -118.01427459716797,
      "logps/rejected": -256.4695739746094,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.280319929122925,
      "rewards/margins": 10.511781692504883,
      "rewards/rejected": -8.231461524963379,
      "step": 450
    },
    {
      "epoch": 0.18233387358184766,
      "eval_logits/chosen": -1.091180443763733,
      "eval_logits/rejected": -1.5023369789123535,
      "eval_logps/chosen": -118.44429016113281,
      "eval_logps/rejected": -254.42095947265625,
      "eval_loss": 6.72021706122905e-05,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.2567639350891113,
      "eval_rewards/margins": 10.404584884643555,
      "eval_rewards/rejected": -8.147821426391602,
      "eval_runtime": 658.0769,
      "eval_samples_per_second": 11.251,
      "eval_steps_per_second": 0.626,
      "step": 450
    },
    {
      "epoch": 0.18638573743922204,
      "grad_norm": 0.0027279488276690245,
      "learning_rate": 6.20782726045884e-06,
      "logits/chosen": -1.087437391281128,
      "logits/rejected": -1.5125293731689453,
      "logps/chosen": -118.81121063232422,
      "logps/rejected": -255.43765258789062,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.273927927017212,
      "rewards/margins": 10.494739532470703,
      "rewards/rejected": -8.220812797546387,
      "step": 460
    },
    {
      "epoch": 0.19043760129659643,
      "grad_norm": 0.0034659032244235277,
      "learning_rate": 6.342780026990554e-06,
      "logits/chosen": -1.077628493309021,
      "logits/rejected": -1.4835717678070068,
      "logps/chosen": -117.53942108154297,
      "logps/rejected": -256.7050476074219,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.338557720184326,
      "rewards/margins": 10.676908493041992,
      "rewards/rejected": -8.33835220336914,
      "step": 470
    },
    {
      "epoch": 0.19448946515397084,
      "grad_norm": 0.0018187056994065642,
      "learning_rate": 6.4777327935222675e-06,
      "logits/chosen": -1.0798588991165161,
      "logits/rejected": -1.5022971630096436,
      "logps/chosen": -116.7596664428711,
      "logps/rejected": -258.91400146484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3907530307769775,
      "rewards/margins": 10.954094886779785,
      "rewards/rejected": -8.563342094421387,
      "step": 480
    },
    {
      "epoch": 0.19854132901134522,
      "grad_norm": 0.003787844907492399,
      "learning_rate": 6.612685560053982e-06,
      "logits/chosen": -1.0318020582199097,
      "logits/rejected": -1.4701728820800781,
      "logps/chosen": -118.04106903076172,
      "logps/rejected": -257.62615966796875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.336604118347168,
      "rewards/margins": 10.68603515625,
      "rewards/rejected": -8.349430084228516,
      "step": 490
    },
    {
      "epoch": 0.2025931928687196,
      "grad_norm": 0.001693087164312601,
      "learning_rate": 6.747638326585696e-06,
      "logits/chosen": -1.0626085996627808,
      "logits/rejected": -1.4769538640975952,
      "logps/chosen": -117.07071685791016,
      "logps/rejected": -257.8610534667969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3624086380004883,
      "rewards/margins": 10.87511157989502,
      "rewards/rejected": -8.512703895568848,
      "step": 500
    },
    {
      "epoch": 0.2025931928687196,
      "eval_logits/chosen": -1.078675627708435,
      "eval_logits/rejected": -1.4860773086547852,
      "eval_logps/chosen": -117.57008361816406,
      "eval_logps/rejected": -258.2732849121094,
      "eval_loss": 4.4150063331471756e-05,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.3441848754882812,
      "eval_rewards/margins": 10.877237319946289,
      "eval_rewards/rejected": -8.533052444458008,
      "eval_runtime": 657.9761,
      "eval_samples_per_second": 11.253,
      "eval_steps_per_second": 0.626,
      "step": 500
    },
    {
      "epoch": 0.206645056726094,
      "grad_norm": 0.001665365183725953,
      "learning_rate": 6.882591093117409e-06,
      "logits/chosen": -1.0459882020950317,
      "logits/rejected": -1.4801123142242432,
      "logps/chosen": -116.1864242553711,
      "logps/rejected": -259.5511779785156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4086875915527344,
      "rewards/margins": 11.01952838897705,
      "rewards/rejected": -8.610840797424316,
      "step": 510
    },
    {
      "epoch": 0.2106969205834684,
      "grad_norm": 0.0014464287087321281,
      "learning_rate": 7.017543859649123e-06,
      "logits/chosen": -1.0493347644805908,
      "logits/rejected": -1.4791755676269531,
      "logps/chosen": -115.95133972167969,
      "logps/rejected": -260.8875427246094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4290733337402344,
      "rewards/margins": 11.176859855651855,
      "rewards/rejected": -8.747786521911621,
      "step": 520
    },
    {
      "epoch": 0.21474878444084278,
      "grad_norm": 0.0018413382349535823,
      "learning_rate": 7.152496626180837e-06,
      "logits/chosen": -1.0302752256393433,
      "logits/rejected": -1.4574908018112183,
      "logps/chosen": -116.60781860351562,
      "logps/rejected": -261.29876708984375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.466017484664917,
      "rewards/margins": 11.22760009765625,
      "rewards/rejected": -8.761584281921387,
      "step": 530
    },
    {
      "epoch": 0.2188006482982172,
      "grad_norm": 0.0014453129842877388,
      "learning_rate": 7.2874493927125516e-06,
      "logits/chosen": -1.05780827999115,
      "logits/rejected": -1.4844599962234497,
      "logps/chosen": -116.33468627929688,
      "logps/rejected": -260.7745666503906,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.44503116607666,
      "rewards/margins": 11.185885429382324,
      "rewards/rejected": -8.74085521697998,
      "step": 540
    },
    {
      "epoch": 0.22285251215559157,
      "grad_norm": 0.002703328849747777,
      "learning_rate": 7.422402159244265e-06,
      "logits/chosen": -1.0520823001861572,
      "logits/rejected": -1.4584662914276123,
      "logps/chosen": -116.45873260498047,
      "logps/rejected": -262.7840576171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4690232276916504,
      "rewards/margins": 11.33821964263916,
      "rewards/rejected": -8.869195938110352,
      "step": 550
    },
    {
      "epoch": 0.22285251215559157,
      "eval_logits/chosen": -1.0702204704284668,
      "eval_logits/rejected": -1.4801982641220093,
      "eval_logps/chosen": -116.69920349121094,
      "eval_logps/rejected": -261.5146789550781,
      "eval_loss": 3.066386852879077e-05,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.4312729835510254,
      "eval_rewards/margins": 11.288468360900879,
      "eval_rewards/rejected": -8.857195854187012,
      "eval_runtime": 657.8882,
      "eval_samples_per_second": 11.254,
      "eval_steps_per_second": 0.626,
      "step": 550
    },
    {
      "epoch": 0.22690437601296595,
      "grad_norm": 0.0012216470204293728,
      "learning_rate": 7.557354925775979e-06,
      "logits/chosen": -1.0269840955734253,
      "logits/rejected": -1.4592022895812988,
      "logps/chosen": -116.34583282470703,
      "logps/rejected": -263.69110107421875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5144259929656982,
      "rewards/margins": 11.517362594604492,
      "rewards/rejected": -9.002935409545898,
      "step": 560
    },
    {
      "epoch": 0.23095623987034036,
      "grad_norm": 0.0007370927487500012,
      "learning_rate": 7.692307692307694e-06,
      "logits/chosen": -1.0158203840255737,
      "logits/rejected": -1.447105050086975,
      "logps/chosen": -116.02507781982422,
      "logps/rejected": -264.2305603027344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.493821382522583,
      "rewards/margins": 11.59961986541748,
      "rewards/rejected": -9.105795860290527,
      "step": 570
    },
    {
      "epoch": 0.23500810372771475,
      "grad_norm": 0.0022627676371484995,
      "learning_rate": 7.827260458839407e-06,
      "logits/chosen": -1.0068782567977905,
      "logits/rejected": -1.4721413850784302,
      "logps/chosen": -116.01976776123047,
      "logps/rejected": -264.0223083496094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.528806209564209,
      "rewards/margins": 11.57716178894043,
      "rewards/rejected": -9.048355102539062,
      "step": 580
    },
    {
      "epoch": 0.23905996758508913,
      "grad_norm": 0.0015734812477603555,
      "learning_rate": 7.96221322537112e-06,
      "logits/chosen": -1.008754014968872,
      "logits/rejected": -1.4133723974227905,
      "logps/chosen": -116.37065124511719,
      "logps/rejected": -264.7450866699219,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.539128065109253,
      "rewards/margins": 11.731771469116211,
      "rewards/rejected": -9.192643165588379,
      "step": 590
    },
    {
      "epoch": 0.24311183144246354,
      "grad_norm": 0.0008875586208887398,
      "learning_rate": 8.097165991902834e-06,
      "logits/chosen": -1.0271269083023071,
      "logits/rejected": -1.4361445903778076,
      "logps/chosen": -115.80826568603516,
      "logps/rejected": -264.06683349609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.54843807220459,
      "rewards/margins": 11.632083892822266,
      "rewards/rejected": -9.083645820617676,
      "step": 600
    },
    {
      "epoch": 0.24311183144246354,
      "eval_logits/chosen": -1.0300260782241821,
      "eval_logits/rejected": -1.443122386932373,
      "eval_logps/chosen": -115.9371109008789,
      "eval_logps/rejected": -264.74847412109375,
      "eval_loss": 2.165735168091487e-05,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.507481813430786,
      "eval_rewards/margins": 11.688055038452148,
      "eval_rewards/rejected": -9.180574417114258,
      "eval_runtime": 658.0461,
      "eval_samples_per_second": 11.251,
      "eval_steps_per_second": 0.626,
      "step": 600
    },
    {
      "epoch": 0.24716369529983792,
      "grad_norm": 0.0007235104567371309,
      "learning_rate": 8.232118758434549e-06,
      "logits/chosen": -1.0256165266036987,
      "logits/rejected": -1.4475750923156738,
      "logps/chosen": -114.85009765625,
      "logps/rejected": -264.8584289550781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5647478103637695,
      "rewards/margins": 11.745806694030762,
      "rewards/rejected": -9.181057929992676,
      "step": 610
    },
    {
      "epoch": 0.25121555915721233,
      "grad_norm": 0.0009592686546966434,
      "learning_rate": 8.367071524966263e-06,
      "logits/chosen": -0.9948481917381287,
      "logits/rejected": -1.4419747591018677,
      "logps/chosen": -116.20551300048828,
      "logps/rejected": -267.3464660644531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.584946870803833,
      "rewards/margins": 11.9420747756958,
      "rewards/rejected": -9.357129096984863,
      "step": 620
    },
    {
      "epoch": 0.2552674230145867,
      "grad_norm": 0.0018740741070359945,
      "learning_rate": 8.502024291497976e-06,
      "logits/chosen": -1.0022295713424683,
      "logits/rejected": -1.404227614402771,
      "logps/chosen": -115.06513214111328,
      "logps/rejected": -266.9801025390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.606013774871826,
      "rewards/margins": 11.995136260986328,
      "rewards/rejected": -9.389123916625977,
      "step": 630
    },
    {
      "epoch": 0.2593192868719611,
      "grad_norm": 0.0010919544147327542,
      "learning_rate": 8.636977058029691e-06,
      "logits/chosen": -0.9951332211494446,
      "logits/rejected": -1.448172926902771,
      "logps/chosen": -114.10124969482422,
      "logps/rejected": -268.3458557128906,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6123576164245605,
      "rewards/margins": 12.068544387817383,
      "rewards/rejected": -9.45618724822998,
      "step": 640
    },
    {
      "epoch": 0.2633711507293355,
      "grad_norm": 0.0008402303792536259,
      "learning_rate": 8.771929824561405e-06,
      "logits/chosen": -1.0138694047927856,
      "logits/rejected": -1.4006102085113525,
      "logps/chosen": -115.1278076171875,
      "logps/rejected": -268.4206237792969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.596374750137329,
      "rewards/margins": 12.014070510864258,
      "rewards/rejected": -9.417695999145508,
      "step": 650
    },
    {
      "epoch": 0.2633711507293355,
      "eval_logits/chosen": -1.031700849533081,
      "eval_logits/rejected": -1.4289768934249878,
      "eval_logps/chosen": -115.24848175048828,
      "eval_logps/rejected": -267.67767333984375,
      "eval_loss": 1.5523668480454944e-05,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.576345682144165,
      "eval_rewards/margins": 12.049837112426758,
      "eval_rewards/rejected": -9.473491668701172,
      "eval_runtime": 658.0307,
      "eval_samples_per_second": 11.252,
      "eval_steps_per_second": 0.626,
      "step": 650
    },
    {
      "epoch": 0.26742301458670986,
      "grad_norm": 0.0010203344281762838,
      "learning_rate": 8.906882591093118e-06,
      "logits/chosen": -0.9803736209869385,
      "logits/rejected": -1.3996490240097046,
      "logps/chosen": -113.33806610107422,
      "logps/rejected": -269.2818603515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6770434379577637,
      "rewards/margins": 12.208734512329102,
      "rewards/rejected": -9.53169059753418,
      "step": 660
    },
    {
      "epoch": 0.2714748784440843,
      "grad_norm": 0.0009641316719353199,
      "learning_rate": 9.041835357624831e-06,
      "logits/chosen": -1.0069340467453003,
      "logits/rejected": -1.416864275932312,
      "logps/chosen": -114.13117218017578,
      "logps/rejected": -268.98675537109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6602096557617188,
      "rewards/margins": 12.218779563903809,
      "rewards/rejected": -9.558568954467773,
      "step": 670
    },
    {
      "epoch": 0.2755267423014587,
      "grad_norm": 0.0004945212276652455,
      "learning_rate": 9.176788124156547e-06,
      "logits/chosen": -1.000825047492981,
      "logits/rejected": -1.411185622215271,
      "logps/chosen": -114.98800659179688,
      "logps/rejected": -272.5845031738281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.673659563064575,
      "rewards/margins": 12.43789005279541,
      "rewards/rejected": -9.764229774475098,
      "step": 680
    },
    {
      "epoch": 0.27957860615883307,
      "grad_norm": 0.0016324473544955254,
      "learning_rate": 9.31174089068826e-06,
      "logits/chosen": -0.980940043926239,
      "logits/rejected": -1.3893826007843018,
      "logps/chosen": -114.27484893798828,
      "logps/rejected": -269.0587158203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6675384044647217,
      "rewards/margins": 12.248590469360352,
      "rewards/rejected": -9.58105182647705,
      "step": 690
    },
    {
      "epoch": 0.28363047001620745,
      "grad_norm": 0.0005120071582496166,
      "learning_rate": 9.446693657219973e-06,
      "logits/chosen": -0.9979413747787476,
      "logits/rejected": -1.3880137205123901,
      "logps/chosen": -113.4884262084961,
      "logps/rejected": -270.5970764160156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6926565170288086,
      "rewards/margins": 12.41943359375,
      "rewards/rejected": -9.726777076721191,
      "step": 700
    },
    {
      "epoch": 0.28363047001620745,
      "eval_logits/chosen": -1.0087049007415771,
      "eval_logits/rejected": -1.3951642513275146,
      "eval_logps/chosen": -114.66770935058594,
      "eval_logps/rejected": -270.48291015625,
      "eval_loss": 1.1520413863763679e-05,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.6344213485717773,
      "eval_rewards/margins": 12.388434410095215,
      "eval_rewards/rejected": -9.754013061523438,
      "eval_runtime": 658.0487,
      "eval_samples_per_second": 11.251,
      "eval_steps_per_second": 0.626,
      "step": 700
    },
    {
      "epoch": 0.28768233387358183,
      "grad_norm": 0.0006853205850347877,
      "learning_rate": 9.581646423751689e-06,
      "logits/chosen": -0.9878454208374023,
      "logits/rejected": -1.3959782123565674,
      "logps/chosen": -113.78047943115234,
      "logps/rejected": -272.41021728515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.711303949356079,
      "rewards/margins": 12.54239559173584,
      "rewards/rejected": -9.831093788146973,
      "step": 710
    },
    {
      "epoch": 0.2917341977309562,
      "grad_norm": 0.0004263566224835813,
      "learning_rate": 9.7165991902834e-06,
      "logits/chosen": -0.9797188639640808,
      "logits/rejected": -1.3997111320495605,
      "logps/chosen": -113.36839294433594,
      "logps/rejected": -272.79583740234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.742356300354004,
      "rewards/margins": 12.54971981048584,
      "rewards/rejected": -9.807363510131836,
      "step": 720
    },
    {
      "epoch": 0.29578606158833065,
      "grad_norm": 0.00023152201902121305,
      "learning_rate": 9.851551956815116e-06,
      "logits/chosen": -0.9830971956253052,
      "logits/rejected": -1.3845362663269043,
      "logps/chosen": -113.41996765136719,
      "logps/rejected": -273.1960144042969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.721712350845337,
      "rewards/margins": 12.736370086669922,
      "rewards/rejected": -10.014657974243164,
      "step": 730
    },
    {
      "epoch": 0.29983792544570503,
      "grad_norm": 0.0005608124774880707,
      "learning_rate": 9.986504723346829e-06,
      "logits/chosen": -0.9742939472198486,
      "logits/rejected": -1.3444586992263794,
      "logps/chosen": -113.43490600585938,
      "logps/rejected": -274.0641174316406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6840059757232666,
      "rewards/margins": 12.688366889953613,
      "rewards/rejected": -10.00436019897461,
      "step": 740
    },
    {
      "epoch": 0.3038897893030794,
      "grad_norm": 0.0005869499291293323,
      "learning_rate": 9.986492570914004e-06,
      "logits/chosen": -0.9667108654975891,
      "logits/rejected": -1.383889079093933,
      "logps/chosen": -113.077392578125,
      "logps/rejected": -273.8304138183594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.772247552871704,
      "rewards/margins": 12.869462966918945,
      "rewards/rejected": -10.09721565246582,
      "step": 750
    },
    {
      "epoch": 0.3038897893030794,
      "eval_logits/chosen": -1.0030393600463867,
      "eval_logits/rejected": -1.3840956687927246,
      "eval_logps/chosen": -114.08601379394531,
      "eval_logps/rejected": -272.99749755859375,
      "eval_loss": 8.76898684509797e-06,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.6925902366638184,
      "eval_rewards/margins": 12.6980619430542,
      "eval_rewards/rejected": -10.005472183227539,
      "eval_runtime": 657.9195,
      "eval_samples_per_second": 11.254,
      "eval_steps_per_second": 0.626,
      "step": 750
    },
    {
      "epoch": 0.3079416531604538,
      "grad_norm": 0.0002582529850769788,
      "learning_rate": 9.971484316374006e-06,
      "logits/chosen": -0.9930279850959778,
      "logits/rejected": -1.3527050018310547,
      "logps/chosen": -114.37895202636719,
      "logps/rejected": -273.5732421875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.73421573638916,
      "rewards/margins": 12.774086952209473,
      "rewards/rejected": -10.039870262145996,
      "step": 760
    },
    {
      "epoch": 0.3119935170178282,
      "grad_norm": 0.00047567905858159065,
      "learning_rate": 9.95647606183401e-06,
      "logits/chosen": -0.9919410943984985,
      "logits/rejected": -1.3644182682037354,
      "logps/chosen": -113.54793548583984,
      "logps/rejected": -275.1521911621094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.78049373626709,
      "rewards/margins": 12.89228343963623,
      "rewards/rejected": -10.11178970336914,
      "step": 770
    },
    {
      "epoch": 0.3160453808752026,
      "grad_norm": 0.0004460848867893219,
      "learning_rate": 9.941467807294013e-06,
      "logits/chosen": -0.9861297011375427,
      "logits/rejected": -1.3743053674697876,
      "logps/chosen": -112.3475570678711,
      "logps/rejected": -275.0509948730469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.791991710662842,
      "rewards/margins": 12.899449348449707,
      "rewards/rejected": -10.107457160949707,
      "step": 780
    },
    {
      "epoch": 0.320097244732577,
      "grad_norm": 0.00035315827699378133,
      "learning_rate": 9.926459552754015e-06,
      "logits/chosen": -0.9596935510635376,
      "logits/rejected": -1.3788365125656128,
      "logps/chosen": -113.31448364257812,
      "logps/rejected": -276.55328369140625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.783583641052246,
      "rewards/margins": 13.183259963989258,
      "rewards/rejected": -10.399676322937012,
      "step": 790
    },
    {
      "epoch": 0.3241491085899514,
      "grad_norm": 0.0005995644023641944,
      "learning_rate": 9.911451298214018e-06,
      "logits/chosen": -0.9586429595947266,
      "logits/rejected": -1.3155739307403564,
      "logps/chosen": -112.5149917602539,
      "logps/rejected": -275.69073486328125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.829101324081421,
      "rewards/margins": 13.077914237976074,
      "rewards/rejected": -10.248811721801758,
      "step": 800
    },
    {
      "epoch": 0.3241491085899514,
      "eval_logits/chosen": -1.0101659297943115,
      "eval_logits/rejected": -1.3674921989440918,
      "eval_logps/chosen": -113.71437072753906,
      "eval_logps/rejected": -275.37432861328125,
      "eval_loss": 6.814159860368818e-06,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.72975492477417,
      "eval_rewards/margins": 12.972909927368164,
      "eval_rewards/rejected": -10.243154525756836,
      "eval_runtime": 660.0291,
      "eval_samples_per_second": 11.218,
      "eval_steps_per_second": 0.624,
      "step": 800
    },
    {
      "epoch": 0.32820097244732577,
      "grad_norm": 0.0001659575937082991,
      "learning_rate": 9.896443043674022e-06,
      "logits/chosen": -0.9594641327857971,
      "logits/rejected": -1.3676989078521729,
      "logps/chosen": -111.76673889160156,
      "logps/rejected": -277.4734802246094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8681840896606445,
      "rewards/margins": 13.216644287109375,
      "rewards/rejected": -10.34846019744873,
      "step": 810
    },
    {
      "epoch": 0.33225283630470015,
      "grad_norm": 0.00022199955128598958,
      "learning_rate": 9.881434789134025e-06,
      "logits/chosen": -0.979736328125,
      "logits/rejected": -1.3501309156417847,
      "logps/chosen": -112.58365631103516,
      "logps/rejected": -277.4759521484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.794842481613159,
      "rewards/margins": 13.254104614257812,
      "rewards/rejected": -10.459260940551758,
      "step": 820
    },
    {
      "epoch": 0.33630470016207453,
      "grad_norm": 0.0002384011313552037,
      "learning_rate": 9.866426534594028e-06,
      "logits/chosen": -0.970429539680481,
      "logits/rejected": -1.355580449104309,
      "logps/chosen": -112.8162612915039,
      "logps/rejected": -277.3512268066406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7844314575195312,
      "rewards/margins": 13.164278030395508,
      "rewards/rejected": -10.379846572875977,
      "step": 830
    },
    {
      "epoch": 0.34035656401944897,
      "grad_norm": 0.0003860573924612254,
      "learning_rate": 9.85141828005403e-06,
      "logits/chosen": -0.9769508242607117,
      "logits/rejected": -1.3295193910598755,
      "logps/chosen": -113.16212463378906,
      "logps/rejected": -277.39874267578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.831162691116333,
      "rewards/margins": 13.187525749206543,
      "rewards/rejected": -10.356362342834473,
      "step": 840
    },
    {
      "epoch": 0.34440842787682335,
      "grad_norm": 0.00017003035463858396,
      "learning_rate": 9.836410025514034e-06,
      "logits/chosen": -0.9663823246955872,
      "logits/rejected": -1.3480101823806763,
      "logps/chosen": -112.63446807861328,
      "logps/rejected": -278.4540100097656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.832660675048828,
      "rewards/margins": 13.384835243225098,
      "rewards/rejected": -10.552173614501953,
      "step": 850
    },
    {
      "epoch": 0.34440842787682335,
      "eval_logits/chosen": -1.0045629739761353,
      "eval_logits/rejected": -1.3488807678222656,
      "eval_logps/chosen": -113.40267181396484,
      "eval_logps/rejected": -277.4642333984375,
      "eval_loss": 5.4946622185525484e-06,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.760925531387329,
      "eval_rewards/margins": 13.213071823120117,
      "eval_rewards/rejected": -10.452144622802734,
      "eval_runtime": 658.0453,
      "eval_samples_per_second": 11.252,
      "eval_steps_per_second": 0.626,
      "step": 850
    },
    {
      "epoch": 0.34846029173419774,
      "grad_norm": 0.000273724872386083,
      "learning_rate": 9.821401770974035e-06,
      "logits/chosen": -0.9551014304161072,
      "logits/rejected": -1.3251674175262451,
      "logps/chosen": -112.65969848632812,
      "logps/rejected": -278.6461486816406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8454368114471436,
      "rewards/margins": 13.34626293182373,
      "rewards/rejected": -10.500825881958008,
      "step": 860
    },
    {
      "epoch": 0.3525121555915721,
      "grad_norm": 0.0002813730388879776,
      "learning_rate": 9.806393516434039e-06,
      "logits/chosen": -0.9453781247138977,
      "logits/rejected": -1.3513928651809692,
      "logps/chosen": -112.64397430419922,
      "logps/rejected": -279.1123352050781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8162314891815186,
      "rewards/margins": 13.449708938598633,
      "rewards/rejected": -10.633478164672852,
      "step": 870
    },
    {
      "epoch": 0.3565640194489465,
      "grad_norm": 0.00038633422809652984,
      "learning_rate": 9.791385261894044e-06,
      "logits/chosen": -0.9620229005813599,
      "logits/rejected": -1.3113970756530762,
      "logps/chosen": -112.11418914794922,
      "logps/rejected": -280.71514892578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.852627754211426,
      "rewards/margins": 13.574984550476074,
      "rewards/rejected": -10.722359657287598,
      "step": 880
    },
    {
      "epoch": 0.3606158833063209,
      "grad_norm": 0.00038599190884269774,
      "learning_rate": 9.776377007354046e-06,
      "logits/chosen": -0.9710432887077332,
      "logits/rejected": -1.3242309093475342,
      "logps/chosen": -112.74330139160156,
      "logps/rejected": -279.4281005859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8442838191986084,
      "rewards/margins": 13.384805679321289,
      "rewards/rejected": -10.540522575378418,
      "step": 890
    },
    {
      "epoch": 0.3646677471636953,
      "grad_norm": 0.0003790074260905385,
      "learning_rate": 9.761368752814049e-06,
      "logits/chosen": -0.9823834896087646,
      "logits/rejected": -1.3220099210739136,
      "logps/chosen": -113.65370178222656,
      "logps/rejected": -279.15252685546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.85807204246521,
      "rewards/margins": 13.438652992248535,
      "rewards/rejected": -10.580580711364746,
      "step": 900
    },
    {
      "epoch": 0.3646677471636953,
      "eval_logits/chosen": -0.9927268028259277,
      "eval_logits/rejected": -1.3295562267303467,
      "eval_logps/chosen": -113.10031127929688,
      "eval_logps/rejected": -279.3315124511719,
      "eval_loss": 4.538566827250179e-06,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.7911629676818848,
      "eval_rewards/margins": 13.430037498474121,
      "eval_rewards/rejected": -10.638875007629395,
      "eval_runtime": 659.9746,
      "eval_samples_per_second": 11.219,
      "eval_steps_per_second": 0.624,
      "step": 900
    },
    {
      "epoch": 0.3687196110210697,
      "grad_norm": 0.00014131773787084967,
      "learning_rate": 9.74636049827405e-06,
      "logits/chosen": -0.9639739990234375,
      "logits/rejected": -1.318598985671997,
      "logps/chosen": -112.04338836669922,
      "logps/rejected": -279.8641357421875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8556644916534424,
      "rewards/margins": 13.507773399353027,
      "rewards/rejected": -10.652108192443848,
      "step": 910
    },
    {
      "epoch": 0.3727714748784441,
      "grad_norm": 0.00025860543246380985,
      "learning_rate": 9.731352243734054e-06,
      "logits/chosen": -0.9789298176765442,
      "logits/rejected": -1.3201037645339966,
      "logps/chosen": -112.76766204833984,
      "logps/rejected": -280.4600830078125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8445730209350586,
      "rewards/margins": 13.659768104553223,
      "rewards/rejected": -10.815194129943848,
      "step": 920
    },
    {
      "epoch": 0.37682333873581847,
      "grad_norm": 0.00032551950425840914,
      "learning_rate": 9.716343989194058e-06,
      "logits/chosen": -0.9662618637084961,
      "logits/rejected": -1.3028830289840698,
      "logps/chosen": -111.63435363769531,
      "logps/rejected": -280.16644287109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8973257541656494,
      "rewards/margins": 13.563043594360352,
      "rewards/rejected": -10.665717124938965,
      "step": 930
    },
    {
      "epoch": 0.38087520259319285,
      "grad_norm": 0.0003041207673959434,
      "learning_rate": 9.701335734654061e-06,
      "logits/chosen": -0.9609310030937195,
      "logits/rejected": -1.3017886877059937,
      "logps/chosen": -112.3936996459961,
      "logps/rejected": -281.9355163574219,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8747782707214355,
      "rewards/margins": 13.720125198364258,
      "rewards/rejected": -10.84534740447998,
      "step": 940
    },
    {
      "epoch": 0.38492706645056723,
      "grad_norm": 0.0007778910803608596,
      "learning_rate": 9.686327480114064e-06,
      "logits/chosen": -0.9676986932754517,
      "logits/rejected": -1.3104395866394043,
      "logps/chosen": -112.07537078857422,
      "logps/rejected": -281.6306457519531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8806912899017334,
      "rewards/margins": 13.744512557983398,
      "rewards/rejected": -10.863818168640137,
      "step": 950
    },
    {
      "epoch": 0.38492706645056723,
      "eval_logits/chosen": -0.9823371171951294,
      "eval_logits/rejected": -1.3137986660003662,
      "eval_logps/chosen": -112.68508911132812,
      "eval_logps/rejected": -281.2151794433594,
      "eval_loss": 3.7383849758043652e-06,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.83268404006958,
      "eval_rewards/margins": 13.659927368164062,
      "eval_rewards/rejected": -10.827244758605957,
      "eval_runtime": 660.2661,
      "eval_samples_per_second": 11.214,
      "eval_steps_per_second": 0.624,
      "step": 950
    },
    {
      "epoch": 0.3889789303079417,
      "grad_norm": 0.00016004052304197103,
      "learning_rate": 9.671319225574066e-06,
      "logits/chosen": -0.9678320288658142,
      "logits/rejected": -1.3027371168136597,
      "logps/chosen": -112.4660415649414,
      "logps/rejected": -281.6878662109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.886528491973877,
      "rewards/margins": 13.726334571838379,
      "rewards/rejected": -10.839804649353027,
      "step": 960
    },
    {
      "epoch": 0.39303079416531606,
      "grad_norm": 0.0001520434016129002,
      "learning_rate": 9.65631097103407e-06,
      "logits/chosen": -0.9582164883613586,
      "logits/rejected": -1.2870689630508423,
      "logps/chosen": -112.67086029052734,
      "logps/rejected": -283.65972900390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.869288682937622,
      "rewards/margins": 13.861096382141113,
      "rewards/rejected": -10.991806983947754,
      "step": 970
    },
    {
      "epoch": 0.39708265802269044,
      "grad_norm": 0.00021535011183004826,
      "learning_rate": 9.641302716494071e-06,
      "logits/chosen": -0.9589084386825562,
      "logits/rejected": -1.2887883186340332,
      "logps/chosen": -111.85392761230469,
      "logps/rejected": -281.7579345703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8492848873138428,
      "rewards/margins": 13.705706596374512,
      "rewards/rejected": -10.856423377990723,
      "step": 980
    },
    {
      "epoch": 0.4011345218800648,
      "grad_norm": 0.00020488310838118196,
      "learning_rate": 9.626294461954075e-06,
      "logits/chosen": -0.9616044759750366,
      "logits/rejected": -1.2965401411056519,
      "logps/chosen": -111.1251449584961,
      "logps/rejected": -283.02923583984375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.951281785964966,
      "rewards/margins": 13.97263240814209,
      "rewards/rejected": -11.021352767944336,
      "step": 990
    },
    {
      "epoch": 0.4051863857374392,
      "grad_norm": 0.00014300347538664937,
      "learning_rate": 9.611286207414078e-06,
      "logits/chosen": -0.957852303981781,
      "logits/rejected": -1.2917855978012085,
      "logps/chosen": -111.81720733642578,
      "logps/rejected": -283.7075500488281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.923614263534546,
      "rewards/margins": 13.97872257232666,
      "rewards/rejected": -11.055109024047852,
      "step": 1000
    },
    {
      "epoch": 0.4051863857374392,
      "eval_logits/chosen": -0.9789116382598877,
      "eval_logits/rejected": -1.3010200262069702,
      "eval_logps/chosen": -112.41659545898438,
      "eval_logps/rejected": -282.9280090332031,
      "eval_loss": 3.1251206564775202e-06,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.8595333099365234,
      "eval_rewards/margins": 13.858057975769043,
      "eval_rewards/rejected": -10.998525619506836,
      "eval_runtime": 660.269,
      "eval_samples_per_second": 11.214,
      "eval_steps_per_second": 0.624,
      "step": 1000
    },
    {
      "epoch": 0.40923824959481364,
      "grad_norm": 0.0005220123566687107,
      "learning_rate": 9.596277952874082e-06,
      "logits/chosen": -0.9391909241676331,
      "logits/rejected": -1.297073483467102,
      "logps/chosen": -111.75109100341797,
      "logps/rejected": -283.1003112792969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9314379692077637,
      "rewards/margins": 13.907914161682129,
      "rewards/rejected": -10.97647762298584,
      "step": 1010
    },
    {
      "epoch": 0.413290113452188,
      "grad_norm": 0.0003276396309956908,
      "learning_rate": 9.581269698334085e-06,
      "logits/chosen": -0.960158109664917,
      "logits/rejected": -1.3214308023452759,
      "logps/chosen": -111.71788787841797,
      "logps/rejected": -283.8362121582031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.924668550491333,
      "rewards/margins": 13.982831001281738,
      "rewards/rejected": -11.0581636428833,
      "step": 1020
    },
    {
      "epoch": 0.4173419773095624,
      "grad_norm": 0.0001340145245194435,
      "learning_rate": 9.566261443794087e-06,
      "logits/chosen": -0.974898099899292,
      "logits/rejected": -1.3104703426361084,
      "logps/chosen": -111.79582214355469,
      "logps/rejected": -285.63092041015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9361088275909424,
      "rewards/margins": 14.022080421447754,
      "rewards/rejected": -11.08597183227539,
      "step": 1030
    },
    {
      "epoch": 0.4213938411669368,
      "grad_norm": 0.00012633968435693532,
      "learning_rate": 9.55125318925409e-06,
      "logits/chosen": -0.9730085730552673,
      "logits/rejected": -1.2769588232040405,
      "logps/chosen": -110.78377532958984,
      "logps/rejected": -284.2268981933594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9252431392669678,
      "rewards/margins": 14.092554092407227,
      "rewards/rejected": -11.167312622070312,
      "step": 1040
    },
    {
      "epoch": 0.42544570502431117,
      "grad_norm": 0.00010579601803328842,
      "learning_rate": 9.536244934714094e-06,
      "logits/chosen": -0.96152663230896,
      "logits/rejected": -1.2916251420974731,
      "logps/chosen": -109.97510528564453,
      "logps/rejected": -284.11175537109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9813199043273926,
      "rewards/margins": 14.068181037902832,
      "rewards/rejected": -11.086858749389648,
      "step": 1050
    },
    {
      "epoch": 0.42544570502431117,
      "eval_logits/chosen": -0.9753500819206238,
      "eval_logits/rejected": -1.287696361541748,
      "eval_logps/chosen": -112.18319702148438,
      "eval_logps/rejected": -284.8260803222656,
      "eval_loss": 2.570784999988973e-06,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.8828725814819336,
      "eval_rewards/margins": 14.071202278137207,
      "eval_rewards/rejected": -11.18833065032959,
      "eval_runtime": 660.368,
      "eval_samples_per_second": 11.212,
      "eval_steps_per_second": 0.624,
      "step": 1050
    },
    {
      "epoch": 0.42949756888168555,
      "grad_norm": 0.0001430579723091796,
      "learning_rate": 9.521236680174097e-06,
      "logits/chosen": -0.9688810110092163,
      "logits/rejected": -1.2829087972640991,
      "logps/chosen": -112.16279602050781,
      "logps/rejected": -285.31756591796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9200236797332764,
      "rewards/margins": 14.141681671142578,
      "rewards/rejected": -11.221658706665039,
      "step": 1060
    },
    {
      "epoch": 0.43354943273906,
      "grad_norm": 6.991266855038702e-05,
      "learning_rate": 9.5062284256341e-06,
      "logits/chosen": -0.9442747235298157,
      "logits/rejected": -1.262116551399231,
      "logps/chosen": -111.14962768554688,
      "logps/rejected": -285.7078552246094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9615018367767334,
      "rewards/margins": 14.187654495239258,
      "rewards/rejected": -11.226153373718262,
      "step": 1070
    },
    {
      "epoch": 0.4376012965964344,
      "grad_norm": 0.00022211392933968455,
      "learning_rate": 9.491220171094102e-06,
      "logits/chosen": -0.9678923487663269,
      "logits/rejected": -1.2818944454193115,
      "logps/chosen": -111.69573211669922,
      "logps/rejected": -287.5627136230469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9129486083984375,
      "rewards/margins": 14.332964897155762,
      "rewards/rejected": -11.420016288757324,
      "step": 1080
    },
    {
      "epoch": 0.44165316045380876,
      "grad_norm": 0.00014934873615857214,
      "learning_rate": 9.476211916554106e-06,
      "logits/chosen": -0.9637159705162048,
      "logits/rejected": -1.263195276260376,
      "logps/chosen": -112.36006164550781,
      "logps/rejected": -286.6079406738281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8961293697357178,
      "rewards/margins": 14.229351997375488,
      "rewards/rejected": -11.333223342895508,
      "step": 1090
    },
    {
      "epoch": 0.44570502431118314,
      "grad_norm": 0.00023202145530376583,
      "learning_rate": 9.461203662014109e-06,
      "logits/chosen": -0.9612440466880798,
      "logits/rejected": -1.2696155309677124,
      "logps/chosen": -112.3745346069336,
      "logps/rejected": -287.69012451171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9029252529144287,
      "rewards/margins": 14.36845588684082,
      "rewards/rejected": -11.465531349182129,
      "step": 1100
    },
    {
      "epoch": 0.44570502431118314,
      "eval_logits/chosen": -0.973721981048584,
      "eval_logits/rejected": -1.2801706790924072,
      "eval_logps/chosen": -112.02223205566406,
      "eval_logps/rejected": -286.2309875488281,
      "eval_loss": 2.2304900539893424e-06,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.8989689350128174,
      "eval_rewards/margins": 14.227792739868164,
      "eval_rewards/rejected": -11.32882308959961,
      "eval_runtime": 660.246,
      "eval_samples_per_second": 11.214,
      "eval_steps_per_second": 0.624,
      "step": 1100
    },
    {
      "epoch": 0.4497568881685575,
      "grad_norm": 8.071884803939611e-05,
      "learning_rate": 9.446195407474111e-06,
      "logits/chosen": -0.9535583853721619,
      "logits/rejected": -1.287434458732605,
      "logps/chosen": -111.23747253417969,
      "logps/rejected": -286.5403747558594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9855968952178955,
      "rewards/margins": 14.391644477844238,
      "rewards/rejected": -11.406046867370605,
      "step": 1110
    },
    {
      "epoch": 0.4538087520259319,
      "grad_norm": 0.0004843325004912913,
      "learning_rate": 9.431187152934114e-06,
      "logits/chosen": -0.9513693451881409,
      "logits/rejected": -1.2654842138290405,
      "logps/chosen": -110.70319366455078,
      "logps/rejected": -287.89312744140625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.96667218208313,
      "rewards/margins": 14.396854400634766,
      "rewards/rejected": -11.430181503295898,
      "step": 1120
    },
    {
      "epoch": 0.45786061588330634,
      "grad_norm": 0.00020084546122234315,
      "learning_rate": 9.416178898394118e-06,
      "logits/chosen": -0.9512994289398193,
      "logits/rejected": -1.2680131196975708,
      "logps/chosen": -111.81349182128906,
      "logps/rejected": -286.9837341308594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.921030282974243,
      "rewards/margins": 14.246526718139648,
      "rewards/rejected": -11.325496673583984,
      "step": 1130
    },
    {
      "epoch": 0.4619124797406807,
      "grad_norm": 7.329774962272495e-05,
      "learning_rate": 9.401170643854121e-06,
      "logits/chosen": -0.9566375017166138,
      "logits/rejected": -1.278363585472107,
      "logps/chosen": -110.75605010986328,
      "logps/rejected": -288.46820068359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9941275119781494,
      "rewards/margins": 14.503043174743652,
      "rewards/rejected": -11.508919715881348,
      "step": 1140
    },
    {
      "epoch": 0.4659643435980551,
      "grad_norm": 8.835856715450063e-05,
      "learning_rate": 9.386162389314125e-06,
      "logits/chosen": -0.9616451263427734,
      "logits/rejected": -1.2581592798233032,
      "logps/chosen": -110.53977966308594,
      "logps/rejected": -288.35699462890625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9831340312957764,
      "rewards/margins": 14.568155288696289,
      "rewards/rejected": -11.585020065307617,
      "step": 1150
    },
    {
      "epoch": 0.4659643435980551,
      "eval_logits/chosen": -0.9735735058784485,
      "eval_logits/rejected": -1.2739366292953491,
      "eval_logps/chosen": -111.88179016113281,
      "eval_logps/rejected": -287.6111755371094,
      "eval_loss": 1.9441790755081456e-06,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.9130136966705322,
      "eval_rewards/margins": 14.379854202270508,
      "eval_rewards/rejected": -11.466840744018555,
      "eval_runtime": 660.2556,
      "eval_samples_per_second": 11.214,
      "eval_steps_per_second": 0.624,
      "step": 1150
    },
    {
      "epoch": 0.4700162074554295,
      "grad_norm": 8.497169619658962e-05,
      "learning_rate": 9.371154134774126e-06,
      "logits/chosen": -0.9573142528533936,
      "logits/rejected": -1.2469346523284912,
      "logps/chosen": -111.91331481933594,
      "logps/rejected": -288.5337219238281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.956511974334717,
      "rewards/margins": 14.44264030456543,
      "rewards/rejected": -11.486127853393555,
      "step": 1160
    },
    {
      "epoch": 0.4740680713128039,
      "grad_norm": 0.00022965000243857503,
      "learning_rate": 9.35614588023413e-06,
      "logits/chosen": -0.9571576714515686,
      "logits/rejected": -1.271868109703064,
      "logps/chosen": -110.45994567871094,
      "logps/rejected": -288.6113586425781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0243794918060303,
      "rewards/margins": 14.56635856628418,
      "rewards/rejected": -11.54197883605957,
      "step": 1170
    },
    {
      "epoch": 0.47811993517017826,
      "grad_norm": 0.0003564605431165546,
      "learning_rate": 9.341137625694131e-06,
      "logits/chosen": -0.9566469192504883,
      "logits/rejected": -1.26055908203125,
      "logps/chosen": -110.68568420410156,
      "logps/rejected": -288.16619873046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9829514026641846,
      "rewards/margins": 14.538333892822266,
      "rewards/rejected": -11.555380821228027,
      "step": 1180
    },
    {
      "epoch": 0.4821717990275527,
      "grad_norm": 8.522019197698683e-05,
      "learning_rate": 9.326129371154135e-06,
      "logits/chosen": -0.961935818195343,
      "logits/rejected": -1.2638294696807861,
      "logps/chosen": -111.3947525024414,
      "logps/rejected": -289.1004943847656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9743118286132812,
      "rewards/margins": 14.57591438293457,
      "rewards/rejected": -11.601603507995605,
      "step": 1190
    },
    {
      "epoch": 0.4862236628849271,
      "grad_norm": 0.00014217206626199186,
      "learning_rate": 9.311121116614138e-06,
      "logits/chosen": -0.9611042141914368,
      "logits/rejected": -1.2734590768814087,
      "logps/chosen": -111.36783599853516,
      "logps/rejected": -290.3299255371094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9741716384887695,
      "rewards/margins": 14.585290908813477,
      "rewards/rejected": -11.611119270324707,
      "step": 1200
    },
    {
      "epoch": 0.4862236628849271,
      "eval_logits/chosen": -0.9973398447036743,
      "eval_logits/rejected": -1.2864365577697754,
      "eval_logps/chosen": -111.73997497558594,
      "eval_logps/rejected": -289.3089294433594,
      "eval_loss": 1.631402483326383e-06,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.9271950721740723,
      "eval_rewards/margins": 14.563812255859375,
      "eval_rewards/rejected": -11.636616706848145,
      "eval_runtime": 660.2673,
      "eval_samples_per_second": 11.214,
      "eval_steps_per_second": 0.624,
      "step": 1200
    },
    {
      "epoch": 0.49027552674230146,
      "grad_norm": 8.652352698845789e-05,
      "learning_rate": 9.296112862074142e-06,
      "logits/chosen": -0.962738573551178,
      "logits/rejected": -1.257235050201416,
      "logps/chosen": -111.7171859741211,
      "logps/rejected": -290.6097106933594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.958895206451416,
      "rewards/margins": 14.682781219482422,
      "rewards/rejected": -11.723886489868164,
      "step": 1210
    },
    {
      "epoch": 0.49432739059967584,
      "grad_norm": 5.8049598010256886e-05,
      "learning_rate": 9.281104607534145e-06,
      "logits/chosen": -0.9487282037734985,
      "logits/rejected": -1.249599575996399,
      "logps/chosen": -109.90348052978516,
      "logps/rejected": -290.8208312988281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0212409496307373,
      "rewards/margins": 14.790689468383789,
      "rewards/rejected": -11.769448280334473,
      "step": 1220
    },
    {
      "epoch": 0.4983792544570502,
      "grad_norm": 9.261705417884514e-05,
      "learning_rate": 9.266096352994147e-06,
      "logits/chosen": -0.9655988812446594,
      "logits/rejected": -1.2366749048233032,
      "logps/chosen": -111.27457427978516,
      "logps/rejected": -290.5723876953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.993466377258301,
      "rewards/margins": 14.69091510772705,
      "rewards/rejected": -11.697447776794434,
      "step": 1230
    },
    {
      "epoch": 0.5024311183144247,
      "grad_norm": 0.00016064975352492183,
      "learning_rate": 9.25108809845415e-06,
      "logits/chosen": -0.9459314346313477,
      "logits/rejected": -1.2686599493026733,
      "logps/chosen": -111.96553039550781,
      "logps/rejected": -291.0901184082031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.974982976913452,
      "rewards/margins": 14.71306324005127,
      "rewards/rejected": -11.738081932067871,
      "step": 1240
    },
    {
      "epoch": 0.506482982171799,
      "grad_norm": 8.82770837051794e-05,
      "learning_rate": 9.236079843914154e-06,
      "logits/chosen": -0.9558974504470825,
      "logits/rejected": -1.2635960578918457,
      "logps/chosen": -110.29508209228516,
      "logps/rejected": -290.46405029296875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.025198459625244,
      "rewards/margins": 14.70692253112793,
      "rewards/rejected": -11.681722640991211,
      "step": 1250
    },
    {
      "epoch": 0.506482982171799,
      "eval_logits/chosen": -0.9870012402534485,
      "eval_logits/rejected": -1.2686045169830322,
      "eval_logps/chosen": -111.6612548828125,
      "eval_logps/rejected": -290.4904479980469,
      "eval_loss": 1.4487125099549303e-06,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.9350662231445312,
      "eval_rewards/margins": 14.689835548400879,
      "eval_rewards/rejected": -11.754766464233398,
      "eval_runtime": 660.3028,
      "eval_samples_per_second": 11.213,
      "eval_steps_per_second": 0.624,
      "step": 1250
    },
    {
      "epoch": 0.5105348460291734,
      "grad_norm": 0.0001483349915361032,
      "learning_rate": 9.221071589374157e-06,
      "logits/chosen": -0.9602691531181335,
      "logits/rejected": -1.2540396451950073,
      "logps/chosen": -110.69275665283203,
      "logps/rejected": -290.8595275878906,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0299112796783447,
      "rewards/margins": 14.76969051361084,
      "rewards/rejected": -11.739777565002441,
      "step": 1260
    },
    {
      "epoch": 0.5145867098865479,
      "grad_norm": 6.636109901592135e-05,
      "learning_rate": 9.20606333483416e-06,
      "logits/chosen": -0.954090416431427,
      "logits/rejected": -1.258052110671997,
      "logps/chosen": -109.84585571289062,
      "logps/rejected": -292.0687561035156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.058316946029663,
      "rewards/margins": 14.921138763427734,
      "rewards/rejected": -11.862822532653809,
      "step": 1270
    },
    {
      "epoch": 0.5186385737439222,
      "grad_norm": 0.00013109849533066154,
      "learning_rate": 9.191055080294162e-06,
      "logits/chosen": -0.9620006680488586,
      "logits/rejected": -1.2502659559249878,
      "logps/chosen": -111.36671447753906,
      "logps/rejected": -291.8580627441406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0132246017456055,
      "rewards/margins": 14.834657669067383,
      "rewards/rejected": -11.821431159973145,
      "step": 1280
    },
    {
      "epoch": 0.5226904376012966,
      "grad_norm": 5.540152778849006e-05,
      "learning_rate": 9.176046825754166e-06,
      "logits/chosen": -0.9571611881256104,
      "logits/rejected": -1.2665116786956787,
      "logps/chosen": -110.1987075805664,
      "logps/rejected": -293.12457275390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0355942249298096,
      "rewards/margins": 14.982242584228516,
      "rewards/rejected": -11.946649551391602,
      "step": 1290
    },
    {
      "epoch": 0.526742301458671,
      "grad_norm": 8.706421067472547e-05,
      "learning_rate": 9.161038571214168e-06,
      "logits/chosen": -0.956296980381012,
      "logits/rejected": -1.2626007795333862,
      "logps/chosen": -109.61963653564453,
      "logps/rejected": -292.52056884765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0315158367156982,
      "rewards/margins": 14.91031265258789,
      "rewards/rejected": -11.87879753112793,
      "step": 1300
    },
    {
      "epoch": 0.526742301458671,
      "eval_logits/chosen": -0.9715558290481567,
      "eval_logits/rejected": -1.2562642097473145,
      "eval_logps/chosen": -111.44157409667969,
      "eval_logps/rejected": -291.3668518066406,
      "eval_loss": 1.3256604916023207e-06,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.9570353031158447,
      "eval_rewards/margins": 14.799443244934082,
      "eval_rewards/rejected": -11.842409133911133,
      "eval_runtime": 660.2464,
      "eval_samples_per_second": 11.214,
      "eval_steps_per_second": 0.624,
      "step": 1300
    },
    {
      "epoch": 0.5307941653160454,
      "grad_norm": 6.619278428843245e-05,
      "learning_rate": 9.146030316674171e-06,
      "logits/chosen": -0.9809182286262512,
      "logits/rejected": -1.2584463357925415,
      "logps/chosen": -111.23551940917969,
      "logps/rejected": -290.51751708984375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.009885311126709,
      "rewards/margins": 14.80660629272461,
      "rewards/rejected": -11.79671859741211,
      "step": 1310
    },
    {
      "epoch": 0.5348460291734197,
      "grad_norm": 6.740832759533077e-05,
      "learning_rate": 9.131022062134174e-06,
      "logits/chosen": -0.9744822978973389,
      "logits/rejected": -1.2522770166397095,
      "logps/chosen": -110.54716491699219,
      "logps/rejected": -293.0761413574219,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0305418968200684,
      "rewards/margins": 14.981204986572266,
      "rewards/rejected": -11.950662612915039,
      "step": 1320
    },
    {
      "epoch": 0.5388978930307942,
      "grad_norm": 4.477183392737061e-05,
      "learning_rate": 9.116013807594178e-06,
      "logits/chosen": -0.953049898147583,
      "logits/rejected": -1.261191964149475,
      "logps/chosen": -110.87936401367188,
      "logps/rejected": -292.4115295410156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.050781726837158,
      "rewards/margins": 14.959651947021484,
      "rewards/rejected": -11.908870697021484,
      "step": 1330
    },
    {
      "epoch": 0.5429497568881686,
      "grad_norm": 5.2174516895320266e-05,
      "learning_rate": 9.101005553054181e-06,
      "logits/chosen": -0.960311770439148,
      "logits/rejected": -1.246680736541748,
      "logps/chosen": -111.20600891113281,
      "logps/rejected": -291.935546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0119898319244385,
      "rewards/margins": 14.858885765075684,
      "rewards/rejected": -11.846895217895508,
      "step": 1340
    },
    {
      "epoch": 0.5470016207455429,
      "grad_norm": 0.00010178080992773175,
      "learning_rate": 9.085997298514183e-06,
      "logits/chosen": -0.9307942986488342,
      "logits/rejected": -1.2356764078140259,
      "logps/chosen": -110.38352966308594,
      "logps/rejected": -291.8614501953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0522117614746094,
      "rewards/margins": 14.910490989685059,
      "rewards/rejected": -11.85827922821045,
      "step": 1350
    },
    {
      "epoch": 0.5470016207455429,
      "eval_logits/chosen": -0.9828789234161377,
      "eval_logits/rejected": -1.2572388648986816,
      "eval_logps/chosen": -111.41586303710938,
      "eval_logps/rejected": -292.48114013671875,
      "eval_loss": 1.1854280046463828e-06,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.9596071243286133,
      "eval_rewards/margins": 14.913446426391602,
      "eval_rewards/rejected": -11.953838348388672,
      "eval_runtime": 660.2565,
      "eval_samples_per_second": 11.214,
      "eval_steps_per_second": 0.624,
      "step": 1350
    },
    {
      "epoch": 0.5510534846029174,
      "grad_norm": 0.00022050880943425,
      "learning_rate": 9.070989043974186e-06,
      "logits/chosen": -0.9654724597930908,
      "logits/rejected": -1.2213767766952515,
      "logps/chosen": -110.69276428222656,
      "logps/rejected": -292.5841064453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0184366703033447,
      "rewards/margins": 14.926412582397461,
      "rewards/rejected": -11.907977104187012,
      "step": 1360
    },
    {
      "epoch": 0.5551053484602917,
      "grad_norm": 9.077700087800622e-05,
      "learning_rate": 9.05598078943419e-06,
      "logits/chosen": -0.9437484741210938,
      "logits/rejected": -1.2385755777359009,
      "logps/chosen": -110.29655456542969,
      "logps/rejected": -293.9698486328125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.072600841522217,
      "rewards/margins": 15.177414894104004,
      "rewards/rejected": -12.104813575744629,
      "step": 1370
    },
    {
      "epoch": 0.5591572123176661,
      "grad_norm": 6.290877354331315e-05,
      "learning_rate": 9.040972534894192e-06,
      "logits/chosen": -0.9499402642250061,
      "logits/rejected": -1.262143611907959,
      "logps/chosen": -109.24268341064453,
      "logps/rejected": -294.36883544921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.056851625442505,
      "rewards/margins": 15.167757987976074,
      "rewards/rejected": -12.110904693603516,
      "step": 1380
    },
    {
      "epoch": 0.5632090761750406,
      "grad_norm": 0.0001979395019588992,
      "learning_rate": 9.025964280354197e-06,
      "logits/chosen": -0.9501563310623169,
      "logits/rejected": -1.2465254068374634,
      "logps/chosen": -109.71662902832031,
      "logps/rejected": -294.7782897949219,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.016782522201538,
      "rewards/margins": 15.195131301879883,
      "rewards/rejected": -12.17834758758545,
      "step": 1390
    },
    {
      "epoch": 0.5672609400324149,
      "grad_norm": 9.179364860756323e-05,
      "learning_rate": 9.010956025814198e-06,
      "logits/chosen": -0.9448656439781189,
      "logits/rejected": -1.2433806657791138,
      "logps/chosen": -109.48567199707031,
      "logps/rejected": -294.6673889160156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0524191856384277,
      "rewards/margins": 15.18110179901123,
      "rewards/rejected": -12.128682136535645,
      "step": 1400
    },
    {
      "epoch": 0.5672609400324149,
      "eval_logits/chosen": -0.9666537642478943,
      "eval_logits/rejected": -1.245568871498108,
      "eval_logps/chosen": -111.20926666259766,
      "eval_logps/rejected": -293.1695251464844,
      "eval_loss": 1.1026525044144364e-06,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.9802653789520264,
      "eval_rewards/margins": 15.00294017791748,
      "eval_rewards/rejected": -12.022675514221191,
      "eval_runtime": 657.9972,
      "eval_samples_per_second": 11.252,
      "eval_steps_per_second": 0.626,
      "step": 1400
    },
    {
      "epoch": 0.5713128038897893,
      "grad_norm": 7.402592018479481e-05,
      "learning_rate": 8.995947771274202e-06,
      "logits/chosen": -0.9521650671958923,
      "logits/rejected": -1.2431867122650146,
      "logps/chosen": -110.92412567138672,
      "logps/rejected": -295.1664733886719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0659775733947754,
      "rewards/margins": 15.19791030883789,
      "rewards/rejected": -12.131933212280273,
      "step": 1410
    },
    {
      "epoch": 0.5753646677471637,
      "grad_norm": 5.878585216123611e-05,
      "learning_rate": 8.980939516734205e-06,
      "logits/chosen": -0.9558809399604797,
      "logits/rejected": -1.237985610961914,
      "logps/chosen": -110.80642700195312,
      "logps/rejected": -294.6929931640625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.028352975845337,
      "rewards/margins": 15.155354499816895,
      "rewards/rejected": -12.127002716064453,
      "step": 1420
    },
    {
      "epoch": 0.5794165316045381,
      "grad_norm": 7.350226223934442e-05,
      "learning_rate": 8.965931262194207e-06,
      "logits/chosen": -0.9748159646987915,
      "logits/rejected": -1.2480131387710571,
      "logps/chosen": -111.68008422851562,
      "logps/rejected": -294.0392761230469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0166327953338623,
      "rewards/margins": 15.060970306396484,
      "rewards/rejected": -12.044337272644043,
      "step": 1430
    },
    {
      "epoch": 0.5834683954619124,
      "grad_norm": 0.00010062765795737505,
      "learning_rate": 8.95092300765421e-06,
      "logits/chosen": -0.956743597984314,
      "logits/rejected": -1.2324788570404053,
      "logps/chosen": -111.09513854980469,
      "logps/rejected": -294.4328308105469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.027946710586548,
      "rewards/margins": 15.142075538635254,
      "rewards/rejected": -12.114130973815918,
      "step": 1440
    },
    {
      "epoch": 0.5875202593192869,
      "grad_norm": 3.6476340028457344e-05,
      "learning_rate": 8.935914753114214e-06,
      "logits/chosen": -0.93390291929245,
      "logits/rejected": -1.2643619775772095,
      "logps/chosen": -110.3382339477539,
      "logps/rejected": -295.6949157714844,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0454061031341553,
      "rewards/margins": 15.326750755310059,
      "rewards/rejected": -12.281344413757324,
      "step": 1450
    },
    {
      "epoch": 0.5875202593192869,
      "eval_logits/chosen": -0.9941796660423279,
      "eval_logits/rejected": -1.2666490077972412,
      "eval_logps/chosen": -111.17910766601562,
      "eval_logps/rejected": -294.23052978515625,
      "eval_loss": 9.906432296702405e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.983280897140503,
      "eval_rewards/margins": 15.112060546875,
      "eval_rewards/rejected": -12.128779411315918,
      "eval_runtime": 660.0408,
      "eval_samples_per_second": 11.217,
      "eval_steps_per_second": 0.624,
      "step": 1450
    },
    {
      "epoch": 0.5915721231766613,
      "grad_norm": 3.417656262172386e-05,
      "learning_rate": 8.920906498574217e-06,
      "logits/chosen": -0.9628769159317017,
      "logits/rejected": -1.2284659147262573,
      "logps/chosen": -111.88285827636719,
      "logps/rejected": -295.775390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.999023914337158,
      "rewards/margins": 15.234338760375977,
      "rewards/rejected": -12.23531436920166,
      "step": 1460
    },
    {
      "epoch": 0.5956239870340356,
      "grad_norm": 3.768848910112865e-05,
      "learning_rate": 8.90589824403422e-06,
      "logits/chosen": -0.9420002698898315,
      "logits/rejected": -1.2281954288482666,
      "logps/chosen": -110.00930786132812,
      "logps/rejected": -294.4632873535156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0742979049682617,
      "rewards/margins": 15.22895336151123,
      "rewards/rejected": -12.154653549194336,
      "step": 1470
    },
    {
      "epoch": 0.5996758508914101,
      "grad_norm": 4.7602912673028186e-05,
      "learning_rate": 8.890889989494222e-06,
      "logits/chosen": -0.9583718180656433,
      "logits/rejected": -1.2395317554473877,
      "logps/chosen": -109.17208099365234,
      "logps/rejected": -295.4866638183594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.069913625717163,
      "rewards/margins": 15.283977508544922,
      "rewards/rejected": -12.214064598083496,
      "step": 1480
    },
    {
      "epoch": 0.6037277147487844,
      "grad_norm": 3.677717177197337e-05,
      "learning_rate": 8.875881734954226e-06,
      "logits/chosen": -0.9670850038528442,
      "logits/rejected": -1.2482547760009766,
      "logps/chosen": -111.20230102539062,
      "logps/rejected": -296.07672119140625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0110421180725098,
      "rewards/margins": 15.339611053466797,
      "rewards/rejected": -12.328569412231445,
      "step": 1490
    },
    {
      "epoch": 0.6077795786061588,
      "grad_norm": 6.113961717346683e-05,
      "learning_rate": 8.860873480414228e-06,
      "logits/chosen": -0.9546039700508118,
      "logits/rejected": -1.239587426185608,
      "logps/chosen": -110.67524719238281,
      "logps/rejected": -295.7835388183594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0471036434173584,
      "rewards/margins": 15.220789909362793,
      "rewards/rejected": -12.173686027526855,
      "step": 1500
    },
    {
      "epoch": 0.6077795786061588,
      "eval_logits/chosen": -0.9696044921875,
      "eval_logits/rejected": -1.242756962776184,
      "eval_logps/chosen": -111.04248809814453,
      "eval_logps/rejected": -294.9416198730469,
      "eval_loss": 9.284163979828008e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.9969444274902344,
      "eval_rewards/margins": 15.196829795837402,
      "eval_rewards/rejected": -12.199886322021484,
      "eval_runtime": 659.9648,
      "eval_samples_per_second": 11.219,
      "eval_steps_per_second": 0.624,
      "step": 1500
    },
    {
      "epoch": 0.6118314424635333,
      "grad_norm": 0.00010565384582150728,
      "learning_rate": 8.845865225874231e-06,
      "logits/chosen": -0.9540724158287048,
      "logits/rejected": -1.2362232208251953,
      "logps/chosen": -110.95648956298828,
      "logps/rejected": -295.4747314453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0460703372955322,
      "rewards/margins": 15.277571678161621,
      "rewards/rejected": -12.231502532958984,
      "step": 1510
    },
    {
      "epoch": 0.6158833063209076,
      "grad_norm": 6.378133548423648e-05,
      "learning_rate": 8.830856971334234e-06,
      "logits/chosen": -0.954107940196991,
      "logits/rejected": -1.2532161474227905,
      "logps/chosen": -108.83919525146484,
      "logps/rejected": -296.8812255859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1111133098602295,
      "rewards/margins": 15.391324043273926,
      "rewards/rejected": -12.2802095413208,
      "step": 1520
    },
    {
      "epoch": 0.619935170178282,
      "grad_norm": 4.6979050239315256e-05,
      "learning_rate": 8.815848716794238e-06,
      "logits/chosen": -0.9490892291069031,
      "logits/rejected": -1.237484335899353,
      "logps/chosen": -109.17033386230469,
      "logps/rejected": -296.7481994628906,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.074795961380005,
      "rewards/margins": 15.35445499420166,
      "rewards/rejected": -12.279657363891602,
      "step": 1530
    },
    {
      "epoch": 0.6239870340356564,
      "grad_norm": 6.864364695502445e-05,
      "learning_rate": 8.800840462254241e-06,
      "logits/chosen": -0.9532595872879028,
      "logits/rejected": -1.2381584644317627,
      "logps/chosen": -110.54261016845703,
      "logps/rejected": -297.6771545410156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.061347246170044,
      "rewards/margins": 15.484623908996582,
      "rewards/rejected": -12.4232759475708,
      "step": 1540
    },
    {
      "epoch": 0.6280388978930308,
      "grad_norm": 5.509885886567645e-05,
      "learning_rate": 8.785832207714243e-06,
      "logits/chosen": -0.9511829614639282,
      "logits/rejected": -1.2367159128189087,
      "logps/chosen": -109.2529525756836,
      "logps/rejected": -298.1808776855469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0861127376556396,
      "rewards/margins": 15.517443656921387,
      "rewards/rejected": -12.431331634521484,
      "step": 1550
    },
    {
      "epoch": 0.6280388978930308,
      "eval_logits/chosen": -0.9912152290344238,
      "eval_logits/rejected": -1.259311318397522,
      "eval_logps/chosen": -111.05461120605469,
      "eval_logps/rejected": -295.8396911621094,
      "eval_loss": 8.481291047246486e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 2.9957311153411865,
      "eval_rewards/margins": 15.28542423248291,
      "eval_rewards/rejected": -12.289692878723145,
      "eval_runtime": 660.0901,
      "eval_samples_per_second": 11.217,
      "eval_steps_per_second": 0.624,
      "step": 1550
    },
    {
      "epoch": 0.6320907617504052,
      "grad_norm": 7.277915574377403e-05,
      "learning_rate": 8.770823953174246e-06,
      "logits/chosen": -0.9516242742538452,
      "logits/rejected": -1.2391493320465088,
      "logps/chosen": -111.23517608642578,
      "logps/rejected": -296.1101379394531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0449342727661133,
      "rewards/margins": 15.367486000061035,
      "rewards/rejected": -12.322551727294922,
      "step": 1560
    },
    {
      "epoch": 0.6361426256077796,
      "grad_norm": 4.478929258766584e-05,
      "learning_rate": 8.75581569863425e-06,
      "logits/chosen": -0.9553163051605225,
      "logits/rejected": -1.2346341609954834,
      "logps/chosen": -110.90254211425781,
      "logps/rejected": -296.3621826171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0853469371795654,
      "rewards/margins": 15.453842163085938,
      "rewards/rejected": -12.368496894836426,
      "step": 1570
    },
    {
      "epoch": 0.640194489465154,
      "grad_norm": 4.4538293877849355e-05,
      "learning_rate": 8.740807444094253e-06,
      "logits/chosen": -0.9542552828788757,
      "logits/rejected": -1.2168352603912354,
      "logps/chosen": -111.39136505126953,
      "logps/rejected": -295.93505859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.039647340774536,
      "rewards/margins": 15.247894287109375,
      "rewards/rejected": -12.208246231079102,
      "step": 1580
    },
    {
      "epoch": 0.6442463533225283,
      "grad_norm": 4.4035768951289356e-05,
      "learning_rate": 8.725799189554257e-06,
      "logits/chosen": -0.9646400809288025,
      "logits/rejected": -1.2394483089447021,
      "logps/chosen": -110.61803436279297,
      "logps/rejected": -298.1299743652344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0604629516601562,
      "rewards/margins": 15.54178237915039,
      "rewards/rejected": -12.481318473815918,
      "step": 1590
    },
    {
      "epoch": 0.6482982171799028,
      "grad_norm": 4.055317185702734e-05,
      "learning_rate": 8.710790935014258e-06,
      "logits/chosen": -0.9540335536003113,
      "logits/rejected": -1.226391077041626,
      "logps/chosen": -110.3465347290039,
      "logps/rejected": -296.5976867675781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0967838764190674,
      "rewards/margins": 15.452139854431152,
      "rewards/rejected": -12.355356216430664,
      "step": 1600
    },
    {
      "epoch": 0.6482982171799028,
      "eval_logits/chosen": -0.9963459372520447,
      "eval_logits/rejected": -1.2614251375198364,
      "eval_logps/chosen": -110.98289489746094,
      "eval_logps/rejected": -296.5805358886719,
      "eval_loss": 7.882362069722149e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.0029025077819824,
      "eval_rewards/margins": 15.366681098937988,
      "eval_rewards/rejected": -12.363778114318848,
      "eval_runtime": 660.0614,
      "eval_samples_per_second": 11.217,
      "eval_steps_per_second": 0.624,
      "step": 1600
    },
    {
      "epoch": 0.6523500810372771,
      "grad_norm": 3.0179848181433044e-05,
      "learning_rate": 8.695782680474262e-06,
      "logits/chosen": -0.9439318776130676,
      "logits/rejected": -1.216618299484253,
      "logps/chosen": -110.74729919433594,
      "logps/rejected": -296.43804931640625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.056190252304077,
      "rewards/margins": 15.361733436584473,
      "rewards/rejected": -12.3055419921875,
      "step": 1610
    },
    {
      "epoch": 0.6564019448946515,
      "grad_norm": 0.00011598259152378887,
      "learning_rate": 8.680774425934264e-06,
      "logits/chosen": -0.9612037539482117,
      "logits/rejected": -1.2288869619369507,
      "logps/chosen": -110.16704559326172,
      "logps/rejected": -297.1379699707031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.057806968688965,
      "rewards/margins": 15.488568305969238,
      "rewards/rejected": -12.430760383605957,
      "step": 1620
    },
    {
      "epoch": 0.660453808752026,
      "grad_norm": 4.8609501391183585e-05,
      "learning_rate": 8.665766171394267e-06,
      "logits/chosen": -0.9516136050224304,
      "logits/rejected": -1.2299059629440308,
      "logps/chosen": -108.99888610839844,
      "logps/rejected": -299.0168762207031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1037096977233887,
      "rewards/margins": 15.597698211669922,
      "rewards/rejected": -12.493988990783691,
      "step": 1630
    },
    {
      "epoch": 0.6645056726094003,
      "grad_norm": 6.331945769488811e-05,
      "learning_rate": 8.65075791685427e-06,
      "logits/chosen": -0.9660440683364868,
      "logits/rejected": -1.2430554628372192,
      "logps/chosen": -110.30313110351562,
      "logps/rejected": -297.8039245605469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0342953205108643,
      "rewards/margins": 15.4889554977417,
      "rewards/rejected": -12.454659461975098,
      "step": 1640
    },
    {
      "epoch": 0.6685575364667747,
      "grad_norm": 6.663906970061362e-05,
      "learning_rate": 8.635749662314274e-06,
      "logits/chosen": -0.9538125395774841,
      "logits/rejected": -1.2279130220413208,
      "logps/chosen": -109.44841766357422,
      "logps/rejected": -297.01837158203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.114189863204956,
      "rewards/margins": 15.4672212600708,
      "rewards/rejected": -12.353031158447266,
      "step": 1650
    },
    {
      "epoch": 0.6685575364667747,
      "eval_logits/chosen": -0.9842001795768738,
      "eval_logits/rejected": -1.2457659244537354,
      "eval_logps/chosen": -110.90794372558594,
      "eval_logps/rejected": -297.2918395996094,
      "eval_loss": 7.317187851185736e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.0103981494903564,
      "eval_rewards/margins": 15.445306777954102,
      "eval_rewards/rejected": -12.434906959533691,
      "eval_runtime": 660.1054,
      "eval_samples_per_second": 11.216,
      "eval_steps_per_second": 0.624,
      "step": 1650
    },
    {
      "epoch": 0.6726094003241491,
      "grad_norm": 4.348358925199136e-05,
      "learning_rate": 8.620741407774277e-06,
      "logits/chosen": -0.9552786946296692,
      "logits/rejected": -1.2268904447555542,
      "logps/chosen": -111.0877914428711,
      "logps/rejected": -296.8569030761719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0642521381378174,
      "rewards/margins": 15.461289405822754,
      "rewards/rejected": -12.397037506103516,
      "step": 1660
    },
    {
      "epoch": 0.6766612641815235,
      "grad_norm": 0.00014763206127099693,
      "learning_rate": 8.605733153234279e-06,
      "logits/chosen": -0.9552820920944214,
      "logits/rejected": -1.2488830089569092,
      "logps/chosen": -110.29084014892578,
      "logps/rejected": -299.09515380859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.092834949493408,
      "rewards/margins": 15.636582374572754,
      "rewards/rejected": -12.54374885559082,
      "step": 1670
    },
    {
      "epoch": 0.6807131280388979,
      "grad_norm": 5.1606773922685534e-05,
      "learning_rate": 8.590724898694282e-06,
      "logits/chosen": -0.9457971453666687,
      "logits/rejected": -1.206853985786438,
      "logps/chosen": -110.04716491699219,
      "logps/rejected": -296.72747802734375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1184566020965576,
      "rewards/margins": 15.559531211853027,
      "rewards/rejected": -12.441073417663574,
      "step": 1680
    },
    {
      "epoch": 0.6847649918962723,
      "grad_norm": 3.092627230216749e-05,
      "learning_rate": 8.575716644154286e-06,
      "logits/chosen": -0.9578930139541626,
      "logits/rejected": -1.231552243232727,
      "logps/chosen": -110.51782989501953,
      "logps/rejected": -298.4883728027344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.064654588699341,
      "rewards/margins": 15.591686248779297,
      "rewards/rejected": -12.527032852172852,
      "step": 1690
    },
    {
      "epoch": 0.6888168557536467,
      "grad_norm": 0.00016064252122305334,
      "learning_rate": 8.560708389614288e-06,
      "logits/chosen": -0.9631405472755432,
      "logits/rejected": -1.2213727235794067,
      "logps/chosen": -110.19290161132812,
      "logps/rejected": -298.48150634765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.097748279571533,
      "rewards/margins": 15.561808586120605,
      "rewards/rejected": -12.464061737060547,
      "step": 1700
    },
    {
      "epoch": 0.6888168557536467,
      "eval_logits/chosen": -0.9839051365852356,
      "eval_logits/rejected": -1.2429444789886475,
      "eval_logps/chosen": -110.84707641601562,
      "eval_logps/rejected": -298.07794189453125,
      "eval_loss": 6.767778586436179e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.0164856910705566,
      "eval_rewards/margins": 15.530004501342773,
      "eval_rewards/rejected": -12.513520240783691,
      "eval_runtime": 660.1439,
      "eval_samples_per_second": 11.216,
      "eval_steps_per_second": 0.624,
      "step": 1700
    },
    {
      "epoch": 0.692868719611021,
      "grad_norm": 4.53038519481197e-05,
      "learning_rate": 8.545700135074291e-06,
      "logits/chosen": -0.963180661201477,
      "logits/rejected": -1.2384047508239746,
      "logps/chosen": -110.7230224609375,
      "logps/rejected": -298.0184020996094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0918118953704834,
      "rewards/margins": 15.59386157989502,
      "rewards/rejected": -12.502050399780273,
      "step": 1710
    },
    {
      "epoch": 0.6969205834683955,
      "grad_norm": 1.722994420561008e-05,
      "learning_rate": 8.530691880534295e-06,
      "logits/chosen": -0.9652491211891174,
      "logits/rejected": -1.2031772136688232,
      "logps/chosen": -111.2859115600586,
      "logps/rejected": -299.26641845703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0548810958862305,
      "rewards/margins": 15.71076774597168,
      "rewards/rejected": -12.655885696411133,
      "step": 1720
    },
    {
      "epoch": 0.7009724473257699,
      "grad_norm": 5.003461774322204e-05,
      "learning_rate": 8.515683625994298e-06,
      "logits/chosen": -0.9635353088378906,
      "logits/rejected": -1.227256178855896,
      "logps/chosen": -109.7396240234375,
      "logps/rejected": -297.9764709472656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0842108726501465,
      "rewards/margins": 15.59292984008789,
      "rewards/rejected": -12.508719444274902,
      "step": 1730
    },
    {
      "epoch": 0.7050243111831442,
      "grad_norm": 4.6853474486852065e-05,
      "learning_rate": 8.500675371454301e-06,
      "logits/chosen": -0.9571965932846069,
      "logits/rejected": -1.2346524000167847,
      "logps/chosen": -110.17761993408203,
      "logps/rejected": -300.9472961425781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.07942795753479,
      "rewards/margins": 15.799840927124023,
      "rewards/rejected": -12.720414161682129,
      "step": 1740
    },
    {
      "epoch": 0.7090761750405187,
      "grad_norm": 7.191787153715268e-05,
      "learning_rate": 8.485667116914303e-06,
      "logits/chosen": -0.9528246521949768,
      "logits/rejected": -1.2193206548690796,
      "logps/chosen": -110.28775787353516,
      "logps/rejected": -299.3614807128906,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.083807945251465,
      "rewards/margins": 15.71611499786377,
      "rewards/rejected": -12.632307052612305,
      "step": 1750
    },
    {
      "epoch": 0.7090761750405187,
      "eval_logits/chosen": -0.9769449830055237,
      "eval_logits/rejected": -1.2357670068740845,
      "eval_logps/chosen": -110.7610855102539,
      "eval_logps/rejected": -298.78265380859375,
      "eval_loss": 6.295198318184703e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.025084972381592,
      "eval_rewards/margins": 15.609077453613281,
      "eval_rewards/rejected": -12.583992958068848,
      "eval_runtime": 660.0968,
      "eval_samples_per_second": 11.217,
      "eval_steps_per_second": 0.624,
      "step": 1750
    },
    {
      "epoch": 0.713128038897893,
      "grad_norm": 6.214807945070788e-05,
      "learning_rate": 8.470658862374307e-06,
      "logits/chosen": -0.9671065807342529,
      "logits/rejected": -1.2267494201660156,
      "logps/chosen": -111.212646484375,
      "logps/rejected": -297.99432373046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.069645643234253,
      "rewards/margins": 15.604259490966797,
      "rewards/rejected": -12.534612655639648,
      "step": 1760
    },
    {
      "epoch": 0.7171799027552674,
      "grad_norm": 2.1496705812751316e-05,
      "learning_rate": 8.45565060783431e-06,
      "logits/chosen": -0.9392750859260559,
      "logits/rejected": -1.2306023836135864,
      "logps/chosen": -109.73208618164062,
      "logps/rejected": -300.1722412109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0873124599456787,
      "rewards/margins": 15.78653335571289,
      "rewards/rejected": -12.699220657348633,
      "step": 1770
    },
    {
      "epoch": 0.7212317666126418,
      "grad_norm": 4.5025444705970585e-05,
      "learning_rate": 8.440642353294313e-06,
      "logits/chosen": -0.9593215584754944,
      "logits/rejected": -1.2293932437896729,
      "logps/chosen": -109.81471252441406,
      "logps/rejected": -299.7237854003906,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0805766582489014,
      "rewards/margins": 15.711773872375488,
      "rewards/rejected": -12.631196022033691,
      "step": 1780
    },
    {
      "epoch": 0.7252836304700162,
      "grad_norm": 9.227264672517776e-05,
      "learning_rate": 8.425634098754315e-06,
      "logits/chosen": -0.950941801071167,
      "logits/rejected": -1.231359839439392,
      "logps/chosen": -110.4205093383789,
      "logps/rejected": -301.0591735839844,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0956263542175293,
      "rewards/margins": 15.80740737915039,
      "rewards/rejected": -12.711782455444336,
      "step": 1790
    },
    {
      "epoch": 0.7293354943273906,
      "grad_norm": 2.5774721507332288e-05,
      "learning_rate": 8.410625844214319e-06,
      "logits/chosen": -0.9582427144050598,
      "logits/rejected": -1.2166783809661865,
      "logps/chosen": -111.40494537353516,
      "logps/rejected": -300.53167724609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.071667432785034,
      "rewards/margins": 15.72705364227295,
      "rewards/rejected": -12.65538501739502,
      "step": 1800
    },
    {
      "epoch": 0.7293354943273906,
      "eval_logits/chosen": -0.9738713502883911,
      "eval_logits/rejected": -1.2305481433868408,
      "eval_logps/chosen": -110.67764282226562,
      "eval_logps/rejected": -299.36517333984375,
      "eval_loss": 5.93142544857983e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.0334293842315674,
      "eval_rewards/margins": 15.67567253112793,
      "eval_rewards/rejected": -12.642244338989258,
      "eval_runtime": 660.1429,
      "eval_samples_per_second": 11.216,
      "eval_steps_per_second": 0.624,
      "step": 1800
    },
    {
      "epoch": 0.733387358184765,
      "grad_norm": 1.6181831597350538e-05,
      "learning_rate": 8.395617589674322e-06,
      "logits/chosen": -0.9676950573921204,
      "logits/rejected": -1.2319369316101074,
      "logps/chosen": -109.94116973876953,
      "logps/rejected": -300.9438781738281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0895590782165527,
      "rewards/margins": 15.79028034210205,
      "rewards/rejected": -12.70072078704834,
      "step": 1810
    },
    {
      "epoch": 0.7374392220421394,
      "grad_norm": 2.0902380128973164e-05,
      "learning_rate": 8.380609335134324e-06,
      "logits/chosen": -0.9366939663887024,
      "logits/rejected": -1.229324460029602,
      "logps/chosen": -109.44600677490234,
      "logps/rejected": -302.6067810058594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.142228841781616,
      "rewards/margins": 16.04191017150879,
      "rewards/rejected": -12.899679183959961,
      "step": 1820
    },
    {
      "epoch": 0.7414910858995137,
      "grad_norm": 2.0663153918576427e-05,
      "learning_rate": 8.365601080594327e-06,
      "logits/chosen": -0.9395460486412048,
      "logits/rejected": -1.2137764692306519,
      "logps/chosen": -109.63568878173828,
      "logps/rejected": -301.1596984863281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.148167133331299,
      "rewards/margins": 15.912049293518066,
      "rewards/rejected": -12.763882637023926,
      "step": 1830
    },
    {
      "epoch": 0.7455429497568882,
      "grad_norm": 2.4156342988135293e-05,
      "learning_rate": 8.35059282605433e-06,
      "logits/chosen": -0.9509289860725403,
      "logits/rejected": -1.2320055961608887,
      "logps/chosen": -109.17674255371094,
      "logps/rejected": -300.7659606933594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.152684450149536,
      "rewards/margins": 15.885882377624512,
      "rewards/rejected": -12.733199119567871,
      "step": 1840
    },
    {
      "epoch": 0.7495948136142626,
      "grad_norm": 5.195208723307587e-05,
      "learning_rate": 8.335584571514334e-06,
      "logits/chosen": -0.9608786702156067,
      "logits/rejected": -1.2239487171173096,
      "logps/chosen": -111.0685043334961,
      "logps/rejected": -300.0669860839844,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0595474243164062,
      "rewards/margins": 15.727933883666992,
      "rewards/rejected": -12.668384552001953,
      "step": 1850
    },
    {
      "epoch": 0.7495948136142626,
      "eval_logits/chosen": -0.9690695405006409,
      "eval_logits/rejected": -1.2296942472457886,
      "eval_logps/chosen": -110.52428436279297,
      "eval_logps/rejected": -299.7806091308594,
      "eval_loss": 5.65465313684399e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.04876446723938,
      "eval_rewards/margins": 15.732549667358398,
      "eval_rewards/rejected": -12.683784484863281,
      "eval_runtime": 658.0929,
      "eval_samples_per_second": 11.251,
      "eval_steps_per_second": 0.626,
      "step": 1850
    },
    {
      "epoch": 0.7536466774716369,
      "grad_norm": 2.637179204612039e-05,
      "learning_rate": 8.320576316974337e-06,
      "logits/chosen": -0.9759941697120667,
      "logits/rejected": -1.222524642944336,
      "logps/chosen": -110.23280334472656,
      "logps/rejected": -301.49127197265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.082556962966919,
      "rewards/margins": 15.849783897399902,
      "rewards/rejected": -12.767226219177246,
      "step": 1860
    },
    {
      "epoch": 0.7576985413290114,
      "grad_norm": 2.6323505153413862e-05,
      "learning_rate": 8.305568062434339e-06,
      "logits/chosen": -0.9682739973068237,
      "logits/rejected": -1.2260493040084839,
      "logps/chosen": -111.04216766357422,
      "logps/rejected": -301.8972473144531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0877504348754883,
      "rewards/margins": 15.785056114196777,
      "rewards/rejected": -12.697306632995605,
      "step": 1870
    },
    {
      "epoch": 0.7617504051863857,
      "grad_norm": 2.6471077944734134e-05,
      "learning_rate": 8.290559807894343e-06,
      "logits/chosen": -0.9587245583534241,
      "logits/rejected": -1.235076904296875,
      "logps/chosen": -109.97119140625,
      "logps/rejected": -301.52105712890625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1070733070373535,
      "rewards/margins": 15.876703262329102,
      "rewards/rejected": -12.76962947845459,
      "step": 1880
    },
    {
      "epoch": 0.7658022690437601,
      "grad_norm": 2.7889856937690638e-05,
      "learning_rate": 8.275551553354344e-06,
      "logits/chosen": -0.9489023685455322,
      "logits/rejected": -1.22849440574646,
      "logps/chosen": -107.74160766601562,
      "logps/rejected": -300.9273986816406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1654727458953857,
      "rewards/margins": 15.985053062438965,
      "rewards/rejected": -12.819581031799316,
      "step": 1890
    },
    {
      "epoch": 0.7698541329011345,
      "grad_norm": 2.9925871785962954e-05,
      "learning_rate": 8.26054329881435e-06,
      "logits/chosen": -0.9606993198394775,
      "logits/rejected": -1.2304047346115112,
      "logps/chosen": -110.7623062133789,
      "logps/rejected": -301.9601745605469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0822935104370117,
      "rewards/margins": 15.915146827697754,
      "rewards/rejected": -12.832853317260742,
      "step": 1900
    },
    {
      "epoch": 0.7698541329011345,
      "eval_logits/chosen": -0.9868106245994568,
      "eval_logits/rejected": -1.241007924079895,
      "eval_logps/chosen": -110.56230926513672,
      "eval_logps/rejected": -300.6956481933594,
      "eval_loss": 5.184895144338952e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.044961929321289,
      "eval_rewards/margins": 15.820250511169434,
      "eval_rewards/rejected": -12.775288581848145,
      "eval_runtime": 658.0376,
      "eval_samples_per_second": 11.252,
      "eval_steps_per_second": 0.626,
      "step": 1900
    },
    {
      "epoch": 0.7739059967585089,
      "grad_norm": 4.860399349126965e-05,
      "learning_rate": 8.245535044274353e-06,
      "logits/chosen": -0.9716636538505554,
      "logits/rejected": -1.2283477783203125,
      "logps/chosen": -110.34529876708984,
      "logps/rejected": -301.6912841796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0592539310455322,
      "rewards/margins": 15.871448516845703,
      "rewards/rejected": -12.812193870544434,
      "step": 1910
    },
    {
      "epoch": 0.7779578606158833,
      "grad_norm": 6.4569860114716e-05,
      "learning_rate": 8.230526789734355e-06,
      "logits/chosen": -0.9652984738349915,
      "logits/rejected": -1.2083947658538818,
      "logps/chosen": -110.9874496459961,
      "logps/rejected": -302.92138671875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.103862762451172,
      "rewards/margins": 16.010520935058594,
      "rewards/rejected": -12.906659126281738,
      "step": 1920
    },
    {
      "epoch": 0.7820097244732577,
      "grad_norm": 4.364985943539068e-05,
      "learning_rate": 8.215518535194358e-06,
      "logits/chosen": -0.9482593536376953,
      "logits/rejected": -1.2342857122421265,
      "logps/chosen": -108.68612670898438,
      "logps/rejected": -301.5144958496094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1161887645721436,
      "rewards/margins": 15.925652503967285,
      "rewards/rejected": -12.809463500976562,
      "step": 1930
    },
    {
      "epoch": 0.7860615883306321,
      "grad_norm": 5.059886825620197e-05,
      "learning_rate": 8.20051028065436e-06,
      "logits/chosen": -0.9467249512672424,
      "logits/rejected": -1.2305415868759155,
      "logps/chosen": -110.29265594482422,
      "logps/rejected": -301.26641845703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.125284433364868,
      "rewards/margins": 15.913708686828613,
      "rewards/rejected": -12.788424491882324,
      "step": 1940
    },
    {
      "epoch": 0.7901134521880064,
      "grad_norm": 3.123753413092345e-05,
      "learning_rate": 8.185502026114363e-06,
      "logits/chosen": -0.9409447312355042,
      "logits/rejected": -1.2227845191955566,
      "logps/chosen": -109.94808197021484,
      "logps/rejected": -302.04449462890625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1201517581939697,
      "rewards/margins": 16.04159927368164,
      "rewards/rejected": -12.92144775390625,
      "step": 1950
    },
    {
      "epoch": 0.7901134521880064,
      "eval_logits/chosen": -0.983956515789032,
      "eval_logits/rejected": -1.2356351613998413,
      "eval_logps/chosen": -110.52811431884766,
      "eval_logps/rejected": -301.3548278808594,
      "eval_loss": 4.869766598858405e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.048381805419922,
      "eval_rewards/margins": 15.889588356018066,
      "eval_rewards/rejected": -12.841206550598145,
      "eval_runtime": 660.0123,
      "eval_samples_per_second": 11.218,
      "eval_steps_per_second": 0.624,
      "step": 1950
    },
    {
      "epoch": 0.7941653160453809,
      "grad_norm": 1.6401634638896212e-05,
      "learning_rate": 8.170493771574367e-06,
      "logits/chosen": -0.9526603817939758,
      "logits/rejected": -1.2315199375152588,
      "logps/chosen": -110.46824645996094,
      "logps/rejected": -302.6124267578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1375341415405273,
      "rewards/margins": 16.068687438964844,
      "rewards/rejected": -12.931153297424316,
      "step": 1960
    },
    {
      "epoch": 0.7982171799027553,
      "grad_norm": 3.981655390816741e-05,
      "learning_rate": 8.15548551703437e-06,
      "logits/chosen": -0.9453114867210388,
      "logits/rejected": -1.215904951095581,
      "logps/chosen": -108.2788314819336,
      "logps/rejected": -302.3946533203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.17598032951355,
      "rewards/margins": 16.017545700073242,
      "rewards/rejected": -12.841565132141113,
      "step": 1970
    },
    {
      "epoch": 0.8022690437601296,
      "grad_norm": 1.9739898561965674e-05,
      "learning_rate": 8.140477262494373e-06,
      "logits/chosen": -0.9491396546363831,
      "logits/rejected": -1.2109198570251465,
      "logps/chosen": -109.99365997314453,
      "logps/rejected": -302.4257507324219,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1579694747924805,
      "rewards/margins": 16.02589225769043,
      "rewards/rejected": -12.86792278289795,
      "step": 1980
    },
    {
      "epoch": 0.8063209076175041,
      "grad_norm": 3.2942043617367744e-05,
      "learning_rate": 8.125469007954375e-06,
      "logits/chosen": -0.941801905632019,
      "logits/rejected": -1.2268625497817993,
      "logps/chosen": -109.67646026611328,
      "logps/rejected": -302.0210266113281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1409685611724854,
      "rewards/margins": 16.054298400878906,
      "rewards/rejected": -12.913330078125,
      "step": 1990
    },
    {
      "epoch": 0.8103727714748784,
      "grad_norm": 3.328973980387673e-05,
      "learning_rate": 8.110460753414379e-06,
      "logits/chosen": -0.9555728435516357,
      "logits/rejected": -1.2261112928390503,
      "logps/chosen": -110.30085754394531,
      "logps/rejected": -303.8353271484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1162924766540527,
      "rewards/margins": 16.052085876464844,
      "rewards/rejected": -12.935792922973633,
      "step": 2000
    },
    {
      "epoch": 0.8103727714748784,
      "eval_logits/chosen": -0.9666898250579834,
      "eval_logits/rejected": -1.2228957414627075,
      "eval_logps/chosen": -110.36251068115234,
      "eval_logps/rejected": -301.76934814453125,
      "eval_loss": 4.659783314764354e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.06494140625,
      "eval_rewards/margins": 15.947602272033691,
      "eval_rewards/rejected": -12.882659912109375,
      "eval_runtime": 660.0284,
      "eval_samples_per_second": 11.218,
      "eval_steps_per_second": 0.624,
      "step": 2000
    },
    {
      "epoch": 0.8144246353322528,
      "grad_norm": 1.4460420061368495e-05,
      "learning_rate": 8.095452498874382e-06,
      "logits/chosen": -0.9715719819068909,
      "logits/rejected": -1.2192199230194092,
      "logps/chosen": -109.86674499511719,
      "logps/rejected": -301.76434326171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1414902210235596,
      "rewards/margins": 16.00307846069336,
      "rewards/rejected": -12.861587524414062,
      "step": 2010
    },
    {
      "epoch": 0.8184764991896273,
      "grad_norm": 6.478391878772527e-05,
      "learning_rate": 8.080444244334384e-06,
      "logits/chosen": -0.9564737677574158,
      "logits/rejected": -1.2045562267303467,
      "logps/chosen": -108.61962890625,
      "logps/rejected": -303.49786376953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1600611209869385,
      "rewards/margins": 16.183496475219727,
      "rewards/rejected": -13.023435592651367,
      "step": 2020
    },
    {
      "epoch": 0.8225283630470016,
      "grad_norm": 2.9472497772076167e-05,
      "learning_rate": 8.065435989794387e-06,
      "logits/chosen": -0.9578773379325867,
      "logits/rejected": -1.223385214805603,
      "logps/chosen": -109.77428436279297,
      "logps/rejected": -303.5888671875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1244020462036133,
      "rewards/margins": 16.179670333862305,
      "rewards/rejected": -13.055269241333008,
      "step": 2030
    },
    {
      "epoch": 0.826580226904376,
      "grad_norm": 5.243470150162466e-05,
      "learning_rate": 8.05042773525439e-06,
      "logits/chosen": -0.9719753265380859,
      "logits/rejected": -1.2280404567718506,
      "logps/chosen": -110.10773468017578,
      "logps/rejected": -303.7828674316406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.111752986907959,
      "rewards/margins": 16.16956901550293,
      "rewards/rejected": -13.057815551757812,
      "step": 2040
    },
    {
      "epoch": 0.8306320907617504,
      "grad_norm": 3.237457713112235e-05,
      "learning_rate": 8.035419480714394e-06,
      "logits/chosen": -0.9367961883544922,
      "logits/rejected": -1.209633708000183,
      "logps/chosen": -109.77885437011719,
      "logps/rejected": -304.89691162109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1424176692962646,
      "rewards/margins": 16.200634002685547,
      "rewards/rejected": -13.058215141296387,
      "step": 2050
    },
    {
      "epoch": 0.8306320907617504,
      "eval_logits/chosen": -0.9693470001220703,
      "eval_logits/rejected": -1.2243363857269287,
      "eval_logps/chosen": -110.29119873046875,
      "eval_logps/rejected": -302.1759338378906,
      "eval_loss": 4.4409716792870313e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.0720736980438232,
      "eval_rewards/margins": 15.995390892028809,
      "eval_rewards/rejected": -12.92331600189209,
      "eval_runtime": 659.9755,
      "eval_samples_per_second": 11.219,
      "eval_steps_per_second": 0.624,
      "step": 2050
    },
    {
      "epoch": 0.8346839546191248,
      "grad_norm": 1.5255555808835197e-05,
      "learning_rate": 8.020411226174396e-06,
      "logits/chosen": -0.9544581174850464,
      "logits/rejected": -1.2138450145721436,
      "logps/chosen": -108.7799301147461,
      "logps/rejected": -302.1813659667969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1372766494750977,
      "rewards/margins": 16.07466697692871,
      "rewards/rejected": -12.937390327453613,
      "step": 2060
    },
    {
      "epoch": 0.8387358184764991,
      "grad_norm": 3.847632615361363e-05,
      "learning_rate": 8.0054029716344e-06,
      "logits/chosen": -0.9481338858604431,
      "logits/rejected": -1.2043477296829224,
      "logps/chosen": -110.19457244873047,
      "logps/rejected": -302.8457946777344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1281635761260986,
      "rewards/margins": 16.106060028076172,
      "rewards/rejected": -12.97789478302002,
      "step": 2070
    },
    {
      "epoch": 0.8427876823338736,
      "grad_norm": 4.2611958633642644e-05,
      "learning_rate": 7.990394717094403e-06,
      "logits/chosen": -0.9538971781730652,
      "logits/rejected": -1.2121487855911255,
      "logps/chosen": -109.85386657714844,
      "logps/rejected": -303.5600891113281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.137418270111084,
      "rewards/margins": 16.140491485595703,
      "rewards/rejected": -13.003071784973145,
      "step": 2080
    },
    {
      "epoch": 0.846839546191248,
      "grad_norm": 4.119348886888474e-05,
      "learning_rate": 7.975386462554406e-06,
      "logits/chosen": -0.9455956220626831,
      "logits/rejected": -1.2411589622497559,
      "logps/chosen": -108.50851440429688,
      "logps/rejected": -304.57586669921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2042062282562256,
      "rewards/margins": 16.30094337463379,
      "rewards/rejected": -13.096736907958984,
      "step": 2090
    },
    {
      "epoch": 0.8508914100486223,
      "grad_norm": 4.610854739439674e-05,
      "learning_rate": 7.96037820801441e-06,
      "logits/chosen": -0.9589834809303284,
      "logits/rejected": -1.2162960767745972,
      "logps/chosen": -110.89701080322266,
      "logps/rejected": -302.62799072265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0954887866973877,
      "rewards/margins": 16.030658721923828,
      "rewards/rejected": -12.935168266296387,
      "step": 2100
    },
    {
      "epoch": 0.8508914100486223,
      "eval_logits/chosen": -0.9647515416145325,
      "eval_logits/rejected": -1.2190096378326416,
      "eval_logps/chosen": -110.22238159179688,
      "eval_logps/rejected": -302.7558288574219,
      "eval_loss": 4.1841948927867634e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.0789549350738525,
      "eval_rewards/margins": 16.06026268005371,
      "eval_rewards/rejected": -12.981307029724121,
      "eval_runtime": 659.9091,
      "eval_samples_per_second": 11.22,
      "eval_steps_per_second": 0.624,
      "step": 2100
    },
    {
      "epoch": 0.8549432739059968,
      "grad_norm": 7.719548011664301e-05,
      "learning_rate": 7.945369953474411e-06,
      "logits/chosen": -0.9446741938591003,
      "logits/rejected": -1.22459876537323,
      "logps/chosen": -110.51274108886719,
      "logps/rejected": -303.3446044921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.129234790802002,
      "rewards/margins": 16.153383255004883,
      "rewards/rejected": -13.024148941040039,
      "step": 2110
    },
    {
      "epoch": 0.8589951377633711,
      "grad_norm": 2.026069341809489e-05,
      "learning_rate": 7.930361698934415e-06,
      "logits/chosen": -0.9670036435127258,
      "logits/rejected": -1.215293288230896,
      "logps/chosen": -109.58710479736328,
      "logps/rejected": -304.4532775878906,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.140310764312744,
      "rewards/margins": 16.27900505065918,
      "rewards/rejected": -13.138696670532227,
      "step": 2120
    },
    {
      "epoch": 0.8630470016207455,
      "grad_norm": 4.710930079454556e-05,
      "learning_rate": 7.915353444394418e-06,
      "logits/chosen": -0.9606796503067017,
      "logits/rejected": -1.200211763381958,
      "logps/chosen": -108.72264099121094,
      "logps/rejected": -304.9479064941406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1467487812042236,
      "rewards/margins": 16.292722702026367,
      "rewards/rejected": -13.145975112915039,
      "step": 2130
    },
    {
      "epoch": 0.86709886547812,
      "grad_norm": 4.4419884943636134e-05,
      "learning_rate": 7.90034518985442e-06,
      "logits/chosen": -0.9616580605506897,
      "logits/rejected": -1.2213574647903442,
      "logps/chosen": -110.62073516845703,
      "logps/rejected": -304.69097900390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0965681076049805,
      "rewards/margins": 16.208520889282227,
      "rewards/rejected": -13.111952781677246,
      "step": 2140
    },
    {
      "epoch": 0.8711507293354943,
      "grad_norm": 4.5354092435445637e-05,
      "learning_rate": 7.885336935314423e-06,
      "logits/chosen": -0.9539754986763,
      "logits/rejected": -1.2153164148330688,
      "logps/chosen": -110.38399505615234,
      "logps/rejected": -303.0393981933594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.09737491607666,
      "rewards/margins": 16.079133987426758,
      "rewards/rejected": -12.981756210327148,
      "step": 2150
    },
    {
      "epoch": 0.8711507293354943,
      "eval_logits/chosen": -0.964377224445343,
      "eval_logits/rejected": -1.217590093612671,
      "eval_logps/chosen": -110.18504333496094,
      "eval_logps/rejected": -303.2809753417969,
      "eval_loss": 3.976499272084766e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.082688808441162,
      "eval_rewards/margins": 16.11651039123535,
      "eval_rewards/rejected": -13.033820152282715,
      "eval_runtime": 658.0257,
      "eval_samples_per_second": 11.252,
      "eval_steps_per_second": 0.626,
      "step": 2150
    },
    {
      "epoch": 0.8752025931928687,
      "grad_norm": 6.480027514044195e-05,
      "learning_rate": 7.870328680774427e-06,
      "logits/chosen": -0.9567605257034302,
      "logits/rejected": -1.2163498401641846,
      "logps/chosen": -108.91773223876953,
      "logps/rejected": -304.4596252441406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.154085636138916,
      "rewards/margins": 16.183382034301758,
      "rewards/rejected": -13.029295921325684,
      "step": 2160
    },
    {
      "epoch": 0.8792544570502431,
      "grad_norm": 2.1783509509987198e-05,
      "learning_rate": 7.85532042623443e-06,
      "logits/chosen": -0.9555526375770569,
      "logits/rejected": -1.2262533903121948,
      "logps/chosen": -110.84054565429688,
      "logps/rejected": -306.3641357421875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1060667037963867,
      "rewards/margins": 16.345033645629883,
      "rewards/rejected": -13.23896598815918,
      "step": 2170
    },
    {
      "epoch": 0.8833063209076175,
      "grad_norm": 4.148611333221197e-05,
      "learning_rate": 7.840312171694434e-06,
      "logits/chosen": -0.9608704447746277,
      "logits/rejected": -1.2355401515960693,
      "logps/chosen": -108.211669921875,
      "logps/rejected": -306.52606201171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.148042917251587,
      "rewards/margins": 16.463624954223633,
      "rewards/rejected": -13.315583229064941,
      "step": 2180
    },
    {
      "epoch": 0.8873581847649918,
      "grad_norm": 1.6699072148185223e-05,
      "learning_rate": 7.825303917154435e-06,
      "logits/chosen": -0.9467074871063232,
      "logits/rejected": -1.2218189239501953,
      "logps/chosen": -107.47708129882812,
      "logps/rejected": -306.47119140625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.200939178466797,
      "rewards/margins": 16.48932647705078,
      "rewards/rejected": -13.288387298583984,
      "step": 2190
    },
    {
      "epoch": 0.8914100486223663,
      "grad_norm": 3.9296108298003674e-05,
      "learning_rate": 7.810295662614439e-06,
      "logits/chosen": -0.9578278660774231,
      "logits/rejected": -1.1927484273910522,
      "logps/chosen": -109.21939849853516,
      "logps/rejected": -303.5795593261719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1450209617614746,
      "rewards/margins": 16.233585357666016,
      "rewards/rejected": -13.088563919067383,
      "step": 2200
    },
    {
      "epoch": 0.8914100486223663,
      "eval_logits/chosen": -0.9689859747886658,
      "eval_logits/rejected": -1.220730185508728,
      "eval_logps/chosen": -110.15634155273438,
      "eval_logps/rejected": -303.7755126953125,
      "eval_loss": 3.7930050211798516e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.0855586528778076,
      "eval_rewards/margins": 16.16883659362793,
      "eval_rewards/rejected": -13.083277702331543,
      "eval_runtime": 657.9854,
      "eval_samples_per_second": 11.253,
      "eval_steps_per_second": 0.626,
      "step": 2200
    },
    {
      "epoch": 0.8954619124797407,
      "grad_norm": 9.78490697889356e-06,
      "learning_rate": 7.79528740807444e-06,
      "logits/chosen": -0.9606353044509888,
      "logits/rejected": -1.2140560150146484,
      "logps/chosen": -109.4519271850586,
      "logps/rejected": -303.8180847167969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1682541370391846,
      "rewards/margins": 16.253660202026367,
      "rewards/rejected": -13.085405349731445,
      "step": 2210
    },
    {
      "epoch": 0.899513776337115,
      "grad_norm": 3.433896563365124e-05,
      "learning_rate": 7.780279153534444e-06,
      "logits/chosen": -0.9603633284568787,
      "logits/rejected": -1.2174922227859497,
      "logps/chosen": -108.57514190673828,
      "logps/rejected": -304.82806396484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1801135540008545,
      "rewards/margins": 16.351207733154297,
      "rewards/rejected": -13.17109489440918,
      "step": 2220
    },
    {
      "epoch": 0.9035656401944895,
      "grad_norm": 1.7691954781184904e-05,
      "learning_rate": 7.765270898994447e-06,
      "logits/chosen": -0.95579594373703,
      "logits/rejected": -1.2212696075439453,
      "logps/chosen": -110.44479370117188,
      "logps/rejected": -305.1944274902344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.132660150527954,
      "rewards/margins": 16.284616470336914,
      "rewards/rejected": -13.151957511901855,
      "step": 2230
    },
    {
      "epoch": 0.9076175040518638,
      "grad_norm": 2.412088360870257e-05,
      "learning_rate": 7.75026264445445e-06,
      "logits/chosen": -0.9635848999023438,
      "logits/rejected": -1.2182646989822388,
      "logps/chosen": -109.63471984863281,
      "logps/rejected": -304.3304443359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.115788459777832,
      "rewards/margins": 16.186290740966797,
      "rewards/rejected": -13.070501327514648,
      "step": 2240
    },
    {
      "epoch": 0.9116693679092382,
      "grad_norm": 2.194743683503475e-05,
      "learning_rate": 7.735254389914454e-06,
      "logits/chosen": -0.9533272981643677,
      "logits/rejected": -1.2021334171295166,
      "logps/chosen": -110.09785461425781,
      "logps/rejected": -305.65771484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.130030393600464,
      "rewards/margins": 16.36699104309082,
      "rewards/rejected": -13.236960411071777,
      "step": 2250
    },
    {
      "epoch": 0.9116693679092382,
      "eval_logits/chosen": -0.9838148951530457,
      "eval_logits/rejected": -1.2295185327529907,
      "eval_logps/chosen": -110.23644256591797,
      "eval_logps/rejected": -304.6207580566406,
      "eval_loss": 3.502078413930576e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.0775492191314697,
      "eval_rewards/margins": 16.245349884033203,
      "eval_rewards/rejected": -13.167800903320312,
      "eval_runtime": 657.9779,
      "eval_samples_per_second": 11.253,
      "eval_steps_per_second": 0.626,
      "step": 2250
    },
    {
      "epoch": 0.9157212317666127,
      "grad_norm": 1.3532581760955509e-05,
      "learning_rate": 7.720246135374456e-06,
      "logits/chosen": -0.9631421566009521,
      "logits/rejected": -1.1869605779647827,
      "logps/chosen": -110.19988250732422,
      "logps/rejected": -303.62103271484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.128401041030884,
      "rewards/margins": 16.225181579589844,
      "rewards/rejected": -13.096781730651855,
      "step": 2260
    },
    {
      "epoch": 0.919773095623987,
      "grad_norm": 2.268419848405756e-05,
      "learning_rate": 7.70523788083446e-06,
      "logits/chosen": -0.9476529955863953,
      "logits/rejected": -1.217498779296875,
      "logps/chosen": -108.96920776367188,
      "logps/rejected": -305.6793212890625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1439383029937744,
      "rewards/margins": 16.39657974243164,
      "rewards/rejected": -13.252642631530762,
      "step": 2270
    },
    {
      "epoch": 0.9238249594813615,
      "grad_norm": 7.45236175134778e-05,
      "learning_rate": 7.690229626294463e-06,
      "logits/chosen": -0.9605175256729126,
      "logits/rejected": -1.2135826349258423,
      "logps/chosen": -110.78817749023438,
      "logps/rejected": -306.30999755859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1067616939544678,
      "rewards/margins": 16.265913009643555,
      "rewards/rejected": -13.159149169921875,
      "step": 2280
    },
    {
      "epoch": 0.9278768233387358,
      "grad_norm": 1.7361502614221536e-05,
      "learning_rate": 7.675221371754466e-06,
      "logits/chosen": -0.9507707953453064,
      "logits/rejected": -1.2035835981369019,
      "logps/chosen": -109.37841796875,
      "logps/rejected": -305.3598937988281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1481592655181885,
      "rewards/margins": 16.325180053710938,
      "rewards/rejected": -13.177021026611328,
      "step": 2290
    },
    {
      "epoch": 0.9319286871961102,
      "grad_norm": 7.1184422267833725e-06,
      "learning_rate": 7.66021311721447e-06,
      "logits/chosen": -0.9573420286178589,
      "logits/rejected": -1.236305594444275,
      "logps/chosen": -109.24344635009766,
      "logps/rejected": -307.2495422363281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.155404806137085,
      "rewards/margins": 16.50580406188965,
      "rewards/rejected": -13.3503999710083,
      "step": 2300
    },
    {
      "epoch": 0.9319286871961102,
      "eval_logits/chosen": -0.9896643161773682,
      "eval_logits/rejected": -1.2349047660827637,
      "eval_logps/chosen": -110.19889831542969,
      "eval_logps/rejected": -305.12225341796875,
      "eval_loss": 3.330087849917618e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.0813028812408447,
      "eval_rewards/margins": 16.299251556396484,
      "eval_rewards/rejected": -13.217947959899902,
      "eval_runtime": 658.0447,
      "eval_samples_per_second": 11.252,
      "eval_steps_per_second": 0.626,
      "step": 2300
    },
    {
      "epoch": 0.9359805510534847,
      "grad_norm": 1.445604630134767e-05,
      "learning_rate": 7.645204862674471e-06,
      "logits/chosen": -0.9550139307975769,
      "logits/rejected": -1.2085530757904053,
      "logps/chosen": -108.17741394042969,
      "logps/rejected": -305.9125671386719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1891822814941406,
      "rewards/margins": 16.470483779907227,
      "rewards/rejected": -13.281301498413086,
      "step": 2310
    },
    {
      "epoch": 0.940032414910859,
      "grad_norm": 4.8019915993791074e-05,
      "learning_rate": 7.630196608134475e-06,
      "logits/chosen": -0.9634786248207092,
      "logits/rejected": -1.2144436836242676,
      "logps/chosen": -110.15174865722656,
      "logps/rejected": -306.0100402832031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1174349784851074,
      "rewards/margins": 16.30495834350586,
      "rewards/rejected": -13.18752384185791,
      "step": 2320
    },
    {
      "epoch": 0.9440842787682334,
      "grad_norm": 1.4580664355889894e-05,
      "learning_rate": 7.615188353594477e-06,
      "logits/chosen": -0.942629873752594,
      "logits/rejected": -1.216013789176941,
      "logps/chosen": -108.79090881347656,
      "logps/rejected": -306.5632629394531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1543760299682617,
      "rewards/margins": 16.500843048095703,
      "rewards/rejected": -13.34646987915039,
      "step": 2330
    },
    {
      "epoch": 0.9481361426256077,
      "grad_norm": 1.3364463484322187e-05,
      "learning_rate": 7.600180099054481e-06,
      "logits/chosen": -0.9466074705123901,
      "logits/rejected": -1.2142013311386108,
      "logps/chosen": -108.32075500488281,
      "logps/rejected": -307.10748291015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.168116807937622,
      "rewards/margins": 16.500261306762695,
      "rewards/rejected": -13.33214282989502,
      "step": 2340
    },
    {
      "epoch": 0.9521880064829822,
      "grad_norm": 3.1726067390991375e-05,
      "learning_rate": 7.585171844514484e-06,
      "logits/chosen": -0.9811385869979858,
      "logits/rejected": -1.2290759086608887,
      "logps/chosen": -110.23688507080078,
      "logps/rejected": -306.6099548339844,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.13777232170105,
      "rewards/margins": 16.42414093017578,
      "rewards/rejected": -13.286368370056152,
      "step": 2350
    },
    {
      "epoch": 0.9521880064829822,
      "eval_logits/chosen": -0.9722626805305481,
      "eval_logits/rejected": -1.2209999561309814,
      "eval_logps/chosen": -110.04175567626953,
      "eval_logps/rejected": -305.3106994628906,
      "eval_loss": 3.2545648309678654e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.097018003463745,
      "eval_rewards/margins": 16.333810806274414,
      "eval_rewards/rejected": -13.236793518066406,
      "eval_runtime": 658.0383,
      "eval_samples_per_second": 11.252,
      "eval_steps_per_second": 0.626,
      "step": 2350
    },
    {
      "epoch": 0.9562398703403565,
      "grad_norm": 2.971366666315589e-05,
      "learning_rate": 7.570163589974486e-06,
      "logits/chosen": -0.9537125825881958,
      "logits/rejected": -1.2295702695846558,
      "logps/chosen": -109.1019515991211,
      "logps/rejected": -306.8534240722656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.152003765106201,
      "rewards/margins": 16.544950485229492,
      "rewards/rejected": -13.39294719696045,
      "step": 2360
    },
    {
      "epoch": 0.960291734197731,
      "grad_norm": 9.768622840056196e-05,
      "learning_rate": 7.55515533543449e-06,
      "logits/chosen": -0.98313307762146,
      "logits/rejected": -1.2179185152053833,
      "logps/chosen": -111.0000228881836,
      "logps/rejected": -305.8616638183594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.125744104385376,
      "rewards/margins": 16.362468719482422,
      "rewards/rejected": -13.236722946166992,
      "step": 2370
    },
    {
      "epoch": 0.9643435980551054,
      "grad_norm": 8.749291009735316e-05,
      "learning_rate": 7.540147080894492e-06,
      "logits/chosen": -0.9686611294746399,
      "logits/rejected": -1.219423532485962,
      "logps/chosen": -109.52857208251953,
      "logps/rejected": -306.1866455078125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1496975421905518,
      "rewards/margins": 16.51058578491211,
      "rewards/rejected": -13.360889434814453,
      "step": 2380
    },
    {
      "epoch": 0.9683954619124797,
      "grad_norm": 1.0122907951881643e-05,
      "learning_rate": 7.525138826354495e-06,
      "logits/chosen": -0.9360008835792542,
      "logits/rejected": -1.204994797706604,
      "logps/chosen": -108.57238006591797,
      "logps/rejected": -307.67620849609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.18050217628479,
      "rewards/margins": 16.660018920898438,
      "rewards/rejected": -13.479516983032227,
      "step": 2390
    },
    {
      "epoch": 0.9724473257698542,
      "grad_norm": 1.0658640348992776e-05,
      "learning_rate": 7.510130571814499e-06,
      "logits/chosen": -0.9478757977485657,
      "logits/rejected": -1.1838241815567017,
      "logps/chosen": -108.75774383544922,
      "logps/rejected": -307.07232666015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1749796867370605,
      "rewards/margins": 16.550107955932617,
      "rewards/rejected": -13.375128746032715,
      "step": 2400
    },
    {
      "epoch": 0.9724473257698542,
      "eval_logits/chosen": -0.9685700535774231,
      "eval_logits/rejected": -1.216765284538269,
      "eval_logps/chosen": -109.97445678710938,
      "eval_logps/rejected": -305.79486083984375,
      "eval_loss": 3.088084667979274e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.103747844696045,
      "eval_rewards/margins": 16.38895606994629,
      "eval_rewards/rejected": -13.285209655761719,
      "eval_runtime": 658.1341,
      "eval_samples_per_second": 11.25,
      "eval_steps_per_second": 0.626,
      "step": 2400
    },
    {
      "epoch": 0.9764991896272285,
      "grad_norm": 3.388383629499003e-05,
      "learning_rate": 7.495122317274501e-06,
      "logits/chosen": -0.960583508014679,
      "logits/rejected": -1.2182528972625732,
      "logps/chosen": -109.86973571777344,
      "logps/rejected": -306.9446716308594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.136913537979126,
      "rewards/margins": 16.53013038635254,
      "rewards/rejected": -13.393216133117676,
      "step": 2410
    },
    {
      "epoch": 0.9805510534846029,
      "grad_norm": 1.2524588782980572e-05,
      "learning_rate": 7.480114062734505e-06,
      "logits/chosen": -0.9616130590438843,
      "logits/rejected": -1.2080204486846924,
      "logps/chosen": -108.43402862548828,
      "logps/rejected": -308.57305908203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1996264457702637,
      "rewards/margins": 16.611858367919922,
      "rewards/rejected": -13.412230491638184,
      "step": 2420
    },
    {
      "epoch": 0.9846029173419774,
      "grad_norm": 3.357211244292557e-05,
      "learning_rate": 7.465105808194507e-06,
      "logits/chosen": -0.9700466394424438,
      "logits/rejected": -1.2211939096450806,
      "logps/chosen": -108.138427734375,
      "logps/rejected": -309.7015686035156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.188143014907837,
      "rewards/margins": 16.78921127319336,
      "rewards/rejected": -13.601068496704102,
      "step": 2430
    },
    {
      "epoch": 0.9886547811993517,
      "grad_norm": 2.0380715795909055e-05,
      "learning_rate": 7.450097553654511e-06,
      "logits/chosen": -0.9535197019577026,
      "logits/rejected": -1.2204744815826416,
      "logps/chosen": -109.47471618652344,
      "logps/rejected": -306.65875244140625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1321237087249756,
      "rewards/margins": 16.443172454833984,
      "rewards/rejected": -13.31104850769043,
      "step": 2440
    },
    {
      "epoch": 0.9927066450567261,
      "grad_norm": 1.2417739526426885e-05,
      "learning_rate": 7.435089299114514e-06,
      "logits/chosen": -0.9472233057022095,
      "logits/rejected": -1.2015880346298218,
      "logps/chosen": -108.79183197021484,
      "logps/rejected": -305.70318603515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1425604820251465,
      "rewards/margins": 16.415618896484375,
      "rewards/rejected": -13.273058891296387,
      "step": 2450
    },
    {
      "epoch": 0.9927066450567261,
      "eval_logits/chosen": -0.9706429839134216,
      "eval_logits/rejected": -1.217729926109314,
      "eval_logps/chosen": -109.95721435546875,
      "eval_logps/rejected": -306.3364562988281,
      "eval_loss": 2.9294906767063367e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1054720878601074,
      "eval_rewards/margins": 16.444839477539062,
      "eval_rewards/rejected": -13.339369773864746,
      "eval_runtime": 660.0731,
      "eval_samples_per_second": 11.217,
      "eval_steps_per_second": 0.624,
      "step": 2450
    },
    {
      "epoch": 0.9967585089141004,
      "grad_norm": 1.7658723663771525e-05,
      "learning_rate": 7.420081044574517e-06,
      "logits/chosen": -0.9635002017021179,
      "logits/rejected": -1.2199903726577759,
      "logps/chosen": -108.45741271972656,
      "logps/rejected": -308.1318359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1865954399108887,
      "rewards/margins": 16.701438903808594,
      "rewards/rejected": -13.51484489440918,
      "step": 2460
    },
    {
      "epoch": 1.0008103727714748,
      "grad_norm": 1.2052866622980218e-05,
      "learning_rate": 7.40507279003452e-06,
      "logits/chosen": -0.9616323113441467,
      "logits/rejected": -1.208625316619873,
      "logps/chosen": -109.19210815429688,
      "logps/rejected": -307.1117858886719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1796886920928955,
      "rewards/margins": 16.61495590209961,
      "rewards/rejected": -13.435267448425293,
      "step": 2470
    },
    {
      "epoch": 1.0048622366288493,
      "grad_norm": 7.082668162183836e-05,
      "learning_rate": 7.390064535494522e-06,
      "logits/chosen": -0.9665688872337341,
      "logits/rejected": -1.2241923809051514,
      "logps/chosen": -109.77351379394531,
      "logps/rejected": -307.30645751953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.18113112449646,
      "rewards/margins": 16.553733825683594,
      "rewards/rejected": -13.372599601745605,
      "step": 2480
    },
    {
      "epoch": 1.0089141004862237,
      "grad_norm": 9.002913429867476e-06,
      "learning_rate": 7.375056280954525e-06,
      "logits/chosen": -0.945849597454071,
      "logits/rejected": -1.2262566089630127,
      "logps/chosen": -109.76206970214844,
      "logps/rejected": -308.98834228515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1569628715515137,
      "rewards/margins": 16.612316131591797,
      "rewards/rejected": -13.455352783203125,
      "step": 2490
    },
    {
      "epoch": 1.012965964343598,
      "grad_norm": 2.3271795726031996e-05,
      "learning_rate": 7.360048026414529e-06,
      "logits/chosen": -0.9565450549125671,
      "logits/rejected": -1.2092770338058472,
      "logps/chosen": -109.36769104003906,
      "logps/rejected": -307.6745910644531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1507160663604736,
      "rewards/margins": 16.6374454498291,
      "rewards/rejected": -13.486729621887207,
      "step": 2500
    },
    {
      "epoch": 1.012965964343598,
      "eval_logits/chosen": -0.9681739211082458,
      "eval_logits/rejected": -1.2141157388687134,
      "eval_logps/chosen": -109.94417572021484,
      "eval_logps/rejected": -306.7958984375,
      "eval_loss": 2.7970889959760825e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.106775999069214,
      "eval_rewards/margins": 16.492088317871094,
      "eval_rewards/rejected": -13.385313034057617,
      "eval_runtime": 660.045,
      "eval_samples_per_second": 11.217,
      "eval_steps_per_second": 0.624,
      "step": 2500
    },
    {
      "epoch": 1.0170178282009725,
      "grad_norm": 6.652682350249961e-05,
      "learning_rate": 7.345039771874531e-06,
      "logits/chosen": -0.9601606130599976,
      "logits/rejected": -1.192555546760559,
      "logps/chosen": -110.04460906982422,
      "logps/rejected": -307.0443115234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1392316818237305,
      "rewards/margins": 16.51374053955078,
      "rewards/rejected": -13.37450885772705,
      "step": 2510
    },
    {
      "epoch": 1.0210696920583469,
      "grad_norm": 1.2063643225701526e-05,
      "learning_rate": 7.330031517334535e-06,
      "logits/chosen": -0.9684518575668335,
      "logits/rejected": -1.202974557876587,
      "logps/chosen": -110.22441864013672,
      "logps/rejected": -306.74822998046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1343910694122314,
      "rewards/margins": 16.446535110473633,
      "rewards/rejected": -13.312140464782715,
      "step": 2520
    },
    {
      "epoch": 1.0251215559157212,
      "grad_norm": 1.1835069017251953e-05,
      "learning_rate": 7.315023262794537e-06,
      "logits/chosen": -0.9609777331352234,
      "logits/rejected": -1.1967358589172363,
      "logps/chosen": -108.42552947998047,
      "logps/rejected": -308.50885009765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1840617656707764,
      "rewards/margins": 16.66929817199707,
      "rewards/rejected": -13.485235214233398,
      "step": 2530
    },
    {
      "epoch": 1.0291734197730957,
      "grad_norm": 5.298601081449306e-06,
      "learning_rate": 7.300015008254541e-06,
      "logits/chosen": -0.9574286937713623,
      "logits/rejected": -1.218118667602539,
      "logps/chosen": -109.4346694946289,
      "logps/rejected": -308.8258056640625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.176945686340332,
      "rewards/margins": 16.675168991088867,
      "rewards/rejected": -13.498223304748535,
      "step": 2540
    },
    {
      "epoch": 1.03322528363047,
      "grad_norm": 1.3952515473647509e-05,
      "learning_rate": 7.285006753714544e-06,
      "logits/chosen": -0.9570482969284058,
      "logits/rejected": -1.2174618244171143,
      "logps/chosen": -110.29024505615234,
      "logps/rejected": -308.82611083984375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.126805067062378,
      "rewards/margins": 16.703540802001953,
      "rewards/rejected": -13.57673454284668,
      "step": 2550
    },
    {
      "epoch": 1.03322528363047,
      "eval_logits/chosen": -0.9698132276535034,
      "eval_logits/rejected": -1.2148218154907227,
      "eval_logps/chosen": -109.87054443359375,
      "eval_logps/rejected": -307.2029724121094,
      "eval_loss": 2.692640350687725e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1141395568847656,
      "eval_rewards/margins": 16.540159225463867,
      "eval_rewards/rejected": -13.426016807556152,
      "eval_runtime": 657.969,
      "eval_samples_per_second": 11.253,
      "eval_steps_per_second": 0.626,
      "step": 2550
    },
    {
      "epoch": 1.0372771474878444,
      "grad_norm": 9.740067980601452e-06,
      "learning_rate": 7.269998499174547e-06,
      "logits/chosen": -0.9624655246734619,
      "logits/rejected": -1.219292163848877,
      "logps/chosen": -109.78215789794922,
      "logps/rejected": -309.5098876953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1512138843536377,
      "rewards/margins": 16.796560287475586,
      "rewards/rejected": -13.645344734191895,
      "step": 2560
    },
    {
      "epoch": 1.0413290113452187,
      "grad_norm": 4.33393579442054e-05,
      "learning_rate": 7.25499024463455e-06,
      "logits/chosen": -0.961388885974884,
      "logits/rejected": -1.2088836431503296,
      "logps/chosen": -108.20442199707031,
      "logps/rejected": -307.0656433105469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2014071941375732,
      "rewards/margins": 16.71923065185547,
      "rewards/rejected": -13.517821311950684,
      "step": 2570
    },
    {
      "epoch": 1.0453808752025933,
      "grad_norm": 1.0012409802584443e-05,
      "learning_rate": 7.239981990094552e-06,
      "logits/chosen": -0.9583010077476501,
      "logits/rejected": -1.2122553586959839,
      "logps/chosen": -108.27478790283203,
      "logps/rejected": -309.6136474609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2165255546569824,
      "rewards/margins": 16.80967903137207,
      "rewards/rejected": -13.59315299987793,
      "step": 2580
    },
    {
      "epoch": 1.0494327390599676,
      "grad_norm": 3.054554690606892e-05,
      "learning_rate": 7.224973735554555e-06,
      "logits/chosen": -0.9547958374023438,
      "logits/rejected": -1.217245101928711,
      "logps/chosen": -110.09384155273438,
      "logps/rejected": -306.6957702636719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.198312282562256,
      "rewards/margins": 16.49526596069336,
      "rewards/rejected": -13.296950340270996,
      "step": 2590
    },
    {
      "epoch": 1.053484602917342,
      "grad_norm": 1.1041483958251774e-05,
      "learning_rate": 7.209965481014558e-06,
      "logits/chosen": -0.9645355939865112,
      "logits/rejected": -1.234001636505127,
      "logps/chosen": -108.32767486572266,
      "logps/rejected": -308.9895324707031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.186584711074829,
      "rewards/margins": 16.76895523071289,
      "rewards/rejected": -13.582369804382324,
      "step": 2600
    },
    {
      "epoch": 1.053484602917342,
      "eval_logits/chosen": -0.9692774415016174,
      "eval_logits/rejected": -1.2138540744781494,
      "eval_logps/chosen": -109.85295867919922,
      "eval_logps/rejected": -307.5146484375,
      "eval_loss": 2.5956160243367776e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1158974170684814,
      "eval_rewards/margins": 16.57308578491211,
      "eval_rewards/rejected": -13.457188606262207,
      "eval_runtime": 660.0129,
      "eval_samples_per_second": 11.218,
      "eval_steps_per_second": 0.624,
      "step": 2600
    },
    {
      "epoch": 1.0575364667747165,
      "grad_norm": 1.8728651411947794e-05,
      "learning_rate": 7.194957226474561e-06,
      "logits/chosen": -0.9509859085083008,
      "logits/rejected": -1.2229541540145874,
      "logps/chosen": -109.80980682373047,
      "logps/rejected": -307.8365173339844,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1706836223602295,
      "rewards/margins": 16.656150817871094,
      "rewards/rejected": -13.485466957092285,
      "step": 2610
    },
    {
      "epoch": 1.0615883306320908,
      "grad_norm": 1.7515225408715196e-05,
      "learning_rate": 7.179948971934565e-06,
      "logits/chosen": -0.9719070196151733,
      "logits/rejected": -1.2121365070343018,
      "logps/chosen": -109.24164581298828,
      "logps/rejected": -309.3529968261719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.152325391769409,
      "rewards/margins": 16.653779983520508,
      "rewards/rejected": -13.50145435333252,
      "step": 2620
    },
    {
      "epoch": 1.0656401944894651,
      "grad_norm": 3.066382487304509e-05,
      "learning_rate": 7.1649407173945674e-06,
      "logits/chosen": -0.9582745432853699,
      "logits/rejected": -1.2080187797546387,
      "logps/chosen": -110.07678985595703,
      "logps/rejected": -308.70703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.156878709793091,
      "rewards/margins": 16.607290267944336,
      "rewards/rejected": -13.450408935546875,
      "step": 2630
    },
    {
      "epoch": 1.0696920583468394,
      "grad_norm": 1.7575619494891725e-05,
      "learning_rate": 7.149932462854571e-06,
      "logits/chosen": -0.960273802280426,
      "logits/rejected": -1.197909951210022,
      "logps/chosen": -109.26360321044922,
      "logps/rejected": -310.2976379394531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1560423374176025,
      "rewards/margins": 16.926837921142578,
      "rewards/rejected": -13.770796775817871,
      "step": 2640
    },
    {
      "epoch": 1.073743922204214,
      "grad_norm": 1.885088022390846e-05,
      "learning_rate": 7.1349242083145734e-06,
      "logits/chosen": -0.9642795324325562,
      "logits/rejected": -1.2084158658981323,
      "logps/chosen": -109.43455505371094,
      "logps/rejected": -308.03277587890625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1592416763305664,
      "rewards/margins": 16.610139846801758,
      "rewards/rejected": -13.450898170471191,
      "step": 2650
    },
    {
      "epoch": 1.073743922204214,
      "eval_logits/chosen": -0.9705721139907837,
      "eval_logits/rejected": -1.213883638381958,
      "eval_logps/chosen": -109.81368255615234,
      "eval_logps/rejected": -308.0562744140625,
      "eval_loss": 2.4671899723216484e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.119823932647705,
      "eval_rewards/margins": 16.631174087524414,
      "eval_rewards/rejected": -13.511351585388184,
      "eval_runtime": 660.1147,
      "eval_samples_per_second": 11.216,
      "eval_steps_per_second": 0.624,
      "step": 2650
    },
    {
      "epoch": 1.0777957860615883,
      "grad_norm": 1.9579389118007384e-05,
      "learning_rate": 7.119915953774577e-06,
      "logits/chosen": -0.9679809808731079,
      "logits/rejected": -1.21659517288208,
      "logps/chosen": -109.89112854003906,
      "logps/rejected": -309.0220947265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.161513328552246,
      "rewards/margins": 16.76967430114746,
      "rewards/rejected": -13.608160018920898,
      "step": 2660
    },
    {
      "epoch": 1.0818476499189627,
      "grad_norm": 7.737518899375573e-06,
      "learning_rate": 7.10490769923458e-06,
      "logits/chosen": -0.9529644846916199,
      "logits/rejected": -1.1993566751480103,
      "logps/chosen": -108.52783966064453,
      "logps/rejected": -310.7772521972656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.216148853302002,
      "rewards/margins": 16.948780059814453,
      "rewards/rejected": -13.73263168334961,
      "step": 2670
    },
    {
      "epoch": 1.0858995137763372,
      "grad_norm": 1.2826951206079684e-05,
      "learning_rate": 7.089899444694582e-06,
      "logits/chosen": -0.9718688130378723,
      "logits/rejected": -1.2110012769699097,
      "logps/chosen": -109.75894165039062,
      "logps/rejected": -309.4825134277344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.167945623397827,
      "rewards/margins": 16.783924102783203,
      "rewards/rejected": -13.61597728729248,
      "step": 2680
    },
    {
      "epoch": 1.0899513776337115,
      "grad_norm": 1.1608887689362746e-05,
      "learning_rate": 7.0748911901545855e-06,
      "logits/chosen": -0.966856837272644,
      "logits/rejected": -1.2022080421447754,
      "logps/chosen": -110.1280288696289,
      "logps/rejected": -307.17724609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1565988063812256,
      "rewards/margins": 16.637836456298828,
      "rewards/rejected": -13.48123550415039,
      "step": 2690
    },
    {
      "epoch": 1.0940032414910859,
      "grad_norm": 7.56221852498129e-05,
      "learning_rate": 7.059882935614588e-06,
      "logits/chosen": -0.9639146327972412,
      "logits/rejected": -1.2138209342956543,
      "logps/chosen": -109.44894409179688,
      "logps/rejected": -308.7769775390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.192052125930786,
      "rewards/margins": 16.716800689697266,
      "rewards/rejected": -13.524748802185059,
      "step": 2700
    },
    {
      "epoch": 1.0940032414910859,
      "eval_logits/chosen": -0.9828658699989319,
      "eval_logits/rejected": -1.2210161685943604,
      "eval_logps/chosen": -109.9037094116211,
      "eval_logps/rejected": -308.7952880859375,
      "eval_loss": 2.3104557556052896e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1108219623565674,
      "eval_rewards/margins": 16.69607162475586,
      "eval_rewards/rejected": -13.585249900817871,
      "eval_runtime": 660.189,
      "eval_samples_per_second": 11.215,
      "eval_steps_per_second": 0.624,
      "step": 2700
    },
    {
      "epoch": 1.0980551053484602,
      "grad_norm": 2.3086533474270254e-05,
      "learning_rate": 7.0448746810745915e-06,
      "logits/chosen": -0.9712207913398743,
      "logits/rejected": -1.207464575767517,
      "logps/chosen": -109.36491394042969,
      "logps/rejected": -310.7517395019531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1938600540161133,
      "rewards/margins": 16.914466857910156,
      "rewards/rejected": -13.72060775756836,
      "step": 2710
    },
    {
      "epoch": 1.1021069692058347,
      "grad_norm": 8.873008482623845e-06,
      "learning_rate": 7.029866426534595e-06,
      "logits/chosen": -0.9603047966957092,
      "logits/rejected": -1.2173120975494385,
      "logps/chosen": -108.4090347290039,
      "logps/rejected": -309.2873840332031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.239743232727051,
      "rewards/margins": 16.840497970581055,
      "rewards/rejected": -13.600753784179688,
      "step": 2720
    },
    {
      "epoch": 1.106158833063209,
      "grad_norm": 9.48353590501938e-06,
      "learning_rate": 7.0148581719945975e-06,
      "logits/chosen": -0.9486533403396606,
      "logits/rejected": -1.2198814153671265,
      "logps/chosen": -109.09212493896484,
      "logps/rejected": -310.3663635253906,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.19498348236084,
      "rewards/margins": 16.885324478149414,
      "rewards/rejected": -13.690340042114258,
      "step": 2730
    },
    {
      "epoch": 1.1102106969205834,
      "grad_norm": 1.4153183656162582e-05,
      "learning_rate": 6.999849917454601e-06,
      "logits/chosen": -0.9536285996437073,
      "logits/rejected": -1.2051887512207031,
      "logps/chosen": -109.12940979003906,
      "logps/rejected": -311.1226806640625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.199876070022583,
      "rewards/margins": 16.80828857421875,
      "rewards/rejected": -13.608412742614746,
      "step": 2740
    },
    {
      "epoch": 1.114262560777958,
      "grad_norm": 8.834592335915659e-06,
      "learning_rate": 6.9848416629146035e-06,
      "logits/chosen": -0.9559447169303894,
      "logits/rejected": -1.213094711303711,
      "logps/chosen": -107.90196228027344,
      "logps/rejected": -310.5301818847656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.203704595565796,
      "rewards/margins": 16.849332809448242,
      "rewards/rejected": -13.645628929138184,
      "step": 2750
    },
    {
      "epoch": 1.114262560777958,
      "eval_logits/chosen": -0.9668858647346497,
      "eval_logits/rejected": -1.2091883420944214,
      "eval_logps/chosen": -109.76824951171875,
      "eval_logps/rejected": -308.951904296875,
      "eval_loss": 2.2648006847703073e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1243679523468018,
      "eval_rewards/margins": 16.725284576416016,
      "eval_rewards/rejected": -13.60091495513916,
      "eval_runtime": 660.0675,
      "eval_samples_per_second": 11.217,
      "eval_steps_per_second": 0.624,
      "step": 2750
    },
    {
      "epoch": 1.1183144246353323,
      "grad_norm": 9.556223631079774e-06,
      "learning_rate": 6.969833408374607e-06,
      "logits/chosen": -0.9617080092430115,
      "logits/rejected": -1.2125369310379028,
      "logps/chosen": -108.94612884521484,
      "logps/rejected": -309.7747497558594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1654481887817383,
      "rewards/margins": 16.90652084350586,
      "rewards/rejected": -13.741071701049805,
      "step": 2760
    },
    {
      "epoch": 1.1223662884927066,
      "grad_norm": 1.3097763257974293e-05,
      "learning_rate": 6.95482515383461e-06,
      "logits/chosen": -0.9512226581573486,
      "logits/rejected": -1.2244887351989746,
      "logps/chosen": -108.65338897705078,
      "logps/rejected": -310.4426574707031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.196953296661377,
      "rewards/margins": 16.88877296447754,
      "rewards/rejected": -13.691821098327637,
      "step": 2770
    },
    {
      "epoch": 1.1264181523500811,
      "grad_norm": 4.03745298171998e-06,
      "learning_rate": 6.939816899294612e-06,
      "logits/chosen": -0.9613186717033386,
      "logits/rejected": -1.2134889364242554,
      "logps/chosen": -108.67975616455078,
      "logps/rejected": -311.4523010253906,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2251639366149902,
      "rewards/margins": 16.996583938598633,
      "rewards/rejected": -13.7714204788208,
      "step": 2780
    },
    {
      "epoch": 1.1304700162074555,
      "grad_norm": 1.7077558368328027e-05,
      "learning_rate": 6.9248086447546155e-06,
      "logits/chosen": -0.9708888530731201,
      "logits/rejected": -1.2140629291534424,
      "logps/chosen": -110.174072265625,
      "logps/rejected": -309.85662841796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1481001377105713,
      "rewards/margins": 16.778745651245117,
      "rewards/rejected": -13.630645751953125,
      "step": 2790
    },
    {
      "epoch": 1.1345218800648298,
      "grad_norm": 8.170194632839411e-06,
      "learning_rate": 6.909800390214618e-06,
      "logits/chosen": -0.9594457149505615,
      "logits/rejected": -1.1950657367706299,
      "logps/chosen": -109.31538391113281,
      "logps/rejected": -309.1491394042969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.198590040206909,
      "rewards/margins": 16.764400482177734,
      "rewards/rejected": -13.565810203552246,
      "step": 2800
    },
    {
      "epoch": 1.1345218800648298,
      "eval_logits/chosen": -0.9978938698768616,
      "eval_logits/rejected": -1.2356668710708618,
      "eval_logps/chosen": -109.79161071777344,
      "eval_logps/rejected": -309.6037292480469,
      "eval_loss": 2.1162637153793185e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1220312118530273,
      "eval_rewards/margins": 16.788131713867188,
      "eval_rewards/rejected": -13.66610050201416,
      "eval_runtime": 661.7278,
      "eval_samples_per_second": 11.189,
      "eval_steps_per_second": 0.623,
      "step": 2800
    },
    {
      "epoch": 1.1385737439222041,
      "grad_norm": 1.399219036102295e-05,
      "learning_rate": 6.8947921356746215e-06,
      "logits/chosen": -0.9749470949172974,
      "logits/rejected": -1.2154656648635864,
      "logps/chosen": -109.16936492919922,
      "logps/rejected": -310.18341064453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1595356464385986,
      "rewards/margins": 16.84941864013672,
      "rewards/rejected": -13.689882278442383,
      "step": 2810
    },
    {
      "epoch": 1.1426256077795787,
      "grad_norm": 2.8051452318322845e-05,
      "learning_rate": 6.879783881134625e-06,
      "logits/chosen": -0.9578418135643005,
      "logits/rejected": -1.2031115293502808,
      "logps/chosen": -109.2094497680664,
      "logps/rejected": -311.97772216796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.177567958831787,
      "rewards/margins": 16.95680046081543,
      "rewards/rejected": -13.779232025146484,
      "step": 2820
    },
    {
      "epoch": 1.146677471636953,
      "grad_norm": 2.0797868273803033e-05,
      "learning_rate": 6.8647756265946275e-06,
      "logits/chosen": -0.9547550678253174,
      "logits/rejected": -1.1948775053024292,
      "logps/chosen": -108.92700958251953,
      "logps/rejected": -310.82525634765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2154428958892822,
      "rewards/margins": 16.930280685424805,
      "rewards/rejected": -13.714838027954102,
      "step": 2830
    },
    {
      "epoch": 1.1507293354943273,
      "grad_norm": 1.0841012226592284e-05,
      "learning_rate": 6.849767372054631e-06,
      "logits/chosen": -0.9574143290519714,
      "logits/rejected": -1.2079883813858032,
      "logps/chosen": -109.1623306274414,
      "logps/rejected": -311.0281677246094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.185199499130249,
      "rewards/margins": 16.914926528930664,
      "rewards/rejected": -13.72972583770752,
      "step": 2840
    },
    {
      "epoch": 1.1547811993517019,
      "grad_norm": 1.7846401533461176e-05,
      "learning_rate": 6.8347591175146335e-06,
      "logits/chosen": -0.9691399931907654,
      "logits/rejected": -1.219811201095581,
      "logps/chosen": -108.34465789794922,
      "logps/rejected": -310.8151550292969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2249460220336914,
      "rewards/margins": 17.007930755615234,
      "rewards/rejected": -13.782984733581543,
      "step": 2850
    },
    {
      "epoch": 1.1547811993517019,
      "eval_logits/chosen": -0.9882211685180664,
      "eval_logits/rejected": -1.2233705520629883,
      "eval_logps/chosen": -109.82547760009766,
      "eval_logps/rejected": -310.0188293457031,
      "eval_loss": 2.0442119819108484e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1186447143554688,
      "eval_rewards/margins": 16.82625389099121,
      "eval_rewards/rejected": -13.707608222961426,
      "eval_runtime": 676.0561,
      "eval_samples_per_second": 10.952,
      "eval_steps_per_second": 0.609,
      "step": 2850
    },
    {
      "epoch": 1.1588330632090762,
      "grad_norm": 3.249685687478632e-05,
      "learning_rate": 6.819750862974637e-06,
      "logits/chosen": -0.9430721402168274,
      "logits/rejected": -1.2220674753189087,
      "logps/chosen": -107.54280090332031,
      "logps/rejected": -311.5782470703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.235630750656128,
      "rewards/margins": 17.058828353881836,
      "rewards/rejected": -13.823197364807129,
      "step": 2860
    },
    {
      "epoch": 1.1628849270664505,
      "grad_norm": 2.4229198970715515e-05,
      "learning_rate": 6.804742608434639e-06,
      "logits/chosen": -0.9694324135780334,
      "logits/rejected": -1.2041586637496948,
      "logps/chosen": -110.08727264404297,
      "logps/rejected": -310.7659912109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.178534984588623,
      "rewards/margins": 16.909570693969727,
      "rewards/rejected": -13.731036186218262,
      "step": 2870
    },
    {
      "epoch": 1.1669367909238249,
      "grad_norm": 4.381558028399013e-05,
      "learning_rate": 6.789734353894643e-06,
      "logits/chosen": -0.9603563547134399,
      "logits/rejected": -1.2125976085662842,
      "logps/chosen": -109.15208435058594,
      "logps/rejected": -309.8031311035156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.184885263442993,
      "rewards/margins": 16.767210006713867,
      "rewards/rejected": -13.582324028015137,
      "step": 2880
    },
    {
      "epoch": 1.1709886547811994,
      "grad_norm": 1.5234799320751335e-05,
      "learning_rate": 6.774726099354646e-06,
      "logits/chosen": -0.9676933884620667,
      "logits/rejected": -1.2179750204086304,
      "logps/chosen": -109.15142059326172,
      "logps/rejected": -310.5060729980469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2184736728668213,
      "rewards/margins": 17.02947425842285,
      "rewards/rejected": -13.81100082397461,
      "step": 2890
    },
    {
      "epoch": 1.1750405186385737,
      "grad_norm": 9.855923963186797e-06,
      "learning_rate": 6.759717844814648e-06,
      "logits/chosen": -0.9541704654693604,
      "logits/rejected": -1.201403021812439,
      "logps/chosen": -109.80725860595703,
      "logps/rejected": -310.9417724609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.17207932472229,
      "rewards/margins": 16.8778018951416,
      "rewards/rejected": -13.705724716186523,
      "step": 2900
    },
    {
      "epoch": 1.1750405186385737,
      "eval_logits/chosen": -0.9700528979301453,
      "eval_logits/rejected": -1.2112122774124146,
      "eval_logps/chosen": -109.64909362792969,
      "eval_logps/rejected": -310.08612060546875,
      "eval_loss": 2.0142599055361643e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1362838745117188,
      "eval_rewards/margins": 16.850624084472656,
      "eval_rewards/rejected": -13.714340209960938,
      "eval_runtime": 689.3816,
      "eval_samples_per_second": 10.74,
      "eval_steps_per_second": 0.598,
      "step": 2900
    },
    {
      "epoch": 1.179092382495948,
      "grad_norm": 3.834591552731581e-05,
      "learning_rate": 6.7447095902746515e-06,
      "logits/chosen": -0.9695179462432861,
      "logits/rejected": -1.1922355890274048,
      "logps/chosen": -109.74037170410156,
      "logps/rejected": -311.2341003417969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.206512689590454,
      "rewards/margins": 16.96829605102539,
      "rewards/rejected": -13.761781692504883,
      "step": 2910
    },
    {
      "epoch": 1.1831442463533226,
      "grad_norm": 3.295067654107697e-05,
      "learning_rate": 6.729701335734654e-06,
      "logits/chosen": -0.9576932191848755,
      "logits/rejected": -1.2104113101959229,
      "logps/chosen": -107.79331970214844,
      "logps/rejected": -310.0033874511719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.241929531097412,
      "rewards/margins": 16.90349578857422,
      "rewards/rejected": -13.661564826965332,
      "step": 2920
    },
    {
      "epoch": 1.187196110210697,
      "grad_norm": 3.6701349017675966e-05,
      "learning_rate": 6.7146930811946575e-06,
      "logits/chosen": -0.9661968350410461,
      "logits/rejected": -1.218201994895935,
      "logps/chosen": -108.45980072021484,
      "logps/rejected": -311.50604248046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2053275108337402,
      "rewards/margins": 17.048315048217773,
      "rewards/rejected": -13.842986106872559,
      "step": 2930
    },
    {
      "epoch": 1.1912479740680713,
      "grad_norm": 1.0536626177781727e-05,
      "learning_rate": 6.699684826654661e-06,
      "logits/chosen": -0.9455077648162842,
      "logits/rejected": -1.207572340965271,
      "logps/chosen": -108.63707733154297,
      "logps/rejected": -312.3814392089844,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2204668521881104,
      "rewards/margins": 17.038225173950195,
      "rewards/rejected": -13.817757606506348,
      "step": 2940
    },
    {
      "epoch": 1.1952998379254458,
      "grad_norm": 1.333241834799992e-05,
      "learning_rate": 6.6846765721146636e-06,
      "logits/chosen": -0.9706599116325378,
      "logits/rejected": -1.2194017171859741,
      "logps/chosen": -109.55686950683594,
      "logps/rejected": -310.6762390136719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1767683029174805,
      "rewards/margins": 16.9427433013916,
      "rewards/rejected": -13.765974998474121,
      "step": 2950
    },
    {
      "epoch": 1.1952998379254458,
      "eval_logits/chosen": -0.9654695987701416,
      "eval_logits/rejected": -1.2049760818481445,
      "eval_logps/chosen": -109.65506744384766,
      "eval_logps/rejected": -310.4198303222656,
      "eval_loss": 1.9477491264296987e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1356868743896484,
      "eval_rewards/margins": 16.883392333984375,
      "eval_rewards/rejected": -13.747706413269043,
      "eval_runtime": 657.3556,
      "eval_samples_per_second": 11.263,
      "eval_steps_per_second": 0.627,
      "step": 2950
    },
    {
      "epoch": 1.1993517017828201,
      "grad_norm": 1.7855747501016594e-05,
      "learning_rate": 6.669668317574667e-06,
      "logits/chosen": -0.978232204914093,
      "logits/rejected": -1.2113639116287231,
      "logps/chosen": -108.95658111572266,
      "logps/rejected": -311.7427978515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1936254501342773,
      "rewards/margins": 17.014652252197266,
      "rewards/rejected": -13.821026802062988,
      "step": 2960
    },
    {
      "epoch": 1.2034035656401945,
      "grad_norm": 7.74248837842606e-06,
      "learning_rate": 6.6546600630346696e-06,
      "logits/chosen": -0.9706756472587585,
      "logits/rejected": -1.2182252407073975,
      "logps/chosen": -109.6180648803711,
      "logps/rejected": -311.6573486328125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.184460163116455,
      "rewards/margins": 17.020050048828125,
      "rewards/rejected": -13.835589408874512,
      "step": 2970
    },
    {
      "epoch": 1.2074554294975688,
      "grad_norm": 2.9659451683983207e-05,
      "learning_rate": 6.639651808494673e-06,
      "logits/chosen": -0.9520205855369568,
      "logits/rejected": -1.2165043354034424,
      "logps/chosen": -108.52173614501953,
      "logps/rejected": -312.7424621582031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.274117946624756,
      "rewards/margins": 17.17621612548828,
      "rewards/rejected": -13.902097702026367,
      "step": 2980
    },
    {
      "epoch": 1.2115072933549433,
      "grad_norm": 3.948523590224795e-05,
      "learning_rate": 6.624643553954676e-06,
      "logits/chosen": -0.9564474821090698,
      "logits/rejected": -1.2092334032058716,
      "logps/chosen": -106.98612976074219,
      "logps/rejected": -312.2510070800781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.25746488571167,
      "rewards/margins": 17.082439422607422,
      "rewards/rejected": -13.82497501373291,
      "step": 2990
    },
    {
      "epoch": 1.2155591572123177,
      "grad_norm": 1.3657989256898873e-05,
      "learning_rate": 6.609635299414678e-06,
      "logits/chosen": -0.9624108672142029,
      "logits/rejected": -1.1985855102539062,
      "logps/chosen": -109.48686218261719,
      "logps/rejected": -310.64849853515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1811981201171875,
      "rewards/margins": 16.845518112182617,
      "rewards/rejected": -13.664319038391113,
      "step": 3000
    },
    {
      "epoch": 1.2155591572123177,
      "eval_logits/chosen": -0.9728745222091675,
      "eval_logits/rejected": -1.211281418800354,
      "eval_logps/chosen": -109.64574432373047,
      "eval_logps/rejected": -310.9217529296875,
      "eval_loss": 1.861751570686465e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.136619806289673,
      "eval_rewards/margins": 16.934520721435547,
      "eval_rewards/rejected": -13.797901153564453,
      "eval_runtime": 668.7462,
      "eval_samples_per_second": 11.071,
      "eval_steps_per_second": 0.616,
      "step": 3000
    },
    {
      "epoch": 1.219611021069692,
      "grad_norm": 6.40754951746203e-05,
      "learning_rate": 6.5946270448746816e-06,
      "logits/chosen": -0.9686071276664734,
      "logits/rejected": -1.2071101665496826,
      "logps/chosen": -110.0566177368164,
      "logps/rejected": -312.8505554199219,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1576898097991943,
      "rewards/margins": 17.086423873901367,
      "rewards/rejected": -13.928732872009277,
      "step": 3010
    },
    {
      "epoch": 1.2236628849270665,
      "grad_norm": 2.9892496968386695e-05,
      "learning_rate": 6.579618790334684e-06,
      "logits/chosen": -0.9622131586074829,
      "logits/rejected": -1.193557858467102,
      "logps/chosen": -109.8584976196289,
      "logps/rejected": -311.5018310546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1661813259124756,
      "rewards/margins": 16.959264755249023,
      "rewards/rejected": -13.793082237243652,
      "step": 3020
    },
    {
      "epoch": 1.2277147487844409,
      "grad_norm": 3.853169982903637e-05,
      "learning_rate": 6.564610535794688e-06,
      "logits/chosen": -0.9711416959762573,
      "logits/rejected": -1.2121407985687256,
      "logps/chosen": -110.18315124511719,
      "logps/rejected": -310.7025451660156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1713101863861084,
      "rewards/margins": 16.95589256286621,
      "rewards/rejected": -13.784581184387207,
      "step": 3030
    },
    {
      "epoch": 1.2317666126418152,
      "grad_norm": 2.0132029021624476e-05,
      "learning_rate": 6.549602281254691e-06,
      "logits/chosen": -0.9588344097137451,
      "logits/rejected": -1.2041640281677246,
      "logps/chosen": -108.35846710205078,
      "logps/rejected": -312.06689453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2102229595184326,
      "rewards/margins": 17.149709701538086,
      "rewards/rejected": -13.939485549926758,
      "step": 3040
    },
    {
      "epoch": 1.2358184764991895,
      "grad_norm": 7.818654921720736e-06,
      "learning_rate": 6.534594026714694e-06,
      "logits/chosen": -0.9599415063858032,
      "logits/rejected": -1.2277412414550781,
      "logps/chosen": -108.01351928710938,
      "logps/rejected": -313.9948425292969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.244805097579956,
      "rewards/margins": 17.302661895751953,
      "rewards/rejected": -14.057855606079102,
      "step": 3050
    },
    {
      "epoch": 1.2358184764991895,
      "eval_logits/chosen": -0.9753085374832153,
      "eval_logits/rejected": -1.2139606475830078,
      "eval_logps/chosen": -109.60204315185547,
      "eval_logps/rejected": -311.3223571777344,
      "eval_loss": 1.7824810072397668e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.140988826751709,
      "eval_rewards/margins": 16.97894859313965,
      "eval_rewards/rejected": -13.837959289550781,
      "eval_runtime": 683.1407,
      "eval_samples_per_second": 10.838,
      "eval_steps_per_second": 0.603,
      "step": 3050
    },
    {
      "epoch": 1.239870340356564,
      "grad_norm": 7.812601324985735e-06,
      "learning_rate": 6.519585772174697e-06,
      "logits/chosen": -0.9611976146697998,
      "logits/rejected": -1.2113627195358276,
      "logps/chosen": -108.8201675415039,
      "logps/rejected": -312.1944274902344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2425155639648438,
      "rewards/margins": 17.1594295501709,
      "rewards/rejected": -13.916913032531738,
      "step": 3060
    },
    {
      "epoch": 1.2439222042139384,
      "grad_norm": 7.239479145937366e-06,
      "learning_rate": 6.5045775176347e-06,
      "logits/chosen": -0.9635852575302124,
      "logits/rejected": -1.214555025100708,
      "logps/chosen": -108.35509490966797,
      "logps/rejected": -312.2405090332031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.231051206588745,
      "rewards/margins": 17.1818790435791,
      "rewards/rejected": -13.950826644897461,
      "step": 3070
    },
    {
      "epoch": 1.2479740680713127,
      "grad_norm": 2.3521504772361368e-05,
      "learning_rate": 6.489569263094703e-06,
      "logits/chosen": -0.9610281586647034,
      "logits/rejected": -1.218076229095459,
      "logps/chosen": -109.09011840820312,
      "logps/rejected": -311.3946533203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1831178665161133,
      "rewards/margins": 17.014116287231445,
      "rewards/rejected": -13.830997467041016,
      "step": 3080
    },
    {
      "epoch": 1.2520259319286873,
      "grad_norm": 9.943691111402586e-06,
      "learning_rate": 6.4745610085547065e-06,
      "logits/chosen": -0.9672773480415344,
      "logits/rejected": -1.20703125,
      "logps/chosen": -108.8431625366211,
      "logps/rejected": -313.36944580078125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2411019802093506,
      "rewards/margins": 17.166112899780273,
      "rewards/rejected": -13.925012588500977,
      "step": 3090
    },
    {
      "epoch": 1.2560777957860616,
      "grad_norm": 9.48311026149895e-06,
      "learning_rate": 6.459552754014708e-06,
      "logits/chosen": -0.9685242772102356,
      "logits/rejected": -1.2248913049697876,
      "logps/chosen": -109.63764190673828,
      "logps/rejected": -314.3175964355469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.185171604156494,
      "rewards/margins": 17.155221939086914,
      "rewards/rejected": -13.970049858093262,
      "step": 3100
    },
    {
      "epoch": 1.2560777957860616,
      "eval_logits/chosen": -0.9879578351974487,
      "eval_logits/rejected": -1.2205256223678589,
      "eval_logps/chosen": -109.65383911132812,
      "eval_logps/rejected": -311.7254333496094,
      "eval_loss": 1.7127376850112341e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1358084678649902,
      "eval_rewards/margins": 17.014074325561523,
      "eval_rewards/rejected": -13.878265380859375,
      "eval_runtime": 670.5917,
      "eval_samples_per_second": 11.041,
      "eval_steps_per_second": 0.614,
      "step": 3100
    },
    {
      "epoch": 1.260129659643436,
      "grad_norm": 5.6280673561559524e-06,
      "learning_rate": 6.444544499474712e-06,
      "logits/chosen": -0.9702467322349548,
      "logits/rejected": -1.2132569551467896,
      "logps/chosen": -110.29287719726562,
      "logps/rejected": -313.4588623046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.165759563446045,
      "rewards/margins": 17.14694595336914,
      "rewards/rejected": -13.98118782043457,
      "step": 3110
    },
    {
      "epoch": 1.2641815235008105,
      "grad_norm": 2.7766443963628262e-05,
      "learning_rate": 6.429536244934714e-06,
      "logits/chosen": -0.9578903317451477,
      "logits/rejected": -1.2012211084365845,
      "logps/chosen": -109.4847183227539,
      "logps/rejected": -311.3115234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1903371810913086,
      "rewards/margins": 17.070411682128906,
      "rewards/rejected": -13.88007640838623,
      "step": 3120
    },
    {
      "epoch": 1.2682333873581848,
      "grad_norm": 8.8372298705508e-06,
      "learning_rate": 6.414527990394718e-06,
      "logits/chosen": -0.9595220685005188,
      "logits/rejected": -1.2112177610397339,
      "logps/chosen": -108.19755554199219,
      "logps/rejected": -312.2407531738281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.194171190261841,
      "rewards/margins": 17.092443466186523,
      "rewards/rejected": -13.898271560668945,
      "step": 3130
    },
    {
      "epoch": 1.2722852512155591,
      "grad_norm": 5.094006610306678e-06,
      "learning_rate": 6.39951973585472e-06,
      "logits/chosen": -0.9771420955657959,
      "logits/rejected": -1.2134661674499512,
      "logps/chosen": -109.30355834960938,
      "logps/rejected": -312.1795959472656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2299623489379883,
      "rewards/margins": 17.149782180786133,
      "rewards/rejected": -13.919818878173828,
      "step": 3140
    },
    {
      "epoch": 1.2763371150729337,
      "grad_norm": 3.5263063182355836e-05,
      "learning_rate": 6.384511481314724e-06,
      "logits/chosen": -0.9556463956832886,
      "logits/rejected": -1.201846957206726,
      "logps/chosen": -109.28461456298828,
      "logps/rejected": -311.3385925292969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1858484745025635,
      "rewards/margins": 16.945899963378906,
      "rewards/rejected": -13.760053634643555,
      "step": 3150
    },
    {
      "epoch": 1.2763371150729337,
      "eval_logits/chosen": -0.9694511890411377,
      "eval_logits/rejected": -1.2067033052444458,
      "eval_logps/chosen": -109.51664733886719,
      "eval_logps/rejected": -311.8854064941406,
      "eval_loss": 1.675977472359591e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1495275497436523,
      "eval_rewards/margins": 17.043792724609375,
      "eval_rewards/rejected": -13.894266128540039,
      "eval_runtime": 667.8441,
      "eval_samples_per_second": 11.086,
      "eval_steps_per_second": 0.617,
      "step": 3150
    },
    {
      "epoch": 1.280388978930308,
      "grad_norm": 8.296900887216907e-06,
      "learning_rate": 6.369503226774727e-06,
      "logits/chosen": -0.9759266376495361,
      "logits/rejected": -1.2088817358016968,
      "logps/chosen": -108.7704086303711,
      "logps/rejected": -313.3003845214844,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2279977798461914,
      "rewards/margins": 17.15936279296875,
      "rewards/rejected": -13.931365966796875,
      "step": 3160
    },
    {
      "epoch": 1.2844408427876823,
      "grad_norm": 2.008057890634518e-05,
      "learning_rate": 6.35449497223473e-06,
      "logits/chosen": -0.9693723320960999,
      "logits/rejected": -1.2205625772476196,
      "logps/chosen": -108.16254425048828,
      "logps/rejected": -312.3143005371094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.24662446975708,
      "rewards/margins": 17.193058013916016,
      "rewards/rejected": -13.94643497467041,
      "step": 3170
    },
    {
      "epoch": 1.2884927066450567,
      "grad_norm": 7.960902621562127e-06,
      "learning_rate": 6.339486717694733e-06,
      "logits/chosen": -0.9644364714622498,
      "logits/rejected": -1.2073167562484741,
      "logps/chosen": -109.11122131347656,
      "logps/rejected": -314.1689453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.24955153465271,
      "rewards/margins": 17.34427833557129,
      "rewards/rejected": -14.09472942352295,
      "step": 3180
    },
    {
      "epoch": 1.292544570502431,
      "grad_norm": 1.7171289073303342e-05,
      "learning_rate": 6.324478463154735e-06,
      "logits/chosen": -0.9464921355247498,
      "logits/rejected": -1.191055417060852,
      "logps/chosen": -109.27963256835938,
      "logps/rejected": -312.8675842285156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2031736373901367,
      "rewards/margins": 17.16599464416504,
      "rewards/rejected": -13.962823867797852,
      "step": 3190
    },
    {
      "epoch": 1.2965964343598055,
      "grad_norm": 1.7018590369843878e-05,
      "learning_rate": 6.309470208614738e-06,
      "logits/chosen": -0.9641759991645813,
      "logits/rejected": -1.2251524925231934,
      "logps/chosen": -106.86421203613281,
      "logps/rejected": -313.9484558105469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2740588188171387,
      "rewards/margins": 17.36844825744629,
      "rewards/rejected": -14.094389915466309,
      "step": 3200
    },
    {
      "epoch": 1.2965964343598055,
      "eval_logits/chosen": -0.9669303297996521,
      "eval_logits/rejected": -1.2042489051818848,
      "eval_logps/chosen": -109.52879333496094,
      "eval_logps/rejected": -312.3120422363281,
      "eval_loss": 1.6180324280412606e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1483137607574463,
      "eval_rewards/margins": 17.085241317749023,
      "eval_rewards/rejected": -13.936927795410156,
      "eval_runtime": 681.2481,
      "eval_samples_per_second": 10.868,
      "eval_steps_per_second": 0.605,
      "step": 3200
    },
    {
      "epoch": 1.3006482982171799,
      "grad_norm": 9.484376278123818e-06,
      "learning_rate": 6.294461954074742e-06,
      "logits/chosen": -0.9570868611335754,
      "logits/rejected": -1.188042402267456,
      "logps/chosen": -108.84678649902344,
      "logps/rejected": -312.2694396972656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.214073896408081,
      "rewards/margins": 17.14763832092285,
      "rewards/rejected": -13.933562278747559,
      "step": 3210
    },
    {
      "epoch": 1.3047001620745542,
      "grad_norm": 4.871696091868216e-06,
      "learning_rate": 6.279453699534744e-06,
      "logits/chosen": -0.9526418447494507,
      "logits/rejected": -1.220396637916565,
      "logps/chosen": -108.03825378417969,
      "logps/rejected": -314.1305236816406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2451412677764893,
      "rewards/margins": 17.401790618896484,
      "rewards/rejected": -14.156647682189941,
      "step": 3220
    },
    {
      "epoch": 1.3087520259319287,
      "grad_norm": 6.676681550743524e-06,
      "learning_rate": 6.264445444994748e-06,
      "logits/chosen": -0.9678587317466736,
      "logits/rejected": -1.2017152309417725,
      "logps/chosen": -109.80901336669922,
      "logps/rejected": -313.3891906738281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1907846927642822,
      "rewards/margins": 17.13675880432129,
      "rewards/rejected": -13.945975303649902,
      "step": 3230
    },
    {
      "epoch": 1.312803889789303,
      "grad_norm": 1.5305944543797523e-05,
      "learning_rate": 6.24943719045475e-06,
      "logits/chosen": -0.9755159020423889,
      "logits/rejected": -1.1885570287704468,
      "logps/chosen": -110.71768188476562,
      "logps/rejected": -313.28118896484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.169814109802246,
      "rewards/margins": 17.160905838012695,
      "rewards/rejected": -13.991090774536133,
      "step": 3240
    },
    {
      "epoch": 1.3168557536466774,
      "grad_norm": 8.812554369796999e-06,
      "learning_rate": 6.234428935914754e-06,
      "logits/chosen": -0.9581737518310547,
      "logits/rejected": -1.1885933876037598,
      "logps/chosen": -109.94810485839844,
      "logps/rejected": -313.65667724609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.169693946838379,
      "rewards/margins": 17.15659523010254,
      "rewards/rejected": -13.98690128326416,
      "step": 3250
    },
    {
      "epoch": 1.3168557536466774,
      "eval_logits/chosen": -0.9879531264305115,
      "eval_logits/rejected": -1.217955470085144,
      "eval_logps/chosen": -109.622802734375,
      "eval_logps/rejected": -313.0201721191406,
      "eval_loss": 1.512936904646267e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.138913154602051,
      "eval_rewards/margins": 17.146656036376953,
      "eval_rewards/rejected": -14.007742881774902,
      "eval_runtime": 662.6363,
      "eval_samples_per_second": 11.174,
      "eval_steps_per_second": 0.622,
      "step": 3250
    },
    {
      "epoch": 1.320907617504052,
      "grad_norm": 1.7330803530057892e-05,
      "learning_rate": 6.219420681374757e-06,
      "logits/chosen": -0.9541265368461609,
      "logits/rejected": -1.212295413017273,
      "logps/chosen": -108.89168548583984,
      "logps/rejected": -313.27557373046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2004942893981934,
      "rewards/margins": 17.252246856689453,
      "rewards/rejected": -14.051752090454102,
      "step": 3260
    },
    {
      "epoch": 1.3249594813614263,
      "grad_norm": 1.0267608558933716e-05,
      "learning_rate": 6.20441242683476e-06,
      "logits/chosen": -0.959732174873352,
      "logits/rejected": -1.187722086906433,
      "logps/chosen": -108.71839141845703,
      "logps/rejected": -313.2086486816406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.211418628692627,
      "rewards/margins": 17.164676666259766,
      "rewards/rejected": -13.953258514404297,
      "step": 3270
    },
    {
      "epoch": 1.3290113452188006,
      "grad_norm": 1.5800977053004317e-05,
      "learning_rate": 6.189404172294763e-06,
      "logits/chosen": -0.9553423523902893,
      "logits/rejected": -1.1899616718292236,
      "logps/chosen": -109.203369140625,
      "logps/rejected": -314.1956787109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.207944631576538,
      "rewards/margins": 17.303342819213867,
      "rewards/rejected": -14.095399856567383,
      "step": 3280
    },
    {
      "epoch": 1.3330632090761751,
      "grad_norm": 1.0804586963786278e-05,
      "learning_rate": 6.174395917754765e-06,
      "logits/chosen": -0.9881101250648499,
      "logits/rejected": -1.1937135457992554,
      "logps/chosen": -109.8092269897461,
      "logps/rejected": -314.1241149902344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.205904960632324,
      "rewards/margins": 17.211772918701172,
      "rewards/rejected": -14.005867958068848,
      "step": 3290
    },
    {
      "epoch": 1.3371150729335495,
      "grad_norm": 1.2788170351996087e-05,
      "learning_rate": 6.159387663214768e-06,
      "logits/chosen": -0.9444805383682251,
      "logits/rejected": -1.2099872827529907,
      "logps/chosen": -108.5473403930664,
      "logps/rejected": -314.2455749511719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.225715160369873,
      "rewards/margins": 17.341238021850586,
      "rewards/rejected": -14.115520477294922,
      "step": 3300
    },
    {
      "epoch": 1.3371150729335495,
      "eval_logits/chosen": -0.9682581424713135,
      "eval_logits/rejected": -1.2041715383529663,
      "eval_logps/chosen": -109.43754577636719,
      "eval_logps/rejected": -313.0159606933594,
      "eval_loss": 1.4919004343028064e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.157438039779663,
      "eval_rewards/margins": 17.164758682250977,
      "eval_rewards/rejected": -14.007320404052734,
      "eval_runtime": 660.6432,
      "eval_samples_per_second": 11.207,
      "eval_steps_per_second": 0.624,
      "step": 3300
    },
    {
      "epoch": 1.3411669367909238,
      "grad_norm": 7.93843537394423e-06,
      "learning_rate": 6.1443794086747725e-06,
      "logits/chosen": -0.9528685808181763,
      "logits/rejected": -1.20863938331604,
      "logps/chosen": -108.23860931396484,
      "logps/rejected": -312.99127197265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.248526096343994,
      "rewards/margins": 17.247163772583008,
      "rewards/rejected": -13.998639106750488,
      "step": 3310
    },
    {
      "epoch": 1.3452188006482984,
      "grad_norm": 8.834734217089135e-06,
      "learning_rate": 6.129371154134774e-06,
      "logits/chosen": -0.9572389125823975,
      "logits/rejected": -1.2056647539138794,
      "logps/chosen": -108.65901947021484,
      "logps/rejected": -314.50128173828125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1915442943573,
      "rewards/margins": 17.344852447509766,
      "rewards/rejected": -14.153307914733887,
      "step": 3320
    },
    {
      "epoch": 1.3492706645056727,
      "grad_norm": 3.8810932892374694e-05,
      "learning_rate": 6.114362899594778e-06,
      "logits/chosen": -0.9432046413421631,
      "logits/rejected": -1.2089511156082153,
      "logps/chosen": -108.8655776977539,
      "logps/rejected": -315.0828857421875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2165868282318115,
      "rewards/margins": 17.300975799560547,
      "rewards/rejected": -14.084390640258789,
      "step": 3330
    },
    {
      "epoch": 1.353322528363047,
      "grad_norm": 4.887206614512252e-06,
      "learning_rate": 6.09935464505478e-06,
      "logits/chosen": -0.9624608159065247,
      "logits/rejected": -1.2240010499954224,
      "logps/chosen": -108.18408966064453,
      "logps/rejected": -315.2415466308594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.214587688446045,
      "rewards/margins": 17.451108932495117,
      "rewards/rejected": -14.236522674560547,
      "step": 3340
    },
    {
      "epoch": 1.3573743922204213,
      "grad_norm": 5.749606771132676e-06,
      "learning_rate": 6.084346390514784e-06,
      "logits/chosen": -0.9573169946670532,
      "logits/rejected": -1.190148115158081,
      "logps/chosen": -109.55158233642578,
      "logps/rejected": -312.7564392089844,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2026069164276123,
      "rewards/margins": 17.133691787719727,
      "rewards/rejected": -13.931085586547852,
      "step": 3350
    },
    {
      "epoch": 1.3573743922204213,
      "eval_logits/chosen": -0.970223605632782,
      "eval_logits/rejected": -1.2049980163574219,
      "eval_logps/chosen": -109.43147277832031,
      "eval_logps/rejected": -313.37371826171875,
      "eval_loss": 1.445255435328363e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1580469608306885,
      "eval_rewards/margins": 17.201139450073242,
      "eval_rewards/rejected": -14.043091773986816,
      "eval_runtime": 660.5803,
      "eval_samples_per_second": 11.208,
      "eval_steps_per_second": 0.624,
      "step": 3350
    },
    {
      "epoch": 1.3614262560777957,
      "grad_norm": 6.225182005437091e-06,
      "learning_rate": 6.069338135974787e-06,
      "logits/chosen": -0.9479268193244934,
      "logits/rejected": -1.1831086874008179,
      "logps/chosen": -108.24063873291016,
      "logps/rejected": -314.0235900878906,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.241534471511841,
      "rewards/margins": 17.33696174621582,
      "rewards/rejected": -14.095426559448242,
      "step": 3360
    },
    {
      "epoch": 1.3654781199351702,
      "grad_norm": 3.8037185277062235e-06,
      "learning_rate": 6.05432988143479e-06,
      "logits/chosen": -0.9577153325080872,
      "logits/rejected": -1.196504831314087,
      "logps/chosen": -109.3306655883789,
      "logps/rejected": -315.1477966308594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2319626808166504,
      "rewards/margins": 17.362394332885742,
      "rewards/rejected": -14.130433082580566,
      "step": 3370
    },
    {
      "epoch": 1.3695299837925445,
      "grad_norm": 7.264603027579142e-06,
      "learning_rate": 6.039321626894793e-06,
      "logits/chosen": -0.960659384727478,
      "logits/rejected": -1.21016526222229,
      "logps/chosen": -109.74238586425781,
      "logps/rejected": -314.30889892578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1629416942596436,
      "rewards/margins": 17.332054138183594,
      "rewards/rejected": -14.169112205505371,
      "step": 3380
    },
    {
      "epoch": 1.3735818476499189,
      "grad_norm": 5.798762686026748e-06,
      "learning_rate": 6.024313372354796e-06,
      "logits/chosen": -0.9659271240234375,
      "logits/rejected": -1.2181833982467651,
      "logps/chosen": -108.92655944824219,
      "logps/rejected": -314.48583984375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2449417114257812,
      "rewards/margins": 17.421527862548828,
      "rewards/rejected": -14.17658519744873,
      "step": 3390
    },
    {
      "epoch": 1.3776337115072934,
      "grad_norm": 9.220641004503705e-06,
      "learning_rate": 6.009305117814799e-06,
      "logits/chosen": -0.9584363102912903,
      "logits/rejected": -1.1954525709152222,
      "logps/chosen": -108.27457427978516,
      "logps/rejected": -315.4519348144531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2615156173706055,
      "rewards/margins": 17.52600860595703,
      "rewards/rejected": -14.264492988586426,
      "step": 3400
    },
    {
      "epoch": 1.3776337115072934,
      "eval_logits/chosen": -0.9918527007102966,
      "eval_logits/rejected": -1.220714807510376,
      "eval_logps/chosen": -109.54891204833984,
      "eval_logps/rejected": -314.1236877441406,
      "eval_loss": 1.3570124224315805e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.146301746368408,
      "eval_rewards/margins": 17.26439094543457,
      "eval_rewards/rejected": -14.118091583251953,
      "eval_runtime": 662.5143,
      "eval_samples_per_second": 11.176,
      "eval_steps_per_second": 0.622,
      "step": 3400
    },
    {
      "epoch": 1.3816855753646677,
      "grad_norm": 6.401640348485671e-06,
      "learning_rate": 5.994296863274801e-06,
      "logits/chosen": -0.9543588757514954,
      "logits/rejected": -1.2096937894821167,
      "logps/chosen": -108.40690612792969,
      "logps/rejected": -316.5176086425781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.250415563583374,
      "rewards/margins": 17.588228225708008,
      "rewards/rejected": -14.337810516357422,
      "step": 3410
    },
    {
      "epoch": 1.385737439222042,
      "grad_norm": 7.610854027007008e-06,
      "learning_rate": 5.979288608734804e-06,
      "logits/chosen": -0.9576588273048401,
      "logits/rejected": -1.1955324411392212,
      "logps/chosen": -109.1751708984375,
      "logps/rejected": -314.9342346191406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2158362865448,
      "rewards/margins": 17.415836334228516,
      "rewards/rejected": -14.199999809265137,
      "step": 3420
    },
    {
      "epoch": 1.3897893030794166,
      "grad_norm": 4.906746198685141e-06,
      "learning_rate": 5.964280354194808e-06,
      "logits/chosen": -0.9672905206680298,
      "logits/rejected": -1.1918128728866577,
      "logps/chosen": -109.35330963134766,
      "logps/rejected": -315.4357604980469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1970880031585693,
      "rewards/margins": 17.4501895904541,
      "rewards/rejected": -14.253100395202637,
      "step": 3430
    },
    {
      "epoch": 1.393841166936791,
      "grad_norm": 1.7268190276809037e-05,
      "learning_rate": 5.94927209965481e-06,
      "logits/chosen": -0.9625003933906555,
      "logits/rejected": -1.2034894227981567,
      "logps/chosen": -108.31163787841797,
      "logps/rejected": -316.0549621582031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.240849733352661,
      "rewards/margins": 17.41853904724121,
      "rewards/rejected": -14.177687644958496,
      "step": 3440
    },
    {
      "epoch": 1.3978930307941653,
      "grad_norm": 7.347574864979833e-06,
      "learning_rate": 5.934263845114814e-06,
      "logits/chosen": -0.9486963152885437,
      "logits/rejected": -1.216882348060608,
      "logps/chosen": -108.84146881103516,
      "logps/rejected": -315.2445068359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2250006198883057,
      "rewards/margins": 17.36940574645996,
      "rewards/rejected": -14.144404411315918,
      "step": 3450
    },
    {
      "epoch": 1.3978930307941653,
      "eval_logits/chosen": -0.9700472950935364,
      "eval_logits/rejected": -1.2043304443359375,
      "eval_logps/chosen": -109.39541625976562,
      "eval_logps/rejected": -314.0571594238281,
      "eval_loss": 1.3514242880319216e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.161651849746704,
      "eval_rewards/margins": 17.273088455200195,
      "eval_rewards/rejected": -14.111435890197754,
      "eval_runtime": 663.1123,
      "eval_samples_per_second": 11.166,
      "eval_steps_per_second": 0.621,
      "step": 3450
    },
    {
      "epoch": 1.4019448946515398,
      "grad_norm": 8.939105100580491e-06,
      "learning_rate": 5.919255590574816e-06,
      "logits/chosen": -0.9727906584739685,
      "logits/rejected": -1.19076406955719,
      "logps/chosen": -109.75849914550781,
      "logps/rejected": -313.2750549316406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1896746158599854,
      "rewards/margins": 17.198753356933594,
      "rewards/rejected": -14.009076118469238,
      "step": 3460
    },
    {
      "epoch": 1.4059967585089141,
      "grad_norm": 8.345450623892248e-06,
      "learning_rate": 5.90424733603482e-06,
      "logits/chosen": -0.9543637633323669,
      "logits/rejected": -1.2029889822006226,
      "logps/chosen": -108.40011596679688,
      "logps/rejected": -314.6285095214844,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2274868488311768,
      "rewards/margins": 17.273284912109375,
      "rewards/rejected": -14.045796394348145,
      "step": 3470
    },
    {
      "epoch": 1.4100486223662885,
      "grad_norm": 4.251101927366108e-06,
      "learning_rate": 5.889239081494823e-06,
      "logits/chosen": -0.9565185308456421,
      "logits/rejected": -1.1892540454864502,
      "logps/chosen": -108.2246322631836,
      "logps/rejected": -315.8067932128906,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2416796684265137,
      "rewards/margins": 17.44449806213379,
      "rewards/rejected": -14.20281982421875,
      "step": 3480
    },
    {
      "epoch": 1.414100486223663,
      "grad_norm": 6.270918674999848e-06,
      "learning_rate": 5.874230826954826e-06,
      "logits/chosen": -0.9648173451423645,
      "logits/rejected": -1.2057849168777466,
      "logps/chosen": -108.76558685302734,
      "logps/rejected": -315.7758483886719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2343640327453613,
      "rewards/margins": 17.45017433166504,
      "rewards/rejected": -14.21580982208252,
      "step": 3490
    },
    {
      "epoch": 1.4181523500810373,
      "grad_norm": 6.242263225431088e-06,
      "learning_rate": 5.859222572414829e-06,
      "logits/chosen": -0.9528531432151794,
      "logits/rejected": -1.2007869482040405,
      "logps/chosen": -108.47574615478516,
      "logps/rejected": -317.7001953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.273150682449341,
      "rewards/margins": 17.647972106933594,
      "rewards/rejected": -14.3748197555542,
      "step": 3500
    },
    {
      "epoch": 1.4181523500810373,
      "eval_logits/chosen": -0.9920565485954285,
      "eval_logits/rejected": -1.2199008464813232,
      "eval_logps/chosen": -109.49647521972656,
      "eval_logps/rejected": -314.68231201171875,
      "eval_loss": 1.28018996292667e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1515462398529053,
      "eval_rewards/margins": 17.325502395629883,
      "eval_rewards/rejected": -14.173955917358398,
      "eval_runtime": 662.9428,
      "eval_samples_per_second": 11.168,
      "eval_steps_per_second": 0.621,
      "step": 3500
    },
    {
      "epoch": 1.4222042139384117,
      "grad_norm": 8.54422523843823e-06,
      "learning_rate": 5.844214317874831e-06,
      "logits/chosen": -0.9673510789871216,
      "logits/rejected": -1.2166383266448975,
      "logps/chosen": -108.13471984863281,
      "logps/rejected": -315.3136901855469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2391507625579834,
      "rewards/margins": 17.3529052734375,
      "rewards/rejected": -14.113754272460938,
      "step": 3510
    },
    {
      "epoch": 1.426256077795786,
      "grad_norm": 9.1289157353458e-06,
      "learning_rate": 5.829206063334834e-06,
      "logits/chosen": -0.9765715003013611,
      "logits/rejected": -1.208653211593628,
      "logps/chosen": -109.50243377685547,
      "logps/rejected": -316.7020568847656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2334234714508057,
      "rewards/margins": 17.64775848388672,
      "rewards/rejected": -14.414335250854492,
      "step": 3520
    },
    {
      "epoch": 1.4303079416531603,
      "grad_norm": 1.0726081200118642e-05,
      "learning_rate": 5.814197808794838e-06,
      "logits/chosen": -0.9649112224578857,
      "logits/rejected": -1.216942310333252,
      "logps/chosen": -109.17703247070312,
      "logps/rejected": -315.6427917480469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2589776515960693,
      "rewards/margins": 17.56110382080078,
      "rewards/rejected": -14.302124977111816,
      "step": 3530
    },
    {
      "epoch": 1.4343598055105349,
      "grad_norm": 7.605322480230825e-06,
      "learning_rate": 5.79918955425484e-06,
      "logits/chosen": -0.9684303402900696,
      "logits/rejected": -1.2046997547149658,
      "logps/chosen": -108.52332305908203,
      "logps/rejected": -315.15814208984375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2342731952667236,
      "rewards/margins": 17.40003776550293,
      "rewards/rejected": -14.165764808654785,
      "step": 3540
    },
    {
      "epoch": 1.4384116693679092,
      "grad_norm": 1.0980385923176073e-05,
      "learning_rate": 5.784181299714844e-06,
      "logits/chosen": -0.9674128293991089,
      "logits/rejected": -1.2049371004104614,
      "logps/chosen": -107.88756561279297,
      "logps/rejected": -315.3216857910156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.239750862121582,
      "rewards/margins": 17.433073043823242,
      "rewards/rejected": -14.193323135375977,
      "step": 3550
    },
    {
      "epoch": 1.4384116693679092,
      "eval_logits/chosen": -0.9761521220207214,
      "eval_logits/rejected": -1.2098267078399658,
      "eval_logps/chosen": -109.37076568603516,
      "eval_logps/rejected": -314.67626953125,
      "eval_loss": 1.2733954690702376e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.164116144180298,
      "eval_rewards/margins": 17.337467193603516,
      "eval_rewards/rejected": -14.173351287841797,
      "eval_runtime": 662.9539,
      "eval_samples_per_second": 11.168,
      "eval_steps_per_second": 0.621,
      "step": 3550
    },
    {
      "epoch": 1.4424635332252835,
      "grad_norm": 6.137217496871017e-06,
      "learning_rate": 5.769173045174846e-06,
      "logits/chosen": -0.9557913541793823,
      "logits/rejected": -1.2017483711242676,
      "logps/chosen": -107.53488159179688,
      "logps/rejected": -316.9659118652344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2536418437957764,
      "rewards/margins": 17.57965087890625,
      "rewards/rejected": -14.326007843017578,
      "step": 3560
    },
    {
      "epoch": 1.446515397082658,
      "grad_norm": 3.537427255650982e-05,
      "learning_rate": 5.75416479063485e-06,
      "logits/chosen": -0.9414838552474976,
      "logits/rejected": -1.2034283876419067,
      "logps/chosen": -107.01081085205078,
      "logps/rejected": -316.0235595703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.271057605743408,
      "rewards/margins": 17.381750106811523,
      "rewards/rejected": -14.110692977905273,
      "step": 3570
    },
    {
      "epoch": 1.4505672609400324,
      "grad_norm": 6.594872047571698e-06,
      "learning_rate": 5.739156536094853e-06,
      "logits/chosen": -0.9697946906089783,
      "logits/rejected": -1.1990686655044556,
      "logps/chosen": -108.61802673339844,
      "logps/rejected": -316.5038757324219,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.230477809906006,
      "rewards/margins": 17.491981506347656,
      "rewards/rejected": -14.261506080627441,
      "step": 3580
    },
    {
      "epoch": 1.4546191247974067,
      "grad_norm": 1.4138061487756204e-05,
      "learning_rate": 5.724148281554856e-06,
      "logits/chosen": -0.9630403518676758,
      "logits/rejected": -1.2050740718841553,
      "logps/chosen": -108.44572448730469,
      "logps/rejected": -315.1803894042969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2415361404418945,
      "rewards/margins": 17.399463653564453,
      "rewards/rejected": -14.157925605773926,
      "step": 3590
    },
    {
      "epoch": 1.4586709886547813,
      "grad_norm": 7.2224402174470015e-06,
      "learning_rate": 5.709140027014859e-06,
      "logits/chosen": -0.9400904774665833,
      "logits/rejected": -1.1875113248825073,
      "logps/chosen": -108.74930572509766,
      "logps/rejected": -315.8128356933594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.231011152267456,
      "rewards/margins": 17.45760726928711,
      "rewards/rejected": -14.226595878601074,
      "step": 3600
    },
    {
      "epoch": 1.4586709886547813,
      "eval_logits/chosen": -0.9722992181777954,
      "eval_logits/rejected": -1.2060984373092651,
      "eval_logps/chosen": -109.30361938476562,
      "eval_logps/rejected": -314.8688659667969,
      "eval_loss": 1.2434720986220782e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1708314418792725,
      "eval_rewards/margins": 17.36344337463379,
      "eval_rewards/rejected": -14.192611694335938,
      "eval_runtime": 663.4166,
      "eval_samples_per_second": 11.16,
      "eval_steps_per_second": 0.621,
      "step": 3600
    },
    {
      "epoch": 1.4627228525121556,
      "grad_norm": 6.3852685343590565e-06,
      "learning_rate": 5.694131772474861e-06,
      "logits/chosen": -0.9639496207237244,
      "logits/rejected": -1.2003190517425537,
      "logps/chosen": -108.68683624267578,
      "logps/rejected": -316.43585205078125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2357661724090576,
      "rewards/margins": 17.586694717407227,
      "rewards/rejected": -14.350927352905273,
      "step": 3610
    },
    {
      "epoch": 1.46677471636953,
      "grad_norm": 9.98025916487677e-06,
      "learning_rate": 5.679123517934864e-06,
      "logits/chosen": -0.9612903594970703,
      "logits/rejected": -1.1824277639389038,
      "logps/chosen": -107.56800079345703,
      "logps/rejected": -314.85516357421875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2589564323425293,
      "rewards/margins": 17.432729721069336,
      "rewards/rejected": -14.173775672912598,
      "step": 3620
    },
    {
      "epoch": 1.4708265802269045,
      "grad_norm": 2.0052910258527845e-05,
      "learning_rate": 5.664115263394868e-06,
      "logits/chosen": -0.9558570384979248,
      "logits/rejected": -1.2040566205978394,
      "logps/chosen": -108.70698547363281,
      "logps/rejected": -315.31787109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.264512062072754,
      "rewards/margins": 17.43938636779785,
      "rewards/rejected": -14.174872398376465,
      "step": 3630
    },
    {
      "epoch": 1.4748784440842788,
      "grad_norm": 7.068727427395061e-06,
      "learning_rate": 5.64910700885487e-06,
      "logits/chosen": -0.9596124887466431,
      "logits/rejected": -1.2153346538543701,
      "logps/chosen": -108.03771209716797,
      "logps/rejected": -318.3464660644531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.210784912109375,
      "rewards/margins": 17.65658950805664,
      "rewards/rejected": -14.44580364227295,
      "step": 3640
    },
    {
      "epoch": 1.4789303079416531,
      "grad_norm": 8.767375220486429e-06,
      "learning_rate": 5.634098754314874e-06,
      "logits/chosen": -0.9488992691040039,
      "logits/rejected": -1.1887593269348145,
      "logps/chosen": -107.88094329833984,
      "logps/rejected": -315.5525817871094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.292868137359619,
      "rewards/margins": 17.515554428100586,
      "rewards/rejected": -14.222685813903809,
      "step": 3650
    },
    {
      "epoch": 1.4789303079416531,
      "eval_logits/chosen": -0.9733116626739502,
      "eval_logits/rejected": -1.2053263187408447,
      "eval_logps/chosen": -109.34339141845703,
      "eval_logps/rejected": -315.2198791503906,
      "eval_loss": 1.2054697151597793e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1668541431427,
      "eval_rewards/margins": 17.39456558227539,
      "eval_rewards/rejected": -14.22771167755127,
      "eval_runtime": 663.5102,
      "eval_samples_per_second": 11.159,
      "eval_steps_per_second": 0.621,
      "step": 3650
    },
    {
      "epoch": 1.4829821717990275,
      "grad_norm": 7.54353550291853e-06,
      "learning_rate": 5.619090499774876e-06,
      "logits/chosen": -0.9498493075370789,
      "logits/rejected": -1.2062077522277832,
      "logps/chosen": -109.1008071899414,
      "logps/rejected": -315.29022216796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1963202953338623,
      "rewards/margins": 17.379905700683594,
      "rewards/rejected": -14.183587074279785,
      "step": 3660
    },
    {
      "epoch": 1.487034035656402,
      "grad_norm": 2.701729727050406e-06,
      "learning_rate": 5.60408224523488e-06,
      "logits/chosen": -0.954522967338562,
      "logits/rejected": -1.197543740272522,
      "logps/chosen": -108.44230651855469,
      "logps/rejected": -318.29608154296875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.25152587890625,
      "rewards/margins": 17.643075942993164,
      "rewards/rejected": -14.39155101776123,
      "step": 3670
    },
    {
      "epoch": 1.4910858995137763,
      "grad_norm": 5.08013636135729e-06,
      "learning_rate": 5.589073990694882e-06,
      "logits/chosen": -0.9814316630363464,
      "logits/rejected": -1.2127686738967896,
      "logps/chosen": -109.36546325683594,
      "logps/rejected": -315.67999267578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.18919038772583,
      "rewards/margins": 17.470264434814453,
      "rewards/rejected": -14.281076431274414,
      "step": 3680
    },
    {
      "epoch": 1.4951377633711507,
      "grad_norm": 8.137954864650965e-06,
      "learning_rate": 5.574065736154886e-06,
      "logits/chosen": -0.9642794728279114,
      "logits/rejected": -1.1955642700195312,
      "logps/chosen": -108.89013671875,
      "logps/rejected": -316.7543029785156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2404532432556152,
      "rewards/margins": 17.62625503540039,
      "rewards/rejected": -14.385802268981934,
      "step": 3690
    },
    {
      "epoch": 1.499189627228525,
      "grad_norm": 2.0117735402891412e-05,
      "learning_rate": 5.559057481614889e-06,
      "logits/chosen": -0.9436579942703247,
      "logits/rejected": -1.1839931011199951,
      "logps/chosen": -108.5077133178711,
      "logps/rejected": -316.2707214355469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2324440479278564,
      "rewards/margins": 17.492176055908203,
      "rewards/rejected": -14.259733200073242,
      "step": 3700
    },
    {
      "epoch": 1.499189627228525,
      "eval_logits/chosen": -0.974548876285553,
      "eval_logits/rejected": -1.2070963382720947,
      "eval_logps/chosen": -109.29476928710938,
      "eval_logps/rejected": -315.4743347167969,
      "eval_loss": 1.1724890924824649e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.171715497970581,
      "eval_rewards/margins": 17.42487335205078,
      "eval_rewards/rejected": -14.253158569335938,
      "eval_runtime": 663.4897,
      "eval_samples_per_second": 11.159,
      "eval_steps_per_second": 0.621,
      "step": 3700
    },
    {
      "epoch": 1.5032414910858996,
      "grad_norm": 4.992085450794548e-06,
      "learning_rate": 5.544049227074891e-06,
      "logits/chosen": -0.9674142003059387,
      "logits/rejected": -1.1863280534744263,
      "logps/chosen": -109.25318908691406,
      "logps/rejected": -314.8880615234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2352867126464844,
      "rewards/margins": 17.398120880126953,
      "rewards/rejected": -14.162833213806152,
      "step": 3710
    },
    {
      "epoch": 1.5072933549432739,
      "grad_norm": 4.062382231495576e-06,
      "learning_rate": 5.529040972534894e-06,
      "logits/chosen": -0.9425821304321289,
      "logits/rejected": -1.1931798458099365,
      "logps/chosen": -108.00586700439453,
      "logps/rejected": -315.8692626953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2697389125823975,
      "rewards/margins": 17.538049697875977,
      "rewards/rejected": -14.268309593200684,
      "step": 3720
    },
    {
      "epoch": 1.5113452188006482,
      "grad_norm": 1.9489334590616636e-05,
      "learning_rate": 5.514032717994897e-06,
      "logits/chosen": -0.9636507630348206,
      "logits/rejected": -1.2020808458328247,
      "logps/chosen": -109.30130004882812,
      "logps/rejected": -316.9945373535156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1893112659454346,
      "rewards/margins": 17.497333526611328,
      "rewards/rejected": -14.308022499084473,
      "step": 3730
    },
    {
      "epoch": 1.5153970826580228,
      "grad_norm": 1.9115646864520386e-05,
      "learning_rate": 5.4990244634549004e-06,
      "logits/chosen": -0.9532636404037476,
      "logits/rejected": -1.192227840423584,
      "logps/chosen": -108.51616668701172,
      "logps/rejected": -315.6667175292969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.240192174911499,
      "rewards/margins": 17.530908584594727,
      "rewards/rejected": -14.290714263916016,
      "step": 3740
    },
    {
      "epoch": 1.519448946515397,
      "grad_norm": 5.272378075460438e-06,
      "learning_rate": 5.484016208914904e-06,
      "logits/chosen": -0.9705985188484192,
      "logits/rejected": -1.2012253999710083,
      "logps/chosen": -108.23621368408203,
      "logps/rejected": -317.6721496582031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2313127517700195,
      "rewards/margins": 17.620288848876953,
      "rewards/rejected": -14.388975143432617,
      "step": 3750
    },
    {
      "epoch": 1.519448946515397,
      "eval_logits/chosen": -0.9684827327728271,
      "eval_logits/rejected": -1.2004016637802124,
      "eval_logps/chosen": -109.24481964111328,
      "eval_logps/rejected": -315.8060302734375,
      "eval_loss": 1.130395119730565e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1767098903656006,
      "eval_rewards/margins": 17.46303939819336,
      "eval_rewards/rejected": -14.28632926940918,
      "eval_runtime": 663.4432,
      "eval_samples_per_second": 11.16,
      "eval_steps_per_second": 0.621,
      "step": 3750
    },
    {
      "epoch": 1.5235008103727714,
      "grad_norm": 6.1954165175848175e-06,
      "learning_rate": 5.4690079543749064e-06,
      "logits/chosen": -0.9727413654327393,
      "logits/rejected": -1.2027579545974731,
      "logps/chosen": -108.85786437988281,
      "logps/rejected": -317.4558410644531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.218287229537964,
      "rewards/margins": 17.62915802001953,
      "rewards/rejected": -14.410872459411621,
      "step": 3760
    },
    {
      "epoch": 1.527552674230146,
      "grad_norm": 5.180510925129056e-06,
      "learning_rate": 5.45399969983491e-06,
      "logits/chosen": -0.9645715951919556,
      "logits/rejected": -1.1870073080062866,
      "logps/chosen": -109.94966125488281,
      "logps/rejected": -316.37982177734375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1916048526763916,
      "rewards/margins": 17.476436614990234,
      "rewards/rejected": -14.284832954406738,
      "step": 3770
    },
    {
      "epoch": 1.5316045380875203,
      "grad_norm": 7.327073490159819e-06,
      "learning_rate": 5.4389914452949124e-06,
      "logits/chosen": -0.9505946040153503,
      "logits/rejected": -1.202714443206787,
      "logps/chosen": -108.26834869384766,
      "logps/rejected": -316.9022216796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.238265037536621,
      "rewards/margins": 17.610530853271484,
      "rewards/rejected": -14.37226676940918,
      "step": 3780
    },
    {
      "epoch": 1.5356564019448946,
      "grad_norm": 7.510065188398585e-06,
      "learning_rate": 5.423983190754916e-06,
      "logits/chosen": -0.9567342400550842,
      "logits/rejected": -1.1898601055145264,
      "logps/chosen": -107.89736938476562,
      "logps/rejected": -316.4672546386719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2650341987609863,
      "rewards/margins": 17.62150001525879,
      "rewards/rejected": -14.356466293334961,
      "step": 3790
    },
    {
      "epoch": 1.5397082658022692,
      "grad_norm": 6.228626261872705e-06,
      "learning_rate": 5.408974936214919e-06,
      "logits/chosen": -0.9419568777084351,
      "logits/rejected": -1.1964390277862549,
      "logps/chosen": -107.32307434082031,
      "logps/rejected": -318.02191162109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2831625938415527,
      "rewards/margins": 17.69293212890625,
      "rewards/rejected": -14.409770965576172,
      "step": 3800
    },
    {
      "epoch": 1.5397082658022692,
      "eval_logits/chosen": -0.9724193811416626,
      "eval_logits/rejected": -1.20346999168396,
      "eval_logps/chosen": -109.28492736816406,
      "eval_logps/rejected": -316.20904541015625,
      "eval_loss": 1.0901329261514547e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.172699213027954,
      "eval_rewards/margins": 17.49932861328125,
      "eval_rewards/rejected": -14.326629638671875,
      "eval_runtime": 663.4698,
      "eval_samples_per_second": 11.16,
      "eval_steps_per_second": 0.621,
      "step": 3800
    },
    {
      "epoch": 1.5437601296596435,
      "grad_norm": 1.283068831980927e-05,
      "learning_rate": 5.393966681674921e-06,
      "logits/chosen": -0.9620767831802368,
      "logits/rejected": -1.1938879489898682,
      "logps/chosen": -108.77943420410156,
      "logps/rejected": -318.4185485839844,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.229729175567627,
      "rewards/margins": 17.725805282592773,
      "rewards/rejected": -14.496077537536621,
      "step": 3810
    },
    {
      "epoch": 1.5478119935170178,
      "grad_norm": 2.9260332667035982e-05,
      "learning_rate": 5.378958427134925e-06,
      "logits/chosen": -0.9631065726280212,
      "logits/rejected": -1.2083336114883423,
      "logps/chosen": -108.52242279052734,
      "logps/rejected": -317.4120178222656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.296027660369873,
      "rewards/margins": 17.68168830871582,
      "rewards/rejected": -14.385659217834473,
      "step": 3820
    },
    {
      "epoch": 1.5518638573743924,
      "grad_norm": 4.707081188826123e-06,
      "learning_rate": 5.363950172594927e-06,
      "logits/chosen": -0.9571763277053833,
      "logits/rejected": -1.2118968963623047,
      "logps/chosen": -108.25720977783203,
      "logps/rejected": -318.5433654785156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.246180295944214,
      "rewards/margins": 17.719322204589844,
      "rewards/rejected": -14.47314167022705,
      "step": 3830
    },
    {
      "epoch": 1.5559157212317665,
      "grad_norm": 1.0459262739459518e-05,
      "learning_rate": 5.3489419180549305e-06,
      "logits/chosen": -0.977500855922699,
      "logits/rejected": -1.1893442869186401,
      "logps/chosen": -109.2089614868164,
      "logps/rejected": -319.8080749511719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.246575117111206,
      "rewards/margins": 17.779726028442383,
      "rewards/rejected": -14.533150672912598,
      "step": 3840
    },
    {
      "epoch": 1.559967585089141,
      "grad_norm": 3.85352041121223e-06,
      "learning_rate": 5.333933663514934e-06,
      "logits/chosen": -0.9658922553062439,
      "logits/rejected": -1.201623558998108,
      "logps/chosen": -109.0021743774414,
      "logps/rejected": -317.1396484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.187291383743286,
      "rewards/margins": 17.594905853271484,
      "rewards/rejected": -14.407611846923828,
      "step": 3850
    },
    {
      "epoch": 1.559967585089141,
      "eval_logits/chosen": -0.9734312891960144,
      "eval_logits/rejected": -1.204223394393921,
      "eval_logps/chosen": -109.30084991455078,
      "eval_logps/rejected": -316.61053466796875,
      "eval_loss": 1.0523393711991957e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.171109437942505,
      "eval_rewards/margins": 17.537887573242188,
      "eval_rewards/rejected": -14.366777420043945,
      "eval_runtime": 663.7938,
      "eval_samples_per_second": 11.154,
      "eval_steps_per_second": 0.621,
      "step": 3850
    },
    {
      "epoch": 1.5640194489465153,
      "grad_norm": 3.579104486561846e-06,
      "learning_rate": 5.3189254089749365e-06,
      "logits/chosen": -0.9664371609687805,
      "logits/rejected": -1.2130407094955444,
      "logps/chosen": -108.39588165283203,
      "logps/rejected": -317.6293029785156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.246037721633911,
      "rewards/margins": 17.729427337646484,
      "rewards/rejected": -14.483388900756836,
      "step": 3860
    },
    {
      "epoch": 1.5680713128038897,
      "grad_norm": 6.614993253606372e-06,
      "learning_rate": 5.30391715443494e-06,
      "logits/chosen": -0.9652690291404724,
      "logits/rejected": -1.2071861028671265,
      "logps/chosen": -108.62710571289062,
      "logps/rejected": -317.63916015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.22904896736145,
      "rewards/margins": 17.600061416625977,
      "rewards/rejected": -14.371011734008789,
      "step": 3870
    },
    {
      "epoch": 1.5721231766612642,
      "grad_norm": 4.290187007427448e-06,
      "learning_rate": 5.2889088998949425e-06,
      "logits/chosen": -0.9534419775009155,
      "logits/rejected": -1.1890816688537598,
      "logps/chosen": -108.34811401367188,
      "logps/rejected": -317.65673828125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2479617595672607,
      "rewards/margins": 17.63799476623535,
      "rewards/rejected": -14.390031814575195,
      "step": 3880
    },
    {
      "epoch": 1.5761750405186385,
      "grad_norm": 5.77761829845258e-06,
      "learning_rate": 5.273900645354946e-06,
      "logits/chosen": -0.9653971791267395,
      "logits/rejected": -1.1733673810958862,
      "logps/chosen": -109.12783813476562,
      "logps/rejected": -316.7720642089844,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.249236583709717,
      "rewards/margins": 17.553876876831055,
      "rewards/rejected": -14.304642677307129,
      "step": 3890
    },
    {
      "epoch": 1.5802269043760129,
      "grad_norm": 5.493533080880297e-06,
      "learning_rate": 5.258892390814949e-06,
      "logits/chosen": -0.9704580903053284,
      "logits/rejected": -1.2150682210922241,
      "logps/chosen": -108.91256713867188,
      "logps/rejected": -319.23419189453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2321040630340576,
      "rewards/margins": 17.845125198364258,
      "rewards/rejected": -14.613018989562988,
      "step": 3900
    },
    {
      "epoch": 1.5802269043760129,
      "eval_logits/chosen": -0.9785429835319519,
      "eval_logits/rejected": -1.2089550495147705,
      "eval_logps/chosen": -109.25596618652344,
      "eval_logps/rejected": -316.824951171875,
      "eval_loss": 1.0255480731302669e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1755969524383545,
      "eval_rewards/margins": 17.56381607055664,
      "eval_rewards/rejected": -14.388218879699707,
      "eval_runtime": 663.5303,
      "eval_samples_per_second": 11.158,
      "eval_steps_per_second": 0.621,
      "step": 3900
    },
    {
      "epoch": 1.5842787682333874,
      "grad_norm": 5.203523869568016e-06,
      "learning_rate": 5.243884136274952e-06,
      "logits/chosen": -0.9706067442893982,
      "logits/rejected": -1.2097996473312378,
      "logps/chosen": -107.60671997070312,
      "logps/rejected": -318.22271728515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2841460704803467,
      "rewards/margins": 17.731903076171875,
      "rewards/rejected": -14.447755813598633,
      "step": 3910
    },
    {
      "epoch": 1.5883306320907618,
      "grad_norm": 6.18762533122208e-06,
      "learning_rate": 5.228875881734955e-06,
      "logits/chosen": -0.9635491967201233,
      "logits/rejected": -1.1880855560302734,
      "logps/chosen": -107.89752960205078,
      "logps/rejected": -318.7754211425781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2739787101745605,
      "rewards/margins": 17.841236114501953,
      "rewards/rejected": -14.567258834838867,
      "step": 3920
    },
    {
      "epoch": 1.592382495948136,
      "grad_norm": 1.798323319235351e-05,
      "learning_rate": 5.213867627194957e-06,
      "logits/chosen": -0.973612368106842,
      "logits/rejected": -1.1960965394973755,
      "logps/chosen": -109.48294830322266,
      "logps/rejected": -318.3647766113281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.19464111328125,
      "rewards/margins": 17.724042892456055,
      "rewards/rejected": -14.529403686523438,
      "step": 3930
    },
    {
      "epoch": 1.5964343598055106,
      "grad_norm": 1.4453021321969572e-05,
      "learning_rate": 5.1988593726549605e-06,
      "logits/chosen": -0.9594123959541321,
      "logits/rejected": -1.2001526355743408,
      "logps/chosen": -108.56301879882812,
      "logps/rejected": -318.62384033203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.201404094696045,
      "rewards/margins": 17.75627899169922,
      "rewards/rejected": -14.554872512817383,
      "step": 3940
    },
    {
      "epoch": 1.600486223662885,
      "grad_norm": 1.7161020878120326e-05,
      "learning_rate": 5.183851118114963e-06,
      "logits/chosen": -0.9724298715591431,
      "logits/rejected": -1.2018600702285767,
      "logps/chosen": -109.44165802001953,
      "logps/rejected": -317.9696960449219,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2142207622528076,
      "rewards/margins": 17.690942764282227,
      "rewards/rejected": -14.47671890258789,
      "step": 3950
    },
    {
      "epoch": 1.600486223662885,
      "eval_logits/chosen": -0.977683961391449,
      "eval_logits/rejected": -1.2075324058532715,
      "eval_logps/chosen": -109.19850158691406,
      "eval_logps/rejected": -317.0234680175781,
      "eval_loss": 1.0080263734835171e-07,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1813430786132812,
      "eval_rewards/margins": 17.589412689208984,
      "eval_rewards/rejected": -14.40807056427002,
      "eval_runtime": 673.7,
      "eval_samples_per_second": 10.99,
      "eval_steps_per_second": 0.612,
      "step": 3950
    },
    {
      "epoch": 1.6045380875202593,
      "grad_norm": 1.2696204066742212e-05,
      "learning_rate": 5.1688428635749665e-06,
      "logits/chosen": -0.9766330718994141,
      "logits/rejected": -1.2089101076126099,
      "logps/chosen": -107.4908676147461,
      "logps/rejected": -317.90374755859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.255730152130127,
      "rewards/margins": 17.775943756103516,
      "rewards/rejected": -14.52021312713623,
      "step": 3960
    },
    {
      "epoch": 1.6085899513776338,
      "grad_norm": 7.823682608432136e-06,
      "learning_rate": 5.15383460903497e-06,
      "logits/chosen": -0.9714142084121704,
      "logits/rejected": -1.1973127126693726,
      "logps/chosen": -108.08839416503906,
      "logps/rejected": -316.92938232421875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2525742053985596,
      "rewards/margins": 17.704811096191406,
      "rewards/rejected": -14.452237129211426,
      "step": 3970
    },
    {
      "epoch": 1.6126418152350082,
      "grad_norm": 6.02427917328896e-06,
      "learning_rate": 5.1388263544949725e-06,
      "logits/chosen": -0.9654116034507751,
      "logits/rejected": -1.2230839729309082,
      "logps/chosen": -108.58455657958984,
      "logps/rejected": -319.99609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2604827880859375,
      "rewards/margins": 17.88028907775879,
      "rewards/rejected": -14.619808197021484,
      "step": 3980
    },
    {
      "epoch": 1.6166936790923825,
      "grad_norm": 8.140964382619131e-06,
      "learning_rate": 5.123818099954976e-06,
      "logits/chosen": -0.9634442925453186,
      "logits/rejected": -1.199520468711853,
      "logps/chosen": -109.39273071289062,
      "logps/rejected": -317.8063049316406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.23472261428833,
      "rewards/margins": 17.63693618774414,
      "rewards/rejected": -14.402215957641602,
      "step": 3990
    },
    {
      "epoch": 1.620745542949757,
      "grad_norm": 8.955745215644129e-06,
      "learning_rate": 5.1088098454149785e-06,
      "logits/chosen": -0.9667535424232483,
      "logits/rejected": -1.2081085443496704,
      "logps/chosen": -107.45613861083984,
      "logps/rejected": -320.45849609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.265660047531128,
      "rewards/margins": 17.913358688354492,
      "rewards/rejected": -14.647696495056152,
      "step": 4000
    },
    {
      "epoch": 1.620745542949757,
      "eval_logits/chosen": -0.9739937782287598,
      "eval_logits/rejected": -1.2036855220794678,
      "eval_logps/chosen": -109.220947265625,
      "eval_logps/rejected": -317.4233093261719,
      "eval_loss": 9.71496092461166e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.179098129272461,
      "eval_rewards/margins": 17.627151489257812,
      "eval_rewards/rejected": -14.448055267333984,
      "eval_runtime": 660.0616,
      "eval_samples_per_second": 11.217,
      "eval_steps_per_second": 0.624,
      "step": 4000
    },
    {
      "epoch": 1.6247974068071311,
      "grad_norm": 3.5037126053794054e-06,
      "learning_rate": 5.093801590874982e-06,
      "logits/chosen": -0.9621725082397461,
      "logits/rejected": -1.1956188678741455,
      "logps/chosen": -109.2527084350586,
      "logps/rejected": -318.2307434082031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2300877571105957,
      "rewards/margins": 17.69159698486328,
      "rewards/rejected": -14.461508750915527,
      "step": 4010
    },
    {
      "epoch": 1.6288492706645057,
      "grad_norm": 4.9843401939142495e-06,
      "learning_rate": 5.078793336334985e-06,
      "logits/chosen": -0.9534133076667786,
      "logits/rejected": -1.1932679414749146,
      "logps/chosen": -107.59471893310547,
      "logps/rejected": -319.98193359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.285687208175659,
      "rewards/margins": 17.901277542114258,
      "rewards/rejected": -14.615589141845703,
      "step": 4020
    },
    {
      "epoch": 1.63290113452188,
      "grad_norm": 2.6889072159974603e-06,
      "learning_rate": 5.063785081794987e-06,
      "logits/chosen": -0.9574196934700012,
      "logits/rejected": -1.197640299797058,
      "logps/chosen": -107.38705444335938,
      "logps/rejected": -317.0834655761719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.266223907470703,
      "rewards/margins": 17.728179931640625,
      "rewards/rejected": -14.461956977844238,
      "step": 4030
    },
    {
      "epoch": 1.6369529983792543,
      "grad_norm": 2.5947690573957516e-06,
      "learning_rate": 5.0487768272549905e-06,
      "logits/chosen": -0.9642937779426575,
      "logits/rejected": -1.2198621034622192,
      "logps/chosen": -107.37706756591797,
      "logps/rejected": -318.87896728515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.257359266281128,
      "rewards/margins": 17.830778121948242,
      "rewards/rejected": -14.573415756225586,
      "step": 4040
    },
    {
      "epoch": 1.641004862236629,
      "grad_norm": 8.20930927147856e-06,
      "learning_rate": 5.033768572714993e-06,
      "logits/chosen": -0.958414614200592,
      "logits/rejected": -1.1930384635925293,
      "logps/chosen": -107.63899230957031,
      "logps/rejected": -319.4289245605469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2218847274780273,
      "rewards/margins": 17.83708381652832,
      "rewards/rejected": -14.615198135375977,
      "step": 4050
    },
    {
      "epoch": 1.641004862236629,
      "eval_logits/chosen": -0.975097119808197,
      "eval_logits/rejected": -1.2044233083724976,
      "eval_logps/chosen": -109.19059753417969,
      "eval_logps/rejected": -317.51708984375,
      "eval_loss": 9.533959399732339e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1821327209472656,
      "eval_rewards/margins": 17.63956642150879,
      "eval_rewards/rejected": -14.457432746887207,
      "eval_runtime": 660.5812,
      "eval_samples_per_second": 11.208,
      "eval_steps_per_second": 0.624,
      "step": 4050
    },
    {
      "epoch": 1.6450567260940032,
      "grad_norm": 1.4627960808866192e-05,
      "learning_rate": 5.0187603181749965e-06,
      "logits/chosen": -0.9663168787956238,
      "logits/rejected": -1.1901664733886719,
      "logps/chosen": -107.59746551513672,
      "logps/rejected": -318.67193603515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2689335346221924,
      "rewards/margins": 17.76458168029785,
      "rewards/rejected": -14.495648384094238,
      "step": 4060
    },
    {
      "epoch": 1.6491085899513775,
      "grad_norm": 4.622088908945443e-06,
      "learning_rate": 5.003752063635e-06,
      "logits/chosen": -0.9560664892196655,
      "logits/rejected": -1.1980023384094238,
      "logps/chosen": -108.25928497314453,
      "logps/rejected": -317.87664794921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.250584602355957,
      "rewards/margins": 17.7623348236084,
      "rewards/rejected": -14.511751174926758,
      "step": 4070
    },
    {
      "epoch": 1.653160453808752,
      "grad_norm": 2.349050009797793e-05,
      "learning_rate": 4.9887438090950026e-06,
      "logits/chosen": -0.9641364216804504,
      "logits/rejected": -1.1876428127288818,
      "logps/chosen": -107.60128784179688,
      "logps/rejected": -319.0780334472656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2614548206329346,
      "rewards/margins": 17.83684730529785,
      "rewards/rejected": -14.575393676757812,
      "step": 4080
    },
    {
      "epoch": 1.6572123176661264,
      "grad_norm": 5.146464900462888e-06,
      "learning_rate": 4.973735554555006e-06,
      "logits/chosen": -0.9657267928123474,
      "logits/rejected": -1.1887136697769165,
      "logps/chosen": -108.01840209960938,
      "logps/rejected": -318.5270690917969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2487587928771973,
      "rewards/margins": 17.866899490356445,
      "rewards/rejected": -14.618142127990723,
      "step": 4090
    },
    {
      "epoch": 1.6612641815235007,
      "grad_norm": 1.2196490388305392e-05,
      "learning_rate": 4.9587273000150086e-06,
      "logits/chosen": -0.9598487615585327,
      "logits/rejected": -1.201452612876892,
      "logps/chosen": -107.60063171386719,
      "logps/rejected": -318.6435852050781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.257800817489624,
      "rewards/margins": 17.75874137878418,
      "rewards/rejected": -14.50094223022461,
      "step": 4100
    },
    {
      "epoch": 1.6612641815235007,
      "eval_logits/chosen": -0.9671822190284729,
      "eval_logits/rejected": -1.1948264837265015,
      "eval_logps/chosen": -109.20098114013672,
      "eval_logps/rejected": -317.9280090332031,
      "eval_loss": 9.195412076223874e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1810953617095947,
      "eval_rewards/margins": 17.67962074279785,
      "eval_rewards/rejected": -14.498525619506836,
      "eval_runtime": 663.6517,
      "eval_samples_per_second": 11.156,
      "eval_steps_per_second": 0.621,
      "step": 4100
    },
    {
      "epoch": 1.6653160453808753,
      "grad_norm": 2.757898528216174e-06,
      "learning_rate": 4.943719045475012e-06,
      "logits/chosen": -0.9675645232200623,
      "logits/rejected": -1.189418911933899,
      "logps/chosen": -107.36319732666016,
      "logps/rejected": -318.67498779296875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.253856658935547,
      "rewards/margins": 17.851980209350586,
      "rewards/rejected": -14.598124504089355,
      "step": 4110
    },
    {
      "epoch": 1.6693679092382496,
      "grad_norm": 3.7102433907421073e-06,
      "learning_rate": 4.9287107909350146e-06,
      "logits/chosen": -0.9603763222694397,
      "logits/rejected": -1.1864895820617676,
      "logps/chosen": -107.8660659790039,
      "logps/rejected": -319.4140930175781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.261554002761841,
      "rewards/margins": 17.922758102416992,
      "rewards/rejected": -14.66120433807373,
      "step": 4120
    },
    {
      "epoch": 1.673419773095624,
      "grad_norm": 1.7009240764309652e-05,
      "learning_rate": 4.913702536395017e-06,
      "logits/chosen": -0.966614842414856,
      "logits/rejected": -1.1995640993118286,
      "logps/chosen": -108.76692962646484,
      "logps/rejected": -319.5960388183594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.229149103164673,
      "rewards/margins": 17.841875076293945,
      "rewards/rejected": -14.612726211547852,
      "step": 4130
    },
    {
      "epoch": 1.6774716369529985,
      "grad_norm": 5.367153789848089e-06,
      "learning_rate": 4.898694281855021e-06,
      "logits/chosen": -0.9619210958480835,
      "logits/rejected": -1.194135308265686,
      "logps/chosen": -108.61856079101562,
      "logps/rejected": -317.7147521972656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.242100954055786,
      "rewards/margins": 17.756872177124023,
      "rewards/rejected": -14.5147705078125,
      "step": 4140
    },
    {
      "epoch": 1.6815235008103726,
      "grad_norm": 3.283277465015999e-06,
      "learning_rate": 4.883686027315024e-06,
      "logits/chosen": -0.9638760089874268,
      "logits/rejected": -1.205974817276001,
      "logps/chosen": -107.50321960449219,
      "logps/rejected": -317.22576904296875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.268812894821167,
      "rewards/margins": 17.68256950378418,
      "rewards/rejected": -14.413758277893066,
      "step": 4150
    },
    {
      "epoch": 1.6815235008103726,
      "eval_logits/chosen": -0.9770015478134155,
      "eval_logits/rejected": -1.2045894861221313,
      "eval_logps/chosen": -109.17327117919922,
      "eval_logps/rejected": -318.26959228515625,
      "eval_loss": 8.916208571463358e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.183866024017334,
      "eval_rewards/margins": 17.716548919677734,
      "eval_rewards/rejected": -14.532684326171875,
      "eval_runtime": 663.031,
      "eval_samples_per_second": 11.167,
      "eval_steps_per_second": 0.621,
      "step": 4150
    },
    {
      "epoch": 1.6855753646677472,
      "grad_norm": 5.341072665032698e-06,
      "learning_rate": 4.868677772775027e-06,
      "logits/chosen": -0.966580331325531,
      "logits/rejected": -1.2173370122909546,
      "logps/chosen": -108.13761138916016,
      "logps/rejected": -322.0377197265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2559990882873535,
      "rewards/margins": 18.029720306396484,
      "rewards/rejected": -14.773722648620605,
      "step": 4160
    },
    {
      "epoch": 1.6896272285251217,
      "grad_norm": 5.1339147830731235e-06,
      "learning_rate": 4.85366951823503e-06,
      "logits/chosen": -0.9749870300292969,
      "logits/rejected": -1.2149051427841187,
      "logps/chosen": -107.59915924072266,
      "logps/rejected": -320.6737060546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2866246700286865,
      "rewards/margins": 17.952587127685547,
      "rewards/rejected": -14.665961265563965,
      "step": 4170
    },
    {
      "epoch": 1.6936790923824958,
      "grad_norm": 4.617202648660168e-06,
      "learning_rate": 4.838661263695033e-06,
      "logits/chosen": -0.966048538684845,
      "logits/rejected": -1.20872163772583,
      "logps/chosen": -107.75770568847656,
      "logps/rejected": -320.81640625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.31170916557312,
      "rewards/margins": 18.048131942749023,
      "rewards/rejected": -14.73642349243164,
      "step": 4180
    },
    {
      "epoch": 1.6977309562398704,
      "grad_norm": 8.695402357261628e-06,
      "learning_rate": 4.823653009155035e-06,
      "logits/chosen": -0.9720503091812134,
      "logits/rejected": -1.2081489562988281,
      "logps/chosen": -108.02511596679688,
      "logps/rejected": -320.03839111328125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2493691444396973,
      "rewards/margins": 17.917095184326172,
      "rewards/rejected": -14.667726516723633,
      "step": 4190
    },
    {
      "epoch": 1.7017828200972447,
      "grad_norm": 6.944351753190858e-06,
      "learning_rate": 4.808644754615039e-06,
      "logits/chosen": -0.9694951176643372,
      "logits/rejected": -1.2126491069793701,
      "logps/chosen": -107.50739288330078,
      "logps/rejected": -320.3431701660156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3132131099700928,
      "rewards/margins": 18.086727142333984,
      "rewards/rejected": -14.773515701293945,
      "step": 4200
    },
    {
      "epoch": 1.7017828200972447,
      "eval_logits/chosen": -0.9932587146759033,
      "eval_logits/rejected": -1.215240240097046,
      "eval_logps/chosen": -109.29456329345703,
      "eval_logps/rejected": -318.77752685546875,
      "eval_loss": 8.563549158679962e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1717369556427,
      "eval_rewards/margins": 17.75521469116211,
      "eval_rewards/rejected": -14.583476066589355,
      "eval_runtime": 664.1116,
      "eval_samples_per_second": 11.149,
      "eval_steps_per_second": 0.62,
      "step": 4200
    },
    {
      "epoch": 1.705834683954619,
      "grad_norm": 8.203344805224333e-06,
      "learning_rate": 4.793636500075042e-06,
      "logits/chosen": -0.962543249130249,
      "logits/rejected": -1.1963212490081787,
      "logps/chosen": -107.7632064819336,
      "logps/rejected": -319.73724365234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.301147222518921,
      "rewards/margins": 17.937274932861328,
      "rewards/rejected": -14.636128425598145,
      "step": 4210
    },
    {
      "epoch": 1.7098865478119936,
      "grad_norm": 1.5218613953038584e-05,
      "learning_rate": 4.778628245535045e-06,
      "logits/chosen": -0.9696997404098511,
      "logits/rejected": -1.1978899240493774,
      "logps/chosen": -107.52715301513672,
      "logps/rejected": -319.97869873046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2426681518554688,
      "rewards/margins": 17.97463035583496,
      "rewards/rejected": -14.731962203979492,
      "step": 4220
    },
    {
      "epoch": 1.713938411669368,
      "grad_norm": 2.6728112061391585e-05,
      "learning_rate": 4.763619990995047e-06,
      "logits/chosen": -0.9696226716041565,
      "logits/rejected": -1.2055093050003052,
      "logps/chosen": -107.90619659423828,
      "logps/rejected": -319.2203674316406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2442867755889893,
      "rewards/margins": 17.835180282592773,
      "rewards/rejected": -14.590893745422363,
      "step": 4230
    },
    {
      "epoch": 1.7179902755267422,
      "grad_norm": 4.324649125919677e-06,
      "learning_rate": 4.748611736455051e-06,
      "logits/chosen": -0.9708512425422668,
      "logits/rejected": -1.2109978199005127,
      "logps/chosen": -108.79696655273438,
      "logps/rejected": -319.5374755859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.270660877227783,
      "rewards/margins": 17.86390495300293,
      "rewards/rejected": -14.593246459960938,
      "step": 4240
    },
    {
      "epoch": 1.7220421393841168,
      "grad_norm": 6.176253918965813e-06,
      "learning_rate": 4.733603481915054e-06,
      "logits/chosen": -0.9561964273452759,
      "logits/rejected": -1.1975693702697754,
      "logps/chosen": -108.15457916259766,
      "logps/rejected": -320.9573974609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2581639289855957,
      "rewards/margins": 18.00908660888672,
      "rewards/rejected": -14.750924110412598,
      "step": 4250
    },
    {
      "epoch": 1.7220421393841168,
      "eval_logits/chosen": -0.9719837307929993,
      "eval_logits/rejected": -1.1987754106521606,
      "eval_logps/chosen": -109.13520050048828,
      "eval_logps/rejected": -318.714111328125,
      "eval_loss": 8.510431115382744e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1876730918884277,
      "eval_rewards/margins": 17.76481056213379,
      "eval_rewards/rejected": -14.57713794708252,
      "eval_runtime": 665.5016,
      "eval_samples_per_second": 11.125,
      "eval_steps_per_second": 0.619,
      "step": 4250
    },
    {
      "epoch": 1.726094003241491,
      "grad_norm": 7.230630217236467e-06,
      "learning_rate": 4.718595227375057e-06,
      "logits/chosen": -0.9625142216682434,
      "logits/rejected": -1.1945154666900635,
      "logps/chosen": -107.9782485961914,
      "logps/rejected": -318.8968505859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.266047954559326,
      "rewards/margins": 17.839750289916992,
      "rewards/rejected": -14.57370376586914,
      "step": 4260
    },
    {
      "epoch": 1.7301458670988654,
      "grad_norm": 6.9535558395728e-06,
      "learning_rate": 4.70358697283506e-06,
      "logits/chosen": -0.9719042181968689,
      "logits/rejected": -1.2067571878433228,
      "logps/chosen": -108.28577423095703,
      "logps/rejected": -319.3832702636719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.296572685241699,
      "rewards/margins": 17.96100425720215,
      "rewards/rejected": -14.66443157196045,
      "step": 4270
    },
    {
      "epoch": 1.73419773095624,
      "grad_norm": 4.54012160844286e-06,
      "learning_rate": 4.688578718295063e-06,
      "logits/chosen": -0.9618276357650757,
      "logits/rejected": -1.1899430751800537,
      "logps/chosen": -108.38196563720703,
      "logps/rejected": -320.1663513183594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2343204021453857,
      "rewards/margins": 17.94451332092285,
      "rewards/rejected": -14.710190773010254,
      "step": 4280
    },
    {
      "epoch": 1.7382495948136143,
      "grad_norm": 4.2299789129174314e-06,
      "learning_rate": 4.673570463755065e-06,
      "logits/chosen": -0.9642791748046875,
      "logits/rejected": -1.2018933296203613,
      "logps/chosen": -107.65156555175781,
      "logps/rejected": -319.62750244140625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2711496353149414,
      "rewards/margins": 17.844547271728516,
      "rewards/rejected": -14.573396682739258,
      "step": 4290
    },
    {
      "epoch": 1.7423014586709886,
      "grad_norm": 1.1666396858345252e-05,
      "learning_rate": 4.658562209215069e-06,
      "logits/chosen": -0.9570120573043823,
      "logits/rejected": -1.187798023223877,
      "logps/chosen": -108.56417846679688,
      "logps/rejected": -319.67279052734375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2457265853881836,
      "rewards/margins": 17.93968391418457,
      "rewards/rejected": -14.69395923614502,
      "step": 4300
    },
    {
      "epoch": 1.7423014586709886,
      "eval_logits/chosen": -0.9662920832633972,
      "eval_logits/rejected": -1.1940561532974243,
      "eval_logps/chosen": -109.10293579101562,
      "eval_logps/rejected": -318.8600158691406,
      "eval_loss": 8.35769000673281e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1909005641937256,
      "eval_rewards/margins": 17.782625198364258,
      "eval_rewards/rejected": -14.59172534942627,
      "eval_runtime": 686.8154,
      "eval_samples_per_second": 10.78,
      "eval_steps_per_second": 0.6,
      "step": 4300
    },
    {
      "epoch": 1.7463533225283632,
      "grad_norm": 8.781639735389035e-06,
      "learning_rate": 4.643553954675072e-06,
      "logits/chosen": -0.9702736139297485,
      "logits/rejected": -1.2177287340164185,
      "logps/chosen": -109.19371032714844,
      "logps/rejected": -319.2940673828125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.216906785964966,
      "rewards/margins": 17.830280303955078,
      "rewards/rejected": -14.613371849060059,
      "step": 4310
    },
    {
      "epoch": 1.7504051863857373,
      "grad_norm": 5.20419280292117e-06,
      "learning_rate": 4.628545700135075e-06,
      "logits/chosen": -0.9795908331871033,
      "logits/rejected": -1.2105401754379272,
      "logps/chosen": -108.30752563476562,
      "logps/rejected": -319.4550476074219,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2215776443481445,
      "rewards/margins": 17.895877838134766,
      "rewards/rejected": -14.67430305480957,
      "step": 4320
    },
    {
      "epoch": 1.7544570502431118,
      "grad_norm": 2.9489826829376398e-06,
      "learning_rate": 4.613537445595078e-06,
      "logits/chosen": -0.9719784259796143,
      "logits/rejected": -1.1975128650665283,
      "logps/chosen": -107.42411041259766,
      "logps/rejected": -321.16790771484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3067071437835693,
      "rewards/margins": 18.113649368286133,
      "rewards/rejected": -14.806941986083984,
      "step": 4330
    },
    {
      "epoch": 1.7585089141004864,
      "grad_norm": 1.4856002962915227e-05,
      "learning_rate": 4.598529191055081e-06,
      "logits/chosen": -0.9707692265510559,
      "logits/rejected": -1.192682147026062,
      "logps/chosen": -109.07689666748047,
      "logps/rejected": -320.1982116699219,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2610487937927246,
      "rewards/margins": 17.966022491455078,
      "rewards/rejected": -14.704972267150879,
      "step": 4340
    },
    {
      "epoch": 1.7625607779578605,
      "grad_norm": 8.642625289212447e-06,
      "learning_rate": 4.583520936515083e-06,
      "logits/chosen": -0.9686830639839172,
      "logits/rejected": -1.1929088830947876,
      "logps/chosen": -108.37218475341797,
      "logps/rejected": -320.23455810546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2672200202941895,
      "rewards/margins": 17.942583084106445,
      "rewards/rejected": -14.675362586975098,
      "step": 4350
    },
    {
      "epoch": 1.7625607779578605,
      "eval_logits/chosen": -0.9792025089263916,
      "eval_logits/rejected": -1.2058296203613281,
      "eval_logps/chosen": -109.13961029052734,
      "eval_logps/rejected": -319.3219299316406,
      "eval_loss": 8.068646906167487e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1872334480285645,
      "eval_rewards/margins": 17.825153350830078,
      "eval_rewards/rejected": -14.637920379638672,
      "eval_runtime": 660.0292,
      "eval_samples_per_second": 11.218,
      "eval_steps_per_second": 0.624,
      "step": 4350
    },
    {
      "epoch": 1.766612641815235,
      "grad_norm": 6.106893579271855e-06,
      "learning_rate": 4.568512681975087e-06,
      "logits/chosen": -0.9581748247146606,
      "logits/rejected": -1.1933001279830933,
      "logps/chosen": -107.26266479492188,
      "logps/rejected": -319.28619384765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3186635971069336,
      "rewards/margins": 18.03749656677246,
      "rewards/rejected": -14.718830108642578,
      "step": 4360
    },
    {
      "epoch": 1.7706645056726094,
      "grad_norm": 3.562696065273485e-06,
      "learning_rate": 4.55350442743509e-06,
      "logits/chosen": -0.9714043140411377,
      "logits/rejected": -1.2021480798721313,
      "logps/chosen": -109.37030792236328,
      "logps/rejected": -321.1347961425781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2486050128936768,
      "rewards/margins": 18.014080047607422,
      "rewards/rejected": -14.76547622680664,
      "step": 4370
    },
    {
      "epoch": 1.7747163695299837,
      "grad_norm": 1.0784347068693023e-05,
      "learning_rate": 4.538496172895093e-06,
      "logits/chosen": -0.9613971710205078,
      "logits/rejected": -1.178953766822815,
      "logps/chosen": -108.04612731933594,
      "logps/rejected": -320.51702880859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.295865058898926,
      "rewards/margins": 18.03044891357422,
      "rewards/rejected": -14.73458480834961,
      "step": 4380
    },
    {
      "epoch": 1.7787682333873582,
      "grad_norm": 3.7314644032448996e-06,
      "learning_rate": 4.523487918355095e-06,
      "logits/chosen": -0.9538978934288025,
      "logits/rejected": -1.1952236890792847,
      "logps/chosen": -107.91067504882812,
      "logps/rejected": -320.8572692871094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2640373706817627,
      "rewards/margins": 17.994760513305664,
      "rewards/rejected": -14.730725288391113,
      "step": 4390
    },
    {
      "epoch": 1.7828200972447326,
      "grad_norm": 7.644155630259775e-06,
      "learning_rate": 4.508479663815099e-06,
      "logits/chosen": -0.9672717452049255,
      "logits/rejected": -1.2072142362594604,
      "logps/chosen": -108.7569351196289,
      "logps/rejected": -319.16082763671875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2584824562072754,
      "rewards/margins": 17.830446243286133,
      "rewards/rejected": -14.571964263916016,
      "step": 4400
    },
    {
      "epoch": 1.7828200972447326,
      "eval_logits/chosen": -0.978135883808136,
      "eval_logits/rejected": -1.204634189605713,
      "eval_logps/chosen": -109.13714599609375,
      "eval_logps/rejected": -319.49810791015625,
      "eval_loss": 7.920481692735848e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.18747878074646,
      "eval_rewards/margins": 17.843013763427734,
      "eval_rewards/rejected": -14.655535697937012,
      "eval_runtime": 660.3223,
      "eval_samples_per_second": 11.213,
      "eval_steps_per_second": 0.624,
      "step": 4400
    },
    {
      "epoch": 1.7868719611021069,
      "grad_norm": 2.2184389308677055e-05,
      "learning_rate": 4.493471409275102e-06,
      "logits/chosen": -0.9621807336807251,
      "logits/rejected": -1.1890910863876343,
      "logps/chosen": -107.69747924804688,
      "logps/rejected": -319.6680908203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2675540447235107,
      "rewards/margins": 17.856769561767578,
      "rewards/rejected": -14.589217185974121,
      "step": 4410
    },
    {
      "epoch": 1.7909238249594814,
      "grad_norm": 4.1998664528364316e-06,
      "learning_rate": 4.478463154735105e-06,
      "logits/chosen": -0.9652020335197449,
      "logits/rejected": -1.2166922092437744,
      "logps/chosen": -109.41399383544922,
      "logps/rejected": -321.5539245605469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2054331302642822,
      "rewards/margins": 17.88554573059082,
      "rewards/rejected": -14.680112838745117,
      "step": 4420
    },
    {
      "epoch": 1.7949756888168558,
      "grad_norm": 7.153143542382168e-06,
      "learning_rate": 4.463454900195108e-06,
      "logits/chosen": -0.9790211915969849,
      "logits/rejected": -1.2030084133148193,
      "logps/chosen": -108.37751007080078,
      "logps/rejected": -322.23651123046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2648351192474365,
      "rewards/margins": 18.033552169799805,
      "rewards/rejected": -14.768719673156738,
      "step": 4430
    },
    {
      "epoch": 1.79902755267423,
      "grad_norm": 3.3711171454342548e-06,
      "learning_rate": 4.448446645655111e-06,
      "logits/chosen": -0.9689657688140869,
      "logits/rejected": -1.2139739990234375,
      "logps/chosen": -107.65309143066406,
      "logps/rejected": -323.7328796386719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2879128456115723,
      "rewards/margins": 18.25405502319336,
      "rewards/rejected": -14.966139793395996,
      "step": 4440
    },
    {
      "epoch": 1.8030794165316046,
      "grad_norm": 1.208314415634959e-06,
      "learning_rate": 4.433438391115113e-06,
      "logits/chosen": -0.9697353839874268,
      "logits/rejected": -1.2005465030670166,
      "logps/chosen": -107.74067687988281,
      "logps/rejected": -320.22296142578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2932724952697754,
      "rewards/margins": 18.093793869018555,
      "rewards/rejected": -14.800522804260254,
      "step": 4450
    },
    {
      "epoch": 1.8030794165316046,
      "eval_logits/chosen": -0.9763341546058655,
      "eval_logits/rejected": -1.2022850513458252,
      "eval_logps/chosen": -109.07799530029297,
      "eval_logps/rejected": -319.63885498046875,
      "eval_loss": 7.777131116881719e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1933939456939697,
      "eval_rewards/margins": 17.86300277709961,
      "eval_rewards/rejected": -14.669608116149902,
      "eval_runtime": 660.4874,
      "eval_samples_per_second": 11.21,
      "eval_steps_per_second": 0.624,
      "step": 4450
    },
    {
      "epoch": 1.807131280388979,
      "grad_norm": 6.018221938575152e-06,
      "learning_rate": 4.418430136575117e-06,
      "logits/chosen": -0.9695747494697571,
      "logits/rejected": -1.2011383771896362,
      "logps/chosen": -108.23741912841797,
      "logps/rejected": -322.4383850097656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2775845527648926,
      "rewards/margins": 18.15408706665039,
      "rewards/rejected": -14.87650203704834,
      "step": 4460
    },
    {
      "epoch": 1.8111831442463533,
      "grad_norm": 7.861935046094004e-06,
      "learning_rate": 4.40342188203512e-06,
      "logits/chosen": -0.9580012559890747,
      "logits/rejected": -1.2017271518707275,
      "logps/chosen": -109.01265716552734,
      "logps/rejected": -321.8054504394531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.274156332015991,
      "rewards/margins": 18.100004196166992,
      "rewards/rejected": -14.825848579406738,
      "step": 4470
    },
    {
      "epoch": 1.8152350081037278,
      "grad_norm": 4.612465545505984e-06,
      "learning_rate": 4.388413627495123e-06,
      "logits/chosen": -0.9677801132202148,
      "logits/rejected": -1.18340003490448,
      "logps/chosen": -108.96244049072266,
      "logps/rejected": -320.5141296386719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.234553098678589,
      "rewards/margins": 17.92672348022461,
      "rewards/rejected": -14.692172050476074,
      "step": 4480
    },
    {
      "epoch": 1.819286871961102,
      "grad_norm": 2.053400976365083e-06,
      "learning_rate": 4.373405372955125e-06,
      "logits/chosen": -0.9691886901855469,
      "logits/rejected": -1.1938700675964355,
      "logps/chosen": -109.02762603759766,
      "logps/rejected": -320.2586364746094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2385077476501465,
      "rewards/margins": 17.953767776489258,
      "rewards/rejected": -14.715259552001953,
      "step": 4490
    },
    {
      "epoch": 1.8233387358184765,
      "grad_norm": 5.526775566977449e-06,
      "learning_rate": 4.358397118415129e-06,
      "logits/chosen": -0.9505713582038879,
      "logits/rejected": -1.1996839046478271,
      "logps/chosen": -106.17263793945312,
      "logps/rejected": -322.22296142578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2986068725585938,
      "rewards/margins": 18.22445297241211,
      "rewards/rejected": -14.925846099853516,
      "step": 4500
    },
    {
      "epoch": 1.8233387358184765,
      "eval_logits/chosen": -0.9807091951370239,
      "eval_logits/rejected": -1.2058302164077759,
      "eval_logps/chosen": -109.1124267578125,
      "eval_logps/rejected": -320.0847473144531,
      "eval_loss": 7.469225948852909e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1899495124816895,
      "eval_rewards/margins": 17.90414810180664,
      "eval_rewards/rejected": -14.71419906616211,
      "eval_runtime": 660.4248,
      "eval_samples_per_second": 11.211,
      "eval_steps_per_second": 0.624,
      "step": 4500
    },
    {
      "epoch": 1.827390599675851,
      "grad_norm": 7.4045251494681e-06,
      "learning_rate": 4.343388863875131e-06,
      "logits/chosen": -0.9733620882034302,
      "logits/rejected": -1.2026481628417969,
      "logps/chosen": -108.2680435180664,
      "logps/rejected": -322.1900329589844,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2477121353149414,
      "rewards/margins": 18.169279098510742,
      "rewards/rejected": -14.921568870544434,
      "step": 4510
    },
    {
      "epoch": 1.8314424635332252,
      "grad_norm": 3.332833330205176e-06,
      "learning_rate": 4.328380609335135e-06,
      "logits/chosen": -0.966694712638855,
      "logits/rejected": -1.1901875734329224,
      "logps/chosen": -107.82235717773438,
      "logps/rejected": -322.0663757324219,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.270493745803833,
      "rewards/margins": 18.053882598876953,
      "rewards/rejected": -14.783388137817383,
      "step": 4520
    },
    {
      "epoch": 1.8354943273905997,
      "grad_norm": 3.3187377539434237e-06,
      "learning_rate": 4.313372354795138e-06,
      "logits/chosen": -0.9629502892494202,
      "logits/rejected": -1.1885720491409302,
      "logps/chosen": -108.7431869506836,
      "logps/rejected": -322.45867919921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2092270851135254,
      "rewards/margins": 18.075258255004883,
      "rewards/rejected": -14.866032600402832,
      "step": 4530
    },
    {
      "epoch": 1.839546191247974,
      "grad_norm": 4.360450475360267e-06,
      "learning_rate": 4.298364100255141e-06,
      "logits/chosen": -0.974517285823822,
      "logits/rejected": -1.1914209127426147,
      "logps/chosen": -108.77423858642578,
      "logps/rejected": -320.7426452636719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2381083965301514,
      "rewards/margins": 17.959489822387695,
      "rewards/rejected": -14.721381187438965,
      "step": 4540
    },
    {
      "epoch": 1.8435980551053484,
      "grad_norm": 2.29799570661271e-06,
      "learning_rate": 4.283355845715143e-06,
      "logits/chosen": -0.9552022814750671,
      "logits/rejected": -1.2001949548721313,
      "logps/chosen": -107.79722595214844,
      "logps/rejected": -321.5192565917969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.274078369140625,
      "rewards/margins": 18.0383358001709,
      "rewards/rejected": -14.76425838470459,
      "step": 4550
    },
    {
      "epoch": 1.8435980551053484,
      "eval_logits/chosen": -0.9842467904090881,
      "eval_logits/rejected": -1.2096879482269287,
      "eval_logps/chosen": -109.08003234863281,
      "eval_logps/rejected": -320.178466796875,
      "eval_loss": 7.409251878698342e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.193190813064575,
      "eval_rewards/margins": 17.91676139831543,
      "eval_rewards/rejected": -14.723569869995117,
      "eval_runtime": 660.4845,
      "eval_samples_per_second": 11.21,
      "eval_steps_per_second": 0.624,
      "step": 4550
    },
    {
      "epoch": 1.847649918962723,
      "grad_norm": 2.519145255064359e-06,
      "learning_rate": 4.268347591175147e-06,
      "logits/chosen": -0.9676612615585327,
      "logits/rejected": -1.1968157291412354,
      "logps/chosen": -108.7945327758789,
      "logps/rejected": -321.25469970703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.273383378982544,
      "rewards/margins": 17.944278717041016,
      "rewards/rejected": -14.670894622802734,
      "step": 4560
    },
    {
      "epoch": 1.8517017828200972,
      "grad_norm": 1.055786015058402e-05,
      "learning_rate": 4.253339336635149e-06,
      "logits/chosen": -0.9639263153076172,
      "logits/rejected": -1.2146022319793701,
      "logps/chosen": -108.0430908203125,
      "logps/rejected": -322.0449523925781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2333381175994873,
      "rewards/margins": 18.162742614746094,
      "rewards/rejected": -14.929403305053711,
      "step": 4570
    },
    {
      "epoch": 1.8557536466774716,
      "grad_norm": 3.715491629918688e-06,
      "learning_rate": 4.238331082095153e-06,
      "logits/chosen": -0.9830486178398132,
      "logits/rejected": -1.2109137773513794,
      "logps/chosen": -108.03145599365234,
      "logps/rejected": -321.6439514160156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2837610244750977,
      "rewards/margins": 18.121925354003906,
      "rewards/rejected": -14.838165283203125,
      "step": 4580
    },
    {
      "epoch": 1.859805510534846,
      "grad_norm": 3.2888442547118757e-06,
      "learning_rate": 4.223322827555156e-06,
      "logits/chosen": -0.9521467089653015,
      "logits/rejected": -1.2002835273742676,
      "logps/chosen": -108.80928802490234,
      "logps/rejected": -322.5191955566406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2589292526245117,
      "rewards/margins": 18.116209030151367,
      "rewards/rejected": -14.857280731201172,
      "step": 4590
    },
    {
      "epoch": 1.8638573743922204,
      "grad_norm": 3.29683621203003e-06,
      "learning_rate": 4.208314573015159e-06,
      "logits/chosen": -0.9633061289787292,
      "logits/rejected": -1.2022969722747803,
      "logps/chosen": -107.3350601196289,
      "logps/rejected": -320.0856628417969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3103652000427246,
      "rewards/margins": 18.04131507873535,
      "rewards/rejected": -14.730950355529785,
      "step": 4600
    },
    {
      "epoch": 1.8638573743922204,
      "eval_logits/chosen": -0.9786840081214905,
      "eval_logits/rejected": -1.202929139137268,
      "eval_logps/chosen": -109.06305694580078,
      "eval_logps/rejected": -320.3982849121094,
      "eval_loss": 7.233222021341135e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1948869228363037,
      "eval_rewards/margins": 17.940439224243164,
      "eval_rewards/rejected": -14.745550155639648,
      "eval_runtime": 660.4953,
      "eval_samples_per_second": 11.21,
      "eval_steps_per_second": 0.624,
      "step": 4600
    },
    {
      "epoch": 1.8679092382495948,
      "grad_norm": 4.971577709511621e-06,
      "learning_rate": 4.193306318475161e-06,
      "logits/chosen": -0.9493335485458374,
      "logits/rejected": -1.1981040239334106,
      "logps/chosen": -108.59446716308594,
      "logps/rejected": -320.84735107421875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2676663398742676,
      "rewards/margins": 18.01744270324707,
      "rewards/rejected": -14.749775886535645,
      "step": 4610
    },
    {
      "epoch": 1.8719611021069693,
      "grad_norm": 5.5384443840011954e-06,
      "learning_rate": 4.178298063935165e-06,
      "logits/chosen": -0.9665194749832153,
      "logits/rejected": -1.1918801069259644,
      "logps/chosen": -108.53473663330078,
      "logps/rejected": -321.12249755859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.248072385787964,
      "rewards/margins": 18.042972564697266,
      "rewards/rejected": -14.794903755187988,
      "step": 4620
    },
    {
      "epoch": 1.8760129659643436,
      "grad_norm": 6.041650067345472e-06,
      "learning_rate": 4.163289809395168e-06,
      "logits/chosen": -0.9764280915260315,
      "logits/rejected": -1.203399658203125,
      "logps/chosen": -108.33360290527344,
      "logps/rejected": -319.94610595703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.24983811378479,
      "rewards/margins": 17.952869415283203,
      "rewards/rejected": -14.703031539916992,
      "step": 4630
    },
    {
      "epoch": 1.880064829821718,
      "grad_norm": 2.5019787699420704e-06,
      "learning_rate": 4.148281554855171e-06,
      "logits/chosen": -0.9812704920768738,
      "logits/rejected": -1.2009217739105225,
      "logps/chosen": -108.75373077392578,
      "logps/rejected": -320.7171630859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.269096612930298,
      "rewards/margins": 17.954044342041016,
      "rewards/rejected": -14.684948921203613,
      "step": 4640
    },
    {
      "epoch": 1.8841166936790925,
      "grad_norm": 3.9666892917011864e-06,
      "learning_rate": 4.133273300315173e-06,
      "logits/chosen": -0.9929503202438354,
      "logits/rejected": -1.2013704776763916,
      "logps/chosen": -107.94710540771484,
      "logps/rejected": -321.26123046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2891159057617188,
      "rewards/margins": 18.05217742919922,
      "rewards/rejected": -14.7630615234375,
      "step": 4650
    },
    {
      "epoch": 1.8841166936790925,
      "eval_logits/chosen": -0.9783722162246704,
      "eval_logits/rejected": -1.202864170074463,
      "eval_logps/chosen": -109.06450653076172,
      "eval_logps/rejected": -320.7001647949219,
      "eval_loss": 7.021466785772645e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.194742441177368,
      "eval_rewards/margins": 17.970481872558594,
      "eval_rewards/rejected": -14.775740623474121,
      "eval_runtime": 660.4769,
      "eval_samples_per_second": 11.21,
      "eval_steps_per_second": 0.624,
      "step": 4650
    },
    {
      "epoch": 1.8881685575364666,
      "grad_norm": 3.1112033411773155e-06,
      "learning_rate": 4.118265045775177e-06,
      "logits/chosen": -0.967117190361023,
      "logits/rejected": -1.19990873336792,
      "logps/chosen": -108.15768432617188,
      "logps/rejected": -321.8457336425781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.283776044845581,
      "rewards/margins": 18.083831787109375,
      "rewards/rejected": -14.800055503845215,
      "step": 4660
    },
    {
      "epoch": 1.8922204213938412,
      "grad_norm": 2.5846559310593875e-06,
      "learning_rate": 4.103256791235179e-06,
      "logits/chosen": -0.9678542017936707,
      "logits/rejected": -1.1994032859802246,
      "logps/chosen": -107.25688171386719,
      "logps/rejected": -321.7806091308594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.290947437286377,
      "rewards/margins": 18.137258529663086,
      "rewards/rejected": -14.846309661865234,
      "step": 4670
    },
    {
      "epoch": 1.8962722852512157,
      "grad_norm": 5.48166917724302e-06,
      "learning_rate": 4.088248536695183e-06,
      "logits/chosen": -0.9686475396156311,
      "logits/rejected": -1.2068462371826172,
      "logps/chosen": -107.35350036621094,
      "logps/rejected": -321.7884826660156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2941713333129883,
      "rewards/margins": 18.086830139160156,
      "rewards/rejected": -14.7926607131958,
      "step": 4680
    },
    {
      "epoch": 1.9003241491085898,
      "grad_norm": 4.694249128078809e-06,
      "learning_rate": 4.073240282155186e-06,
      "logits/chosen": -0.9648269414901733,
      "logits/rejected": -1.2069343328475952,
      "logps/chosen": -107.65116882324219,
      "logps/rejected": -323.0342102050781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2500650882720947,
      "rewards/margins": 18.1915283203125,
      "rewards/rejected": -14.941463470458984,
      "step": 4690
    },
    {
      "epoch": 1.9043760129659644,
      "grad_norm": 5.962758677924285e-06,
      "learning_rate": 4.058232027615189e-06,
      "logits/chosen": -0.9601716995239258,
      "logits/rejected": -1.1852545738220215,
      "logps/chosen": -108.1944580078125,
      "logps/rejected": -320.3160095214844,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.262619972229004,
      "rewards/margins": 17.96957778930664,
      "rewards/rejected": -14.706958770751953,
      "step": 4700
    },
    {
      "epoch": 1.9043760129659644,
      "eval_logits/chosen": -0.9788442254066467,
      "eval_logits/rejected": -1.2020491361618042,
      "eval_logps/chosen": -109.03036499023438,
      "eval_logps/rejected": -320.9104919433594,
      "eval_loss": 6.850030587202127e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.198155164718628,
      "eval_rewards/margins": 17.994932174682617,
      "eval_rewards/rejected": -14.79677677154541,
      "eval_runtime": 660.5364,
      "eval_samples_per_second": 11.209,
      "eval_steps_per_second": 0.624,
      "step": 4700
    },
    {
      "epoch": 1.9084278768233387,
      "grad_norm": 2.8154815936431987e-06,
      "learning_rate": 4.043223773075191e-06,
      "logits/chosen": -0.9590250253677368,
      "logits/rejected": -1.198261022567749,
      "logps/chosen": -107.40636444091797,
      "logps/rejected": -322.0196533203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.272249937057495,
      "rewards/margins": 18.1776065826416,
      "rewards/rejected": -14.905356407165527,
      "step": 4710
    },
    {
      "epoch": 1.912479740680713,
      "grad_norm": 3.019512178070727e-06,
      "learning_rate": 4.028215518535195e-06,
      "logits/chosen": -0.9678371548652649,
      "logits/rejected": -1.1993069648742676,
      "logps/chosen": -108.16641998291016,
      "logps/rejected": -321.8016357421875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2627205848693848,
      "rewards/margins": 18.09998321533203,
      "rewards/rejected": -14.837262153625488,
      "step": 4720
    },
    {
      "epoch": 1.9165316045380876,
      "grad_norm": 4.8556812544120476e-06,
      "learning_rate": 4.013207263995197e-06,
      "logits/chosen": -0.977064311504364,
      "logits/rejected": -1.1923290491104126,
      "logps/chosen": -108.24919891357422,
      "logps/rejected": -322.5245361328125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2549569606781006,
      "rewards/margins": 18.176767349243164,
      "rewards/rejected": -14.921812057495117,
      "step": 4730
    },
    {
      "epoch": 1.920583468395462,
      "grad_norm": 2.067858531518141e-06,
      "learning_rate": 3.998199009455201e-06,
      "logits/chosen": -0.9402839541435242,
      "logits/rejected": -1.2003926038742065,
      "logps/chosen": -106.45325469970703,
      "logps/rejected": -324.4967956542969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3108747005462646,
      "rewards/margins": 18.355466842651367,
      "rewards/rejected": -15.044590950012207,
      "step": 4740
    },
    {
      "epoch": 1.9246353322528362,
      "grad_norm": 5.996467280056095e-06,
      "learning_rate": 3.983190754915203e-06,
      "logits/chosen": -0.9844788312911987,
      "logits/rejected": -1.195417881011963,
      "logps/chosen": -109.07532501220703,
      "logps/rejected": -322.93695068359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2502410411834717,
      "rewards/margins": 18.149274826049805,
      "rewards/rejected": -14.89903450012207,
      "step": 4750
    },
    {
      "epoch": 1.9246353322528362,
      "eval_logits/chosen": -0.9755988121032715,
      "eval_logits/rejected": -1.1990272998809814,
      "eval_logps/chosen": -109.0657958984375,
      "eval_logps/rejected": -321.15057373046875,
      "eval_loss": 6.730191159931564e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.1946122646331787,
      "eval_rewards/margins": 18.015398025512695,
      "eval_rewards/rejected": -14.820784568786621,
      "eval_runtime": 660.4034,
      "eval_samples_per_second": 11.211,
      "eval_steps_per_second": 0.624,
      "step": 4750
    },
    {
      "epoch": 1.9286871961102108,
      "grad_norm": 3.9852939153206535e-06,
      "learning_rate": 3.968182500375207e-06,
      "logits/chosen": -0.9592065215110779,
      "logits/rejected": -1.2046445608139038,
      "logps/chosen": -107.64582824707031,
      "logps/rejected": -321.4580078125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.260378837585449,
      "rewards/margins": 18.183618545532227,
      "rewards/rejected": -14.923238754272461,
      "step": 4760
    },
    {
      "epoch": 1.932739059967585,
      "grad_norm": 3.716176934176474e-06,
      "learning_rate": 3.953174245835209e-06,
      "logits/chosen": -0.9753321409225464,
      "logits/rejected": -1.2032028436660767,
      "logps/chosen": -108.28763580322266,
      "logps/rejected": -321.4541015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2404139041900635,
      "rewards/margins": 17.986602783203125,
      "rewards/rejected": -14.746188163757324,
      "step": 4770
    },
    {
      "epoch": 1.9367909238249594,
      "grad_norm": 3.635807843238581e-06,
      "learning_rate": 3.938165991295213e-06,
      "logits/chosen": -0.9639159440994263,
      "logits/rejected": -1.196582555770874,
      "logps/chosen": -108.30272674560547,
      "logps/rejected": -323.1302185058594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2664296627044678,
      "rewards/margins": 18.24969482421875,
      "rewards/rejected": -14.98326587677002,
      "step": 4780
    },
    {
      "epoch": 1.940842787682334,
      "grad_norm": 3.2924308470683172e-06,
      "learning_rate": 3.923157736755216e-06,
      "logits/chosen": -0.972676694393158,
      "logits/rejected": -1.176472544670105,
      "logps/chosen": -108.53832244873047,
      "logps/rejected": -322.3914794921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2719764709472656,
      "rewards/margins": 18.195058822631836,
      "rewards/rejected": -14.923080444335938,
      "step": 4790
    },
    {
      "epoch": 1.9448946515397083,
      "grad_norm": 2.680646275621257e-06,
      "learning_rate": 3.908149482215219e-06,
      "logits/chosen": -0.9661304354667664,
      "logits/rejected": -1.1976679563522339,
      "logps/chosen": -107.57020568847656,
      "logps/rejected": -323.94757080078125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2382466793060303,
      "rewards/margins": 18.25577735900879,
      "rewards/rejected": -15.017531394958496,
      "step": 4800
    },
    {
      "epoch": 1.9448946515397083,
      "eval_logits/chosen": -0.9803680181503296,
      "eval_logits/rejected": -1.2040807008743286,
      "eval_logps/chosen": -109.0024185180664,
      "eval_logps/rejected": -321.3273620605469,
      "eval_loss": 6.577239730631845e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2009496688842773,
      "eval_rewards/margins": 18.039411544799805,
      "eval_rewards/rejected": -14.838460922241211,
      "eval_runtime": 660.4735,
      "eval_samples_per_second": 11.21,
      "eval_steps_per_second": 0.624,
      "step": 4800
    },
    {
      "epoch": 1.9489465153970826,
      "grad_norm": 2.6189354684902355e-06,
      "learning_rate": 3.893141227675221e-06,
      "logits/chosen": -0.9728685021400452,
      "logits/rejected": -1.2008436918258667,
      "logps/chosen": -108.86917114257812,
      "logps/rejected": -323.2511901855469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.238666534423828,
      "rewards/margins": 18.227935791015625,
      "rewards/rejected": -14.989269256591797,
      "step": 4810
    },
    {
      "epoch": 1.9529983792544572,
      "grad_norm": 1.0494446541997604e-05,
      "learning_rate": 3.878132973135225e-06,
      "logits/chosen": -0.9686774611473083,
      "logits/rejected": -1.1878249645233154,
      "logps/chosen": -109.44641876220703,
      "logps/rejected": -322.2353210449219,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2077255249023438,
      "rewards/margins": 18.08070182800293,
      "rewards/rejected": -14.872976303100586,
      "step": 4820
    },
    {
      "epoch": 1.9570502431118313,
      "grad_norm": 2.662289489308023e-06,
      "learning_rate": 3.863124718595227e-06,
      "logits/chosen": -0.9683530926704407,
      "logits/rejected": -1.205175518989563,
      "logps/chosen": -108.19351959228516,
      "logps/rejected": -323.36212158203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.245283842086792,
      "rewards/margins": 18.26123046875,
      "rewards/rejected": -15.015947341918945,
      "step": 4830
    },
    {
      "epoch": 1.9611021069692058,
      "grad_norm": 3.1114741432247683e-06,
      "learning_rate": 3.848116464055231e-06,
      "logits/chosen": -0.9755093455314636,
      "logits/rejected": -1.1997181177139282,
      "logps/chosen": -108.98912048339844,
      "logps/rejected": -323.6244201660156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2453114986419678,
      "rewards/margins": 18.16748046875,
      "rewards/rejected": -14.922168731689453,
      "step": 4840
    },
    {
      "epoch": 1.9651539708265804,
      "grad_norm": 5.440684617497027e-06,
      "learning_rate": 3.833108209515234e-06,
      "logits/chosen": -0.961645245552063,
      "logits/rejected": -1.2061594724655151,
      "logps/chosen": -108.36994171142578,
      "logps/rejected": -321.2346496582031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2903635501861572,
      "rewards/margins": 18.091514587402344,
      "rewards/rejected": -14.80115032196045,
      "step": 4850
    },
    {
      "epoch": 1.9651539708265804,
      "eval_logits/chosen": -0.9778537154197693,
      "eval_logits/rejected": -1.200571894645691,
      "eval_logps/chosen": -109.01957702636719,
      "eval_logps/rejected": -321.4977111816406,
      "eval_loss": 6.502763483240415e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.199235439300537,
      "eval_rewards/margins": 18.054733276367188,
      "eval_rewards/rejected": -14.855497360229492,
      "eval_runtime": 660.4953,
      "eval_samples_per_second": 11.21,
      "eval_steps_per_second": 0.624,
      "step": 4850
    },
    {
      "epoch": 1.9692058346839545,
      "grad_norm": 3.1822473829379305e-06,
      "learning_rate": 3.818099954975237e-06,
      "logits/chosen": -0.9647846817970276,
      "logits/rejected": -1.1994272470474243,
      "logps/chosen": -108.41690826416016,
      "logps/rejected": -323.28631591796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.290191173553467,
      "rewards/margins": 18.278629302978516,
      "rewards/rejected": -14.988438606262207,
      "step": 4860
    },
    {
      "epoch": 1.973257698541329,
      "grad_norm": 2.5745391667442163e-06,
      "learning_rate": 3.80309170043524e-06,
      "logits/chosen": -0.9744833111763,
      "logits/rejected": -1.1996198892593384,
      "logps/chosen": -107.52901458740234,
      "logps/rejected": -323.8044128417969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2914023399353027,
      "rewards/margins": 18.269445419311523,
      "rewards/rejected": -14.978041648864746,
      "step": 4870
    },
    {
      "epoch": 1.9773095623987034,
      "grad_norm": 4.465393885766389e-06,
      "learning_rate": 3.7880834458952424e-06,
      "logits/chosen": -0.9575149416923523,
      "logits/rejected": -1.1838562488555908,
      "logps/chosen": -107.7874984741211,
      "logps/rejected": -323.15985107421875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2660937309265137,
      "rewards/margins": 18.230712890625,
      "rewards/rejected": -14.964617729187012,
      "step": 4880
    },
    {
      "epoch": 1.9813614262560777,
      "grad_norm": 2.848352551154676e-06,
      "learning_rate": 3.7730751913552454e-06,
      "logits/chosen": -0.9726042151451111,
      "logits/rejected": -1.202985167503357,
      "logps/chosen": -109.6990737915039,
      "logps/rejected": -323.98907470703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2264673709869385,
      "rewards/margins": 18.20193099975586,
      "rewards/rejected": -14.9754638671875,
      "step": 4890
    },
    {
      "epoch": 1.9854132901134522,
      "grad_norm": 2.8638380626944127e-06,
      "learning_rate": 3.758066936815249e-06,
      "logits/chosen": -0.970777153968811,
      "logits/rejected": -1.1936142444610596,
      "logps/chosen": -108.6916732788086,
      "logps/rejected": -322.7779846191406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.31592059135437,
      "rewards/margins": 18.30429458618164,
      "rewards/rejected": -14.988374710083008,
      "step": 4900
    },
    {
      "epoch": 1.9854132901134522,
      "eval_logits/chosen": -0.9825440049171448,
      "eval_logits/rejected": -1.20473051071167,
      "eval_logps/chosen": -109.0003890991211,
      "eval_logps/rejected": -321.789794921875,
      "eval_loss": 6.325069534796057e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2011539936065674,
      "eval_rewards/margins": 18.08585548400879,
      "eval_rewards/rejected": -14.884700775146484,
      "eval_runtime": 660.5882,
      "eval_samples_per_second": 11.208,
      "eval_steps_per_second": 0.624,
      "step": 4900
    },
    {
      "epoch": 1.9894651539708266,
      "grad_norm": 2.4442233552690595e-05,
      "learning_rate": 3.743058682275252e-06,
      "logits/chosen": -0.9682837128639221,
      "logits/rejected": -1.1897228956222534,
      "logps/chosen": -108.81273651123047,
      "logps/rejected": -321.91217041015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2439818382263184,
      "rewards/margins": 18.088844299316406,
      "rewards/rejected": -14.844861030578613,
      "step": 4910
    },
    {
      "epoch": 1.993517017828201,
      "grad_norm": 2.8480183118517743e-06,
      "learning_rate": 3.728050427735255e-06,
      "logits/chosen": -0.9716948866844177,
      "logits/rejected": -1.217131495475769,
      "logps/chosen": -107.93961334228516,
      "logps/rejected": -322.7872009277344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2950081825256348,
      "rewards/margins": 18.22598648071289,
      "rewards/rejected": -14.930974960327148,
      "step": 4920
    },
    {
      "epoch": 1.9975688816855754,
      "grad_norm": 3.2138655114977155e-06,
      "learning_rate": 3.7130421731952575e-06,
      "logits/chosen": -0.9685840606689453,
      "logits/rejected": -1.2002424001693726,
      "logps/chosen": -109.53565979003906,
      "logps/rejected": -323.6005554199219,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.233421802520752,
      "rewards/margins": 18.15231704711914,
      "rewards/rejected": -14.91889476776123,
      "step": 4930
    },
    {
      "epoch": 2.0016207455429496,
      "grad_norm": 1.2053304999426473e-05,
      "learning_rate": 3.6980339186552605e-06,
      "logits/chosen": -0.9570005536079407,
      "logits/rejected": -1.176553726196289,
      "logps/chosen": -107.43694305419922,
      "logps/rejected": -323.06964111328125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2987098693847656,
      "rewards/margins": 18.21400260925293,
      "rewards/rejected": -14.91529369354248,
      "step": 4940
    },
    {
      "epoch": 2.005672609400324,
      "grad_norm": 3.1240876978699816e-06,
      "learning_rate": 3.683025664115264e-06,
      "logits/chosen": -0.9605010747909546,
      "logits/rejected": -1.1978155374526978,
      "logps/chosen": -107.3221435546875,
      "logps/rejected": -322.78021240234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2937347888946533,
      "rewards/margins": 18.2900390625,
      "rewards/rejected": -14.996304512023926,
      "step": 4950
    },
    {
      "epoch": 2.005672609400324,
      "eval_logits/chosen": -0.9806666970252991,
      "eval_logits/rejected": -1.2025675773620605,
      "eval_logps/chosen": -109.00515747070312,
      "eval_logps/rejected": -322.0491027832031,
      "eval_loss": 6.160802712429359e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2006771564483643,
      "eval_rewards/margins": 18.11131477355957,
      "eval_rewards/rejected": -14.910634994506836,
      "eval_runtime": 660.5404,
      "eval_samples_per_second": 11.209,
      "eval_steps_per_second": 0.624,
      "step": 4950
    },
    {
      "epoch": 2.0097244732576987,
      "grad_norm": 3.8034179397072876e-06,
      "learning_rate": 3.668017409575267e-06,
      "logits/chosen": -0.9739381074905396,
      "logits/rejected": -1.2024341821670532,
      "logps/chosen": -107.70148468017578,
      "logps/rejected": -322.88226318359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.279003620147705,
      "rewards/margins": 18.232358932495117,
      "rewards/rejected": -14.953354835510254,
      "step": 4960
    },
    {
      "epoch": 2.0137763371150728,
      "grad_norm": 4.456906026462093e-06,
      "learning_rate": 3.65300915503527e-06,
      "logits/chosen": -0.9753895998001099,
      "logits/rejected": -1.188686728477478,
      "logps/chosen": -108.8790054321289,
      "logps/rejected": -324.54022216796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.249891996383667,
      "rewards/margins": 18.289875030517578,
      "rewards/rejected": -15.0399808883667,
      "step": 4970
    },
    {
      "epoch": 2.0178282009724473,
      "grad_norm": 4.742184955830453e-06,
      "learning_rate": 3.6380009004952725e-06,
      "logits/chosen": -0.9557638168334961,
      "logits/rejected": -1.2003400325775146,
      "logps/chosen": -107.69667053222656,
      "logps/rejected": -324.84954833984375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2877097129821777,
      "rewards/margins": 18.38192367553711,
      "rewards/rejected": -15.094216346740723,
      "step": 4980
    },
    {
      "epoch": 2.021880064829822,
      "grad_norm": 1.4145489330985583e-05,
      "learning_rate": 3.6229926459552755e-06,
      "logits/chosen": -0.9673196077346802,
      "logits/rejected": -1.2091333866119385,
      "logps/chosen": -107.90608978271484,
      "logps/rejected": -323.47991943359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3074872493743896,
      "rewards/margins": 18.368791580200195,
      "rewards/rejected": -15.06130313873291,
      "step": 4990
    },
    {
      "epoch": 2.025931928687196,
      "grad_norm": 3.324122644698946e-06,
      "learning_rate": 3.6079843914152785e-06,
      "logits/chosen": -0.9694700241088867,
      "logits/rejected": -1.197292685508728,
      "logps/chosen": -108.34423828125,
      "logps/rejected": -322.24957275390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2302982807159424,
      "rewards/margins": 18.173320770263672,
      "rewards/rejected": -14.943023681640625,
      "step": 5000
    },
    {
      "epoch": 2.025931928687196,
      "eval_logits/chosen": -0.9766953587532043,
      "eval_logits/rejected": -1.1986485719680786,
      "eval_logps/chosen": -108.99577331542969,
      "eval_logps/rejected": -322.276123046875,
      "eval_loss": 6.022185061738128e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2016162872314453,
      "eval_rewards/margins": 18.134952545166016,
      "eval_rewards/rejected": -14.933335304260254,
      "eval_runtime": 660.5052,
      "eval_samples_per_second": 11.21,
      "eval_steps_per_second": 0.624,
      "step": 5000
    },
    {
      "epoch": 2.0299837925445705,
      "grad_norm": 2.3767845505062724e-06,
      "learning_rate": 3.592976136875282e-06,
      "logits/chosen": -0.965735137462616,
      "logits/rejected": -1.2081549167633057,
      "logps/chosen": -108.78414916992188,
      "logps/rejected": -323.2518005371094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.245922803878784,
      "rewards/margins": 18.177051544189453,
      "rewards/rejected": -14.931127548217773,
      "step": 5010
    },
    {
      "epoch": 2.034035656401945,
      "grad_norm": 3.4225790841446724e-06,
      "learning_rate": 3.577967882335285e-06,
      "logits/chosen": -0.9643774628639221,
      "logits/rejected": -1.2113484144210815,
      "logps/chosen": -107.67012786865234,
      "logps/rejected": -323.9315185546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3138530254364014,
      "rewards/margins": 18.391572952270508,
      "rewards/rejected": -15.077719688415527,
      "step": 5020
    },
    {
      "epoch": 2.038087520259319,
      "grad_norm": 2.2828062355984002e-06,
      "learning_rate": 3.5629596277952875e-06,
      "logits/chosen": -0.986914873123169,
      "logits/rejected": -1.1938860416412354,
      "logps/chosen": -109.88799285888672,
      "logps/rejected": -323.33282470703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.202239751815796,
      "rewards/margins": 18.149791717529297,
      "rewards/rejected": -14.947552680969238,
      "step": 5030
    },
    {
      "epoch": 2.0421393841166937,
      "grad_norm": 3.3493358841951704e-06,
      "learning_rate": 3.5479513732552905e-06,
      "logits/chosen": -0.9507116079330444,
      "logits/rejected": -1.1881414651870728,
      "logps/chosen": -107.72386932373047,
      "logps/rejected": -323.3999938964844,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3047282695770264,
      "rewards/margins": 18.352764129638672,
      "rewards/rejected": -15.048036575317383,
      "step": 5040
    },
    {
      "epoch": 2.0461912479740683,
      "grad_norm": 2.662564611455309e-06,
      "learning_rate": 3.5329431187152935e-06,
      "logits/chosen": -0.969707727432251,
      "logits/rejected": -1.2067253589630127,
      "logps/chosen": -108.2742691040039,
      "logps/rejected": -325.4738464355469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2910315990448,
      "rewards/margins": 18.40937042236328,
      "rewards/rejected": -15.118340492248535,
      "step": 5050
    },
    {
      "epoch": 2.0461912479740683,
      "eval_logits/chosen": -0.9814975261688232,
      "eval_logits/rejected": -1.2026540040969849,
      "eval_logps/chosen": -109.00423431396484,
      "eval_logps/rejected": -322.4521484375,
      "eval_loss": 5.9376258576548935e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2007694244384766,
      "eval_rewards/margins": 18.151708602905273,
      "eval_rewards/rejected": -14.950939178466797,
      "eval_runtime": 660.3857,
      "eval_samples_per_second": 11.212,
      "eval_steps_per_second": 0.624,
      "step": 5050
    },
    {
      "epoch": 2.0502431118314424,
      "grad_norm": 2.619849965412868e-06,
      "learning_rate": 3.517934864175297e-06,
      "logits/chosen": -0.9573184847831726,
      "logits/rejected": -1.1907227039337158,
      "logps/chosen": -107.8059310913086,
      "logps/rejected": -323.6286315917969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.290532350540161,
      "rewards/margins": 18.276216506958008,
      "rewards/rejected": -14.985685348510742,
      "step": 5060
    },
    {
      "epoch": 2.054294975688817,
      "grad_norm": 4.405702384246979e-06,
      "learning_rate": 3.5029266096353e-06,
      "logits/chosen": -0.967184841632843,
      "logits/rejected": -1.199464201927185,
      "logps/chosen": -107.3981704711914,
      "logps/rejected": -323.74755859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2805373668670654,
      "rewards/margins": 18.303253173828125,
      "rewards/rejected": -15.022714614868164,
      "step": 5070
    },
    {
      "epoch": 2.0583468395461915,
      "grad_norm": 7.713757440797053e-06,
      "learning_rate": 3.487918355095303e-06,
      "logits/chosen": -0.9631035327911377,
      "logits/rejected": -1.2093658447265625,
      "logps/chosen": -108.29039001464844,
      "logps/rejected": -323.6645812988281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2754151821136475,
      "rewards/margins": 18.31420135498047,
      "rewards/rejected": -15.038785934448242,
      "step": 5080
    },
    {
      "epoch": 2.0623987034035656,
      "grad_norm": 1.4878126421535853e-05,
      "learning_rate": 3.4729101005553055e-06,
      "logits/chosen": -0.9696425795555115,
      "logits/rejected": -1.2049078941345215,
      "logps/chosen": -109.19393920898438,
      "logps/rejected": -324.4574890136719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.255068063735962,
      "rewards/margins": 18.34342384338379,
      "rewards/rejected": -15.088356971740723,
      "step": 5090
    },
    {
      "epoch": 2.06645056726094,
      "grad_norm": 2.702791562114726e-06,
      "learning_rate": 3.4579018460153085e-06,
      "logits/chosen": -0.9682046175003052,
      "logits/rejected": -1.2046129703521729,
      "logps/chosen": -107.95789337158203,
      "logps/rejected": -324.50592041015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.274998188018799,
      "rewards/margins": 18.40148162841797,
      "rewards/rejected": -15.126481056213379,
      "step": 5100
    },
    {
      "epoch": 2.06645056726094,
      "eval_logits/chosen": -0.9781537055969238,
      "eval_logits/rejected": -1.2004783153533936,
      "eval_logps/chosen": -108.95247650146484,
      "eval_logps/rejected": -322.5989685058594,
      "eval_loss": 5.821843984676889e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2059435844421387,
      "eval_rewards/margins": 18.17156410217285,
      "eval_rewards/rejected": -14.965620994567871,
      "eval_runtime": 660.3943,
      "eval_samples_per_second": 11.211,
      "eval_steps_per_second": 0.624,
      "step": 5100
    },
    {
      "epoch": 2.0705024311183142,
      "grad_norm": 7.003120117587969e-06,
      "learning_rate": 3.4428935914753115e-06,
      "logits/chosen": -0.9700896143913269,
      "logits/rejected": -1.196824312210083,
      "logps/chosen": -109.01053619384766,
      "logps/rejected": -323.7058410644531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2263805866241455,
      "rewards/margins": 18.2646484375,
      "rewards/rejected": -15.0382661819458,
      "step": 5110
    },
    {
      "epoch": 2.0745542949756888,
      "grad_norm": 4.903948138235137e-06,
      "learning_rate": 3.427885336935315e-06,
      "logits/chosen": -0.9765785336494446,
      "logits/rejected": -1.1949435472488403,
      "logps/chosen": -108.376220703125,
      "logps/rejected": -322.2400817871094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2593820095062256,
      "rewards/margins": 18.187374114990234,
      "rewards/rejected": -14.92799186706543,
      "step": 5120
    },
    {
      "epoch": 2.0786061588330633,
      "grad_norm": 6.223580385267269e-06,
      "learning_rate": 3.412877082395318e-06,
      "logits/chosen": -0.9860423803329468,
      "logits/rejected": -1.2049118280410767,
      "logps/chosen": -108.06117248535156,
      "logps/rejected": -323.1683654785156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.259824514389038,
      "rewards/margins": 18.323781967163086,
      "rewards/rejected": -15.063958168029785,
      "step": 5130
    },
    {
      "epoch": 2.0826580226904374,
      "grad_norm": 5.068562131782528e-06,
      "learning_rate": 3.3978688278553205e-06,
      "logits/chosen": -0.9900915026664734,
      "logits/rejected": -1.203590750694275,
      "logps/chosen": -108.75131225585938,
      "logps/rejected": -322.52978515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.251455783843994,
      "rewards/margins": 18.286420822143555,
      "rewards/rejected": -15.034967422485352,
      "step": 5140
    },
    {
      "epoch": 2.086709886547812,
      "grad_norm": 6.225033757800702e-06,
      "learning_rate": 3.3828605733153235e-06,
      "logits/chosen": -0.9706783294677734,
      "logits/rejected": -1.197087049484253,
      "logps/chosen": -107.81609344482422,
      "logps/rejected": -324.9122009277344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3132641315460205,
      "rewards/margins": 18.430654525756836,
      "rewards/rejected": -15.117389678955078,
      "step": 5150
    },
    {
      "epoch": 2.086709886547812,
      "eval_logits/chosen": -0.9727792143821716,
      "eval_logits/rejected": -1.1945335865020752,
      "eval_logps/chosen": -108.9156723022461,
      "eval_logps/rejected": -322.7347412109375,
      "eval_loss": 5.735015307095637e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.209625005722046,
      "eval_rewards/margins": 18.18882179260254,
      "eval_rewards/rejected": -14.979195594787598,
      "eval_runtime": 660.3492,
      "eval_samples_per_second": 11.212,
      "eval_steps_per_second": 0.624,
      "step": 5150
    },
    {
      "epoch": 2.0907617504051865,
      "grad_norm": 1.959013388841413e-06,
      "learning_rate": 3.3678523187753265e-06,
      "logits/chosen": -0.9617520570755005,
      "logits/rejected": -1.209423303604126,
      "logps/chosen": -107.73186492919922,
      "logps/rejected": -325.5774841308594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.303151845932007,
      "rewards/margins": 18.433210372924805,
      "rewards/rejected": -15.130060195922852,
      "step": 5160
    },
    {
      "epoch": 2.0948136142625606,
      "grad_norm": 1.271650762646459e-05,
      "learning_rate": 3.35284406423533e-06,
      "logits/chosen": -0.9642054438591003,
      "logits/rejected": -1.1936373710632324,
      "logps/chosen": -107.96542358398438,
      "logps/rejected": -324.3305358886719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.314751625061035,
      "rewards/margins": 18.342897415161133,
      "rewards/rejected": -15.028146743774414,
      "step": 5170
    },
    {
      "epoch": 2.098865478119935,
      "grad_norm": 3.4079785109497607e-06,
      "learning_rate": 3.337835809695333e-06,
      "logits/chosen": -0.975328803062439,
      "logits/rejected": -1.1903932094573975,
      "logps/chosen": -108.29900360107422,
      "logps/rejected": -323.8041687011719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.284480571746826,
      "rewards/margins": 18.29322052001953,
      "rewards/rejected": -15.008742332458496,
      "step": 5180
    },
    {
      "epoch": 2.1029173419773097,
      "grad_norm": 5.6164235502365045e-06,
      "learning_rate": 3.3228275551553356e-06,
      "logits/chosen": -0.9733341932296753,
      "logits/rejected": -1.199615478515625,
      "logps/chosen": -108.6965560913086,
      "logps/rejected": -323.5589904785156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.225916624069214,
      "rewards/margins": 18.289762496948242,
      "rewards/rejected": -15.063846588134766,
      "step": 5190
    },
    {
      "epoch": 2.106969205834684,
      "grad_norm": 1.4267629012465477e-05,
      "learning_rate": 3.3078193006153386e-06,
      "logits/chosen": -0.97093266248703,
      "logits/rejected": -1.1911667585372925,
      "logps/chosen": -108.9158706665039,
      "logps/rejected": -323.22509765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.261925458908081,
      "rewards/margins": 18.200176239013672,
      "rewards/rejected": -14.938252449035645,
      "step": 5200
    },
    {
      "epoch": 2.106969205834684,
      "eval_logits/chosen": -0.9828716516494751,
      "eval_logits/rejected": -1.2042107582092285,
      "eval_logps/chosen": -108.9614028930664,
      "eval_logps/rejected": -323.08966064453125,
      "eval_loss": 5.55416228564809e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.205052614212036,
      "eval_rewards/margins": 18.219741821289062,
      "eval_rewards/rejected": -15.014689445495605,
      "eval_runtime": 660.2431,
      "eval_samples_per_second": 11.214,
      "eval_steps_per_second": 0.624,
      "step": 5200
    },
    {
      "epoch": 2.1110210696920584,
      "grad_norm": 6.4765258684928995e-06,
      "learning_rate": 3.2928110460753416e-06,
      "logits/chosen": -0.9671388268470764,
      "logits/rejected": -1.1946061849594116,
      "logps/chosen": -107.32749938964844,
      "logps/rejected": -322.5291442871094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.310100793838501,
      "rewards/margins": 18.20550537109375,
      "rewards/rejected": -14.895402908325195,
      "step": 5210
    },
    {
      "epoch": 2.115072933549433,
      "grad_norm": 4.7925177568686195e-06,
      "learning_rate": 3.277802791535345e-06,
      "logits/chosen": -0.9682778120040894,
      "logits/rejected": -1.2034763097763062,
      "logps/chosen": -107.63131713867188,
      "logps/rejected": -323.91400146484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2772161960601807,
      "rewards/margins": 18.31756591796875,
      "rewards/rejected": -15.040349006652832,
      "step": 5220
    },
    {
      "epoch": 2.119124797406807,
      "grad_norm": 3.860869583149906e-06,
      "learning_rate": 3.262794536995348e-06,
      "logits/chosen": -0.9654537439346313,
      "logits/rejected": -1.1890361309051514,
      "logps/chosen": -107.89448547363281,
      "logps/rejected": -325.5076904296875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2871017456054688,
      "rewards/margins": 18.467273712158203,
      "rewards/rejected": -15.180171966552734,
      "step": 5230
    },
    {
      "epoch": 2.1231766612641816,
      "grad_norm": 4.159672243986279e-06,
      "learning_rate": 3.2477862824553506e-06,
      "logits/chosen": -0.9758961796760559,
      "logits/rejected": -1.1945488452911377,
      "logps/chosen": -108.34893035888672,
      "logps/rejected": -323.9273681640625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.283567190170288,
      "rewards/margins": 18.373409271240234,
      "rewards/rejected": -15.089844703674316,
      "step": 5240
    },
    {
      "epoch": 2.1272285251215557,
      "grad_norm": 6.8004319473402575e-06,
      "learning_rate": 3.2327780279153536e-06,
      "logits/chosen": -0.9748916029930115,
      "logits/rejected": -1.1907340288162231,
      "logps/chosen": -107.96294403076172,
      "logps/rejected": -323.5213317871094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2687954902648926,
      "rewards/margins": 18.268680572509766,
      "rewards/rejected": -14.999883651733398,
      "step": 5250
    },
    {
      "epoch": 2.1272285251215557,
      "eval_logits/chosen": -0.98344886302948,
      "eval_logits/rejected": -1.2038557529449463,
      "eval_logps/chosen": -108.96256256103516,
      "eval_logps/rejected": -323.3043518066406,
      "eval_loss": 5.466652197583244e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.204936981201172,
      "eval_rewards/margins": 18.241092681884766,
      "eval_rewards/rejected": -15.036153793334961,
      "eval_runtime": 660.1991,
      "eval_samples_per_second": 11.215,
      "eval_steps_per_second": 0.624,
      "step": 5250
    },
    {
      "epoch": 2.1312803889789302,
      "grad_norm": 1.8782106963044498e-06,
      "learning_rate": 3.2177697733753566e-06,
      "logits/chosen": -0.966810941696167,
      "logits/rejected": -1.203284502029419,
      "logps/chosen": -107.60391998291016,
      "logps/rejected": -323.45367431640625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.270557165145874,
      "rewards/margins": 18.329856872558594,
      "rewards/rejected": -15.059297561645508,
      "step": 5260
    },
    {
      "epoch": 2.135332252836305,
      "grad_norm": 2.830571929735015e-06,
      "learning_rate": 3.2027615188353596e-06,
      "logits/chosen": -0.9731746315956116,
      "logits/rejected": -1.197896122932434,
      "logps/chosen": -108.33968353271484,
      "logps/rejected": -323.3170166015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.293929100036621,
      "rewards/margins": 18.29970359802246,
      "rewards/rejected": -15.005772590637207,
      "step": 5270
    },
    {
      "epoch": 2.139384116693679,
      "grad_norm": 4.248033746989677e-06,
      "learning_rate": 3.187753264295363e-06,
      "logits/chosen": -0.9772968888282776,
      "logits/rejected": -1.196101427078247,
      "logps/chosen": -108.44367980957031,
      "logps/rejected": -322.9093017578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.244723081588745,
      "rewards/margins": 18.279142379760742,
      "rewards/rejected": -15.034419059753418,
      "step": 5280
    },
    {
      "epoch": 2.1434359805510534,
      "grad_norm": 1.4391961485671345e-05,
      "learning_rate": 3.172745009755366e-06,
      "logits/chosen": -0.9665384888648987,
      "logits/rejected": -1.1853383779525757,
      "logps/chosen": -108.29757690429688,
      "logps/rejected": -324.8900451660156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2725987434387207,
      "rewards/margins": 18.356849670410156,
      "rewards/rejected": -15.084248542785645,
      "step": 5290
    },
    {
      "epoch": 2.147487844408428,
      "grad_norm": 1.2224590136611369e-05,
      "learning_rate": 3.1577367552153686e-06,
      "logits/chosen": -0.9722302556037903,
      "logits/rejected": -1.1998592615127563,
      "logps/chosen": -107.06742858886719,
      "logps/rejected": -324.5314025878906,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.324317455291748,
      "rewards/margins": 18.51911735534668,
      "rewards/rejected": -15.19480037689209,
      "step": 5300
    },
    {
      "epoch": 2.147487844408428,
      "eval_logits/chosen": -0.9770769476890564,
      "eval_logits/rejected": -1.1975880861282349,
      "eval_logps/chosen": -108.89502716064453,
      "eval_logps/rejected": -323.43988037109375,
      "eval_loss": 5.37000168776558e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.211690664291382,
      "eval_rewards/margins": 18.26140022277832,
      "eval_rewards/rejected": -15.049712181091309,
      "eval_runtime": 660.2684,
      "eval_samples_per_second": 11.214,
      "eval_steps_per_second": 0.624,
      "step": 5300
    },
    {
      "epoch": 2.151539708265802,
      "grad_norm": 2.9248267310322262e-06,
      "learning_rate": 3.1427285006753716e-06,
      "logits/chosen": -0.973681628704071,
      "logits/rejected": -1.1873669624328613,
      "logps/chosen": -107.95748901367188,
      "logps/rejected": -325.4280700683594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2351701259613037,
      "rewards/margins": 18.447160720825195,
      "rewards/rejected": -15.211992263793945,
      "step": 5310
    },
    {
      "epoch": 2.1555915721231766,
      "grad_norm": 1.981665036510094e-06,
      "learning_rate": 3.1277202461353746e-06,
      "logits/chosen": -0.9693456888198853,
      "logits/rejected": -1.2035704851150513,
      "logps/chosen": -108.5029296875,
      "logps/rejected": -323.0021667480469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2811708450317383,
      "rewards/margins": 18.27227210998535,
      "rewards/rejected": -14.991100311279297,
      "step": 5320
    },
    {
      "epoch": 2.159643435980551,
      "grad_norm": 1.3219345191828324e-06,
      "learning_rate": 3.112711991595378e-06,
      "logits/chosen": -0.9648329615592957,
      "logits/rejected": -1.1890065670013428,
      "logps/chosen": -108.34164428710938,
      "logps/rejected": -324.6376037597656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.306445360183716,
      "rewards/margins": 18.42658042907715,
      "rewards/rejected": -15.120135307312012,
      "step": 5330
    },
    {
      "epoch": 2.1636952998379253,
      "grad_norm": 3.4673894333536737e-06,
      "learning_rate": 3.097703737055381e-06,
      "logits/chosen": -0.9712263345718384,
      "logits/rejected": -1.203616976737976,
      "logps/chosen": -108.90235137939453,
      "logps/rejected": -324.6282043457031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2427356243133545,
      "rewards/margins": 18.48871612548828,
      "rewards/rejected": -15.24598217010498,
      "step": 5340
    },
    {
      "epoch": 2.1677471636953,
      "grad_norm": 3.3253777473873924e-06,
      "learning_rate": 3.0826954825153836e-06,
      "logits/chosen": -0.964225709438324,
      "logits/rejected": -1.1941708326339722,
      "logps/chosen": -106.68071746826172,
      "logps/rejected": -323.4434509277344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3373284339904785,
      "rewards/margins": 18.364030838012695,
      "rewards/rejected": -15.026701927185059,
      "step": 5350
    },
    {
      "epoch": 2.1677471636953,
      "eval_logits/chosen": -0.9809548854827881,
      "eval_logits/rejected": -1.2015058994293213,
      "eval_logps/chosen": -108.94307708740234,
      "eval_logps/rejected": -323.71051025390625,
      "eval_loss": 5.263171587444049e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2068848609924316,
      "eval_rewards/margins": 18.283662796020508,
      "eval_rewards/rejected": -15.076776504516602,
      "eval_runtime": 660.3412,
      "eval_samples_per_second": 11.212,
      "eval_steps_per_second": 0.624,
      "step": 5350
    },
    {
      "epoch": 2.1717990275526744,
      "grad_norm": 1.5193423905657255e-06,
      "learning_rate": 3.0676872279753866e-06,
      "logits/chosen": -0.9592820405960083,
      "logits/rejected": -1.2157377004623413,
      "logps/chosen": -107.88704681396484,
      "logps/rejected": -324.0573425292969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.274031162261963,
      "rewards/margins": 18.420141220092773,
      "rewards/rejected": -15.146111488342285,
      "step": 5360
    },
    {
      "epoch": 2.1758508914100485,
      "grad_norm": 5.895638878428144e-06,
      "learning_rate": 3.0526789734353896e-06,
      "logits/chosen": -0.9689838290214539,
      "logits/rejected": -1.1949576139450073,
      "logps/chosen": -108.58103942871094,
      "logps/rejected": -323.4375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.242347478866577,
      "rewards/margins": 18.375865936279297,
      "rewards/rejected": -15.133517265319824,
      "step": 5370
    },
    {
      "epoch": 2.179902755267423,
      "grad_norm": 2.4637804472149583e-06,
      "learning_rate": 3.0376707188953926e-06,
      "logits/chosen": -0.9821479916572571,
      "logits/rejected": -1.2009549140930176,
      "logps/chosen": -107.96622467041016,
      "logps/rejected": -325.5393371582031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3129758834838867,
      "rewards/margins": 18.526084899902344,
      "rewards/rejected": -15.213109970092773,
      "step": 5380
    },
    {
      "epoch": 2.1839546191247976,
      "grad_norm": 2.9899647415732034e-06,
      "learning_rate": 3.022662464355396e-06,
      "logits/chosen": -0.9616569876670837,
      "logits/rejected": -1.1876670122146606,
      "logps/chosen": -107.5402603149414,
      "logps/rejected": -326.1862487792969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.279315710067749,
      "rewards/margins": 18.54355239868164,
      "rewards/rejected": -15.264236450195312,
      "step": 5390
    },
    {
      "epoch": 2.1880064829821717,
      "grad_norm": 2.1008820567658404e-06,
      "learning_rate": 3.0076542098153986e-06,
      "logits/chosen": -0.9718561768531799,
      "logits/rejected": -1.2166821956634521,
      "logps/chosen": -107.14775085449219,
      "logps/rejected": -326.3721618652344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.334036111831665,
      "rewards/margins": 18.727115631103516,
      "rewards/rejected": -15.393081665039062,
      "step": 5400
    },
    {
      "epoch": 2.1880064829821717,
      "eval_logits/chosen": -0.9820877313613892,
      "eval_logits/rejected": -1.2017114162445068,
      "eval_logps/chosen": -108.98477172851562,
      "eval_logps/rejected": -323.9667663574219,
      "eval_loss": 5.144721271221897e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2027156352996826,
      "eval_rewards/margins": 18.30511474609375,
      "eval_rewards/rejected": -15.102398872375488,
      "eval_runtime": 660.2732,
      "eval_samples_per_second": 11.214,
      "eval_steps_per_second": 0.624,
      "step": 5400
    },
    {
      "epoch": 2.1920583468395463,
      "grad_norm": 1.4790945215281681e-06,
      "learning_rate": 2.9926459552754016e-06,
      "logits/chosen": -0.9718219041824341,
      "logits/rejected": -1.2032488584518433,
      "logps/chosen": -108.09237670898438,
      "logps/rejected": -326.63763427734375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2692630290985107,
      "rewards/margins": 18.560598373413086,
      "rewards/rejected": -15.291335105895996,
      "step": 5410
    },
    {
      "epoch": 2.1961102106969204,
      "grad_norm": 8.114738193398807e-06,
      "learning_rate": 2.9776377007354046e-06,
      "logits/chosen": -0.9763138890266418,
      "logits/rejected": -1.1922379732131958,
      "logps/chosen": -108.22923278808594,
      "logps/rejected": -324.1781311035156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2659127712249756,
      "rewards/margins": 18.3449649810791,
      "rewards/rejected": -15.07905101776123,
      "step": 5420
    },
    {
      "epoch": 2.200162074554295,
      "grad_norm": 8.105959750537295e-06,
      "learning_rate": 2.9626294461954076e-06,
      "logits/chosen": -0.9747027158737183,
      "logits/rejected": -1.19993257522583,
      "logps/chosen": -107.4315414428711,
      "logps/rejected": -324.6253356933594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.30452299118042,
      "rewards/margins": 18.396106719970703,
      "rewards/rejected": -15.091582298278809,
      "step": 5430
    },
    {
      "epoch": 2.2042139384116695,
      "grad_norm": 2.2363303742167773e-06,
      "learning_rate": 2.947621191655411e-06,
      "logits/chosen": -0.9820188283920288,
      "logits/rejected": -1.1995137929916382,
      "logps/chosen": -108.4027328491211,
      "logps/rejected": -325.0263671875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2489047050476074,
      "rewards/margins": 18.37090492248535,
      "rewards/rejected": -15.121999740600586,
      "step": 5440
    },
    {
      "epoch": 2.2082658022690436,
      "grad_norm": 2.9256420930323657e-06,
      "learning_rate": 2.9326129371154137e-06,
      "logits/chosen": -0.9694913625717163,
      "logits/rejected": -1.2016642093658447,
      "logps/chosen": -108.28238677978516,
      "logps/rejected": -325.5596618652344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.278430461883545,
      "rewards/margins": 18.484956741333008,
      "rewards/rejected": -15.206525802612305,
      "step": 5450
    },
    {
      "epoch": 2.2082658022690436,
      "eval_logits/chosen": -0.978270947933197,
      "eval_logits/rejected": -1.1978684663772583,
      "eval_logps/chosen": -108.904296875,
      "eval_logps/rejected": -323.9855651855469,
      "eval_loss": 5.096885757893688e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2107620239257812,
      "eval_rewards/margins": 18.31504249572754,
      "eval_rewards/rejected": -15.104281425476074,
      "eval_runtime": 660.2106,
      "eval_samples_per_second": 11.215,
      "eval_steps_per_second": 0.624,
      "step": 5450
    },
    {
      "epoch": 2.212317666126418,
      "grad_norm": 8.608948519395199e-06,
      "learning_rate": 2.9176046825754167e-06,
      "logits/chosen": -0.9615970849990845,
      "logits/rejected": -1.193094253540039,
      "logps/chosen": -108.58715057373047,
      "logps/rejected": -324.37799072265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2963144779205322,
      "rewards/margins": 18.419776916503906,
      "rewards/rejected": -15.123462677001953,
      "step": 5460
    },
    {
      "epoch": 2.2163695299837927,
      "grad_norm": 5.135224000696326e-06,
      "learning_rate": 2.9025964280354197e-06,
      "logits/chosen": -0.9636695981025696,
      "logits/rejected": -1.1896305084228516,
      "logps/chosen": -107.08964538574219,
      "logps/rejected": -324.77850341796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3007524013519287,
      "rewards/margins": 18.336854934692383,
      "rewards/rejected": -15.036102294921875,
      "step": 5470
    },
    {
      "epoch": 2.2204213938411668,
      "grad_norm": 6.5549256760277785e-06,
      "learning_rate": 2.8875881734954227e-06,
      "logits/chosen": -0.9712886810302734,
      "logits/rejected": -1.2023524045944214,
      "logps/chosen": -108.814453125,
      "logps/rejected": -326.064453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2594046592712402,
      "rewards/margins": 18.47957992553711,
      "rewards/rejected": -15.220173835754395,
      "step": 5480
    },
    {
      "epoch": 2.2244732576985413,
      "grad_norm": 3.4416129892633762e-06,
      "learning_rate": 2.872579918955426e-06,
      "logits/chosen": -0.9645144939422607,
      "logits/rejected": -1.1906704902648926,
      "logps/chosen": -108.19139862060547,
      "logps/rejected": -325.2495422363281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.277437686920166,
      "rewards/margins": 18.48769760131836,
      "rewards/rejected": -15.210260391235352,
      "step": 5490
    },
    {
      "epoch": 2.228525121555916,
      "grad_norm": 5.491361207532464e-06,
      "learning_rate": 2.8575716644154287e-06,
      "logits/chosen": -0.961846113204956,
      "logits/rejected": -1.190273642539978,
      "logps/chosen": -107.55595397949219,
      "logps/rejected": -326.3609924316406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.284193515777588,
      "rewards/margins": 18.561208724975586,
      "rewards/rejected": -15.277012825012207,
      "step": 5500
    },
    {
      "epoch": 2.228525121555916,
      "eval_logits/chosen": -0.9809632301330566,
      "eval_logits/rejected": -1.2005460262298584,
      "eval_logps/chosen": -108.89672088623047,
      "eval_logps/rejected": -324.2535400390625,
      "eval_loss": 4.9650584088567484e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2115213871002197,
      "eval_rewards/margins": 18.342599868774414,
      "eval_rewards/rejected": -15.131077766418457,
      "eval_runtime": 660.3362,
      "eval_samples_per_second": 11.212,
      "eval_steps_per_second": 0.624,
      "step": 5500
    },
    {
      "epoch": 2.23257698541329,
      "grad_norm": 2.52136965173122e-06,
      "learning_rate": 2.8425634098754317e-06,
      "logits/chosen": -0.9671697616577148,
      "logits/rejected": -1.1808946132659912,
      "logps/chosen": -107.77230072021484,
      "logps/rejected": -325.3393249511719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2875375747680664,
      "rewards/margins": 18.484088897705078,
      "rewards/rejected": -15.196552276611328,
      "step": 5510
    },
    {
      "epoch": 2.2366288492706645,
      "grad_norm": 6.712255071761319e-06,
      "learning_rate": 2.8275551553354347e-06,
      "logits/chosen": -0.9800280928611755,
      "logits/rejected": -1.184509515762329,
      "logps/chosen": -108.50052642822266,
      "logps/rejected": -325.2090759277344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2751975059509277,
      "rewards/margins": 18.468046188354492,
      "rewards/rejected": -15.192850112915039,
      "step": 5520
    },
    {
      "epoch": 2.240680713128039,
      "grad_norm": 5.123366918269312e-06,
      "learning_rate": 2.8125469007954377e-06,
      "logits/chosen": -0.9732328653335571,
      "logits/rejected": -1.1967648267745972,
      "logps/chosen": -108.0207748413086,
      "logps/rejected": -324.44873046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2871508598327637,
      "rewards/margins": 18.425617218017578,
      "rewards/rejected": -15.138466835021973,
      "step": 5530
    },
    {
      "epoch": 2.244732576985413,
      "grad_norm": 1.9682167931023287e-06,
      "learning_rate": 2.7975386462554403e-06,
      "logits/chosen": -0.9738643765449524,
      "logits/rejected": -1.2164760828018188,
      "logps/chosen": -108.87864685058594,
      "logps/rejected": -325.7362976074219,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.29534912109375,
      "rewards/margins": 18.526559829711914,
      "rewards/rejected": -15.231209754943848,
      "step": 5540
    },
    {
      "epoch": 2.2487844408427877,
      "grad_norm": 3.375124833837617e-06,
      "learning_rate": 2.782530391715444e-06,
      "logits/chosen": -0.9724031686782837,
      "logits/rejected": -1.215806484222412,
      "logps/chosen": -108.00154876708984,
      "logps/rejected": -324.9290466308594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.30460262298584,
      "rewards/margins": 18.49542999267578,
      "rewards/rejected": -15.190828323364258,
      "step": 5550
    },
    {
      "epoch": 2.2487844408427877,
      "eval_logits/chosen": -1.0021088123321533,
      "eval_logits/rejected": -1.216044306755066,
      "eval_logps/chosen": -109.04871368408203,
      "eval_logps/rejected": -324.6860656738281,
      "eval_loss": 4.834061329006545e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.196322202682495,
      "eval_rewards/margins": 18.370651245117188,
      "eval_rewards/rejected": -15.174327850341797,
      "eval_runtime": 660.2646,
      "eval_samples_per_second": 11.214,
      "eval_steps_per_second": 0.624,
      "step": 5550
    },
    {
      "epoch": 2.2528363047001623,
      "grad_norm": 4.11156997870421e-06,
      "learning_rate": 2.7675221371754467e-06,
      "logits/chosen": -0.959725022315979,
      "logits/rejected": -1.182191252708435,
      "logps/chosen": -109.10403442382812,
      "logps/rejected": -325.5591125488281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.234382390975952,
      "rewards/margins": 18.426700592041016,
      "rewards/rejected": -15.192317962646484,
      "step": 5560
    },
    {
      "epoch": 2.2568881685575364,
      "grad_norm": 2.655802745721303e-06,
      "learning_rate": 2.7525138826354497e-06,
      "logits/chosen": -0.9779143929481506,
      "logits/rejected": -1.2030993700027466,
      "logps/chosen": -108.81770324707031,
      "logps/rejected": -326.4799499511719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2410504817962646,
      "rewards/margins": 18.449491500854492,
      "rewards/rejected": -15.208441734313965,
      "step": 5570
    },
    {
      "epoch": 2.260940032414911,
      "grad_norm": 4.91537412017351e-06,
      "learning_rate": 2.7375056280954527e-06,
      "logits/chosen": -0.9816739559173584,
      "logits/rejected": -1.1951929330825806,
      "logps/chosen": -108.85587310791016,
      "logps/rejected": -324.6351623535156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.281838893890381,
      "rewards/margins": 18.444608688354492,
      "rewards/rejected": -15.16277027130127,
      "step": 5580
    },
    {
      "epoch": 2.264991896272285,
      "grad_norm": 4.137316409469349e-06,
      "learning_rate": 2.7224973735554557e-06,
      "logits/chosen": -0.9698922634124756,
      "logits/rejected": -1.201400876045227,
      "logps/chosen": -109.29122924804688,
      "logps/rejected": -324.98504638671875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.235358715057373,
      "rewards/margins": 18.49024200439453,
      "rewards/rejected": -15.254883766174316,
      "step": 5590
    },
    {
      "epoch": 2.2690437601296596,
      "grad_norm": 3.5918981211580103e-06,
      "learning_rate": 2.707489119015459e-06,
      "logits/chosen": -0.9636077284812927,
      "logits/rejected": -1.1759008169174194,
      "logps/chosen": -107.51180267333984,
      "logps/rejected": -322.8291320800781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.270862102508545,
      "rewards/margins": 18.282569885253906,
      "rewards/rejected": -15.011709213256836,
      "step": 5600
    },
    {
      "epoch": 2.2690437601296596,
      "eval_logits/chosen": -0.9744959473609924,
      "eval_logits/rejected": -1.1937018632888794,
      "eval_logps/chosen": -108.83885192871094,
      "eval_logps/rejected": -324.4743347167969,
      "eval_loss": 4.8358597126707537e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2173092365264893,
      "eval_rewards/margins": 18.37046241760254,
      "eval_rewards/rejected": -15.153154373168945,
      "eval_runtime": 660.3298,
      "eval_samples_per_second": 11.213,
      "eval_steps_per_second": 0.624,
      "step": 5600
    },
    {
      "epoch": 2.273095623987034,
      "grad_norm": 5.159456122783013e-06,
      "learning_rate": 2.6924808644754617e-06,
      "logits/chosen": -0.9723551273345947,
      "logits/rejected": -1.2006233930587769,
      "logps/chosen": -108.35022735595703,
      "logps/rejected": -326.8168640136719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2803404331207275,
      "rewards/margins": 18.527671813964844,
      "rewards/rejected": -15.247331619262695,
      "step": 5610
    },
    {
      "epoch": 2.2771474878444082,
      "grad_norm": 4.119930508750258e-06,
      "learning_rate": 2.6774726099354647e-06,
      "logits/chosen": -0.9690415859222412,
      "logits/rejected": -1.2194011211395264,
      "logps/chosen": -107.32176208496094,
      "logps/rejected": -327.4947509765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.294969081878662,
      "rewards/margins": 18.68514060974121,
      "rewards/rejected": -15.39017105102539,
      "step": 5620
    },
    {
      "epoch": 2.281199351701783,
      "grad_norm": 3.2292373361997306e-06,
      "learning_rate": 2.6624643553954677e-06,
      "logits/chosen": -0.9724549055099487,
      "logits/rejected": -1.1870734691619873,
      "logps/chosen": -109.0750732421875,
      "logps/rejected": -324.3908996582031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2350621223449707,
      "rewards/margins": 18.347484588623047,
      "rewards/rejected": -15.112425804138184,
      "step": 5630
    },
    {
      "epoch": 2.2852512155591573,
      "grad_norm": 1.92338734450459e-06,
      "learning_rate": 2.6474561008554707e-06,
      "logits/chosen": -0.9774371981620789,
      "logits/rejected": -1.1965054273605347,
      "logps/chosen": -108.20509338378906,
      "logps/rejected": -325.67022705078125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2764956951141357,
      "rewards/margins": 18.494312286376953,
      "rewards/rejected": -15.217818260192871,
      "step": 5640
    },
    {
      "epoch": 2.2893030794165314,
      "grad_norm": 3.3345258998451754e-06,
      "learning_rate": 2.6324478463154733e-06,
      "logits/chosen": -0.9700595736503601,
      "logits/rejected": -1.1860406398773193,
      "logps/chosen": -108.3021469116211,
      "logps/rejected": -325.4642333984375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.295811176300049,
      "rewards/margins": 18.52042579650879,
      "rewards/rejected": -15.224616050720215,
      "step": 5650
    },
    {
      "epoch": 2.2893030794165314,
      "eval_logits/chosen": -0.9985373616218567,
      "eval_logits/rejected": -1.2101703882217407,
      "eval_logps/chosen": -109.0257568359375,
      "eval_logps/rejected": -324.98095703125,
      "eval_loss": 4.685679755311867e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.198617696762085,
      "eval_rewards/margins": 18.40243911743164,
      "eval_rewards/rejected": -15.203819274902344,
      "eval_runtime": 660.2906,
      "eval_samples_per_second": 11.213,
      "eval_steps_per_second": 0.624,
      "step": 5650
    },
    {
      "epoch": 2.293354943273906,
      "grad_norm": 2.9136933790141484e-06,
      "learning_rate": 2.6174395917754767e-06,
      "logits/chosen": -0.9681564569473267,
      "logits/rejected": -1.2030173540115356,
      "logps/chosen": -108.33345031738281,
      "logps/rejected": -326.4482727050781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2615244388580322,
      "rewards/margins": 18.537817001342773,
      "rewards/rejected": -15.27629280090332,
      "step": 5660
    },
    {
      "epoch": 2.2974068071312805,
      "grad_norm": 3.6357732824399136e-06,
      "learning_rate": 2.6024313372354797e-06,
      "logits/chosen": -0.9570192694664001,
      "logits/rejected": -1.1811176538467407,
      "logps/chosen": -107.85875701904297,
      "logps/rejected": -325.16668701171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2770462036132812,
      "rewards/margins": 18.38956069946289,
      "rewards/rejected": -15.112512588500977,
      "step": 5670
    },
    {
      "epoch": 2.3014586709886546,
      "grad_norm": 5.2873547247145325e-06,
      "learning_rate": 2.5874230826954827e-06,
      "logits/chosen": -0.9654983282089233,
      "logits/rejected": -1.2084449529647827,
      "logps/chosen": -107.34917449951172,
      "logps/rejected": -327.28460693359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3275811672210693,
      "rewards/margins": 18.592912673950195,
      "rewards/rejected": -15.265332221984863,
      "step": 5680
    },
    {
      "epoch": 2.305510534846029,
      "grad_norm": 1.9631031591416104e-06,
      "learning_rate": 2.5724148281554857e-06,
      "logits/chosen": -0.9772531390190125,
      "logits/rejected": -1.1914225816726685,
      "logps/chosen": -108.16785430908203,
      "logps/rejected": -325.9184265136719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.30417799949646,
      "rewards/margins": 18.541934967041016,
      "rewards/rejected": -15.23775863647461,
      "step": 5690
    },
    {
      "epoch": 2.3095623987034037,
      "grad_norm": 3.304847723484272e-06,
      "learning_rate": 2.5574065736154883e-06,
      "logits/chosen": -0.9695168733596802,
      "logits/rejected": -1.1988223791122437,
      "logps/chosen": -107.34342193603516,
      "logps/rejected": -326.7132873535156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2983932495117188,
      "rewards/margins": 18.564241409301758,
      "rewards/rejected": -15.265847206115723,
      "step": 5700
    },
    {
      "epoch": 2.3095623987034037,
      "eval_logits/chosen": -0.9801631569862366,
      "eval_logits/rejected": -1.1987709999084473,
      "eval_logps/chosen": -108.88590240478516,
      "eval_logps/rejected": -324.906982421875,
      "eval_loss": 4.6702645306595514e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2126026153564453,
      "eval_rewards/margins": 18.409025192260742,
      "eval_rewards/rejected": -15.19642162322998,
      "eval_runtime": 660.2445,
      "eval_samples_per_second": 11.214,
      "eval_steps_per_second": 0.624,
      "step": 5700
    },
    {
      "epoch": 2.313614262560778,
      "grad_norm": 2.3572217742184876e-06,
      "learning_rate": 2.5423983190754918e-06,
      "logits/chosen": -0.9702841639518738,
      "logits/rejected": -1.180479645729065,
      "logps/chosen": -108.51237487792969,
      "logps/rejected": -324.76800537109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.285172462463379,
      "rewards/margins": 18.476665496826172,
      "rewards/rejected": -15.191494941711426,
      "step": 5710
    },
    {
      "epoch": 2.3176661264181524,
      "grad_norm": 6.990411748120096e-06,
      "learning_rate": 2.5273900645354948e-06,
      "logits/chosen": -0.9684751033782959,
      "logits/rejected": -1.1934318542480469,
      "logps/chosen": -107.60021209716797,
      "logps/rejected": -325.96014404296875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2991745471954346,
      "rewards/margins": 18.551288604736328,
      "rewards/rejected": -15.252111434936523,
      "step": 5720
    },
    {
      "epoch": 2.321717990275527,
      "grad_norm": 2.516685981390765e-06,
      "learning_rate": 2.5123818099954978e-06,
      "logits/chosen": -0.9524480104446411,
      "logits/rejected": -1.1807420253753662,
      "logps/chosen": -107.41260528564453,
      "logps/rejected": -326.22760009765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.293351888656616,
      "rewards/margins": 18.598726272583008,
      "rewards/rejected": -15.305375099182129,
      "step": 5730
    },
    {
      "epoch": 2.325769854132901,
      "grad_norm": 3.3768424145819154e-06,
      "learning_rate": 2.4973735554555008e-06,
      "logits/chosen": -0.9682026505470276,
      "logits/rejected": -1.1985716819763184,
      "logps/chosen": -107.61695098876953,
      "logps/rejected": -327.2942199707031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2812089920043945,
      "rewards/margins": 18.669212341308594,
      "rewards/rejected": -15.388001441955566,
      "step": 5740
    },
    {
      "epoch": 2.3298217179902756,
      "grad_norm": 1.915690290843486e-06,
      "learning_rate": 2.4823653009155038e-06,
      "logits/chosen": -0.9773512482643127,
      "logits/rejected": -1.214605450630188,
      "logps/chosen": -106.96377563476562,
      "logps/rejected": -326.2522888183594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.290473461151123,
      "rewards/margins": 18.603700637817383,
      "rewards/rejected": -15.313227653503418,
      "step": 5750
    },
    {
      "epoch": 2.3298217179902756,
      "eval_logits/chosen": -0.9762807488441467,
      "eval_logits/rejected": -1.1947269439697266,
      "eval_logps/chosen": -108.86436462402344,
      "eval_logps/rejected": -325.0529479980469,
      "eval_loss": 4.620364535412591e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.214757204055786,
      "eval_rewards/margins": 18.4257755279541,
      "eval_rewards/rejected": -15.211020469665527,
      "eval_runtime": 660.4013,
      "eval_samples_per_second": 11.211,
      "eval_steps_per_second": 0.624,
      "step": 5750
    },
    {
      "epoch": 2.3338735818476497,
      "grad_norm": 2.1481794192368397e-06,
      "learning_rate": 2.4673570463755068e-06,
      "logits/chosen": -0.9639620184898376,
      "logits/rejected": -1.1970309019088745,
      "logps/chosen": -108.05836486816406,
      "logps/rejected": -326.10137939453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2734692096710205,
      "rewards/margins": 18.534753799438477,
      "rewards/rejected": -15.261284828186035,
      "step": 5760
    },
    {
      "epoch": 2.3379254457050243,
      "grad_norm": 1.016176520352019e-05,
      "learning_rate": 2.4523487918355098e-06,
      "logits/chosen": -0.9729595184326172,
      "logits/rejected": -1.1997188329696655,
      "logps/chosen": -109.29617309570312,
      "logps/rejected": -326.4288330078125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2463324069976807,
      "rewards/margins": 18.55828857421875,
      "rewards/rejected": -15.311954498291016,
      "step": 5770
    },
    {
      "epoch": 2.341977309562399,
      "grad_norm": 2.3624691039003665e-06,
      "learning_rate": 2.4373405372955128e-06,
      "logits/chosen": -0.9771804809570312,
      "logits/rejected": -1.1958061456680298,
      "logps/chosen": -108.30644989013672,
      "logps/rejected": -326.2595520019531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.318211317062378,
      "rewards/margins": 18.56531524658203,
      "rewards/rejected": -15.24710464477539,
      "step": 5780
    },
    {
      "epoch": 2.346029173419773,
      "grad_norm": 2.9252137210278306e-06,
      "learning_rate": 2.4223322827555158e-06,
      "logits/chosen": -0.9689458012580872,
      "logits/rejected": -1.1969562768936157,
      "logps/chosen": -108.24433898925781,
      "logps/rejected": -326.3622131347656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.266651153564453,
      "rewards/margins": 18.574663162231445,
      "rewards/rejected": -15.308012962341309,
      "step": 5790
    },
    {
      "epoch": 2.3500810372771475,
      "grad_norm": 4.0559393710282166e-06,
      "learning_rate": 2.4073240282155188e-06,
      "logits/chosen": -0.9673594236373901,
      "logits/rejected": -1.1805312633514404,
      "logps/chosen": -107.56311798095703,
      "logps/rejected": -326.7265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2976877689361572,
      "rewards/margins": 18.496929168701172,
      "rewards/rejected": -15.199243545532227,
      "step": 5800
    },
    {
      "epoch": 2.3500810372771475,
      "eval_logits/chosen": -0.9792886972427368,
      "eval_logits/rejected": -1.1983641386032104,
      "eval_logps/chosen": -108.89390563964844,
      "eval_logps/rejected": -325.24957275390625,
      "eval_loss": 4.520941132568623e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2118031978607178,
      "eval_rewards/margins": 18.442487716674805,
      "eval_rewards/rejected": -15.230685234069824,
      "eval_runtime": 660.2343,
      "eval_samples_per_second": 11.214,
      "eval_steps_per_second": 0.624,
      "step": 5800
    },
    {
      "epoch": 2.354132901134522,
      "grad_norm": 4.35237234341912e-06,
      "learning_rate": 2.392315773675522e-06,
      "logits/chosen": -0.9659538865089417,
      "logits/rejected": -1.1955288648605347,
      "logps/chosen": -107.923095703125,
      "logps/rejected": -326.80548095703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2637534141540527,
      "rewards/margins": 18.59938621520996,
      "rewards/rejected": -15.335634231567383,
      "step": 5810
    },
    {
      "epoch": 2.358184764991896,
      "grad_norm": 4.2245274016750045e-06,
      "learning_rate": 2.377307519135525e-06,
      "logits/chosen": -0.9580984115600586,
      "logits/rejected": -1.1906923055648804,
      "logps/chosen": -106.16527557373047,
      "logps/rejected": -327.8320617675781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.35353684425354,
      "rewards/margins": 18.727663040161133,
      "rewards/rejected": -15.374124526977539,
      "step": 5820
    },
    {
      "epoch": 2.3622366288492707,
      "grad_norm": 2.9806258226017235e-06,
      "learning_rate": 2.362299264595528e-06,
      "logits/chosen": -0.9636472463607788,
      "logits/rejected": -1.1913896799087524,
      "logps/chosen": -107.6203842163086,
      "logps/rejected": -326.91571044921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.282270908355713,
      "rewards/margins": 18.575559616088867,
      "rewards/rejected": -15.293288230895996,
      "step": 5830
    },
    {
      "epoch": 2.366288492706645,
      "grad_norm": 2.2583449208468664e-06,
      "learning_rate": 2.347291010055531e-06,
      "logits/chosen": -0.969796895980835,
      "logits/rejected": -1.2006282806396484,
      "logps/chosen": -108.57011413574219,
      "logps/rejected": -326.08673095703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.253269672393799,
      "rewards/margins": 18.5068302154541,
      "rewards/rejected": -15.253559112548828,
      "step": 5840
    },
    {
      "epoch": 2.3703403565640193,
      "grad_norm": 1.3829013596478035e-06,
      "learning_rate": 2.332282755515534e-06,
      "logits/chosen": -0.9795759916305542,
      "logits/rejected": -1.1850019693374634,
      "logps/chosen": -107.93074035644531,
      "logps/rejected": -328.0816650390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3043065071105957,
      "rewards/margins": 18.752573013305664,
      "rewards/rejected": -15.448267936706543,
      "step": 5850
    },
    {
      "epoch": 2.3703403565640193,
      "eval_logits/chosen": -0.9805318117141724,
      "eval_logits/rejected": -1.1991779804229736,
      "eval_logps/chosen": -108.85604858398438,
      "eval_logps/rejected": -325.3434143066406,
      "eval_loss": 4.4704442814236245e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.215588092803955,
      "eval_rewards/margins": 18.455656051635742,
      "eval_rewards/rejected": -15.240068435668945,
      "eval_runtime": 660.3357,
      "eval_samples_per_second": 11.212,
      "eval_steps_per_second": 0.624,
      "step": 5850
    },
    {
      "epoch": 2.374392220421394,
      "grad_norm": 5.7766483223531395e-06,
      "learning_rate": 2.317274500975537e-06,
      "logits/chosen": -0.9744726419448853,
      "logits/rejected": -1.1932746171951294,
      "logps/chosen": -107.19166564941406,
      "logps/rejected": -326.4128723144531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3251421451568604,
      "rewards/margins": 18.652881622314453,
      "rewards/rejected": -15.327739715576172,
      "step": 5860
    },
    {
      "epoch": 2.3784440842787684,
      "grad_norm": 2.2121885194792412e-06,
      "learning_rate": 2.30226624643554e-06,
      "logits/chosen": -0.9748661518096924,
      "logits/rejected": -1.1863662004470825,
      "logps/chosen": -109.22135925292969,
      "logps/rejected": -327.0158996582031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2374141216278076,
      "rewards/margins": 18.587627410888672,
      "rewards/rejected": -15.350215911865234,
      "step": 5870
    },
    {
      "epoch": 2.3824959481361425,
      "grad_norm": 2.1627809019264532e-06,
      "learning_rate": 2.287257991895543e-06,
      "logits/chosen": -0.9746767282485962,
      "logits/rejected": -1.2033764123916626,
      "logps/chosen": -107.47755432128906,
      "logps/rejected": -325.9498596191406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3126156330108643,
      "rewards/margins": 18.64901351928711,
      "rewards/rejected": -15.336398124694824,
      "step": 5880
    },
    {
      "epoch": 2.386547811993517,
      "grad_norm": 3.7836791761947097e-06,
      "learning_rate": 2.272249737355546e-06,
      "logits/chosen": -0.9714968800544739,
      "logits/rejected": -1.1881319284439087,
      "logps/chosen": -108.20919799804688,
      "logps/rejected": -326.4745178222656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2788820266723633,
      "rewards/margins": 18.569578170776367,
      "rewards/rejected": -15.290694236755371,
      "step": 5890
    },
    {
      "epoch": 2.3905996758508916,
      "grad_norm": 3.031146434295806e-06,
      "learning_rate": 2.257241482815549e-06,
      "logits/chosen": -0.9792813658714294,
      "logits/rejected": -1.1891095638275146,
      "logps/chosen": -107.79071807861328,
      "logps/rejected": -328.1285705566406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3049774169921875,
      "rewards/margins": 18.71900749206543,
      "rewards/rejected": -15.414031028747559,
      "step": 5900
    },
    {
      "epoch": 2.3905996758508916,
      "eval_logits/chosen": -0.9776058793067932,
      "eval_logits/rejected": -1.1963112354278564,
      "eval_logps/chosen": -108.84764099121094,
      "eval_logps/rejected": -325.557861328125,
      "eval_loss": 4.389031715845704e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.21642804145813,
      "eval_rewards/margins": 18.477937698364258,
      "eval_rewards/rejected": -15.261509895324707,
      "eval_runtime": 660.2995,
      "eval_samples_per_second": 11.213,
      "eval_steps_per_second": 0.624,
      "step": 5900
    },
    {
      "epoch": 2.3946515397082657,
      "grad_norm": 1.5837250657568802e-06,
      "learning_rate": 2.242233228275552e-06,
      "logits/chosen": -0.9707351922988892,
      "logits/rejected": -1.2112205028533936,
      "logps/chosen": -107.20185089111328,
      "logps/rejected": -329.29925537109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3092434406280518,
      "rewards/margins": 18.771028518676758,
      "rewards/rejected": -15.461783409118652,
      "step": 5910
    },
    {
      "epoch": 2.3987034035656403,
      "grad_norm": 1.4248274737838074e-06,
      "learning_rate": 2.227224973735555e-06,
      "logits/chosen": -0.9622972011566162,
      "logits/rejected": -1.1854596138000488,
      "logps/chosen": -107.843994140625,
      "logps/rejected": -325.9154968261719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3007805347442627,
      "rewards/margins": 18.525949478149414,
      "rewards/rejected": -15.225168228149414,
      "step": 5920
    },
    {
      "epoch": 2.4027552674230144,
      "grad_norm": 2.0499478523561265e-06,
      "learning_rate": 2.212216719195558e-06,
      "logits/chosen": -0.9776632785797119,
      "logits/rejected": -1.2132608890533447,
      "logps/chosen": -107.80943298339844,
      "logps/rejected": -326.85223388671875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3134846687316895,
      "rewards/margins": 18.68709373474121,
      "rewards/rejected": -15.373608589172363,
      "step": 5930
    },
    {
      "epoch": 2.406807131280389,
      "grad_norm": 4.022785560664488e-06,
      "learning_rate": 2.197208464655561e-06,
      "logits/chosen": -0.9736822843551636,
      "logits/rejected": -1.1963258981704712,
      "logps/chosen": -107.04643249511719,
      "logps/rejected": -327.04217529296875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3099985122680664,
      "rewards/margins": 18.62815284729004,
      "rewards/rejected": -15.318156242370605,
      "step": 5940
    },
    {
      "epoch": 2.4108589951377635,
      "grad_norm": 7.282645128725562e-06,
      "learning_rate": 2.182200210115564e-06,
      "logits/chosen": -0.9786582589149475,
      "logits/rejected": -1.195390224456787,
      "logps/chosen": -109.43639373779297,
      "logps/rejected": -327.4029846191406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.279670000076294,
      "rewards/margins": 18.617467880249023,
      "rewards/rejected": -15.337796211242676,
      "step": 5950
    },
    {
      "epoch": 2.4108589951377635,
      "eval_logits/chosen": -0.9759088158607483,
      "eval_logits/rejected": -1.1941173076629639,
      "eval_logps/chosen": -108.82359313964844,
      "eval_logps/rejected": -325.46142578125,
      "eval_loss": 4.3986084108382784e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2188332080841064,
      "eval_rewards/margins": 18.470699310302734,
      "eval_rewards/rejected": -15.25186824798584,
      "eval_runtime": 660.487,
      "eval_samples_per_second": 11.21,
      "eval_steps_per_second": 0.624,
      "step": 5950
    },
    {
      "epoch": 2.4149108589951376,
      "grad_norm": 5.016336672269972e-06,
      "learning_rate": 2.1671919555755664e-06,
      "logits/chosen": -0.9681698083877563,
      "logits/rejected": -1.1931991577148438,
      "logps/chosen": -107.51768493652344,
      "logps/rejected": -327.22802734375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.334961175918579,
      "rewards/margins": 18.64361000061035,
      "rewards/rejected": -15.308646202087402,
      "step": 5960
    },
    {
      "epoch": 2.418962722852512,
      "grad_norm": 8.80508559930604e-06,
      "learning_rate": 2.15218370103557e-06,
      "logits/chosen": -0.9631980061531067,
      "logits/rejected": -1.2213531732559204,
      "logps/chosen": -107.55696105957031,
      "logps/rejected": -327.5552978515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.278275966644287,
      "rewards/margins": 18.78751564025879,
      "rewards/rejected": -15.509241104125977,
      "step": 5970
    },
    {
      "epoch": 2.4230145867098867,
      "grad_norm": 6.065507477615029e-06,
      "learning_rate": 2.137175446495573e-06,
      "logits/chosen": -0.9734798669815063,
      "logits/rejected": -1.1926671266555786,
      "logps/chosen": -108.64668273925781,
      "logps/rejected": -325.3752746582031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2668917179107666,
      "rewards/margins": 18.480018615722656,
      "rewards/rejected": -15.213125228881836,
      "step": 5980
    },
    {
      "epoch": 2.427066450567261,
      "grad_norm": 4.074761818628758e-06,
      "learning_rate": 2.122167191955576e-06,
      "logits/chosen": -0.9733186364173889,
      "logits/rejected": -1.1757533550262451,
      "logps/chosen": -108.77658081054688,
      "logps/rejected": -325.20086669921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.256643772125244,
      "rewards/margins": 18.523258209228516,
      "rewards/rejected": -15.266615867614746,
      "step": 5990
    },
    {
      "epoch": 2.4311183144246353,
      "grad_norm": 1.9911642539227614e-06,
      "learning_rate": 2.107158937415579e-06,
      "logits/chosen": -0.9687005877494812,
      "logits/rejected": -1.2020748853683472,
      "logps/chosen": -107.47844696044922,
      "logps/rejected": -327.5688171386719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2923924922943115,
      "rewards/margins": 18.662755966186523,
      "rewards/rejected": -15.370365142822266,
      "step": 6000
    },
    {
      "epoch": 2.4311183144246353,
      "eval_logits/chosen": -0.9808048605918884,
      "eval_logits/rejected": -1.1985182762145996,
      "eval_logps/chosen": -108.8628921508789,
      "eval_logps/rejected": -325.8635559082031,
      "eval_loss": 4.268249753636155e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2149031162261963,
      "eval_rewards/margins": 18.506982803344727,
      "eval_rewards/rejected": -15.29207992553711,
      "eval_runtime": 660.2726,
      "eval_samples_per_second": 11.214,
      "eval_steps_per_second": 0.624,
      "step": 6000
    },
    {
      "epoch": 2.43517017828201,
      "grad_norm": 3.6586802707461175e-06,
      "learning_rate": 2.0921506828755814e-06,
      "logits/chosen": -0.9805698990821838,
      "logits/rejected": -1.1940364837646484,
      "logps/chosen": -108.60276794433594,
      "logps/rejected": -327.7688903808594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3031599521636963,
      "rewards/margins": 18.747312545776367,
      "rewards/rejected": -15.444150924682617,
      "step": 6010
    },
    {
      "epoch": 2.439222042139384,
      "grad_norm": 2.5668534817668842e-06,
      "learning_rate": 2.077142428335585e-06,
      "logits/chosen": -0.9724539518356323,
      "logits/rejected": -1.1866434812545776,
      "logps/chosen": -108.19477844238281,
      "logps/rejected": -327.85968017578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.309955596923828,
      "rewards/margins": 18.663129806518555,
      "rewards/rejected": -15.35317611694336,
      "step": 6020
    },
    {
      "epoch": 2.4432739059967585,
      "grad_norm": 2.2230321974348044e-06,
      "learning_rate": 2.062134173795588e-06,
      "logits/chosen": -0.9716467261314392,
      "logits/rejected": -1.1950808763504028,
      "logps/chosen": -107.60568237304688,
      "logps/rejected": -326.04302978515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3049800395965576,
      "rewards/margins": 18.607084274291992,
      "rewards/rejected": -15.302105903625488,
      "step": 6030
    },
    {
      "epoch": 2.447325769854133,
      "grad_norm": 4.1039793359232135e-06,
      "learning_rate": 2.0471259192555905e-06,
      "logits/chosen": -0.9728240370750427,
      "logits/rejected": -1.2006126642227173,
      "logps/chosen": -107.84103393554688,
      "logps/rejected": -327.4736633300781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3131470680236816,
      "rewards/margins": 18.78127670288086,
      "rewards/rejected": -15.468131065368652,
      "step": 6040
    },
    {
      "epoch": 2.451377633711507,
      "grad_norm": 1.3657091813001898e-06,
      "learning_rate": 2.032117664715594e-06,
      "logits/chosen": -0.9755275249481201,
      "logits/rejected": -1.1849957704544067,
      "logps/chosen": -107.99736785888672,
      "logps/rejected": -325.5821533203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2840635776519775,
      "rewards/margins": 18.56288719177246,
      "rewards/rejected": -15.278822898864746,
      "step": 6050
    },
    {
      "epoch": 2.451377633711507,
      "eval_logits/chosen": -0.9780051708221436,
      "eval_logits/rejected": -1.1961445808410645,
      "eval_logps/chosen": -108.83138275146484,
      "eval_logps/rejected": -325.86334228515625,
      "eval_loss": 4.2394276533741504e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2180542945861816,
      "eval_rewards/margins": 18.510120391845703,
      "eval_rewards/rejected": -15.29206371307373,
      "eval_runtime": 660.5379,
      "eval_samples_per_second": 11.209,
      "eval_steps_per_second": 0.624,
      "step": 6050
    },
    {
      "epoch": 2.4554294975688817,
      "grad_norm": 1.4931745226931525e-06,
      "learning_rate": 2.017109410175597e-06,
      "logits/chosen": -0.9615519046783447,
      "logits/rejected": -1.1878507137298584,
      "logps/chosen": -107.65083312988281,
      "logps/rejected": -329.0572509765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3118786811828613,
      "rewards/margins": 18.859901428222656,
      "rewards/rejected": -15.54802417755127,
      "step": 6060
    },
    {
      "epoch": 2.4594813614262563,
      "grad_norm": 1.7799682609620504e-05,
      "learning_rate": 2.0021011556356e-06,
      "logits/chosen": -0.96705561876297,
      "logits/rejected": -1.204535961151123,
      "logps/chosen": -108.06282806396484,
      "logps/rejected": -327.4886169433594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.256913661956787,
      "rewards/margins": 18.616802215576172,
      "rewards/rejected": -15.359890937805176,
      "step": 6070
    },
    {
      "epoch": 2.4635332252836304,
      "grad_norm": 1.7383925978720072e-06,
      "learning_rate": 1.987092901095603e-06,
      "logits/chosen": -0.9585224390029907,
      "logits/rejected": -1.1837499141693115,
      "logps/chosen": -107.86884307861328,
      "logps/rejected": -328.6343994140625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.318385601043701,
      "rewards/margins": 18.738054275512695,
      "rewards/rejected": -15.41966724395752,
      "step": 6080
    },
    {
      "epoch": 2.467585089141005,
      "grad_norm": 1.1537390491866972e-06,
      "learning_rate": 1.9720846465556055e-06,
      "logits/chosen": -0.9602194428443909,
      "logits/rejected": -1.1984214782714844,
      "logps/chosen": -106.77111053466797,
      "logps/rejected": -325.6100158691406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.313378095626831,
      "rewards/margins": 18.595895767211914,
      "rewards/rejected": -15.28251838684082,
      "step": 6090
    },
    {
      "epoch": 2.471636952998379,
      "grad_norm": 1.8969084294440108e-06,
      "learning_rate": 1.957076392015609e-06,
      "logits/chosen": -0.9807277917861938,
      "logits/rejected": -1.20936918258667,
      "logps/chosen": -108.0311050415039,
      "logps/rejected": -326.1614685058594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3173441886901855,
      "rewards/margins": 18.628530502319336,
      "rewards/rejected": -15.311185836791992,
      "step": 6100
    },
    {
      "epoch": 2.471636952998379,
      "eval_logits/chosen": -0.9830069541931152,
      "eval_logits/rejected": -1.2014747858047485,
      "eval_logps/chosen": -108.8082504272461,
      "eval_logps/rejected": -326.0628662109375,
      "eval_loss": 4.179652890456964e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.220367908477783,
      "eval_rewards/margins": 18.532384872436523,
      "eval_rewards/rejected": -15.312015533447266,
      "eval_runtime": 660.4553,
      "eval_samples_per_second": 11.21,
      "eval_steps_per_second": 0.624,
      "step": 6100
    },
    {
      "epoch": 2.4756888168557536,
      "grad_norm": 3.357650712132454e-06,
      "learning_rate": 1.942068137475612e-06,
      "logits/chosen": -0.9689719080924988,
      "logits/rejected": -1.1983363628387451,
      "logps/chosen": -107.31026458740234,
      "logps/rejected": -327.8813781738281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3255045413970947,
      "rewards/margins": 18.77232551574707,
      "rewards/rejected": -15.446820259094238,
      "step": 6110
    },
    {
      "epoch": 2.479740680713128,
      "grad_norm": 2.8045988074154593e-06,
      "learning_rate": 1.9270598829356145e-06,
      "logits/chosen": -0.9831250309944153,
      "logits/rejected": -1.2013838291168213,
      "logps/chosen": -109.80864715576172,
      "logps/rejected": -328.51336669921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2356388568878174,
      "rewards/margins": 18.75834083557129,
      "rewards/rejected": -15.522703170776367,
      "step": 6120
    },
    {
      "epoch": 2.4837925445705022,
      "grad_norm": 4.265214556653518e-06,
      "learning_rate": 1.912051628395618e-06,
      "logits/chosen": -0.9854481220245361,
      "logits/rejected": -1.2226250171661377,
      "logps/chosen": -107.80648040771484,
      "logps/rejected": -328.3800354003906,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2958462238311768,
      "rewards/margins": 18.70480728149414,
      "rewards/rejected": -15.408963203430176,
      "step": 6130
    },
    {
      "epoch": 2.487844408427877,
      "grad_norm": 3.7762381452921545e-06,
      "learning_rate": 1.8970433738556207e-06,
      "logits/chosen": -0.9553852677345276,
      "logits/rejected": -1.2117712497711182,
      "logps/chosen": -107.25263977050781,
      "logps/rejected": -327.5824279785156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.309253454208374,
      "rewards/margins": 18.762123107910156,
      "rewards/rejected": -15.45287036895752,
      "step": 6140
    },
    {
      "epoch": 2.4918962722852513,
      "grad_norm": 2.9514881134673487e-06,
      "learning_rate": 1.882035119315624e-06,
      "logits/chosen": -0.9714182615280151,
      "logits/rejected": -1.1771537065505981,
      "logps/chosen": -108.69279479980469,
      "logps/rejected": -324.65771484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3242886066436768,
      "rewards/margins": 18.51775360107422,
      "rewards/rejected": -15.193465232849121,
      "step": 6150
    },
    {
      "epoch": 2.4918962722852513,
      "eval_logits/chosen": -0.9813016057014465,
      "eval_logits/rejected": -1.1988472938537598,
      "eval_logps/chosen": -108.86288452148438,
      "eval_logps/rejected": -326.3122253417969,
      "eval_loss": 4.100524009231776e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.214904546737671,
      "eval_rewards/margins": 18.55185317993164,
      "eval_rewards/rejected": -15.33694839477539,
      "eval_runtime": 660.6059,
      "eval_samples_per_second": 11.208,
      "eval_steps_per_second": 0.624,
      "step": 6150
    },
    {
      "epoch": 2.4959481361426255,
      "grad_norm": 3.1905897230899427e-06,
      "learning_rate": 1.8670268647756267e-06,
      "logits/chosen": -0.9699487090110779,
      "logits/rejected": -1.1912376880645752,
      "logps/chosen": -107.7493896484375,
      "logps/rejected": -327.40380859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2711246013641357,
      "rewards/margins": 18.685203552246094,
      "rewards/rejected": -15.414080619812012,
      "step": 6160
    },
    {
      "epoch": 2.5,
      "grad_norm": 1.7918980574904708e-06,
      "learning_rate": 1.8520186102356297e-06,
      "logits/chosen": -0.9669857025146484,
      "logits/rejected": -1.1998149156570435,
      "logps/chosen": -108.14031219482422,
      "logps/rejected": -327.1600341796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.298938512802124,
      "rewards/margins": 18.699140548706055,
      "rewards/rejected": -15.400202751159668,
      "step": 6170
    },
    {
      "epoch": 2.5040518638573745,
      "grad_norm": 1.860858105828811e-06,
      "learning_rate": 1.837010355695633e-06,
      "logits/chosen": -0.9667934775352478,
      "logits/rejected": -1.1925849914550781,
      "logps/chosen": -107.15023040771484,
      "logps/rejected": -325.98974609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.313432216644287,
      "rewards/margins": 18.61665916442871,
      "rewards/rejected": -15.303226470947266,
      "step": 6180
    },
    {
      "epoch": 2.5081037277147487,
      "grad_norm": 1.3164225265427376e-06,
      "learning_rate": 1.8220021011556357e-06,
      "logits/chosen": -0.9643639326095581,
      "logits/rejected": -1.1708581447601318,
      "logps/chosen": -108.03089141845703,
      "logps/rejected": -327.45391845703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3030054569244385,
      "rewards/margins": 18.699365615844727,
      "rewards/rejected": -15.396356582641602,
      "step": 6190
    },
    {
      "epoch": 2.512155591572123,
      "grad_norm": 4.9158657020598184e-06,
      "learning_rate": 1.8069938466156387e-06,
      "logits/chosen": -0.9764487147331238,
      "logits/rejected": -1.1940850019454956,
      "logps/chosen": -109.26360321044922,
      "logps/rejected": -327.9493103027344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2739195823669434,
      "rewards/margins": 18.715007781982422,
      "rewards/rejected": -15.441088676452637,
      "step": 6200
    },
    {
      "epoch": 2.512155591572123,
      "eval_logits/chosen": -1.0008020401000977,
      "eval_logits/rejected": -1.2112467288970947,
      "eval_logps/chosen": -108.94487762451172,
      "eval_logps/rejected": -326.58551025390625,
      "eval_loss": 3.98079578189936e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.206705093383789,
      "eval_rewards/margins": 18.570980072021484,
      "eval_rewards/rejected": -15.364276885986328,
      "eval_runtime": 660.6193,
      "eval_samples_per_second": 11.208,
      "eval_steps_per_second": 0.624,
      "step": 6200
    },
    {
      "epoch": 2.5162074554294973,
      "grad_norm": 3.4435008728905814e-06,
      "learning_rate": 1.7919855920756417e-06,
      "logits/chosen": -0.9690682888031006,
      "logits/rejected": -1.2002158164978027,
      "logps/chosen": -107.30548858642578,
      "logps/rejected": -326.1789855957031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.261204242706299,
      "rewards/margins": 18.585628509521484,
      "rewards/rejected": -15.32442569732666,
      "step": 6210
    },
    {
      "epoch": 2.520259319286872,
      "grad_norm": 2.629633627293515e-06,
      "learning_rate": 1.7769773375356447e-06,
      "logits/chosen": -0.9658797979354858,
      "logits/rejected": -1.1845813989639282,
      "logps/chosen": -108.27976989746094,
      "logps/rejected": -326.8075256347656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2834341526031494,
      "rewards/margins": 18.606882095336914,
      "rewards/rejected": -15.323447227478027,
      "step": 6220
    },
    {
      "epoch": 2.5243111831442464,
      "grad_norm": 2.3243953819473973e-06,
      "learning_rate": 1.7619690829956475e-06,
      "logits/chosen": -0.9618611931800842,
      "logits/rejected": -1.174437165260315,
      "logps/chosen": -108.25444793701172,
      "logps/rejected": -325.98699951171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2668089866638184,
      "rewards/margins": 18.61248207092285,
      "rewards/rejected": -15.345672607421875,
      "step": 6230
    },
    {
      "epoch": 2.528363047001621,
      "grad_norm": 1.5239019148793886e-06,
      "learning_rate": 1.7469608284556507e-06,
      "logits/chosen": -0.9782617092132568,
      "logits/rejected": -1.1967428922653198,
      "logps/chosen": -108.69795989990234,
      "logps/rejected": -328.13104248046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.244070529937744,
      "rewards/margins": 18.628637313842773,
      "rewards/rejected": -15.384567260742188,
      "step": 6240
    },
    {
      "epoch": 2.532414910858995,
      "grad_norm": 1.023332742988714e-06,
      "learning_rate": 1.7319525739156537e-06,
      "logits/chosen": -0.9639495015144348,
      "logits/rejected": -1.196813702583313,
      "logps/chosen": -108.13853454589844,
      "logps/rejected": -327.58502197265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3000214099884033,
      "rewards/margins": 18.681150436401367,
      "rewards/rejected": -15.381129264831543,
      "step": 6250
    },
    {
      "epoch": 2.532414910858995,
      "eval_logits/chosen": -0.9815783500671387,
      "eval_logits/rejected": -1.1993688344955444,
      "eval_logps/chosen": -108.81712341308594,
      "eval_logps/rejected": -326.4479064941406,
      "eval_loss": 4.022196975483894e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.21947979927063,
      "eval_rewards/margins": 18.569995880126953,
      "eval_rewards/rejected": -15.350517272949219,
      "eval_runtime": 662.1518,
      "eval_samples_per_second": 11.182,
      "eval_steps_per_second": 0.622,
      "step": 6250
    },
    {
      "epoch": 2.5364667747163696,
      "grad_norm": 5.988872089801589e-06,
      "learning_rate": 1.716944319375657e-06,
      "logits/chosen": -0.9757972955703735,
      "logits/rejected": -1.191314458847046,
      "logps/chosen": -107.9080581665039,
      "logps/rejected": -329.70318603515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3165342807769775,
      "rewards/margins": 18.844295501708984,
      "rewards/rejected": -15.52776050567627,
      "step": 6260
    },
    {
      "epoch": 2.5405186385737437,
      "grad_norm": 1.207193508889759e-05,
      "learning_rate": 1.7019360648356598e-06,
      "logits/chosen": -0.9672058820724487,
      "logits/rejected": -1.1961170434951782,
      "logps/chosen": -107.99466705322266,
      "logps/rejected": -327.3460998535156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2853333950042725,
      "rewards/margins": 18.698740005493164,
      "rewards/rejected": -15.413406372070312,
      "step": 6270
    },
    {
      "epoch": 2.5445705024311183,
      "grad_norm": 1.291679382120492e-06,
      "learning_rate": 1.6869278102956628e-06,
      "logits/chosen": -0.9546921253204346,
      "logits/rejected": -1.1888737678527832,
      "logps/chosen": -107.12965393066406,
      "logps/rejected": -327.9710998535156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3467442989349365,
      "rewards/margins": 18.808881759643555,
      "rewards/rejected": -15.462138175964355,
      "step": 6280
    },
    {
      "epoch": 2.548622366288493,
      "grad_norm": 3.3810204058681848e-06,
      "learning_rate": 1.6719195557556658e-06,
      "logits/chosen": -0.9795487523078918,
      "logits/rejected": -1.197702407836914,
      "logps/chosen": -108.23588562011719,
      "logps/rejected": -327.83807373046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2980990409851074,
      "rewards/margins": 18.747577667236328,
      "rewards/rejected": -15.449479103088379,
      "step": 6290
    },
    {
      "epoch": 2.5526742301458674,
      "grad_norm": 3.419266022319789e-06,
      "learning_rate": 1.6569113012156688e-06,
      "logits/chosen": -0.9572386145591736,
      "logits/rejected": -1.1906791925430298,
      "logps/chosen": -107.73524475097656,
      "logps/rejected": -327.8420715332031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2752833366394043,
      "rewards/margins": 18.61233139038086,
      "rewards/rejected": -15.337051391601562,
      "step": 6300
    },
    {
      "epoch": 2.5526742301458674,
      "eval_logits/chosen": -0.9776512980461121,
      "eval_logits/rejected": -1.1944934129714966,
      "eval_logps/chosen": -108.82562255859375,
      "eval_logps/rejected": -326.63250732421875,
      "eval_loss": 3.954363592129084e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2186315059661865,
      "eval_rewards/margins": 18.58760643005371,
      "eval_rewards/rejected": -15.368974685668945,
      "eval_runtime": 670.5606,
      "eval_samples_per_second": 11.042,
      "eval_steps_per_second": 0.614,
      "step": 6300
    },
    {
      "epoch": 2.5567260940032415,
      "grad_norm": 5.7838697102852166e-06,
      "learning_rate": 1.6419030466756716e-06,
      "logits/chosen": -0.9636346697807312,
      "logits/rejected": -1.187281847000122,
      "logps/chosen": -107.17536163330078,
      "logps/rejected": -328.47930908203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3024773597717285,
      "rewards/margins": 18.8435001373291,
      "rewards/rejected": -15.541020393371582,
      "step": 6310
    },
    {
      "epoch": 2.560777957860616,
      "grad_norm": 3.5440295960142976e-06,
      "learning_rate": 1.6268947921356748e-06,
      "logits/chosen": -0.9722870588302612,
      "logits/rejected": -1.1834774017333984,
      "logps/chosen": -109.13838958740234,
      "logps/rejected": -327.3340148925781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2597129344940186,
      "rewards/margins": 18.711063385009766,
      "rewards/rejected": -15.451351165771484,
      "step": 6320
    },
    {
      "epoch": 2.56482982171799,
      "grad_norm": 1.9810686353594065e-06,
      "learning_rate": 1.6118865375956778e-06,
      "logits/chosen": -0.9808294177055359,
      "logits/rejected": -1.2147830724716187,
      "logps/chosen": -108.866943359375,
      "logps/rejected": -329.5595397949219,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.288322925567627,
      "rewards/margins": 18.934534072875977,
      "rewards/rejected": -15.646211624145508,
      "step": 6330
    },
    {
      "epoch": 2.5688816855753647,
      "grad_norm": 3.13096870740992e-06,
      "learning_rate": 1.5968782830556808e-06,
      "logits/chosen": -0.9726601839065552,
      "logits/rejected": -1.1958694458007812,
      "logps/chosen": -108.95462799072266,
      "logps/rejected": -327.8139953613281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2538976669311523,
      "rewards/margins": 18.66844940185547,
      "rewards/rejected": -15.41455078125,
      "step": 6340
    },
    {
      "epoch": 2.572933549432739,
      "grad_norm": 1.2461340475056204e-06,
      "learning_rate": 1.5818700285156838e-06,
      "logits/chosen": -0.9781768918037415,
      "logits/rejected": -1.1878637075424194,
      "logps/chosen": -108.5665054321289,
      "logps/rejected": -328.9209289550781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.26299786567688,
      "rewards/margins": 18.812883377075195,
      "rewards/rejected": -15.549882888793945,
      "step": 6350
    },
    {
      "epoch": 2.572933549432739,
      "eval_logits/chosen": -0.9855466485023499,
      "eval_logits/rejected": -1.202751636505127,
      "eval_logps/chosen": -108.78968811035156,
      "eval_logps/rejected": -326.6907653808594,
      "eval_loss": 3.922154334645711e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.222224235534668,
      "eval_rewards/margins": 18.597021102905273,
      "eval_rewards/rejected": -15.374796867370605,
      "eval_runtime": 660.8039,
      "eval_samples_per_second": 11.205,
      "eval_steps_per_second": 0.623,
      "step": 6350
    },
    {
      "epoch": 2.5769854132901133,
      "grad_norm": 5.123989467392676e-06,
      "learning_rate": 1.5668617739756866e-06,
      "logits/chosen": -0.9808394312858582,
      "logits/rejected": -1.2014094591140747,
      "logps/chosen": -108.37004089355469,
      "logps/rejected": -327.8202209472656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.29618239402771,
      "rewards/margins": 18.780914306640625,
      "rewards/rejected": -15.484735488891602,
      "step": 6360
    },
    {
      "epoch": 2.581037277147488,
      "grad_norm": 1.168843937193742e-06,
      "learning_rate": 1.5518535194356898e-06,
      "logits/chosen": -0.9609886407852173,
      "logits/rejected": -1.1898174285888672,
      "logps/chosen": -108.14075469970703,
      "logps/rejected": -328.1893310546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.315068244934082,
      "rewards/margins": 18.79088592529297,
      "rewards/rejected": -15.47581672668457,
      "step": 6370
    },
    {
      "epoch": 2.585089141004862,
      "grad_norm": 2.1414846287370892e-06,
      "learning_rate": 1.5368452648956928e-06,
      "logits/chosen": -0.9728871583938599,
      "logits/rejected": -1.1902121305465698,
      "logps/chosen": -108.35099792480469,
      "logps/rejected": -327.84222412109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2617249488830566,
      "rewards/margins": 18.661222457885742,
      "rewards/rejected": -15.399496078491211,
      "step": 6380
    },
    {
      "epoch": 2.5891410048622365,
      "grad_norm": 1.1244234201512882e-06,
      "learning_rate": 1.5218370103556956e-06,
      "logits/chosen": -0.9578056931495667,
      "logits/rejected": -1.1797066926956177,
      "logps/chosen": -107.15328216552734,
      "logps/rejected": -328.0039978027344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3418495655059814,
      "rewards/margins": 18.829952239990234,
      "rewards/rejected": -15.488102912902832,
      "step": 6390
    },
    {
      "epoch": 2.593192868719611,
      "grad_norm": 6.752874924131902e-06,
      "learning_rate": 1.5068287558156988e-06,
      "logits/chosen": -0.9773401021957397,
      "logits/rejected": -1.1944974660873413,
      "logps/chosen": -108.75959777832031,
      "logps/rejected": -330.46429443359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2552199363708496,
      "rewards/margins": 18.80958366394043,
      "rewards/rejected": -15.554365158081055,
      "step": 6400
    },
    {
      "epoch": 2.593192868719611,
      "eval_logits/chosen": -0.9779274463653564,
      "eval_logits/rejected": -1.1951502561569214,
      "eval_logps/chosen": -108.78299713134766,
      "eval_logps/rejected": -326.7506103515625,
      "eval_loss": 3.899987888189571e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.222893238067627,
      "eval_rewards/margins": 18.603673934936523,
      "eval_rewards/rejected": -15.380780220031738,
      "eval_runtime": 661.3907,
      "eval_samples_per_second": 11.195,
      "eval_steps_per_second": 0.623,
      "step": 6400
    },
    {
      "epoch": 2.5972447325769856,
      "grad_norm": 1.3012622730457224e-06,
      "learning_rate": 1.4918205012757018e-06,
      "logits/chosen": -0.962655246257782,
      "logits/rejected": -1.1892623901367188,
      "logps/chosen": -106.69053649902344,
      "logps/rejected": -327.73455810546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3535757064819336,
      "rewards/margins": 18.768230438232422,
      "rewards/rejected": -15.414655685424805,
      "step": 6410
    },
    {
      "epoch": 2.6012965964343597,
      "grad_norm": 1.678008970884548e-06,
      "learning_rate": 1.4768122467357048e-06,
      "logits/chosen": -0.9705030918121338,
      "logits/rejected": -1.2016093730926514,
      "logps/chosen": -107.60749053955078,
      "logps/rejected": -329.26025390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.301589250564575,
      "rewards/margins": 18.92018699645996,
      "rewards/rejected": -15.618597030639648,
      "step": 6420
    },
    {
      "epoch": 2.6053484602917343,
      "grad_norm": 1.2321225767664146e-05,
      "learning_rate": 1.4618039921957078e-06,
      "logits/chosen": -0.9854863286018372,
      "logits/rejected": -1.2013943195343018,
      "logps/chosen": -108.88626861572266,
      "logps/rejected": -328.6309814453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.246922254562378,
      "rewards/margins": 18.843786239624023,
      "rewards/rejected": -15.59686279296875,
      "step": 6430
    },
    {
      "epoch": 2.6094003241491084,
      "grad_norm": 4.80731796415057e-06,
      "learning_rate": 1.4467957376557106e-06,
      "logits/chosen": -0.9673572778701782,
      "logits/rejected": -1.2005420923233032,
      "logps/chosen": -106.57898712158203,
      "logps/rejected": -328.31396484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.35111403465271,
      "rewards/margins": 18.79676628112793,
      "rewards/rejected": -15.44565200805664,
      "step": 6440
    },
    {
      "epoch": 2.613452188006483,
      "grad_norm": 2.3030111151456367e-06,
      "learning_rate": 1.4317874831157138e-06,
      "logits/chosen": -0.9817792177200317,
      "logits/rejected": -1.1919387578964233,
      "logps/chosen": -108.42607116699219,
      "logps/rejected": -327.2312316894531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2762937545776367,
      "rewards/margins": 18.632549285888672,
      "rewards/rejected": -15.356255531311035,
      "step": 6450
    },
    {
      "epoch": 2.613452188006483,
      "eval_logits/chosen": -0.9837523698806763,
      "eval_logits/rejected": -1.2007975578308105,
      "eval_logps/chosen": -108.7887191772461,
      "eval_logps/rejected": -327.0396728515625,
      "eval_loss": 3.796148817514222e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2223217487335205,
      "eval_rewards/margins": 18.632015228271484,
      "eval_rewards/rejected": -15.409693717956543,
      "eval_runtime": 659.9782,
      "eval_samples_per_second": 11.219,
      "eval_steps_per_second": 0.624,
      "step": 6450
    },
    {
      "epoch": 2.6175040518638575,
      "grad_norm": 1.7890401068143547e-05,
      "learning_rate": 1.4167792285757168e-06,
      "logits/chosen": -0.9685986638069153,
      "logits/rejected": -1.2094436883926392,
      "logps/chosen": -107.218017578125,
      "logps/rejected": -327.6186218261719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3231217861175537,
      "rewards/margins": 18.779752731323242,
      "rewards/rejected": -15.456629753112793,
      "step": 6460
    },
    {
      "epoch": 2.621555915721232,
      "grad_norm": 2.380106025157147e-06,
      "learning_rate": 1.4017709740357196e-06,
      "logits/chosen": -0.971544623374939,
      "logits/rejected": -1.1954891681671143,
      "logps/chosen": -107.54015350341797,
      "logps/rejected": -327.93792724609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2983453273773193,
      "rewards/margins": 18.725475311279297,
      "rewards/rejected": -15.427130699157715,
      "step": 6470
    },
    {
      "epoch": 2.625607779578606,
      "grad_norm": 2.2546776108356426e-06,
      "learning_rate": 1.3867627194957228e-06,
      "logits/chosen": -0.9820596575737,
      "logits/rejected": -1.1837631464004517,
      "logps/chosen": -108.93296813964844,
      "logps/rejected": -327.3031005859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.26320743560791,
      "rewards/margins": 18.705913543701172,
      "rewards/rejected": -15.442708015441895,
      "step": 6480
    },
    {
      "epoch": 2.6296596434359807,
      "grad_norm": 2.7140899874211755e-06,
      "learning_rate": 1.3717544649557256e-06,
      "logits/chosen": -0.9743025898933411,
      "logits/rejected": -1.2002640962600708,
      "logps/chosen": -108.07935333251953,
      "logps/rejected": -328.34503173828125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.272324562072754,
      "rewards/margins": 18.76234245300293,
      "rewards/rejected": -15.490017890930176,
      "step": 6490
    },
    {
      "epoch": 2.633711507293355,
      "grad_norm": 2.2897106646269094e-06,
      "learning_rate": 1.3567462104157286e-06,
      "logits/chosen": -0.9556276798248291,
      "logits/rejected": -1.1725317239761353,
      "logps/chosen": -107.97649383544922,
      "logps/rejected": -328.3132629394531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3200385570526123,
      "rewards/margins": 18.83300018310547,
      "rewards/rejected": -15.512959480285645,
      "step": 6500
    },
    {
      "epoch": 2.633711507293355,
      "eval_logits/chosen": -0.9754812717437744,
      "eval_logits/rejected": -1.1918588876724243,
      "eval_logps/chosen": -108.743408203125,
      "eval_logps/rejected": -326.9277038574219,
      "eval_loss": 3.817614313561535e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.22685170173645,
      "eval_rewards/margins": 18.62534523010254,
      "eval_rewards/rejected": -15.398492813110352,
      "eval_runtime": 663.7926,
      "eval_samples_per_second": 11.154,
      "eval_steps_per_second": 0.621,
      "step": 6500
    },
    {
      "epoch": 2.6377633711507293,
      "grad_norm": 1.1392465239623562e-05,
      "learning_rate": 1.3417379558757318e-06,
      "logits/chosen": -0.9725505113601685,
      "logits/rejected": -1.1995872259140015,
      "logps/chosen": -108.75624084472656,
      "logps/rejected": -327.79815673828125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2533085346221924,
      "rewards/margins": 18.761568069458008,
      "rewards/rejected": -15.508259773254395,
      "step": 6510
    },
    {
      "epoch": 2.641815235008104,
      "grad_norm": 3.2896393804549007e-06,
      "learning_rate": 1.3267297013357346e-06,
      "logits/chosen": -0.978614091873169,
      "logits/rejected": -1.1826672554016113,
      "logps/chosen": -107.8505859375,
      "logps/rejected": -328.4673156738281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3108980655670166,
      "rewards/margins": 18.78472900390625,
      "rewards/rejected": -15.473831176757812,
      "step": 6520
    },
    {
      "epoch": 2.645867098865478,
      "grad_norm": 2.180285036956775e-06,
      "learning_rate": 1.3117214467957378e-06,
      "logits/chosen": -0.9547456502914429,
      "logits/rejected": -1.1693174839019775,
      "logps/chosen": -107.28286743164062,
      "logps/rejected": -326.9376525878906,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.31911563873291,
      "rewards/margins": 18.735675811767578,
      "rewards/rejected": -15.416559219360352,
      "step": 6530
    },
    {
      "epoch": 2.6499189627228525,
      "grad_norm": 2.427897243251209e-06,
      "learning_rate": 1.2967131922557409e-06,
      "logits/chosen": -0.9694584012031555,
      "logits/rejected": -1.1888294219970703,
      "logps/chosen": -107.25581359863281,
      "logps/rejected": -329.0333251953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3289191722869873,
      "rewards/margins": 18.88922882080078,
      "rewards/rejected": -15.560308456420898,
      "step": 6540
    },
    {
      "epoch": 2.6539708265802266,
      "grad_norm": 5.982232778478647e-06,
      "learning_rate": 1.2817049377157436e-06,
      "logits/chosen": -0.9642473459243774,
      "logits/rejected": -1.1938378810882568,
      "logps/chosen": -108.4251937866211,
      "logps/rejected": -329.1801452636719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.263878107070923,
      "rewards/margins": 18.861995697021484,
      "rewards/rejected": -15.598118782043457,
      "step": 6550
    },
    {
      "epoch": 2.6539708265802266,
      "eval_logits/chosen": -0.9800311326980591,
      "eval_logits/rejected": -1.196731448173523,
      "eval_logps/chosen": -108.75553894042969,
      "eval_logps/rejected": -327.10906982421875,
      "eval_loss": 3.7646959327730656e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2256405353546143,
      "eval_rewards/margins": 18.64227294921875,
      "eval_rewards/rejected": -15.416630744934082,
      "eval_runtime": 660.9228,
      "eval_samples_per_second": 11.203,
      "eval_steps_per_second": 0.623,
      "step": 6550
    },
    {
      "epoch": 2.658022690437601,
      "grad_norm": 6.1209575505927205e-06,
      "learning_rate": 1.2666966831757469e-06,
      "logits/chosen": -0.9827333688735962,
      "logits/rejected": -1.2020137310028076,
      "logps/chosen": -109.26451110839844,
      "logps/rejected": -327.8392333984375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2394943237304688,
      "rewards/margins": 18.677507400512695,
      "rewards/rejected": -15.438010215759277,
      "step": 6560
    },
    {
      "epoch": 2.6620745542949757,
      "grad_norm": 3.7437689570651855e-06,
      "learning_rate": 1.2516884286357497e-06,
      "logits/chosen": -0.9661514759063721,
      "logits/rejected": -1.187090516090393,
      "logps/chosen": -108.47927856445312,
      "logps/rejected": -328.90728759765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3015549182891846,
      "rewards/margins": 18.853607177734375,
      "rewards/rejected": -15.552051544189453,
      "step": 6570
    },
    {
      "epoch": 2.6661264181523503,
      "grad_norm": 2.605788267828757e-06,
      "learning_rate": 1.2366801740957529e-06,
      "logits/chosen": -0.9742575287818909,
      "logits/rejected": -1.186073899269104,
      "logps/chosen": -108.8031997680664,
      "logps/rejected": -326.662109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2990267276763916,
      "rewards/margins": 18.651639938354492,
      "rewards/rejected": -15.352612495422363,
      "step": 6580
    },
    {
      "epoch": 2.6701782820097244,
      "grad_norm": 1.6996804106383934e-06,
      "learning_rate": 1.2216719195557559e-06,
      "logits/chosen": -0.9773194789886475,
      "logits/rejected": -1.1909611225128174,
      "logps/chosen": -107.54708099365234,
      "logps/rejected": -327.1669006347656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2739102840423584,
      "rewards/margins": 18.743066787719727,
      "rewards/rejected": -15.469155311584473,
      "step": 6590
    },
    {
      "epoch": 2.674230145867099,
      "grad_norm": 2.250463694508653e-06,
      "learning_rate": 1.2066636650157587e-06,
      "logits/chosen": -0.9738913774490356,
      "logits/rejected": -1.1955630779266357,
      "logps/chosen": -108.31841278076172,
      "logps/rejected": -328.6866149902344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.345407724380493,
      "rewards/margins": 18.886333465576172,
      "rewards/rejected": -15.54092788696289,
      "step": 6600
    },
    {
      "epoch": 2.674230145867099,
      "eval_logits/chosen": -0.9773862361907959,
      "eval_logits/rejected": -1.1934053897857666,
      "eval_logps/chosen": -108.75808715820312,
      "eval_logps/rejected": -327.1883239746094,
      "eval_loss": 3.73986921431424e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.22538423538208,
      "eval_rewards/margins": 18.649938583374023,
      "eval_rewards/rejected": -15.424555778503418,
      "eval_runtime": 660.0959,
      "eval_samples_per_second": 11.217,
      "eval_steps_per_second": 0.624,
      "step": 6600
    },
    {
      "epoch": 2.678282009724473,
      "grad_norm": 1.7172280422528274e-05,
      "learning_rate": 1.1916554104757617e-06,
      "logits/chosen": -0.9561381936073303,
      "logits/rejected": -1.1864134073257446,
      "logps/chosen": -106.45145416259766,
      "logps/rejected": -327.47332763671875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.330907106399536,
      "rewards/margins": 18.795562744140625,
      "rewards/rejected": -15.464655876159668,
      "step": 6610
    },
    {
      "epoch": 2.6823338735818476,
      "grad_norm": 2.226166770924465e-06,
      "learning_rate": 1.1766471559357649e-06,
      "logits/chosen": -0.9810739159584045,
      "logits/rejected": -1.2017022371292114,
      "logps/chosen": -107.25318145751953,
      "logps/rejected": -327.8432312011719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.287501096725464,
      "rewards/margins": 18.69404411315918,
      "rewards/rejected": -15.406542778015137,
      "step": 6620
    },
    {
      "epoch": 2.686385737439222,
      "grad_norm": 1.5006548892415594e-06,
      "learning_rate": 1.1616389013957679e-06,
      "logits/chosen": -0.9777254462242126,
      "logits/rejected": -1.1854885816574097,
      "logps/chosen": -108.16984558105469,
      "logps/rejected": -328.51971435546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2975127696990967,
      "rewards/margins": 18.87528419494629,
      "rewards/rejected": -15.577771186828613,
      "step": 6630
    },
    {
      "epoch": 2.6904376012965967,
      "grad_norm": 1.0780703405544045e-06,
      "learning_rate": 1.1466306468557707e-06,
      "logits/chosen": -0.9816153645515442,
      "logits/rejected": -1.1960108280181885,
      "logps/chosen": -108.11849212646484,
      "logps/rejected": -328.8614196777344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3232884407043457,
      "rewards/margins": 18.841562271118164,
      "rewards/rejected": -15.51827335357666,
      "step": 6640
    },
    {
      "epoch": 2.694489465153971,
      "grad_norm": 1.7732330661601736e-06,
      "learning_rate": 1.1316223923157737e-06,
      "logits/chosen": -0.9946374297142029,
      "logits/rejected": -1.1932036876678467,
      "logps/chosen": -108.26897430419922,
      "logps/rejected": -327.3341064453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2642621994018555,
      "rewards/margins": 18.655010223388672,
      "rewards/rejected": -15.390748023986816,
      "step": 6650
    },
    {
      "epoch": 2.694489465153971,
      "eval_logits/chosen": -0.9796309471130371,
      "eval_logits/rejected": -1.196200966835022,
      "eval_logps/chosen": -108.73944854736328,
      "eval_logps/rejected": -327.26641845703125,
      "eval_loss": 3.71922439512673e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2272489070892334,
      "eval_rewards/margins": 18.65962028503418,
      "eval_rewards/rejected": -15.432371139526367,
      "eval_runtime": 660.0752,
      "eval_samples_per_second": 11.217,
      "eval_steps_per_second": 0.624,
      "step": 6650
    },
    {
      "epoch": 2.6985413290113454,
      "grad_norm": 4.143021669733571e-06,
      "learning_rate": 1.1166141377757767e-06,
      "logits/chosen": -0.9760237336158752,
      "logits/rejected": -1.1893646717071533,
      "logps/chosen": -108.61851501464844,
      "logps/rejected": -327.9139404296875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.271268367767334,
      "rewards/margins": 18.741382598876953,
      "rewards/rejected": -15.470113754272461,
      "step": 6660
    },
    {
      "epoch": 2.7025931928687195,
      "grad_norm": 2.5221820578735787e-06,
      "learning_rate": 1.10160588323578e-06,
      "logits/chosen": -0.9682928323745728,
      "logits/rejected": -1.1885979175567627,
      "logps/chosen": -107.11003875732422,
      "logps/rejected": -328.7527770996094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.328941822052002,
      "rewards/margins": 18.782878875732422,
      "rewards/rejected": -15.453938484191895,
      "step": 6670
    },
    {
      "epoch": 2.706645056726094,
      "grad_norm": 3.2951190860330826e-06,
      "learning_rate": 1.0865976286957827e-06,
      "logits/chosen": -0.9695104956626892,
      "logits/rejected": -1.1906455755233765,
      "logps/chosen": -107.94957733154297,
      "logps/rejected": -327.61492919921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3171699047088623,
      "rewards/margins": 18.681623458862305,
      "rewards/rejected": -15.364453315734863,
      "step": 6680
    },
    {
      "epoch": 2.7106969205834686,
      "grad_norm": 1.9521098693076055e-06,
      "learning_rate": 1.0715893741557857e-06,
      "logits/chosen": -0.9845353364944458,
      "logits/rejected": -1.1906646490097046,
      "logps/chosen": -109.4488296508789,
      "logps/rejected": -326.2509765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2422358989715576,
      "rewards/margins": 18.503803253173828,
      "rewards/rejected": -15.261566162109375,
      "step": 6690
    },
    {
      "epoch": 2.7147487844408427,
      "grad_norm": 3.5517336982593406e-06,
      "learning_rate": 1.0565811196157887e-06,
      "logits/chosen": -0.9753643870353699,
      "logits/rejected": -1.1850273609161377,
      "logps/chosen": -108.79093933105469,
      "logps/rejected": -327.6737060546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.246598720550537,
      "rewards/margins": 18.710622787475586,
      "rewards/rejected": -15.46402359008789,
      "step": 6700
    },
    {
      "epoch": 2.7147487844408427,
      "eval_logits/chosen": -0.978154182434082,
      "eval_logits/rejected": -1.194498062133789,
      "eval_logps/chosen": -108.75652313232422,
      "eval_logps/rejected": -327.4468994140625,
      "eval_loss": 3.6394169455888914e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2255406379699707,
      "eval_rewards/margins": 18.675954818725586,
      "eval_rewards/rejected": -15.450413703918457,
      "eval_runtime": 660.1153,
      "eval_samples_per_second": 11.216,
      "eval_steps_per_second": 0.624,
      "step": 6700
    },
    {
      "epoch": 2.718800648298217,
      "grad_norm": 4.4465300561569165e-06,
      "learning_rate": 1.041572865075792e-06,
      "logits/chosen": -0.974888265132904,
      "logits/rejected": -1.1830838918685913,
      "logps/chosen": -108.81172180175781,
      "logps/rejected": -328.89105224609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.28983211517334,
      "rewards/margins": 18.738496780395508,
      "rewards/rejected": -15.448664665222168,
      "step": 6710
    },
    {
      "epoch": 2.7228525121555913,
      "grad_norm": 5.820764727104688e-06,
      "learning_rate": 1.0265646105357947e-06,
      "logits/chosen": -0.9631332159042358,
      "logits/rejected": -1.198506474494934,
      "logps/chosen": -108.20296478271484,
      "logps/rejected": -329.89849853515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2834346294403076,
      "rewards/margins": 18.84991455078125,
      "rewards/rejected": -15.566479682922363,
      "step": 6720
    },
    {
      "epoch": 2.726904376012966,
      "grad_norm": 1.8593394770505256e-06,
      "learning_rate": 1.0115563559957977e-06,
      "logits/chosen": -0.9674119353294373,
      "logits/rejected": -1.1959097385406494,
      "logps/chosen": -107.18940734863281,
      "logps/rejected": -329.5528869628906,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3657302856445312,
      "rewards/margins": 18.96901512145996,
      "rewards/rejected": -15.60328483581543,
      "step": 6730
    },
    {
      "epoch": 2.7309562398703404,
      "grad_norm": 2.07598645829421e-06,
      "learning_rate": 9.965481014558007e-07,
      "logits/chosen": -0.9660282135009766,
      "logits/rejected": -1.2024939060211182,
      "logps/chosen": -107.80119323730469,
      "logps/rejected": -329.72637939453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.331458806991577,
      "rewards/margins": 18.926374435424805,
      "rewards/rejected": -15.594915390014648,
      "step": 6740
    },
    {
      "epoch": 2.735008103727715,
      "grad_norm": 9.826493396758451e-07,
      "learning_rate": 9.81539846915804e-07,
      "logits/chosen": -0.9771832227706909,
      "logits/rejected": -1.191651463508606,
      "logps/chosen": -109.6639404296875,
      "logps/rejected": -328.5887145996094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.218618631362915,
      "rewards/margins": 18.78942108154297,
      "rewards/rejected": -15.570802688598633,
      "step": 6750
    },
    {
      "epoch": 2.735008103727715,
      "eval_logits/chosen": -1.0023131370544434,
      "eval_logits/rejected": -1.211345911026001,
      "eval_logps/chosen": -108.87815856933594,
      "eval_logps/rejected": -327.7980041503906,
      "eval_loss": 3.559854278023522e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2133777141571045,
      "eval_rewards/margins": 18.698904037475586,
      "eval_rewards/rejected": -15.48552417755127,
      "eval_runtime": 660.0792,
      "eval_samples_per_second": 11.217,
      "eval_steps_per_second": 0.624,
      "step": 6750
    },
    {
      "epoch": 2.739059967585089,
      "grad_norm": 1.8573822444523103e-06,
      "learning_rate": 9.665315923758067e-07,
      "logits/chosen": -0.9557449221611023,
      "logits/rejected": -1.1949938535690308,
      "logps/chosen": -107.61819458007812,
      "logps/rejected": -330.19622802734375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.296236991882324,
      "rewards/margins": 18.978748321533203,
      "rewards/rejected": -15.682511329650879,
      "step": 6760
    },
    {
      "epoch": 2.7431118314424636,
      "grad_norm": 4.063704636791954e-06,
      "learning_rate": 9.515233378358097e-07,
      "logits/chosen": -0.9694454073905945,
      "logits/rejected": -1.2075059413909912,
      "logps/chosen": -107.1187515258789,
      "logps/rejected": -327.4303283691406,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3184962272644043,
      "rewards/margins": 18.786420822143555,
      "rewards/rejected": -15.467925071716309,
      "step": 6770
    },
    {
      "epoch": 2.7471636952998377,
      "grad_norm": 5.965304808341898e-06,
      "learning_rate": 9.365150832958128e-07,
      "logits/chosen": -0.9721193909645081,
      "logits/rejected": -1.2048670053482056,
      "logps/chosen": -108.52510833740234,
      "logps/rejected": -328.89630126953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2876505851745605,
      "rewards/margins": 18.767431259155273,
      "rewards/rejected": -15.479781150817871,
      "step": 6780
    },
    {
      "epoch": 2.7512155591572123,
      "grad_norm": 1.0380433650425402e-06,
      "learning_rate": 9.215068287558157e-07,
      "logits/chosen": -0.9637234807014465,
      "logits/rejected": -1.194984793663025,
      "logps/chosen": -107.27920532226562,
      "logps/rejected": -328.76885986328125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3212697505950928,
      "rewards/margins": 18.82971954345703,
      "rewards/rejected": -15.508450508117676,
      "step": 6790
    },
    {
      "epoch": 2.755267423014587,
      "grad_norm": 5.067347046860959e-06,
      "learning_rate": 9.064985742158187e-07,
      "logits/chosen": -0.975241482257843,
      "logits/rejected": -1.1849026679992676,
      "logps/chosen": -106.6024169921875,
      "logps/rejected": -328.2080993652344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.352440118789673,
      "rewards/margins": 18.906578063964844,
      "rewards/rejected": -15.554136276245117,
      "step": 6800
    },
    {
      "epoch": 2.755267423014587,
      "eval_logits/chosen": -0.9843966364860535,
      "eval_logits/rejected": -1.2008051872253418,
      "eval_logps/chosen": -108.77916717529297,
      "eval_logps/rejected": -327.68450927734375,
      "eval_loss": 3.5788882968290636e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.223276138305664,
      "eval_rewards/margins": 18.697452545166016,
      "eval_rewards/rejected": -15.474178314208984,
      "eval_runtime": 660.1503,
      "eval_samples_per_second": 11.216,
      "eval_steps_per_second": 0.624,
      "step": 6800
    },
    {
      "epoch": 2.7593192868719614,
      "grad_norm": 6.067515641916543e-06,
      "learning_rate": 8.914903196758217e-07,
      "logits/chosen": -0.9731653332710266,
      "logits/rejected": -1.206840991973877,
      "logps/chosen": -109.11463928222656,
      "logps/rejected": -329.27374267578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2963597774505615,
      "rewards/margins": 18.844810485839844,
      "rewards/rejected": -15.54845142364502,
      "step": 6810
    },
    {
      "epoch": 2.7633711507293355,
      "grad_norm": 2.348689349673805e-06,
      "learning_rate": 8.764820651358249e-07,
      "logits/chosen": -0.986677885055542,
      "logits/rejected": -1.2064417600631714,
      "logps/chosen": -108.65538787841797,
      "logps/rejected": -327.7326965332031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.285518169403076,
      "rewards/margins": 18.810688018798828,
      "rewards/rejected": -15.525167465209961,
      "step": 6820
    },
    {
      "epoch": 2.76742301458671,
      "grad_norm": 5.050430900155334e-06,
      "learning_rate": 8.614738105958278e-07,
      "logits/chosen": -0.9671355485916138,
      "logits/rejected": -1.181667447090149,
      "logps/chosen": -107.4686050415039,
      "logps/rejected": -328.76409912109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.312302589416504,
      "rewards/margins": 18.85050392150879,
      "rewards/rejected": -15.538204193115234,
      "step": 6830
    },
    {
      "epoch": 2.771474878444084,
      "grad_norm": 3.911934072675649e-06,
      "learning_rate": 8.464655560558308e-07,
      "logits/chosen": -0.9895461797714233,
      "logits/rejected": -1.1977813243865967,
      "logps/chosen": -109.29845428466797,
      "logps/rejected": -327.5505065917969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.289475440979004,
      "rewards/margins": 18.74016761779785,
      "rewards/rejected": -15.450695037841797,
      "step": 6840
    },
    {
      "epoch": 2.7755267423014587,
      "grad_norm": 1.965021510841325e-06,
      "learning_rate": 8.314573015158338e-07,
      "logits/chosen": -0.9708042144775391,
      "logits/rejected": -1.1981844902038574,
      "logps/chosen": -106.59725189208984,
      "logps/rejected": -330.02972412109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3511555194854736,
      "rewards/margins": 18.995262145996094,
      "rewards/rejected": -15.644104957580566,
      "step": 6850
    },
    {
      "epoch": 2.7755267423014587,
      "eval_logits/chosen": -1.0009392499923706,
      "eval_logits/rejected": -1.2106529474258423,
      "eval_logps/chosen": -108.86872100830078,
      "eval_logps/rejected": -327.986572265625,
      "eval_loss": 3.468585774157873e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2143208980560303,
      "eval_rewards/margins": 18.71870231628418,
      "eval_rewards/rejected": -15.50438404083252,
      "eval_runtime": 660.0623,
      "eval_samples_per_second": 11.217,
      "eval_steps_per_second": 0.624,
      "step": 6850
    },
    {
      "epoch": 2.7795786061588332,
      "grad_norm": 3.210667728126282e-06,
      "learning_rate": 8.164490469758369e-07,
      "logits/chosen": -0.9684648513793945,
      "logits/rejected": -1.1943256855010986,
      "logps/chosen": -108.81153106689453,
      "logps/rejected": -328.4454345703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2552523612976074,
      "rewards/margins": 18.796825408935547,
      "rewards/rejected": -15.541571617126465,
      "step": 6860
    },
    {
      "epoch": 2.7836304700162073,
      "grad_norm": 1.2010252703475999e-06,
      "learning_rate": 8.014407924358398e-07,
      "logits/chosen": -0.9661147594451904,
      "logits/rejected": -1.2026818990707397,
      "logps/chosen": -108.54756164550781,
      "logps/rejected": -328.1592102050781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2764689922332764,
      "rewards/margins": 18.8625545501709,
      "rewards/rejected": -15.58608627319336,
      "step": 6870
    },
    {
      "epoch": 2.787682333873582,
      "grad_norm": 4.421299763635034e-06,
      "learning_rate": 7.864325378958428e-07,
      "logits/chosen": -0.9725923538208008,
      "logits/rejected": -1.1912277936935425,
      "logps/chosen": -108.68296813964844,
      "logps/rejected": -328.15887451171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2545135021209717,
      "rewards/margins": 18.749614715576172,
      "rewards/rejected": -15.495102882385254,
      "step": 6880
    },
    {
      "epoch": 2.791734197730956,
      "grad_norm": 5.234048330748919e-06,
      "learning_rate": 7.714242833558458e-07,
      "logits/chosen": -0.9834954738616943,
      "logits/rejected": -1.2085795402526855,
      "logps/chosen": -106.94400024414062,
      "logps/rejected": -328.7509765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.296764612197876,
      "rewards/margins": 18.812835693359375,
      "rewards/rejected": -15.516067504882812,
      "step": 6890
    },
    {
      "epoch": 2.7957860615883305,
      "grad_norm": 2.2638657810603036e-06,
      "learning_rate": 7.564160288158488e-07,
      "logits/chosen": -0.9776025414466858,
      "logits/rejected": -1.1949238777160645,
      "logps/chosen": -107.86272430419922,
      "logps/rejected": -327.9671936035156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2815756797790527,
      "rewards/margins": 18.835134506225586,
      "rewards/rejected": -15.553556442260742,
      "step": 6900
    },
    {
      "epoch": 2.7957860615883305,
      "eval_logits/chosen": -0.9994562864303589,
      "eval_logits/rejected": -1.2082576751708984,
      "eval_logps/chosen": -108.87102508544922,
      "eval_logps/rejected": -328.0704650878906,
      "eval_loss": 3.4673938387186354e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.214090347290039,
      "eval_rewards/margins": 18.72686195373535,
      "eval_rewards/rejected": -15.512771606445312,
      "eval_runtime": 660.1232,
      "eval_samples_per_second": 11.216,
      "eval_steps_per_second": 0.624,
      "step": 6900
    },
    {
      "epoch": 2.799837925445705,
      "grad_norm": 1.272898316528881e-06,
      "learning_rate": 7.414077742758517e-07,
      "logits/chosen": -0.9717825651168823,
      "logits/rejected": -1.1962286233901978,
      "logps/chosen": -107.53852081298828,
      "logps/rejected": -329.1290588378906,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3080146312713623,
      "rewards/margins": 18.90289306640625,
      "rewards/rejected": -15.594879150390625,
      "step": 6910
    },
    {
      "epoch": 2.8038897893030796,
      "grad_norm": 2.171340611312189e-06,
      "learning_rate": 7.263995197358548e-07,
      "logits/chosen": -0.9800161719322205,
      "logits/rejected": -1.1981581449508667,
      "logps/chosen": -108.62446594238281,
      "logps/rejected": -328.65447998046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2291815280914307,
      "rewards/margins": 18.77030372619629,
      "rewards/rejected": -15.54112434387207,
      "step": 6920
    },
    {
      "epoch": 2.8079416531604537,
      "grad_norm": 2.7905246042791987e-06,
      "learning_rate": 7.113912651958578e-07,
      "logits/chosen": -0.9674810767173767,
      "logits/rejected": -1.1972414255142212,
      "logps/chosen": -107.71682739257812,
      "logps/rejected": -328.8835754394531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.32235050201416,
      "rewards/margins": 18.848758697509766,
      "rewards/rejected": -15.526408195495605,
      "step": 6930
    },
    {
      "epoch": 2.8119935170178283,
      "grad_norm": 3.5024731914745644e-06,
      "learning_rate": 6.963830106558608e-07,
      "logits/chosen": -0.9778474569320679,
      "logits/rejected": -1.1916078329086304,
      "logps/chosen": -108.36911010742188,
      "logps/rejected": -327.77008056640625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.303269147872925,
      "rewards/margins": 18.71904945373535,
      "rewards/rejected": -15.41578197479248,
      "step": 6940
    },
    {
      "epoch": 2.8160453808752024,
      "grad_norm": 1.0694878938011243e-06,
      "learning_rate": 6.813747561158637e-07,
      "logits/chosen": -0.9754053354263306,
      "logits/rejected": -1.2015597820281982,
      "logps/chosen": -107.3707046508789,
      "logps/rejected": -329.3548889160156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3287465572357178,
      "rewards/margins": 19.000261306762695,
      "rewards/rejected": -15.671514511108398,
      "step": 6950
    },
    {
      "epoch": 2.8160453808752024,
      "eval_logits/chosen": -1.0035643577575684,
      "eval_logits/rejected": -1.2130249738693237,
      "eval_logps/chosen": -108.90617370605469,
      "eval_logps/rejected": -328.15283203125,
      "eval_loss": 3.4547639415904996e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.210576295852661,
      "eval_rewards/margins": 18.731584548950195,
      "eval_rewards/rejected": -15.521010398864746,
      "eval_runtime": 660.1721,
      "eval_samples_per_second": 11.215,
      "eval_steps_per_second": 0.624,
      "step": 6950
    },
    {
      "epoch": 2.820097244732577,
      "grad_norm": 2.2863562207930954e-06,
      "learning_rate": 6.663665015758668e-07,
      "logits/chosen": -0.9680902361869812,
      "logits/rejected": -1.1875684261322021,
      "logps/chosen": -108.46826171875,
      "logps/rejected": -329.11334228515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.278618812561035,
      "rewards/margins": 18.850582122802734,
      "rewards/rejected": -15.571962356567383,
      "step": 6960
    },
    {
      "epoch": 2.8241491085899515,
      "grad_norm": 2.9234643079689704e-06,
      "learning_rate": 6.513582470358698e-07,
      "logits/chosen": -0.9727660417556763,
      "logits/rejected": -1.2107149362564087,
      "logps/chosen": -106.91544342041016,
      "logps/rejected": -329.1871337890625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.328211545944214,
      "rewards/margins": 18.9506778717041,
      "rewards/rejected": -15.622466087341309,
      "step": 6970
    },
    {
      "epoch": 2.828200972447326,
      "grad_norm": 2.178443310185685e-06,
      "learning_rate": 6.363499924958728e-07,
      "logits/chosen": -0.9760904908180237,
      "logits/rejected": -1.1950515508651733,
      "logps/chosen": -108.16962432861328,
      "logps/rejected": -328.8347473144531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.301807403564453,
      "rewards/margins": 18.828533172607422,
      "rewards/rejected": -15.526728630065918,
      "step": 6980
    },
    {
      "epoch": 2.8322528363047,
      "grad_norm": 1.2093514669686556e-06,
      "learning_rate": 6.213417379558758e-07,
      "logits/chosen": -0.9775959253311157,
      "logits/rejected": -1.197165846824646,
      "logps/chosen": -108.63644409179688,
      "logps/rejected": -329.37310791015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3316054344177246,
      "rewards/margins": 18.91165542602539,
      "rewards/rejected": -15.58005142211914,
      "step": 6990
    },
    {
      "epoch": 2.8363047001620747,
      "grad_norm": 3.0348555810633115e-06,
      "learning_rate": 6.063334834158788e-07,
      "logits/chosen": -0.9617421627044678,
      "logits/rejected": -1.1999685764312744,
      "logps/chosen": -108.18900299072266,
      "logps/rejected": -329.33526611328125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2798407077789307,
      "rewards/margins": 18.769933700561523,
      "rewards/rejected": -15.490091323852539,
      "step": 7000
    },
    {
      "epoch": 2.8363047001620747,
      "eval_logits/chosen": -0.9975866079330444,
      "eval_logits/rejected": -1.2070943117141724,
      "eval_logps/chosen": -108.84796905517578,
      "eval_logps/rejected": -328.05340576171875,
      "eval_loss": 3.4508691015844306e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.216395139694214,
      "eval_rewards/margins": 18.727460861206055,
      "eval_rewards/rejected": -15.511064529418945,
      "eval_runtime": 660.1542,
      "eval_samples_per_second": 11.216,
      "eval_steps_per_second": 0.624,
      "step": 7000
    },
    {
      "epoch": 2.840356564019449,
      "grad_norm": 1.0463385478942655e-05,
      "learning_rate": 5.913252288758818e-07,
      "logits/chosen": -0.9831451773643494,
      "logits/rejected": -1.1934568881988525,
      "logps/chosen": -107.50833129882812,
      "logps/rejected": -329.0498046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2817866802215576,
      "rewards/margins": 18.812671661376953,
      "rewards/rejected": -15.530885696411133,
      "step": 7010
    },
    {
      "epoch": 2.8444084278768234,
      "grad_norm": 2.905329438362969e-06,
      "learning_rate": 5.763169743358848e-07,
      "logits/chosen": -0.9958447813987732,
      "logits/rejected": -1.2032657861709595,
      "logps/chosen": -108.56275939941406,
      "logps/rejected": -327.8564758300781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2593607902526855,
      "rewards/margins": 18.71350860595703,
      "rewards/rejected": -15.454147338867188,
      "step": 7020
    },
    {
      "epoch": 2.848460291734198,
      "grad_norm": 1.7165253893836052e-06,
      "learning_rate": 5.613087197958878e-07,
      "logits/chosen": -0.9777650833129883,
      "logits/rejected": -1.19744873046875,
      "logps/chosen": -107.13492584228516,
      "logps/rejected": -328.3821716308594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3172495365142822,
      "rewards/margins": 18.87519645690918,
      "rewards/rejected": -15.557950019836426,
      "step": 7030
    },
    {
      "epoch": 2.852512155591572,
      "grad_norm": 2.8005056265101302e-06,
      "learning_rate": 5.463004652558908e-07,
      "logits/chosen": -0.9703860878944397,
      "logits/rejected": -1.1866936683654785,
      "logps/chosen": -108.95934295654297,
      "logps/rejected": -329.2110595703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.302504062652588,
      "rewards/margins": 18.789155960083008,
      "rewards/rejected": -15.486650466918945,
      "step": 7040
    },
    {
      "epoch": 2.8565640194489466,
      "grad_norm": 8.147613698383793e-06,
      "learning_rate": 5.312922107158938e-07,
      "logits/chosen": -0.967191755771637,
      "logits/rejected": -1.184667706489563,
      "logps/chosen": -108.0377197265625,
      "logps/rejected": -329.3889465332031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3172271251678467,
      "rewards/margins": 18.912214279174805,
      "rewards/rejected": -15.594986915588379,
      "step": 7050
    },
    {
      "epoch": 2.8565640194489466,
      "eval_logits/chosen": -0.9790101051330566,
      "eval_logits/rejected": -1.1940275430679321,
      "eval_logps/chosen": -108.73690032958984,
      "eval_logps/rejected": -327.8504638671875,
      "eval_loss": 3.5015222721312966e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2275032997131348,
      "eval_rewards/margins": 18.71827507019043,
      "eval_rewards/rejected": -15.490771293640137,
      "eval_runtime": 660.0292,
      "eval_samples_per_second": 11.218,
      "eval_steps_per_second": 0.624,
      "step": 7050
    },
    {
      "epoch": 2.8606158833063207,
      "grad_norm": 4.085312411916675e-06,
      "learning_rate": 5.162839561758968e-07,
      "logits/chosen": -0.9736948609352112,
      "logits/rejected": -1.1846613883972168,
      "logps/chosen": -107.7497329711914,
      "logps/rejected": -327.30706787109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.323688507080078,
      "rewards/margins": 18.717166900634766,
      "rewards/rejected": -15.393479347229004,
      "step": 7060
    },
    {
      "epoch": 2.864667747163695,
      "grad_norm": 1.921224566103774e-06,
      "learning_rate": 5.012757016358997e-07,
      "logits/chosen": -0.9630138874053955,
      "logits/rejected": -1.1948740482330322,
      "logps/chosen": -107.47679901123047,
      "logps/rejected": -328.63006591796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3184473514556885,
      "rewards/margins": 18.853111267089844,
      "rewards/rejected": -15.534664154052734,
      "step": 7070
    },
    {
      "epoch": 2.8687196110210698,
      "grad_norm": 2.2141130102681927e-06,
      "learning_rate": 4.862674470959027e-07,
      "logits/chosen": -0.9536378383636475,
      "logits/rejected": -1.1748371124267578,
      "logps/chosen": -108.21116638183594,
      "logps/rejected": -327.82659912109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3113667964935303,
      "rewards/margins": 18.751161575317383,
      "rewards/rejected": -15.43979549407959,
      "step": 7080
    },
    {
      "epoch": 2.8727714748784443,
      "grad_norm": 1.8589283854453242e-06,
      "learning_rate": 4.7125919255590574e-07,
      "logits/chosen": -0.9696993231773376,
      "logits/rejected": -1.186915636062622,
      "logps/chosen": -108.62277221679688,
      "logps/rejected": -328.4823303222656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2551004886627197,
      "rewards/margins": 18.794811248779297,
      "rewards/rejected": -15.539711952209473,
      "step": 7090
    },
    {
      "epoch": 2.8768233387358184,
      "grad_norm": 2.2271383386396337e-06,
      "learning_rate": 4.562509380159088e-07,
      "logits/chosen": -0.965175449848175,
      "logits/rejected": -1.1976407766342163,
      "logps/chosen": -106.50357055664062,
      "logps/rejected": -329.94818115234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.301396131515503,
      "rewards/margins": 18.945039749145508,
      "rewards/rejected": -15.643644332885742,
      "step": 7100
    },
    {
      "epoch": 2.8768233387358184,
      "eval_logits/chosen": -0.9760178327560425,
      "eval_logits/rejected": -1.1918087005615234,
      "eval_logps/chosen": -108.72132110595703,
      "eval_logps/rejected": -327.9184875488281,
      "eval_loss": 3.470502107916218e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.229060411453247,
      "eval_rewards/margins": 18.72663688659668,
      "eval_rewards/rejected": -15.497576713562012,
      "eval_runtime": 660.2318,
      "eval_samples_per_second": 11.214,
      "eval_steps_per_second": 0.624,
      "step": 7100
    },
    {
      "epoch": 2.880875202593193,
      "grad_norm": 1.2730063190247165e-06,
      "learning_rate": 4.4124268347591175e-07,
      "logits/chosen": -0.9811935424804688,
      "logits/rejected": -1.1913543939590454,
      "logps/chosen": -107.50218200683594,
      "logps/rejected": -330.5213623046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3310484886169434,
      "rewards/margins": 19.015869140625,
      "rewards/rejected": -15.684819221496582,
      "step": 7110
    },
    {
      "epoch": 2.884927066450567,
      "grad_norm": 4.248605364409741e-06,
      "learning_rate": 4.262344289359148e-07,
      "logits/chosen": -0.9625168442726135,
      "logits/rejected": -1.1951909065246582,
      "logps/chosen": -108.038818359375,
      "logps/rejected": -329.1755065917969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3060390949249268,
      "rewards/margins": 18.852630615234375,
      "rewards/rejected": -15.546589851379395,
      "step": 7120
    },
    {
      "epoch": 2.8889789303079416,
      "grad_norm": 1.0449755791341886e-05,
      "learning_rate": 4.1122617439591776e-07,
      "logits/chosen": -0.9882882237434387,
      "logits/rejected": -1.2146639823913574,
      "logps/chosen": -108.18170928955078,
      "logps/rejected": -329.3057556152344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3227803707122803,
      "rewards/margins": 18.88628387451172,
      "rewards/rejected": -15.563502311706543,
      "step": 7130
    },
    {
      "epoch": 2.893030794165316,
      "grad_norm": 1.9005619833478704e-06,
      "learning_rate": 3.962179198559208e-07,
      "logits/chosen": -0.9830000996589661,
      "logits/rejected": -1.2070900201797485,
      "logps/chosen": -106.76356506347656,
      "logps/rejected": -328.2959289550781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.338989734649658,
      "rewards/margins": 18.85256004333496,
      "rewards/rejected": -15.513567924499512,
      "step": 7140
    },
    {
      "epoch": 2.8970826580226903,
      "grad_norm": 1.0993232990585966e-06,
      "learning_rate": 3.8120966531592377e-07,
      "logits/chosen": -0.964229941368103,
      "logits/rejected": -1.2030060291290283,
      "logps/chosen": -107.37648010253906,
      "logps/rejected": -328.80023193359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3193697929382324,
      "rewards/margins": 18.8880558013916,
      "rewards/rejected": -15.568682670593262,
      "step": 7150
    },
    {
      "epoch": 2.8970826580226903,
      "eval_logits/chosen": -0.9820052981376648,
      "eval_logits/rejected": -1.1988831758499146,
      "eval_logps/chosen": -108.72232818603516,
      "eval_logps/rejected": -327.9479675292969,
      "eval_loss": 3.464391085117313e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.228959083557129,
      "eval_rewards/margins": 18.729480743408203,
      "eval_rewards/rejected": -15.50052261352539,
      "eval_runtime": 659.9129,
      "eval_samples_per_second": 11.22,
      "eval_steps_per_second": 0.624,
      "step": 7150
    },
    {
      "epoch": 2.901134521880065,
      "grad_norm": 1.8366931726632174e-06,
      "learning_rate": 3.662014107759268e-07,
      "logits/chosen": -0.9971921443939209,
      "logits/rejected": -1.206737995147705,
      "logps/chosen": -108.55390167236328,
      "logps/rejected": -329.0882263183594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2648727893829346,
      "rewards/margins": 18.834712982177734,
      "rewards/rejected": -15.569838523864746,
      "step": 7160
    },
    {
      "epoch": 2.9051863857374394,
      "grad_norm": 7.031257155176718e-06,
      "learning_rate": 3.511931562359298e-07,
      "logits/chosen": -0.960989236831665,
      "logits/rejected": -1.181280255317688,
      "logps/chosen": -108.57195281982422,
      "logps/rejected": -328.79034423828125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.295433521270752,
      "rewards/margins": 18.852262496948242,
      "rewards/rejected": -15.556829452514648,
      "step": 7170
    },
    {
      "epoch": 2.9092382495948135,
      "grad_norm": 5.850887987435271e-07,
      "learning_rate": 3.3618490169593283e-07,
      "logits/chosen": -0.981634259223938,
      "logits/rejected": -1.1981111764907837,
      "logps/chosen": -108.1755599975586,
      "logps/rejected": -329.65478515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3291821479797363,
      "rewards/margins": 18.9754581451416,
      "rewards/rejected": -15.646275520324707,
      "step": 7180
    },
    {
      "epoch": 2.913290113452188,
      "grad_norm": 2.3675963802816113e-06,
      "learning_rate": 3.211766471559358e-07,
      "logits/chosen": -0.9758962392807007,
      "logits/rejected": -1.2017302513122559,
      "logps/chosen": -106.78636169433594,
      "logps/rejected": -328.0531921386719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.330303907394409,
      "rewards/margins": 18.92644691467285,
      "rewards/rejected": -15.59614086151123,
      "step": 7190
    },
    {
      "epoch": 2.9173419773095626,
      "grad_norm": 1.4073903003009036e-06,
      "learning_rate": 3.061683926159388e-07,
      "logits/chosen": -0.9870431423187256,
      "logits/rejected": -1.2174499034881592,
      "logps/chosen": -107.156494140625,
      "logps/rejected": -330.21923828125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3315341472625732,
      "rewards/margins": 18.96674346923828,
      "rewards/rejected": -15.635211944580078,
      "step": 7200
    },
    {
      "epoch": 2.9173419773095626,
      "eval_logits/chosen": -0.9757615327835083,
      "eval_logits/rejected": -1.191585898399353,
      "eval_logps/chosen": -108.72256469726562,
      "eval_logps/rejected": -328.0598449707031,
      "eval_loss": 3.425428474201908e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.2289371490478516,
      "eval_rewards/margins": 18.74064826965332,
      "eval_rewards/rejected": -15.511712074279785,
      "eval_runtime": 660.0554,
      "eval_samples_per_second": 11.217,
      "eval_steps_per_second": 0.624,
      "step": 7200
    },
    {
      "epoch": 2.9213938411669367,
      "grad_norm": 1.2767155794790597e-06,
      "learning_rate": 2.911601380759418e-07,
      "logits/chosen": -0.9767635464668274,
      "logits/rejected": -1.2136743068695068,
      "logps/chosen": -108.12942504882812,
      "logps/rejected": -329.66217041015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.31459379196167,
      "rewards/margins": 18.887969970703125,
      "rewards/rejected": -15.573376655578613,
      "step": 7210
    },
    {
      "epoch": 2.9254457050243112,
      "grad_norm": 1.7550073607708327e-06,
      "learning_rate": 2.761518835359448e-07,
      "logits/chosen": -0.9704033136367798,
      "logits/rejected": -1.2008793354034424,
      "logps/chosen": -108.53214263916016,
      "logps/rejected": -329.1427917480469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2905702590942383,
      "rewards/margins": 18.85349464416504,
      "rewards/rejected": -15.562925338745117,
      "step": 7220
    },
    {
      "epoch": 2.9294975688816853,
      "grad_norm": 3.6414326132216956e-06,
      "learning_rate": 2.611436289959478e-07,
      "logits/chosen": -0.9911782145500183,
      "logits/rejected": -1.2128431797027588,
      "logps/chosen": -108.56273651123047,
      "logps/rejected": -330.3154602050781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.275686025619507,
      "rewards/margins": 18.83922004699707,
      "rewards/rejected": -15.563532829284668,
      "step": 7230
    },
    {
      "epoch": 2.93354943273906,
      "grad_norm": 2.858848802134162e-06,
      "learning_rate": 2.461353744559508e-07,
      "logits/chosen": -0.9635599255561829,
      "logits/rejected": -1.1884490251541138,
      "logps/chosen": -106.62367248535156,
      "logps/rejected": -328.25146484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3598275184631348,
      "rewards/margins": 18.809885025024414,
      "rewards/rejected": -15.450057029724121,
      "step": 7240
    },
    {
      "epoch": 2.9376012965964344,
      "grad_norm": 1.2439679721865105e-06,
      "learning_rate": 2.3112711991595378e-07,
      "logits/chosen": -0.9607151746749878,
      "logits/rejected": -1.1821510791778564,
      "logps/chosen": -106.63765716552734,
      "logps/rejected": -328.87701416015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.344393491744995,
      "rewards/margins": 18.921140670776367,
      "rewards/rejected": -15.576745986938477,
      "step": 7250
    },
    {
      "epoch": 2.9376012965964344,
      "eval_logits/chosen": -0.9811464548110962,
      "eval_logits/rejected": -1.196897268295288,
      "eval_logps/chosen": -108.70677185058594,
      "eval_logps/rejected": -328.0516662597656,
      "eval_loss": 3.425665440204284e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.230515956878662,
      "eval_rewards/margins": 18.741403579711914,
      "eval_rewards/rejected": -15.510887145996094,
      "eval_runtime": 659.9462,
      "eval_samples_per_second": 11.219,
      "eval_steps_per_second": 0.624,
      "step": 7250
    },
    {
      "epoch": 2.941653160453809,
      "grad_norm": 1.9038395748793846e-06,
      "learning_rate": 2.1611886537595678e-07,
      "logits/chosen": -0.9633909463882446,
      "logits/rejected": -1.1942411661148071,
      "logps/chosen": -107.4720230102539,
      "logps/rejected": -328.3365478515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3182427883148193,
      "rewards/margins": 18.803630828857422,
      "rewards/rejected": -15.485390663146973,
      "step": 7260
    },
    {
      "epoch": 2.945705024311183,
      "grad_norm": 1.439689867765992e-06,
      "learning_rate": 2.0111061083595979e-07,
      "logits/chosen": -0.9990869164466858,
      "logits/rejected": -1.2175521850585938,
      "logps/chosen": -107.74435424804688,
      "logps/rejected": -331.8654479980469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2828431129455566,
      "rewards/margins": 19.112253189086914,
      "rewards/rejected": -15.829410552978516,
      "step": 7270
    },
    {
      "epoch": 2.9497568881685576,
      "grad_norm": 1.8770879250951111e-06,
      "learning_rate": 1.861023562959628e-07,
      "logits/chosen": -0.9633672833442688,
      "logits/rejected": -1.185795545578003,
      "logps/chosen": -108.7204818725586,
      "logps/rejected": -329.3869323730469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.327587127685547,
      "rewards/margins": 18.815723419189453,
      "rewards/rejected": -15.488137245178223,
      "step": 7280
    },
    {
      "epoch": 2.9538087520259317,
      "grad_norm": 2.6701447950472357e-06,
      "learning_rate": 1.710941017559658e-07,
      "logits/chosen": -0.9650854468345642,
      "logits/rejected": -1.1963393688201904,
      "logps/chosen": -107.19867706298828,
      "logps/rejected": -328.4620056152344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3577048778533936,
      "rewards/margins": 18.849342346191406,
      "rewards/rejected": -15.491637229919434,
      "step": 7290
    },
    {
      "epoch": 2.9578606158833063,
      "grad_norm": 7.110392743925331e-06,
      "learning_rate": 1.560858472159688e-07,
      "logits/chosen": -0.9818698763847351,
      "logits/rejected": -1.1954624652862549,
      "logps/chosen": -106.72434997558594,
      "logps/rejected": -328.2565612792969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.324738025665283,
      "rewards/margins": 18.763063430786133,
      "rewards/rejected": -15.438325881958008,
      "step": 7300
    },
    {
      "epoch": 2.9578606158833063,
      "eval_logits/chosen": -0.9837020039558411,
      "eval_logits/rejected": -1.1994996070861816,
      "eval_logps/chosen": -108.7364730834961,
      "eval_logps/rejected": -328.1881103515625,
      "eval_loss": 3.400739245762452e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.227546453475952,
      "eval_rewards/margins": 18.752079010009766,
      "eval_rewards/rejected": -15.524532318115234,
      "eval_runtime": 659.4143,
      "eval_samples_per_second": 11.228,
      "eval_steps_per_second": 0.625,
      "step": 7300
    },
    {
      "epoch": 2.961912479740681,
      "grad_norm": 9.207852258441562e-07,
      "learning_rate": 1.410775926759718e-07,
      "logits/chosen": -0.9928681254386902,
      "logits/rejected": -1.2284150123596191,
      "logps/chosen": -106.771240234375,
      "logps/rejected": -330.71417236328125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.34260630607605,
      "rewards/margins": 19.015148162841797,
      "rewards/rejected": -15.6725435256958,
      "step": 7310
    },
    {
      "epoch": 2.965964343598055,
      "grad_norm": 1.3852826441507204e-06,
      "learning_rate": 1.260693381359748e-07,
      "logits/chosen": -0.9731549620628357,
      "logits/rejected": -1.200088620185852,
      "logps/chosen": -107.93476867675781,
      "logps/rejected": -329.5440673828125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.272829532623291,
      "rewards/margins": 18.86159324645996,
      "rewards/rejected": -15.588763236999512,
      "step": 7320
    },
    {
      "epoch": 2.9700162074554295,
      "grad_norm": 2.1593209567072336e-06,
      "learning_rate": 1.110610835959778e-07,
      "logits/chosen": -0.9732111692428589,
      "logits/rejected": -1.1930580139160156,
      "logps/chosen": -108.34236907958984,
      "logps/rejected": -329.2429504394531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3482120037078857,
      "rewards/margins": 18.90915870666504,
      "rewards/rejected": -15.56094741821289,
      "step": 7330
    },
    {
      "epoch": 2.974068071312804,
      "grad_norm": 1.5361877103714505e-06,
      "learning_rate": 9.60528290559808e-08,
      "logits/chosen": -0.977744996547699,
      "logits/rejected": -1.1999133825302124,
      "logps/chosen": -107.92412567138672,
      "logps/rejected": -330.2553405761719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.3247008323669434,
      "rewards/margins": 19.007099151611328,
      "rewards/rejected": -15.682397842407227,
      "step": 7340
    },
    {
      "epoch": 2.978119935170178,
      "grad_norm": 1.2273446373001207e-05,
      "learning_rate": 8.10445745159838e-08,
      "logits/chosen": -0.9679704308509827,
      "logits/rejected": -1.1986968517303467,
      "logps/chosen": -107.73753356933594,
      "logps/rejected": -331.9222106933594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.320899724960327,
      "rewards/margins": 19.121126174926758,
      "rewards/rejected": -15.800227165222168,
      "step": 7350
    },
    {
      "epoch": 2.978119935170178,
      "eval_logits/chosen": -0.9848833680152893,
      "eval_logits/rejected": -1.20034921169281,
      "eval_logps/chosen": -108.74987030029297,
      "eval_logps/rejected": -328.1879577636719,
      "eval_loss": 3.39335244348149e-08,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 3.226205587387085,
      "eval_rewards/margins": 18.750720977783203,
      "eval_rewards/rejected": -15.524516105651855,
      "eval_runtime": 660.1187,
      "eval_samples_per_second": 11.216,
      "eval_steps_per_second": 0.624,
      "step": 7350
    }
  ],
  "logging_steps": 10,
  "max_steps": 7404,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 9,
  "trial_name": null,
  "trial_params": null
}