{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 4.0,
  "eval_steps": 500,
  "global_step": 516,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.07782101167315175,
      "grad_norm": 309.9635314941406,
      "learning_rate": 3.461538461538461e-07,
      "logits/chosen": 2.9046390056610107,
      "logits/rejected": 2.998429775238037,
      "logps/chosen": -250.68215942382812,
      "logps/rejected": -271.45562744140625,
      "loss": 0.8341,
      "rewards/accuracies": 0.534375011920929,
      "rewards/chosen": 0.07394347339868546,
      "rewards/margins": 0.05555616691708565,
      "rewards/rejected": 0.018387287855148315,
      "step": 10
    },
    {
      "epoch": 0.1556420233463035,
      "grad_norm": 180.70631408691406,
      "learning_rate": 7.307692307692307e-07,
      "logits/chosen": 2.893648386001587,
      "logits/rejected": 2.9738540649414062,
      "logps/chosen": -255.9049835205078,
      "logps/rejected": -271.83197021484375,
      "loss": 0.6594,
      "rewards/accuracies": 0.6656249761581421,
      "rewards/chosen": 0.6080068349838257,
      "rewards/margins": 0.7042483687400818,
      "rewards/rejected": -0.09624147415161133,
      "step": 20
    },
    {
      "epoch": 0.23346303501945526,
      "grad_norm": 152.26849365234375,
      "learning_rate": 1.1153846153846154e-06,
      "logits/chosen": 2.938114643096924,
      "logits/rejected": 2.979564666748047,
      "logps/chosen": -260.2184143066406,
      "logps/rejected": -274.0084228515625,
      "loss": 0.4358,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.1423935890197754,
      "rewards/margins": 2.729647159576416,
      "rewards/rejected": -0.5872532725334167,
      "step": 30
    },
    {
      "epoch": 0.311284046692607,
      "grad_norm": 61.16145324707031,
      "learning_rate": 1.5e-06,
      "logits/chosen": 2.962891101837158,
      "logits/rejected": 3.0372633934020996,
      "logps/chosen": -264.4953918457031,
      "logps/rejected": -278.98541259765625,
      "loss": 0.2424,
      "rewards/accuracies": 0.8843749761581421,
      "rewards/chosen": 3.3210906982421875,
      "rewards/margins": 5.616915225982666,
      "rewards/rejected": -2.2958242893218994,
      "step": 40
    },
    {
      "epoch": 0.38910505836575876,
      "grad_norm": 72.79554748535156,
      "learning_rate": 1.8846153846153845e-06,
      "logits/chosen": 2.916743278503418,
      "logits/rejected": 2.9901554584503174,
      "logps/chosen": -242.43307495117188,
      "logps/rejected": -276.2446594238281,
      "loss": 0.0823,
      "rewards/accuracies": 0.9671875238418579,
      "rewards/chosen": 4.046940803527832,
      "rewards/margins": 9.120622634887695,
      "rewards/rejected": -5.0736823081970215,
      "step": 50
    },
    {
      "epoch": 0.4669260700389105,
      "grad_norm": 51.30080795288086,
      "learning_rate": 1.998877080632712e-06,
      "logits/chosen": 2.911020517349243,
      "logits/rejected": 2.9875214099884033,
      "logps/chosen": -252.1035614013672,
      "logps/rejected": -278.92047119140625,
      "loss": 0.0745,
      "rewards/accuracies": 0.973437488079071,
      "rewards/chosen": 3.5658841133117676,
      "rewards/margins": 11.655838966369629,
      "rewards/rejected": -8.089953422546387,
      "step": 60
    },
    {
      "epoch": 0.5447470817120622,
      "grad_norm": 69.88111877441406,
      "learning_rate": 1.9933831375786213e-06,
      "logits/chosen": 2.9574735164642334,
      "logits/rejected": 3.0135583877563477,
      "logps/chosen": -250.85684204101562,
      "logps/rejected": -278.03582763671875,
      "loss": 0.0516,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": 2.9169883728027344,
      "rewards/margins": 13.218663215637207,
      "rewards/rejected": -10.301675796508789,
      "step": 70
    },
    {
      "epoch": 0.622568093385214,
      "grad_norm": 16.288284301757812,
      "learning_rate": 1.9833370645493046e-06,
      "logits/chosen": 2.9377493858337402,
      "logits/rejected": 3.007570743560791,
      "logps/chosen": -253.7940216064453,
      "logps/rejected": -284.8679504394531,
      "loss": 0.0387,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 3.284533977508545,
      "rewards/margins": 16.070585250854492,
      "rewards/rejected": -12.786050796508789,
      "step": 80
    },
    {
      "epoch": 0.7003891050583657,
      "grad_norm": 8.968023300170898,
      "learning_rate": 1.9687848971864386e-06,
      "logits/chosen": 2.9306931495666504,
      "logits/rejected": 2.964722156524658,
      "logps/chosen": -252.9971160888672,
      "logps/rejected": -275.87188720703125,
      "loss": 0.0492,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 3.5943214893341064,
      "rewards/margins": 17.51387596130371,
      "rewards/rejected": -13.9195556640625,
      "step": 90
    },
    {
      "epoch": 0.7782101167315175,
      "grad_norm": 2.590059757232666,
      "learning_rate": 1.9497933200901047e-06,
      "logits/chosen": 2.9647717475891113,
      "logits/rejected": 3.0007081031799316,
      "logps/chosen": -267.41375732421875,
      "logps/rejected": -281.07513427734375,
      "loss": 0.0337,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": 4.1861348152160645,
      "rewards/margins": 18.587297439575195,
      "rewards/rejected": -14.401159286499023,
      "step": 100
    },
    {
      "epoch": 0.8560311284046692,
      "grad_norm": 77.77639770507812,
      "learning_rate": 1.926449361239848e-06,
      "logits/chosen": 2.8810315132141113,
      "logits/rejected": 2.9851112365722656,
      "logps/chosen": -253.496337890625,
      "logps/rejected": -284.06610107421875,
      "loss": 0.0533,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 4.9546074867248535,
      "rewards/margins": 20.860849380493164,
      "rewards/rejected": -15.906242370605469,
      "step": 110
    },
    {
      "epoch": 0.933852140077821,
      "grad_norm": 127.29902648925781,
      "learning_rate": 1.8988599931931866e-06,
      "logits/chosen": 2.8698949813842773,
      "logits/rejected": 2.943873405456543,
      "logps/chosen": -246.2525177001953,
      "logps/rejected": -279.6893310546875,
      "loss": 0.0179,
      "rewards/accuracies": 0.995312511920929,
      "rewards/chosen": 3.602506160736084,
      "rewards/margins": 21.305601119995117,
      "rewards/rejected": -17.703094482421875,
      "step": 120
    },
    {
      "epoch": 1.0077821011673151,
      "grad_norm": 0.10526127368211746,
      "learning_rate": 1.8671516428890646e-06,
      "logits/chosen": 2.960531711578369,
      "logits/rejected": 3.0081727504730225,
      "logps/chosen": -251.44546508789062,
      "logps/rejected": -297.5830383300781,
      "loss": 0.0043,
      "rewards/accuracies": 0.9983552694320679,
      "rewards/chosen": 5.055600166320801,
      "rewards/margins": 22.011268615722656,
      "rewards/rejected": -16.955669403076172,
      "step": 130
    },
    {
      "epoch": 1.0856031128404668,
      "grad_norm": 3.8229660987854004,
      "learning_rate": 1.8314696123025452e-06,
      "logits/chosen": 2.9409918785095215,
      "logits/rejected": 2.994568109512329,
      "logps/chosen": -251.5284423828125,
      "logps/rejected": -283.5034484863281,
      "loss": 0.002,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 5.045574188232422,
      "rewards/margins": 24.76313591003418,
      "rewards/rejected": -19.717559814453125,
      "step": 140
    },
    {
      "epoch": 1.1634241245136188,
      "grad_norm": 0.1067415103316307,
      "learning_rate": 1.7919774126055671e-06,
      "logits/chosen": 2.9014945030212402,
      "logits/rejected": 3.0171058177948,
      "logps/chosen": -244.58154296875,
      "logps/rejected": -300.16827392578125,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.922303199768066,
      "rewards/margins": 27.960514068603516,
      "rewards/rejected": -22.038206100463867,
      "step": 150
    },
    {
      "epoch": 1.2412451361867705,
      "grad_norm": 0.11893969029188156,
      "learning_rate": 1.7488560148849425e-06,
      "logits/chosen": 2.930441379547119,
      "logits/rejected": 2.9720261096954346,
      "logps/chosen": -257.78741455078125,
      "logps/rejected": -283.66986083984375,
      "loss": 0.0044,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 4.894805908203125,
      "rewards/margins": 27.222713470458984,
      "rewards/rejected": -22.32790756225586,
      "step": 160
    },
    {
      "epoch": 1.3190661478599222,
      "grad_norm": 0.019848737865686417,
      "learning_rate": 1.7023030208511418e-06,
      "logits/chosen": 2.9215731620788574,
      "logits/rejected": 2.998058795928955,
      "logps/chosen": -257.8537902832031,
      "logps/rejected": -300.69781494140625,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.8431878089904785,
      "rewards/margins": 27.765335083007812,
      "rewards/rejected": -22.92214584350586,
      "step": 170
    },
    {
      "epoch": 1.3968871595330739,
      "grad_norm": 0.013338716700673103,
      "learning_rate": 1.6525317573380523e-06,
      "logits/chosen": 2.905754566192627,
      "logits/rejected": 2.9750256538391113,
      "logps/chosen": -256.52801513671875,
      "logps/rejected": -292.92559814453125,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.072903633117676,
      "rewards/margins": 28.44639015197754,
      "rewards/rejected": -23.373485565185547,
      "step": 180
    },
    {
      "epoch": 1.4747081712062258,
      "grad_norm": 1.3543317317962646,
      "learning_rate": 1.5997702987431278e-06,
      "logits/chosen": 2.8545498847961426,
      "logits/rejected": 2.93253755569458,
      "logps/chosen": -249.04165649414062,
      "logps/rejected": -290.2785949707031,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.263331413269043,
      "rewards/margins": 29.8680362701416,
      "rewards/rejected": -25.604705810546875,
      "step": 190
    },
    {
      "epoch": 1.5525291828793775,
      "grad_norm": 0.0023172160144895315,
      "learning_rate": 1.5442604218875369e-06,
      "logits/chosen": 2.8726515769958496,
      "logits/rejected": 2.96691632270813,
      "logps/chosen": -235.4767608642578,
      "logps/rejected": -299.6068420410156,
      "loss": 0.0174,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 4.076581001281738,
      "rewards/margins": 29.738256454467773,
      "rewards/rejected": -25.661672592163086,
      "step": 200
    },
    {
      "epoch": 1.6303501945525292,
      "grad_norm": 0.12606221437454224,
      "learning_rate": 1.4862564980856257e-06,
      "logits/chosen": 2.866564989089966,
      "logits/rejected": 2.9289443492889404,
      "logps/chosen": -251.0595245361328,
      "logps/rejected": -291.28009033203125,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.828310012817383,
      "rewards/margins": 29.65151023864746,
      "rewards/rejected": -26.823205947875977,
      "step": 210
    },
    {
      "epoch": 1.708171206225681,
      "grad_norm": 1.1696350574493408,
      "learning_rate": 1.4260243275007265e-06,
      "logits/chosen": 2.876115083694458,
      "logits/rejected": 2.9629712104797363,
      "logps/chosen": -252.755126953125,
      "logps/rejected": -293.724609375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9194681644439697,
      "rewards/margins": 30.779850006103516,
      "rewards/rejected": -27.86037826538086,
      "step": 220
    },
    {
      "epoch": 1.7859922178988326,
      "grad_norm": 0.7943425178527832,
      "learning_rate": 1.3638399211288185e-06,
      "logits/chosen": 2.853001356124878,
      "logits/rejected": 2.91581392288208,
      "logps/chosen": -248.50790405273438,
      "logps/rejected": -303.24749755859375,
      "loss": 0.0009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.9409661293029785,
      "rewards/margins": 31.501480102539062,
      "rewards/rejected": -27.560516357421875,
      "step": 230
    },
    {
      "epoch": 1.8638132295719845,
      "grad_norm": 22.544157028198242,
      "learning_rate": 1.2999882359915478e-06,
      "logits/chosen": 2.838768482208252,
      "logits/rejected": 2.8869760036468506,
      "logps/chosen": -257.80755615234375,
      "logps/rejected": -301.2661437988281,
      "loss": 0.0009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.118237018585205,
      "rewards/margins": 31.00071144104004,
      "rewards/rejected": -26.882476806640625,
      "step": 240
    },
    {
      "epoch": 1.9416342412451362,
      "grad_norm": 0.08505785465240479,
      "learning_rate": 1.2347618693345148e-06,
      "logits/chosen": 2.8469579219818115,
      "logits/rejected": 2.91058349609375,
      "logps/chosen": -274.753173828125,
      "logps/rejected": -310.84820556640625,
      "loss": 0.0097,
      "rewards/accuracies": 0.996874988079071,
      "rewards/chosen": 3.213207960128784,
      "rewards/margins": 32.35531234741211,
      "rewards/rejected": -29.142101287841797,
      "step": 250
    },
    {
      "epoch": 2.0155642023346303,
      "grad_norm": 0.15910857915878296,
      "learning_rate": 1.1684597178146115e-06,
      "logits/chosen": 2.820605516433716,
      "logits/rejected": 2.9221818447113037,
      "logps/chosen": -240.77276611328125,
      "logps/rejected": -284.52947998046875,
      "loss": 0.0095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.895946979522705,
      "rewards/margins": 31.758358001708984,
      "rewards/rejected": -26.862411499023438,
      "step": 260
    },
    {
      "epoch": 2.093385214007782,
      "grad_norm": 0.036736179143190384,
      "learning_rate": 1.1013856078206078e-06,
      "logits/chosen": 2.7885546684265137,
      "logits/rejected": 2.9099419116973877,
      "logps/chosen": -255.7447509765625,
      "logps/rejected": -300.6316223144531,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.1806111335754395,
      "rewards/margins": 32.345706939697266,
      "rewards/rejected": -28.16509437561035,
      "step": 270
    },
    {
      "epoch": 2.1712062256809337,
      "grad_norm": 0.010115005075931549,
      "learning_rate": 1.0338469032034844e-06,
      "logits/chosen": 2.880408763885498,
      "logits/rejected": 2.913715124130249,
      "logps/chosen": -244.33837890625,
      "logps/rejected": -284.74456787109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.522616386413574,
      "rewards/margins": 32.65098571777344,
      "rewards/rejected": -27.128368377685547,
      "step": 280
    },
    {
      "epoch": 2.2490272373540856,
      "grad_norm": 0.0006411768263205886,
      "learning_rate": 9.661530967965155e-07,
      "logits/chosen": 2.8637478351593018,
      "logits/rejected": 2.8776774406433105,
      "logps/chosen": -251.5993194580078,
      "logps/rejected": -285.7321472167969,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.7556328773498535,
      "rewards/margins": 32.75992202758789,
      "rewards/rejected": -27.004287719726562,
      "step": 290
    },
    {
      "epoch": 2.3268482490272375,
      "grad_norm": 0.010583557188510895,
      "learning_rate": 8.986143921793921e-07,
      "logits/chosen": 2.8255324363708496,
      "logits/rejected": 2.884343385696411,
      "logps/chosen": -256.0443115234375,
      "logps/rejected": -298.6083068847656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.289700984954834,
      "rewards/margins": 32.55815887451172,
      "rewards/rejected": -28.26845359802246,
      "step": 300
    },
    {
      "epoch": 2.404669260700389,
      "grad_norm": 0.05514160171151161,
      "learning_rate": 8.315402821853885e-07,
      "logits/chosen": 2.8168251514434814,
      "logits/rejected": 2.911100387573242,
      "logps/chosen": -242.12930297851562,
      "logps/rejected": -311.6902770996094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.304950714111328,
      "rewards/margins": 33.0163459777832,
      "rewards/rejected": -28.71139144897461,
      "step": 310
    },
    {
      "epoch": 2.482490272373541,
      "grad_norm": 0.0014977785758674145,
      "learning_rate": 7.65238130665485e-07,
      "logits/chosen": 2.8739066123962402,
      "logits/rejected": 2.9437661170959473,
      "logps/chosen": -254.6911163330078,
      "logps/rejected": -298.3764953613281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.761944770812988,
      "rewards/margins": 32.24135208129883,
      "rewards/rejected": -27.479406356811523,
      "step": 320
    },
    {
      "epoch": 2.5603112840466924,
      "grad_norm": 0.003176012309268117,
      "learning_rate": 7.000117640084525e-07,
      "logits/chosen": 2.849517822265625,
      "logits/rejected": 2.9144535064697266,
      "logps/chosen": -253.174560546875,
      "logps/rejected": -302.97296142578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.072056770324707,
      "rewards/margins": 33.25294876098633,
      "rewards/rejected": -28.180889129638672,
      "step": 330
    },
    {
      "epoch": 2.6381322957198443,
      "grad_norm": 0.007333577610552311,
      "learning_rate": 6.361600788711815e-07,
      "logits/chosen": 2.801302671432495,
      "logits/rejected": 2.9121482372283936,
      "logps/chosen": -259.02398681640625,
      "logps/rejected": -303.2242431640625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.999932289123535,
      "rewards/margins": 32.66350555419922,
      "rewards/rejected": -27.66357421875,
      "step": 340
    },
    {
      "epoch": 2.7159533073929962,
      "grad_norm": 0.0016177126672118902,
      "learning_rate": 5.739756724992736e-07,
      "logits/chosen": 2.85249662399292,
      "logits/rejected": 2.9361824989318848,
      "logps/chosen": -244.018310546875,
      "logps/rejected": -286.9032897949219,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.892648220062256,
      "rewards/margins": 32.217613220214844,
      "rewards/rejected": -27.324966430664062,
      "step": 350
    },
    {
      "epoch": 2.7937743190661477,
      "grad_norm": 0.3721546530723572,
      "learning_rate": 5.137435019143744e-07,
      "logits/chosen": 2.801509141921997,
      "logits/rejected": 2.8811545372009277,
      "logps/chosen": -257.4078063964844,
      "logps/rejected": -306.99542236328125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.106388092041016,
      "rewards/margins": 33.20668411254883,
      "rewards/rejected": -29.100292205810547,
      "step": 360
    },
    {
      "epoch": 2.8715953307392996,
      "grad_norm": 0.004263872280716896,
      "learning_rate": 4.557395781124631e-07,
      "logits/chosen": 2.853585958480835,
      "logits/rejected": 2.9362740516662598,
      "logps/chosen": -265.7266845703125,
      "logps/rejected": -301.75091552734375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.696002960205078,
      "rewards/margins": 32.666351318359375,
      "rewards/rejected": -27.9703426361084,
      "step": 370
    },
    {
      "epoch": 2.9494163424124515,
      "grad_norm": 6.243278503417969,
      "learning_rate": 4.0022970125687217e-07,
      "logits/chosen": 2.8308067321777344,
      "logits/rejected": 2.924168586730957,
      "logps/chosen": -253.6175537109375,
      "logps/rejected": -304.56488037109375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.487698554992676,
      "rewards/margins": 33.75385284423828,
      "rewards/rejected": -29.26615333557129,
      "step": 380
    },
    {
      "epoch": 3.0233463035019454,
      "grad_norm": 0.010610430501401424,
      "learning_rate": 3.474682426619474e-07,
      "logits/chosen": 2.86039662361145,
      "logits/rejected": 2.955193042755127,
      "logps/chosen": -244.86038208007812,
      "logps/rejected": -298.5942077636719,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.364467144012451,
      "rewards/margins": 33.471588134765625,
      "rewards/rejected": -28.107120513916016,
      "step": 390
    },
    {
      "epoch": 3.1011673151750974,
      "grad_norm": 0.0002077793178614229,
      "learning_rate": 2.976969791488586e-07,
      "logits/chosen": 2.794825792312622,
      "logits/rejected": 2.913062572479248,
      "logps/chosen": -249.8114471435547,
      "logps/rejected": -304.4586181640625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.951796054840088,
      "rewards/margins": 33.71596908569336,
      "rewards/rejected": -28.764175415039062,
      "step": 400
    },
    {
      "epoch": 3.178988326848249,
      "grad_norm": 0.0018190924311056733,
      "learning_rate": 2.511439851150573e-07,
      "logits/chosen": 2.846440076828003,
      "logits/rejected": 2.9116950035095215,
      "logps/chosen": -245.32241821289062,
      "logps/rejected": -299.29278564453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.315257549285889,
      "rewards/margins": 33.424171447753906,
      "rewards/rejected": -29.10890769958496,
      "step": 410
    },
    {
      "epoch": 3.2568093385214008,
      "grad_norm": 0.0022656808141618967,
      "learning_rate": 2.080225873944328e-07,
      "logits/chosen": 2.8200035095214844,
      "logits/rejected": 2.9111595153808594,
      "logps/chosen": -258.51519775390625,
      "logps/rejected": -303.6025390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.961642265319824,
      "rewards/margins": 33.01478958129883,
      "rewards/rejected": -28.053142547607422,
      "step": 420
    },
    {
      "epoch": 3.3346303501945527,
      "grad_norm": 0.006798978429287672,
      "learning_rate": 1.6853038769745465e-07,
      "logits/chosen": 2.8389997482299805,
      "logits/rejected": 2.8933396339416504,
      "logps/chosen": -245.7606658935547,
      "logps/rejected": -295.68115234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.181079387664795,
      "rewards/margins": 32.27184295654297,
      "rewards/rejected": -28.090763092041016,
      "step": 430
    },
    {
      "epoch": 3.412451361867704,
      "grad_norm": 0.00100744201336056,
      "learning_rate": 1.3284835711093535e-07,
      "logits/chosen": 2.8530194759368896,
      "logits/rejected": 2.9206631183624268,
      "logps/chosen": -258.67431640625,
      "logps/rejected": -298.80755615234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.105883598327637,
      "rewards/margins": 32.130775451660156,
      "rewards/rejected": -28.024892807006836,
      "step": 440
    },
    {
      "epoch": 3.490272373540856,
      "grad_norm": 0.0001550566521473229,
      "learning_rate": 1.0114000680681356e-07,
      "logits/chosen": 2.81189227104187,
      "logits/rejected": 2.925079107284546,
      "logps/chosen": -253.63217163085938,
      "logps/rejected": -299.74273681640625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 6.281905174255371,
      "rewards/margins": 33.87714385986328,
      "rewards/rejected": -27.595239639282227,
      "step": 450
    },
    {
      "epoch": 3.5680933852140075,
      "grad_norm": 0.01041385531425476,
      "learning_rate": 7.355063876015199e-08,
      "logits/chosen": 2.877830982208252,
      "logits/rejected": 2.9270596504211426,
      "logps/chosen": -277.89544677734375,
      "logps/rejected": -303.5425109863281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.2588934898376465,
      "rewards/margins": 32.801124572753906,
      "rewards/rejected": -27.5422306060791,
      "step": 460
    },
    {
      "epoch": 3.6459143968871595,
      "grad_norm": 0.01067538931965828,
      "learning_rate": 5.020667990989525e-08,
      "logits/chosen": 2.8863813877105713,
      "logits/rejected": 2.9612936973571777,
      "logps/chosen": -259.4737854003906,
      "logps/rejected": -296.72637939453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.224932670593262,
      "rewards/margins": 32.182594299316406,
      "rewards/rejected": -27.957666397094727,
      "step": 470
    },
    {
      "epoch": 3.7237354085603114,
      "grad_norm": 0.004288141615688801,
      "learning_rate": 3.121510281356143e-08,
      "logits/chosen": 2.824970245361328,
      "logits/rejected": 2.8980326652526855,
      "logps/chosen": -251.7251434326172,
      "logps/rejected": -293.5643615722656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.086753845214844,
      "rewards/margins": 32.74205780029297,
      "rewards/rejected": -27.65530776977539,
      "step": 480
    },
    {
      "epoch": 3.801556420233463,
      "grad_norm": 9.263599349651486e-05,
      "learning_rate": 1.6662935450695416e-08,
      "logits/chosen": 2.7916643619537354,
      "logits/rejected": 2.8788905143737793,
      "logps/chosen": -239.5885467529297,
      "logps/rejected": -293.6639099121094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.8184919357299805,
      "rewards/margins": 32.92034912109375,
      "rewards/rejected": -27.101858139038086,
      "step": 490
    },
    {
      "epoch": 3.8793774319066148,
      "grad_norm": 0.003024149453267455,
      "learning_rate": 6.6168624213784885e-09,
      "logits/chosen": 2.828496217727661,
      "logits/rejected": 2.8872532844543457,
      "logps/chosen": -242.9530792236328,
      "logps/rejected": -291.443359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.576481342315674,
      "rewards/margins": 31.855998992919922,
      "rewards/rejected": -27.279516220092773,
      "step": 500
    },
    {
      "epoch": 3.9571984435797667,
      "grad_norm": 0.003526742337271571,
      "learning_rate": 1.1229193672881444e-09,
      "logits/chosen": 2.8279409408569336,
      "logits/rejected": 2.9110004901885986,
      "logps/chosen": -243.04946899414062,
      "logps/rejected": -305.1340026855469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.348401069641113,
      "rewards/margins": 32.896034240722656,
      "rewards/rejected": -28.547632217407227,
      "step": 510
    }
  ],
  "logging_steps": 10,
  "max_steps": 516,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 4,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}