Invalid JSON:Unexpected token 'N', ..."ad_norm": NaN,
"... is not valid JSON
| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.0035058401453755047, | |
| "eval_steps": 500, | |
| "global_step": 600, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 5.8430669089591743e-05, | |
| "grad_norm": 10.995234489440918, | |
| "learning_rate": 6e-06, | |
| "loss": 36.76142578125, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.00011686133817918349, | |
| "grad_norm": 10.91879653930664, | |
| "learning_rate": 1.2666666666666668e-05, | |
| "loss": 36.63887023925781, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.00017529200726877522, | |
| "grad_norm": 12.004676818847656, | |
| "learning_rate": 1.9333333333333333e-05, | |
| "loss": 36.38578491210937, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.00023372267635836697, | |
| "grad_norm": 11.3691987991333, | |
| "learning_rate": 2.6000000000000002e-05, | |
| "loss": 36.06534118652344, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.0002921533454479587, | |
| "grad_norm": 13.08530044555664, | |
| "learning_rate": 3.266666666666667e-05, | |
| "loss": 35.64937744140625, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.00035058401453755045, | |
| "grad_norm": 13.18602180480957, | |
| "learning_rate": 3.933333333333333e-05, | |
| "loss": 35.051446533203126, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.0004090146836271422, | |
| "grad_norm": 13.574982643127441, | |
| "learning_rate": 4.600000000000001e-05, | |
| "loss": 34.180169677734376, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.00046744535271673395, | |
| "grad_norm": 14.570645332336426, | |
| "learning_rate": 5.266666666666666e-05, | |
| "loss": 33.2185546875, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.0005258760218063257, | |
| "grad_norm": 16.55890464782715, | |
| "learning_rate": 5.9333333333333343e-05, | |
| "loss": 32.12116394042969, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.0005843066908959174, | |
| "grad_norm": 17.331336975097656, | |
| "learning_rate": 6.6e-05, | |
| "loss": 30.47571716308594, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.0006427373599855092, | |
| "grad_norm": 14.68326187133789, | |
| "learning_rate": 7.266666666666667e-05, | |
| "loss": 28.728692626953126, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.0007011680290751009, | |
| "grad_norm": 18.396814346313477, | |
| "learning_rate": 7.933333333333334e-05, | |
| "loss": 26.857958984375, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.0007595986981646927, | |
| "grad_norm": 16.36812973022461, | |
| "learning_rate": 8.6e-05, | |
| "loss": 25.132843017578125, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.0008180293672542844, | |
| "grad_norm": 21.44991111755371, | |
| "learning_rate": 9.266666666666666e-05, | |
| "loss": 23.530233764648436, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.0008764600363438762, | |
| "grad_norm": 22.47516632080078, | |
| "learning_rate": 9.933333333333334e-05, | |
| "loss": 22.332640075683592, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.0009348907054334679, | |
| "grad_norm": 22.053213119506836, | |
| "learning_rate": 0.00010600000000000002, | |
| "loss": 21.743748474121094, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.0009933213745230596, | |
| "grad_norm": 26.53052520751953, | |
| "learning_rate": 0.00011266666666666668, | |
| "loss": 20.7405029296875, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.0010517520436126514, | |
| "grad_norm": 37.71436309814453, | |
| "learning_rate": 0.00011933333333333334, | |
| "loss": 19.994790649414064, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.001110182712702243, | |
| "grad_norm": 33.68248748779297, | |
| "learning_rate": 0.000126, | |
| "loss": 19.59712371826172, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.001168613381791835, | |
| "grad_norm": 29.311405181884766, | |
| "learning_rate": 0.00013266666666666667, | |
| "loss": 19.54248809814453, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.0012270440508814267, | |
| "grad_norm": 27.571697235107422, | |
| "learning_rate": 0.00013933333333333335, | |
| "loss": 18.915852355957032, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.0012854747199710183, | |
| "grad_norm": 50.6077880859375, | |
| "learning_rate": 0.000146, | |
| "loss": 18.705543518066406, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.0013439053890606102, | |
| "grad_norm": 24.939199447631836, | |
| "learning_rate": 0.00015266666666666667, | |
| "loss": 19.156636047363282, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.0014023360581502018, | |
| "grad_norm": 28.88422393798828, | |
| "learning_rate": 0.00015933333333333332, | |
| "loss": 17.941325378417968, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.0014607667272397936, | |
| "grad_norm": 32.721168518066406, | |
| "learning_rate": 0.000166, | |
| "loss": 17.255938720703124, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.0015191973963293855, | |
| "grad_norm": 44.036075592041016, | |
| "learning_rate": 0.00017266666666666667, | |
| "loss": 17.748182678222655, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.001577628065418977, | |
| "grad_norm": 30.866708755493164, | |
| "learning_rate": 0.00017933333333333332, | |
| "loss": 17.471981811523438, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.001636058734508569, | |
| "grad_norm": 58.11023712158203, | |
| "learning_rate": 0.00018600000000000002, | |
| "loss": 17.05304870605469, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.0016944894035981605, | |
| "grad_norm": 94.0334243774414, | |
| "learning_rate": 0.0001926666666666667, | |
| "loss": 16.84721221923828, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.0017529200726877523, | |
| "grad_norm": 75.26679992675781, | |
| "learning_rate": 0.00019933333333333334, | |
| "loss": 16.655764770507812, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.0018113507417773442, | |
| "grad_norm": NaN, | |
| "learning_rate": 0.00019999999996000412, | |
| "loss": 17.352436828613282, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.0018697814108669358, | |
| "grad_norm": NaN, | |
| "learning_rate": 0.00019999999982174674, | |
| "loss": 17.325209045410155, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.0019282120799565276, | |
| "grad_norm": 421.18212890625, | |
| "learning_rate": 0.00019999999958473402, | |
| "loss": 17.76262512207031, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.0019866427490461192, | |
| "grad_norm": Infinity, | |
| "learning_rate": 0.00019999999924896604, | |
| "loss": 17.62756805419922, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.002045073418135711, | |
| "grad_norm": 128304.0234375, | |
| "learning_rate": 0.00019999999881444277, | |
| "loss": 18.896243286132812, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.002103504087225303, | |
| "grad_norm": 1544019.375, | |
| "learning_rate": 0.00019999999828116422, | |
| "loss": 19.169711303710937, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.0021619347563148947, | |
| "grad_norm": 2917.049072265625, | |
| "learning_rate": 0.00019999999764913037, | |
| "loss": 19.900315856933595, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.002220365425404486, | |
| "grad_norm": 507.0084533691406, | |
| "learning_rate": 0.00019999999691834125, | |
| "loss": 20.770042419433594, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.002278796094494078, | |
| "grad_norm": 6932.96435546875, | |
| "learning_rate": 0.00019999999608879683, | |
| "loss": 21.493869018554687, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.00233722676358367, | |
| "grad_norm": 711.9380493164062, | |
| "learning_rate": 0.00019999999516049713, | |
| "loss": 21.287228393554688, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.0023956574326732616, | |
| "grad_norm": 2512.957275390625, | |
| "learning_rate": 0.00019999999413344217, | |
| "loss": 20.687088012695312, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.0024540881017628535, | |
| "grad_norm": 245.29962158203125, | |
| "learning_rate": 0.0001999999930076319, | |
| "loss": 19.545924377441406, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.002512518770852445, | |
| "grad_norm": 873.497314453125, | |
| "learning_rate": 0.00019999999178306637, | |
| "loss": 19.846446228027343, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.0025709494399420367, | |
| "grad_norm": 273.556884765625, | |
| "learning_rate": 0.00019999999045974555, | |
| "loss": 19.4821044921875, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.0026293801090316285, | |
| "grad_norm": 105.1680679321289, | |
| "learning_rate": 0.00019999998903766944, | |
| "loss": 18.83043212890625, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.0026878107781212203, | |
| "grad_norm": 19779.220703125, | |
| "learning_rate": 0.00019999998751683804, | |
| "loss": 18.68498992919922, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.002746241447210812, | |
| "grad_norm": 379.18695068359375, | |
| "learning_rate": 0.0001999999858972514, | |
| "loss": 17.952909851074217, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.0028046721163004036, | |
| "grad_norm": 982.7449340820312, | |
| "learning_rate": 0.00019999998417890945, | |
| "loss": 18.662619018554686, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.0028631027853899954, | |
| "grad_norm": 264.763671875, | |
| "learning_rate": 0.00019999998236181225, | |
| "loss": 18.683988952636717, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.0029215334544795872, | |
| "grad_norm": 1355.505126953125, | |
| "learning_rate": 0.0001999999804459598, | |
| "loss": 18.34948272705078, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.002979964123569179, | |
| "grad_norm": 106.23206329345703, | |
| "learning_rate": 0.00019999997843135202, | |
| "loss": 18.03973388671875, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.003038394792658771, | |
| "grad_norm": 135.76995849609375, | |
| "learning_rate": 0.00019999997631798904, | |
| "loss": 18.432786560058595, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.0030968254617483623, | |
| "grad_norm": 683.4022216796875, | |
| "learning_rate": 0.00019999997410587078, | |
| "loss": 19.00628662109375, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.003155256130837954, | |
| "grad_norm": 1766.309814453125, | |
| "learning_rate": 0.00019999997179499722, | |
| "loss": 18.93055877685547, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 0.003213686799927546, | |
| "grad_norm": 12474.607421875, | |
| "learning_rate": 0.0001999999693853684, | |
| "loss": 19.35320129394531, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.003272117469017138, | |
| "grad_norm": 14713.984375, | |
| "learning_rate": 0.00019999996687698437, | |
| "loss": 19.091952514648437, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 0.0033305481381067296, | |
| "grad_norm": 37941.45703125, | |
| "learning_rate": 0.00019999996426984504, | |
| "loss": 18.91400604248047, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 0.003388978807196321, | |
| "grad_norm": 40131.44140625, | |
| "learning_rate": 0.0001999999615639505, | |
| "loss": 19.18461456298828, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 0.003447409476285913, | |
| "grad_norm": 4507.1064453125, | |
| "learning_rate": 0.00019999995875930068, | |
| "loss": 19.337628173828126, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 0.0035058401453755047, | |
| "grad_norm": 69082.234375, | |
| "learning_rate": 0.00019999995585589562, | |
| "loss": 19.456033325195314, | |
| "step": 600 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 1000000, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 6, | |
| "save_steps": 200, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 4809098082475008.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |