| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.9992337164750957, |
| "eval_steps": 500, |
| "global_step": 652, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.007662835249042145, |
| "grad_norm": 11.775691032409668, |
| "learning_rate": 2.162496240601504e-06, |
| "loss": 0.318, |
| "mean_token_accuracy": 0.9071180582046509, |
| "num_tokens": 1077246.0, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.01532567049808429, |
| "grad_norm": 2.0914440155029297, |
| "learning_rate": 4.865616541353383e-06, |
| "loss": 0.1405, |
| "mean_token_accuracy": 0.9591145873069763, |
| "num_tokens": 2172444.0, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.022988505747126436, |
| "grad_norm": 0.6351454257965088, |
| "learning_rate": 7.568736842105265e-06, |
| "loss": 0.1095, |
| "mean_token_accuracy": 0.9618923664093018, |
| "num_tokens": 3254491.0, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.03065134099616858, |
| "grad_norm": 0.5055913329124451, |
| "learning_rate": 1.0271857142857143e-05, |
| "loss": 0.0994, |
| "mean_token_accuracy": 0.963628476858139, |
| "num_tokens": 4344711.0, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.038314176245210725, |
| "grad_norm": 0.5738006830215454, |
| "learning_rate": 1.2974977443609024e-05, |
| "loss": 0.0936, |
| "mean_token_accuracy": 0.9665798544883728, |
| "num_tokens": 5430861.0, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.04597701149425287, |
| "grad_norm": 0.5167019367218018, |
| "learning_rate": 1.5678097744360906e-05, |
| "loss": 0.0917, |
| "mean_token_accuracy": 0.9660590350627899, |
| "num_tokens": 6522178.0, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.05363984674329502, |
| "grad_norm": 0.7361806035041809, |
| "learning_rate": 1.8381218045112784e-05, |
| "loss": 0.0884, |
| "mean_token_accuracy": 0.9677951395511627, |
| "num_tokens": 7603367.0, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.06130268199233716, |
| "grad_norm": 0.8272627592086792, |
| "learning_rate": 1.8920370473768527e-05, |
| "loss": 0.0851, |
| "mean_token_accuracy": 0.9677951395511627, |
| "num_tokens": 8699280.0, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.06896551724137931, |
| "grad_norm": 0.5785582661628723, |
| "learning_rate": 1.8914393016996354e-05, |
| "loss": 0.0789, |
| "mean_token_accuracy": 0.97109375, |
| "num_tokens": 9799050.0, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.07662835249042145, |
| "grad_norm": 0.8126276135444641, |
| "learning_rate": 1.8903821629022412e-05, |
| "loss": 0.0767, |
| "mean_token_accuracy": 0.9718750178813934, |
| "num_tokens": 10888773.0, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.0842911877394636, |
| "grad_norm": 0.6835556626319885, |
| "learning_rate": 1.8888663161225525e-05, |
| "loss": 0.0794, |
| "mean_token_accuracy": 0.9729166865348816, |
| "num_tokens": 11967768.0, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.09195402298850575, |
| "grad_norm": 0.5534384846687317, |
| "learning_rate": 1.886892743789796e-05, |
| "loss": 0.0771, |
| "mean_token_accuracy": 0.9727430760860443, |
| "num_tokens": 13065561.0, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.09961685823754789, |
| "grad_norm": 0.7145775556564331, |
| "learning_rate": 1.8844627249878266e-05, |
| "loss": 0.0743, |
| "mean_token_accuracy": 0.9715277850627899, |
| "num_tokens": 14152437.0, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.10727969348659004, |
| "grad_norm": 0.44082367420196533, |
| "learning_rate": 1.881577834626145e-05, |
| "loss": 0.08, |
| "mean_token_accuracy": 0.9718750059604645, |
| "num_tokens": 15253862.0, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.11494252873563218, |
| "grad_norm": 0.37960800528526306, |
| "learning_rate": 1.8782399424191886e-05, |
| "loss": 0.0699, |
| "mean_token_accuracy": 0.9753472447395325, |
| "num_tokens": 16351852.0, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.12260536398467432, |
| "grad_norm": 0.41791388392448425, |
| "learning_rate": 1.8744512116745595e-05, |
| "loss": 0.0731, |
| "mean_token_accuracy": 0.9743923783302307, |
| "num_tokens": 17442499.0, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.13026819923371646, |
| "grad_norm": 0.44737985730171204, |
| "learning_rate": 1.8702140978909714e-05, |
| "loss": 0.0699, |
| "mean_token_accuracy": 0.9751736223697662, |
| "num_tokens": 18525035.0, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.13793103448275862, |
| "grad_norm": 0.3966364562511444, |
| "learning_rate": 1.8655313471668277e-05, |
| "loss": 0.0698, |
| "mean_token_accuracy": 0.9746528029441833, |
| "num_tokens": 19631265.0, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.14559386973180077, |
| "grad_norm": 0.4190419316291809, |
| "learning_rate": 1.8604059944204582e-05, |
| "loss": 0.0697, |
| "mean_token_accuracy": 0.9736979186534882, |
| "num_tokens": 20712851.0, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.1532567049808429, |
| "grad_norm": 0.4391917288303375, |
| "learning_rate": 1.8548413614231694e-05, |
| "loss": 0.0689, |
| "mean_token_accuracy": 0.9743923723697663, |
| "num_tokens": 21807456.0, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.16091954022988506, |
| "grad_norm": 0.42839646339416504, |
| "learning_rate": 1.848841054646387e-05, |
| "loss": 0.0723, |
| "mean_token_accuracy": 0.9734375178813934, |
| "num_tokens": 22895468.0, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.1685823754789272, |
| "grad_norm": 0.4990848898887634, |
| "learning_rate": 1.8424089629242794e-05, |
| "loss": 0.0672, |
| "mean_token_accuracy": 0.9744791805744171, |
| "num_tokens": 23989680.0, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.17624521072796934, |
| "grad_norm": 0.4388997554779053, |
| "learning_rate": 1.8355492549333818e-05, |
| "loss": 0.0669, |
| "mean_token_accuracy": 0.9763021051883698, |
| "num_tokens": 25072335.0, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.1839080459770115, |
| "grad_norm": 0.35158392786979675, |
| "learning_rate": 1.8282663764908525e-05, |
| "loss": 0.0688, |
| "mean_token_accuracy": 0.9746527731418609, |
| "num_tokens": 26162308.0, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.19157088122605365, |
| "grad_norm": 0.3553743362426758, |
| "learning_rate": 1.8205650476731113e-05, |
| "loss": 0.0635, |
| "mean_token_accuracy": 0.9765625119209289, |
| "num_tokens": 27254497.0, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.19923371647509577, |
| "grad_norm": 0.48281288146972656, |
| "learning_rate": 1.8124502597567315e-05, |
| "loss": 0.0639, |
| "mean_token_accuracy": 0.9776909708976745, |
| "num_tokens": 28349197.0, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.20689655172413793, |
| "grad_norm": 0.38537609577178955, |
| "learning_rate": 1.8039272719835648e-05, |
| "loss": 0.0661, |
| "mean_token_accuracy": 0.9774305641651153, |
| "num_tokens": 29430193.0, |
| "step": 135 |
| }, |
| { |
| "epoch": 0.21455938697318008, |
| "grad_norm": 0.4320482313632965, |
| "learning_rate": 1.795001608152194e-05, |
| "loss": 0.0688, |
| "mean_token_accuracy": 0.9763020873069763, |
| "num_tokens": 30532672.0, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.2222222222222222, |
| "grad_norm": 0.3345838487148285, |
| "learning_rate": 1.7856790530379266e-05, |
| "loss": 0.0631, |
| "mean_token_accuracy": 0.9763020873069763, |
| "num_tokens": 31627374.0, |
| "step": 145 |
| }, |
| { |
| "epoch": 0.22988505747126436, |
| "grad_norm": 0.46670979261398315, |
| "learning_rate": 1.7759656486436495e-05, |
| "loss": 0.0632, |
| "mean_token_accuracy": 0.9772569477558136, |
| "num_tokens": 32709508.0, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.23754789272030652, |
| "grad_norm": 0.4126276671886444, |
| "learning_rate": 1.76586769028397e-05, |
| "loss": 0.0641, |
| "mean_token_accuracy": 0.9767361402511596, |
| "num_tokens": 33805592.0, |
| "step": 155 |
| }, |
| { |
| "epoch": 0.24521072796934865, |
| "grad_norm": 0.37826210260391235, |
| "learning_rate": 1.755391722505186e-05, |
| "loss": 0.0654, |
| "mean_token_accuracy": 0.977343761920929, |
| "num_tokens": 34901401.0, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.25287356321839083, |
| "grad_norm": 0.33070090413093567, |
| "learning_rate": 1.7445445348437266e-05, |
| "loss": 0.0604, |
| "mean_token_accuracy": 0.9770833611488342, |
| "num_tokens": 35990805.0, |
| "step": 165 |
| }, |
| { |
| "epoch": 0.26053639846743293, |
| "grad_norm": 0.3684611916542053, |
| "learning_rate": 1.7333331574258133e-05, |
| "loss": 0.0633, |
| "mean_token_accuracy": 0.9767361223697663, |
| "num_tokens": 37095539.0, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.2681992337164751, |
| "grad_norm": 0.4362129867076874, |
| "learning_rate": 1.721764856411194e-05, |
| "loss": 0.0635, |
| "mean_token_accuracy": 0.9766493141651154, |
| "num_tokens": 38185400.0, |
| "step": 175 |
| }, |
| { |
| "epoch": 0.27586206896551724, |
| "grad_norm": 0.4917359948158264, |
| "learning_rate": 1.7098471292839025e-05, |
| "loss": 0.0624, |
| "mean_token_accuracy": 0.9771701574325562, |
| "num_tokens": 39270809.0, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.2835249042145594, |
| "grad_norm": 0.3966314494609833, |
| "learning_rate": 1.6975876999930955e-05, |
| "loss": 0.0596, |
| "mean_token_accuracy": 0.9801215350627899, |
| "num_tokens": 40368119.0, |
| "step": 185 |
| }, |
| { |
| "epoch": 0.29118773946360155, |
| "grad_norm": 0.3741203248500824, |
| "learning_rate": 1.684994513947113e-05, |
| "loss": 0.0623, |
| "mean_token_accuracy": 0.9770833373069763, |
| "num_tokens": 41463186.0, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.2988505747126437, |
| "grad_norm": 0.34731847047805786, |
| "learning_rate": 1.6720757328640154e-05, |
| "loss": 0.0608, |
| "mean_token_accuracy": 0.9786458492279053, |
| "num_tokens": 42548414.0, |
| "step": 195 |
| }, |
| { |
| "epoch": 0.3065134099616858, |
| "grad_norm": 0.3592664301395416, |
| "learning_rate": 1.658839729481924e-05, |
| "loss": 0.0612, |
| "mean_token_accuracy": 0.9782117962837219, |
| "num_tokens": 43637070.0, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.31417624521072796, |
| "grad_norm": 0.36953556537628174, |
| "learning_rate": 1.645295082132601e-05, |
| "loss": 0.0574, |
| "mean_token_accuracy": 0.979253488779068, |
| "num_tokens": 44719233.0, |
| "step": 205 |
| }, |
| { |
| "epoch": 0.3218390804597701, |
| "grad_norm": 0.3487086296081543, |
| "learning_rate": 1.631450569181782e-05, |
| "loss": 0.0608, |
| "mean_token_accuracy": 0.9775173604488373, |
| "num_tokens": 45805568.0, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.32950191570881227, |
| "grad_norm": 0.30979305505752563, |
| "learning_rate": 1.6173151633398657e-05, |
| "loss": 0.0626, |
| "mean_token_accuracy": 0.9780382215976715, |
| "num_tokens": 46907262.0, |
| "step": 215 |
| }, |
| { |
| "epoch": 0.3371647509578544, |
| "grad_norm": 0.3208833932876587, |
| "learning_rate": 1.602898025846646e-05, |
| "loss": 0.0625, |
| "mean_token_accuracy": 0.9773437678813934, |
| "num_tokens": 47996982.0, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.3448275862068966, |
| "grad_norm": 0.39099711179733276, |
| "learning_rate": 1.588208500533861e-05, |
| "loss": 0.0617, |
| "mean_token_accuracy": 0.9773437559604645, |
| "num_tokens": 49101799.0, |
| "step": 225 |
| }, |
| { |
| "epoch": 0.3524904214559387, |
| "grad_norm": 0.34100982546806335, |
| "learning_rate": 1.573256107769398e-05, |
| "loss": 0.0635, |
| "mean_token_accuracy": 0.9769097328186035, |
| "num_tokens": 50196721.0, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.36015325670498083, |
| "grad_norm": 0.3107825517654419, |
| "learning_rate": 1.5580505382870888e-05, |
| "loss": 0.0633, |
| "mean_token_accuracy": 0.9769965529441833, |
| "num_tokens": 51290453.0, |
| "step": 235 |
| }, |
| { |
| "epoch": 0.367816091954023, |
| "grad_norm": 0.31218069791793823, |
| "learning_rate": 1.5426016469060878e-05, |
| "loss": 0.0653, |
| "mean_token_accuracy": 0.9751736164093018, |
| "num_tokens": 52373930.0, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.37547892720306514, |
| "grad_norm": 0.3370981216430664, |
| "learning_rate": 1.526919446143905e-05, |
| "loss": 0.0599, |
| "mean_token_accuracy": 0.9768229424953461, |
| "num_tokens": 53467690.0, |
| "step": 245 |
| }, |
| { |
| "epoch": 0.3831417624521073, |
| "grad_norm": 0.3400161862373352, |
| "learning_rate": 1.5110140997272344e-05, |
| "loss": 0.0561, |
| "mean_token_accuracy": 0.9788194596767426, |
| "num_tokens": 54572016.0, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.39080459770114945, |
| "grad_norm": 0.3365417718887329, |
| "learning_rate": 1.494895916004783e-05, |
| "loss": 0.0595, |
| "mean_token_accuracy": 0.9775173664093018, |
| "num_tokens": 55667785.0, |
| "step": 255 |
| }, |
| { |
| "epoch": 0.39846743295019155, |
| "grad_norm": 0.27660876512527466, |
| "learning_rate": 1.4785753412663708e-05, |
| "loss": 0.0601, |
| "mean_token_accuracy": 0.9776041746139527, |
| "num_tokens": 56751073.0, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.4061302681992337, |
| "grad_norm": 0.30515599250793457, |
| "learning_rate": 1.4620629529726256e-05, |
| "loss": 0.0562, |
| "mean_token_accuracy": 0.9784722268581391, |
| "num_tokens": 57843072.0, |
| "step": 265 |
| }, |
| { |
| "epoch": 0.41379310344827586, |
| "grad_norm": 0.4340786039829254, |
| "learning_rate": 1.4453694528996721e-05, |
| "loss": 0.061, |
| "mean_token_accuracy": 0.9785590589046478, |
| "num_tokens": 58943997.0, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.421455938697318, |
| "grad_norm": 0.38081300258636475, |
| "learning_rate": 1.4285056602032446e-05, |
| "loss": 0.0565, |
| "mean_token_accuracy": 0.9790798723697662, |
| "num_tokens": 60045338.0, |
| "step": 275 |
| }, |
| { |
| "epoch": 0.42911877394636017, |
| "grad_norm": 0.28883832693099976, |
| "learning_rate": 1.4114825044067308e-05, |
| "loss": 0.0621, |
| "mean_token_accuracy": 0.9771701335906983, |
| "num_tokens": 61133724.0, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.4367816091954023, |
| "grad_norm": 0.28922542929649353, |
| "learning_rate": 1.3943110183176817e-05, |
| "loss": 0.0593, |
| "mean_token_accuracy": 0.9784722328186035, |
| "num_tokens": 62214177.0, |
| "step": 285 |
| }, |
| { |
| "epoch": 0.4444444444444444, |
| "grad_norm": 0.34836313128471375, |
| "learning_rate": 1.3770023308773852e-05, |
| "loss": 0.0581, |
| "mean_token_accuracy": 0.97734375, |
| "num_tokens": 63298786.0, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.4521072796934866, |
| "grad_norm": 0.31483110785484314, |
| "learning_rate": 1.3595676599481324e-05, |
| "loss": 0.058, |
| "mean_token_accuracy": 0.9768229246139526, |
| "num_tokens": 64384875.0, |
| "step": 295 |
| }, |
| { |
| "epoch": 0.45977011494252873, |
| "grad_norm": 0.4621759355068207, |
| "learning_rate": 1.3420183050428566e-05, |
| "loss": 0.0577, |
| "mean_token_accuracy": 0.97890625, |
| "num_tokens": 65477649.0, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.4674329501915709, |
| "grad_norm": 0.38731318712234497, |
| "learning_rate": 1.324365640001851e-05, |
| "loss": 0.0545, |
| "mean_token_accuracy": 0.980121523141861, |
| "num_tokens": 66557809.0, |
| "step": 305 |
| }, |
| { |
| "epoch": 0.47509578544061304, |
| "grad_norm": 0.3266485035419464, |
| "learning_rate": 1.306621105621317e-05, |
| "loss": 0.0555, |
| "mean_token_accuracy": 0.9790798723697662, |
| "num_tokens": 67651417.0, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.4827586206896552, |
| "grad_norm": 0.3696978688240051, |
| "learning_rate": 1.2887962022385146e-05, |
| "loss": 0.0575, |
| "mean_token_accuracy": 0.9781250059604645, |
| "num_tokens": 68748506.0, |
| "step": 315 |
| }, |
| { |
| "epoch": 0.4904214559386973, |
| "grad_norm": 0.3244847059249878, |
| "learning_rate": 1.2709024822783297e-05, |
| "loss": 0.0597, |
| "mean_token_accuracy": 0.9786458492279053, |
| "num_tokens": 69831103.0, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.49808429118773945, |
| "grad_norm": 0.2741115093231201, |
| "learning_rate": 1.2529515427660755e-05, |
| "loss": 0.054, |
| "mean_token_accuracy": 0.9805555701255798, |
| "num_tokens": 70922262.0, |
| "step": 325 |
| }, |
| { |
| "epoch": 0.5057471264367817, |
| "grad_norm": 0.2509593963623047, |
| "learning_rate": 1.2349550178113955e-05, |
| "loss": 0.0584, |
| "mean_token_accuracy": 0.978038203716278, |
| "num_tokens": 72011671.0, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.5134099616858238, |
| "grad_norm": 0.2601114809513092, |
| "learning_rate": 1.216924571068126e-05, |
| "loss": 0.0557, |
| "mean_token_accuracy": 0.9801215469837189, |
| "num_tokens": 73115781.0, |
| "step": 335 |
| }, |
| { |
| "epoch": 0.5210727969348659, |
| "grad_norm": 0.3080248236656189, |
| "learning_rate": 1.1988718881750141e-05, |
| "loss": 0.0555, |
| "mean_token_accuracy": 0.9796875059604645, |
| "num_tokens": 74197772.0, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.5287356321839081, |
| "grad_norm": 0.44369372725486755, |
| "learning_rate": 1.1808086691821876e-05, |
| "loss": 0.0544, |
| "mean_token_accuracy": 0.9800347208976745, |
| "num_tokens": 75300928.0, |
| "step": 345 |
| }, |
| { |
| "epoch": 0.5363984674329502, |
| "grad_norm": 0.31000152230262756, |
| "learning_rate": 1.1627466209682822e-05, |
| "loss": 0.056, |
| "mean_token_accuracy": 0.9795138955116272, |
| "num_tokens": 76387213.0, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.5440613026819924, |
| "grad_norm": 0.2904181480407715, |
| "learning_rate": 1.1446974496531438e-05, |
| "loss": 0.0561, |
| "mean_token_accuracy": 0.9789930582046509, |
| "num_tokens": 77470917.0, |
| "step": 355 |
| }, |
| { |
| "epoch": 0.5517241379310345, |
| "grad_norm": 0.2386808842420578, |
| "learning_rate": 1.1266728530110266e-05, |
| "loss": 0.062, |
| "mean_token_accuracy": 0.9775173604488373, |
| "num_tokens": 78568625.0, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.5593869731800766, |
| "grad_norm": 0.2828770577907562, |
| "learning_rate": 1.1086845128891916e-05, |
| "loss": 0.0587, |
| "mean_token_accuracy": 0.97890625, |
| "num_tokens": 79650226.0, |
| "step": 365 |
| }, |
| { |
| "epoch": 0.5670498084291188, |
| "grad_norm": 0.2870100736618042, |
| "learning_rate": 1.0907440876368388e-05, |
| "loss": 0.0583, |
| "mean_token_accuracy": 0.9782118260860443, |
| "num_tokens": 80742756.0, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.5747126436781609, |
| "grad_norm": 0.27211377024650574, |
| "learning_rate": 1.0728632045492557e-05, |
| "loss": 0.0551, |
| "mean_token_accuracy": 0.9795138895511627, |
| "num_tokens": 81823129.0, |
| "step": 375 |
| }, |
| { |
| "epoch": 0.5823754789272031, |
| "grad_norm": 0.26771602034568787, |
| "learning_rate": 1.0550534523321044e-05, |
| "loss": 0.0547, |
| "mean_token_accuracy": 0.9801215291023254, |
| "num_tokens": 82912796.0, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.5900383141762452, |
| "grad_norm": 0.3738754689693451, |
| "learning_rate": 1.0373263735907098e-05, |
| "loss": 0.0532, |
| "mean_token_accuracy": 0.9796007096767425, |
| "num_tokens": 83997407.0, |
| "step": 385 |
| }, |
| { |
| "epoch": 0.5977011494252874, |
| "grad_norm": 0.3444598913192749, |
| "learning_rate": 1.0196934573492302e-05, |
| "loss": 0.0546, |
| "mean_token_accuracy": 0.9808159708976746, |
| "num_tokens": 85097209.0, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.6053639846743295, |
| "grad_norm": 0.21151889860630035, |
| "learning_rate": 1.002166131604553e-05, |
| "loss": 0.0538, |
| "mean_token_accuracy": 0.9802083313465119, |
| "num_tokens": 86191701.0, |
| "step": 395 |
| }, |
| { |
| "epoch": 0.6130268199233716, |
| "grad_norm": 0.2923840582370758, |
| "learning_rate": 9.84755755919744e-06, |
| "loss": 0.0589, |
| "mean_token_accuracy": 0.9780381977558136, |
| "num_tokens": 87290541.0, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.6206896551724138, |
| "grad_norm": 0.23207074403762817, |
| "learning_rate": 9.674736140618434e-06, |
| "loss": 0.0606, |
| "mean_token_accuracy": 0.9776041865348816, |
| "num_tokens": 88398792.0, |
| "step": 405 |
| }, |
| { |
| "epoch": 0.6283524904214559, |
| "grad_norm": 0.33359429240226746, |
| "learning_rate": 9.503309066887955e-06, |
| "loss": 0.056, |
| "mean_token_accuracy": 0.979687511920929, |
| "num_tokens": 89486208.0, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.6360153256704981, |
| "grad_norm": 0.2546365559101105, |
| "learning_rate": 9.333387440902325e-06, |
| "loss": 0.0549, |
| "mean_token_accuracy": 0.980381965637207, |
| "num_tokens": 90584558.0, |
| "step": 415 |
| }, |
| { |
| "epoch": 0.6436781609195402, |
| "grad_norm": 0.2766530215740204, |
| "learning_rate": 9.165081389868311e-06, |
| "loss": 0.0577, |
| "mean_token_accuracy": 0.9791666746139527, |
| "num_tokens": 91682674.0, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.6513409961685823, |
| "grad_norm": 0.3182806074619293, |
| "learning_rate": 8.998499993929042e-06, |
| "loss": 0.0566, |
| "mean_token_accuracy": 0.9783854365348816, |
| "num_tokens": 92783006.0, |
| "step": 425 |
| }, |
| { |
| "epoch": 0.6590038314176245, |
| "grad_norm": 0.23961606621742249, |
| "learning_rate": 8.833751215468525e-06, |
| "loss": 0.0554, |
| "mean_token_accuracy": 0.9798611342906952, |
| "num_tokens": 93875140.0, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.6666666666666666, |
| "grad_norm": 0.29564839601516724, |
| "learning_rate": 8.67094182914057e-06, |
| "loss": 0.0535, |
| "mean_token_accuracy": 0.9807291805744172, |
| "num_tokens": 94969575.0, |
| "step": 435 |
| }, |
| { |
| "epoch": 0.6743295019157088, |
| "grad_norm": 0.3239465355873108, |
| "learning_rate": 8.510177352667533e-06, |
| "loss": 0.0546, |
| "mean_token_accuracy": 0.9793403029441834, |
| "num_tokens": 96060549.0, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.6819923371647509, |
| "grad_norm": 0.39116764068603516, |
| "learning_rate": 8.351561978453641e-06, |
| "loss": 0.0567, |
| "mean_token_accuracy": 0.978819465637207, |
| "num_tokens": 97156334.0, |
| "step": 445 |
| }, |
| { |
| "epoch": 0.6896551724137931, |
| "grad_norm": 0.36669617891311646, |
| "learning_rate": 8.195198506057288e-06, |
| "loss": 0.0558, |
| "mean_token_accuracy": 0.9782986104488373, |
| "num_tokens": 98246337.0, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.6973180076628352, |
| "grad_norm": 0.3902563750743866, |
| "learning_rate": 8.041188275566041e-06, |
| "loss": 0.0581, |
| "mean_token_accuracy": 0.9772569537162781, |
| "num_tokens": 99342179.0, |
| "step": 455 |
| }, |
| { |
| "epoch": 0.7049808429118773, |
| "grad_norm": 0.2951866686344147, |
| "learning_rate": 7.889631101917516e-06, |
| "loss": 0.0546, |
| "mean_token_accuracy": 0.978472238779068, |
| "num_tokens": 100444505.0, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.7126436781609196, |
| "grad_norm": 0.3099174499511719, |
| "learning_rate": 7.740625210208746e-06, |
| "loss": 0.0559, |
| "mean_token_accuracy": 0.9793402850627899, |
| "num_tokens": 101533570.0, |
| "step": 465 |
| }, |
| { |
| "epoch": 0.7203065134099617, |
| "grad_norm": 0.25526103377342224, |
| "learning_rate": 7.594267172035906e-06, |
| "loss": 0.0511, |
| "mean_token_accuracy": 0.9812500178813934, |
| "num_tokens": 102633640.0, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.7279693486590039, |
| "grad_norm": 0.29758980870246887, |
| "learning_rate": 7.450651842905699e-06, |
| "loss": 0.053, |
| "mean_token_accuracy": 0.9807291805744172, |
| "num_tokens": 103723814.0, |
| "step": 475 |
| }, |
| { |
| "epoch": 0.735632183908046, |
| "grad_norm": 0.298353374004364, |
| "learning_rate": 7.309872300758933e-06, |
| "loss": 0.054, |
| "mean_token_accuracy": 0.979600703716278, |
| "num_tokens": 104807793.0, |
| "step": 480 |
| }, |
| { |
| "epoch": 0.7432950191570882, |
| "grad_norm": 0.36888647079467773, |
| "learning_rate": 7.172019785646182e-06, |
| "loss": 0.0555, |
| "mean_token_accuracy": 0.9802951574325561, |
| "num_tokens": 105896118.0, |
| "step": 485 |
| }, |
| { |
| "epoch": 0.7509578544061303, |
| "grad_norm": 0.3105420172214508, |
| "learning_rate": 7.037183640594544e-06, |
| "loss": 0.0508, |
| "mean_token_accuracy": 0.9802083313465119, |
| "num_tokens": 106982558.0, |
| "step": 490 |
| }, |
| { |
| "epoch": 0.7586206896551724, |
| "grad_norm": 0.3314758539199829, |
| "learning_rate": 6.905451253703918e-06, |
| "loss": 0.0552, |
| "mean_token_accuracy": 0.9802083373069763, |
| "num_tokens": 108076197.0, |
| "step": 495 |
| }, |
| { |
| "epoch": 0.7662835249042146, |
| "grad_norm": 0.3398136496543884, |
| "learning_rate": 6.776908001510251e-06, |
| "loss": 0.0523, |
| "mean_token_accuracy": 0.98046875, |
| "num_tokens": 109161850.0, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.7739463601532567, |
| "grad_norm": 0.2903926968574524, |
| "learning_rate": 6.651637193652513e-06, |
| "loss": 0.056, |
| "mean_token_accuracy": 0.979340273141861, |
| "num_tokens": 110253260.0, |
| "step": 505 |
| }, |
| { |
| "epoch": 0.7816091954022989, |
| "grad_norm": 0.24769453704357147, |
| "learning_rate": 6.52972001887921e-06, |
| "loss": 0.0597, |
| "mean_token_accuracy": 0.9774305641651153, |
| "num_tokens": 111348891.0, |
| "step": 510 |
| }, |
| { |
| "epoch": 0.789272030651341, |
| "grad_norm": 0.2901223301887512, |
| "learning_rate": 6.411235492429508e-06, |
| "loss": 0.0575, |
| "mean_token_accuracy": 0.97855903506279, |
| "num_tokens": 112437384.0, |
| "step": 515 |
| }, |
| { |
| "epoch": 0.7969348659003831, |
| "grad_norm": 0.41612568497657776, |
| "learning_rate": 6.296260404822958e-06, |
| "loss": 0.055, |
| "mean_token_accuracy": 0.9800347447395324, |
| "num_tokens": 113519456.0, |
| "step": 520 |
| }, |
| { |
| "epoch": 0.8045977011494253, |
| "grad_norm": 0.3250349462032318, |
| "learning_rate": 6.184869272091133e-06, |
| "loss": 0.0551, |
| "mean_token_accuracy": 0.9795139074325562, |
| "num_tokens": 114603903.0, |
| "step": 525 |
| }, |
| { |
| "epoch": 0.8122605363984674, |
| "grad_norm": 0.29371413588523865, |
| "learning_rate": 6.077134287483337e-06, |
| "loss": 0.0547, |
| "mean_token_accuracy": 0.9800347208976745, |
| "num_tokens": 115699227.0, |
| "step": 530 |
| }, |
| { |
| "epoch": 0.8199233716475096, |
| "grad_norm": 0.35530513525009155, |
| "learning_rate": 5.973125274677734e-06, |
| "loss": 0.0551, |
| "mean_token_accuracy": 0.979687511920929, |
| "num_tokens": 116787851.0, |
| "step": 535 |
| }, |
| { |
| "epoch": 0.8275862068965517, |
| "grad_norm": 0.40407872200012207, |
| "learning_rate": 5.872909642528218e-06, |
| "loss": 0.0573, |
| "mean_token_accuracy": 0.979340273141861, |
| "num_tokens": 117873854.0, |
| "step": 540 |
| }, |
| { |
| "epoch": 0.8352490421455939, |
| "grad_norm": 0.2643112540245056, |
| "learning_rate": 5.776552341376333e-06, |
| "loss": 0.0517, |
| "mean_token_accuracy": 0.9805555701255798, |
| "num_tokens": 118967125.0, |
| "step": 545 |
| }, |
| { |
| "epoch": 0.842911877394636, |
| "grad_norm": 0.25208526849746704, |
| "learning_rate": 5.6841158209565735e-06, |
| "loss": 0.0541, |
| "mean_token_accuracy": 0.9786458313465118, |
| "num_tokens": 120042343.0, |
| "step": 550 |
| }, |
| { |
| "epoch": 0.8505747126436781, |
| "grad_norm": 0.324431836605072, |
| "learning_rate": 5.595659989922345e-06, |
| "loss": 0.0573, |
| "mean_token_accuracy": 0.9793402850627899, |
| "num_tokens": 121137305.0, |
| "step": 555 |
| }, |
| { |
| "epoch": 0.8582375478927203, |
| "grad_norm": 0.28662794828414917, |
| "learning_rate": 5.511242177018818e-06, |
| "loss": 0.0526, |
| "mean_token_accuracy": 0.9802083432674408, |
| "num_tokens": 122226343.0, |
| "step": 560 |
| }, |
| { |
| "epoch": 0.8659003831417624, |
| "grad_norm": 0.2520166039466858, |
| "learning_rate": 5.430917093927816e-06, |
| "loss": 0.0572, |
| "mean_token_accuracy": 0.9789930582046509, |
| "num_tokens": 123320384.0, |
| "step": 565 |
| }, |
| { |
| "epoch": 0.8735632183908046, |
| "grad_norm": 0.2630253732204437, |
| "learning_rate": 5.3547367998088715e-06, |
| "loss": 0.0487, |
| "mean_token_accuracy": 0.9818576514720917, |
| "num_tokens": 124404287.0, |
| "step": 570 |
| }, |
| { |
| "epoch": 0.8812260536398467, |
| "grad_norm": 0.24426770210266113, |
| "learning_rate": 5.282750667559361e-06, |
| "loss": 0.0527, |
| "mean_token_accuracy": 0.9807291686534881, |
| "num_tokens": 125499483.0, |
| "step": 575 |
| }, |
| { |
| "epoch": 0.8888888888888888, |
| "grad_norm": 0.2494325190782547, |
| "learning_rate": 5.215005351815651e-06, |
| "loss": 0.0505, |
| "mean_token_accuracy": 0.9824652910232544, |
| "num_tokens": 126595035.0, |
| "step": 580 |
| }, |
| { |
| "epoch": 0.896551724137931, |
| "grad_norm": 0.3279549777507782, |
| "learning_rate": 5.151544758715945e-06, |
| "loss": 0.0511, |
| "mean_token_accuracy": 0.9807291865348816, |
| "num_tokens": 127687237.0, |
| "step": 585 |
| }, |
| { |
| "epoch": 0.9042145593869731, |
| "grad_norm": 0.2407781481742859, |
| "learning_rate": 5.092410017444466e-06, |
| "loss": 0.0507, |
| "mean_token_accuracy": 0.9813368141651153, |
| "num_tokens": 128784622.0, |
| "step": 590 |
| }, |
| { |
| "epoch": 0.9118773946360154, |
| "grad_norm": 0.2758333683013916, |
| "learning_rate": 5.037639453575379e-06, |
| "loss": 0.0514, |
| "mean_token_accuracy": 0.981250011920929, |
| "num_tokens": 129878622.0, |
| "step": 595 |
| }, |
| { |
| "epoch": 0.9195402298850575, |
| "grad_norm": 0.44546186923980713, |
| "learning_rate": 4.987268564233768e-06, |
| "loss": 0.0501, |
| "mean_token_accuracy": 0.9814236283302307, |
| "num_tokens": 130970165.0, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.9272030651340997, |
| "grad_norm": 0.3608750104904175, |
| "learning_rate": 4.941329995089741e-06, |
| "loss": 0.0515, |
| "mean_token_accuracy": 0.9806423664093018, |
| "num_tokens": 132065030.0, |
| "step": 605 |
| }, |
| { |
| "epoch": 0.9348659003831418, |
| "grad_norm": 0.3858118951320648, |
| "learning_rate": 4.899853519200571e-06, |
| "loss": 0.055, |
| "mean_token_accuracy": 0.9790798604488373, |
| "num_tokens": 133161259.0, |
| "step": 610 |
| }, |
| { |
| "epoch": 0.9425287356321839, |
| "grad_norm": 0.23960553109645844, |
| "learning_rate": 4.862866017714614e-06, |
| "loss": 0.0517, |
| "mean_token_accuracy": 0.9822048664093017, |
| "num_tokens": 134253608.0, |
| "step": 615 |
| }, |
| { |
| "epoch": 0.9501915708812261, |
| "grad_norm": 0.3373887538909912, |
| "learning_rate": 4.830391462449465e-06, |
| "loss": 0.0516, |
| "mean_token_accuracy": 0.9809027791023255, |
| "num_tokens": 135348208.0, |
| "step": 620 |
| }, |
| { |
| "epoch": 0.9578544061302682, |
| "grad_norm": 0.3087356388568878, |
| "learning_rate": 4.8024509003556915e-06, |
| "loss": 0.0566, |
| "mean_token_accuracy": 0.9789062678813935, |
| "num_tokens": 136445414.0, |
| "step": 625 |
| }, |
| { |
| "epoch": 0.9655172413793104, |
| "grad_norm": 0.30412212014198303, |
| "learning_rate": 4.77906243987618e-06, |
| "loss": 0.051, |
| "mean_token_accuracy": 0.9806423783302307, |
| "num_tokens": 137537027.0, |
| "step": 630 |
| }, |
| { |
| "epoch": 0.9731800766283525, |
| "grad_norm": 0.3232341706752777, |
| "learning_rate": 4.760241239209941e-06, |
| "loss": 0.0581, |
| "mean_token_accuracy": 0.9768229246139526, |
| "num_tokens": 138621895.0, |
| "step": 635 |
| }, |
| { |
| "epoch": 0.9808429118773946, |
| "grad_norm": 0.28698432445526123, |
| "learning_rate": 4.745999496488e-06, |
| "loss": 0.0486, |
| "mean_token_accuracy": 0.9817708313465119, |
| "num_tokens": 139717205.0, |
| "step": 640 |
| }, |
| { |
| "epoch": 0.9885057471264368, |
| "grad_norm": 0.25341159105300903, |
| "learning_rate": 4.736346441867709e-06, |
| "loss": 0.0533, |
| "mean_token_accuracy": 0.9797743141651154, |
| "num_tokens": 140814683.0, |
| "step": 645 |
| }, |
| { |
| "epoch": 0.9961685823754789, |
| "grad_norm": 0.315576434135437, |
| "learning_rate": 4.731288331550632e-06, |
| "loss": 0.0512, |
| "mean_token_accuracy": 0.9809895813465118, |
| "num_tokens": 141895145.0, |
| "step": 650 |
| } |
| ], |
| "logging_steps": 5, |
| "max_steps": 652, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 8.554477504520258e+18, |
| "train_batch_size": 16, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
