| { |
| "best_global_step": 1020, |
| "best_metric": 0.35731908679008484, |
| "best_model_checkpoint": "/home/longtail/data/outputs/exaone_7b_lora_rtl_coder_27k_ratio0.2/checkpoint-1020", |
| "epoch": 3.8361581920903953, |
| "eval_steps": 170, |
| "global_step": 1020, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 0.491259765625, |
| "epoch": 0.03766478342749529, |
| "grad_norm": 3.595203399658203, |
| "learning_rate": 1.2000000000000002e-06, |
| "loss": 1.580256175994873, |
| "mean_token_accuracy": 0.7325558871030807, |
| "num_tokens": 113431.0, |
| "step": 10 |
| }, |
| { |
| "entropy": 0.489697265625, |
| "epoch": 0.07532956685499058, |
| "grad_norm": 3.255786657333374, |
| "learning_rate": 2.5333333333333338e-06, |
| "loss": 1.5002482414245606, |
| "mean_token_accuracy": 0.7440871745347977, |
| "num_tokens": 230632.0, |
| "step": 20 |
| }, |
| { |
| "entropy": 0.506591796875, |
| "epoch": 0.11299435028248588, |
| "grad_norm": 2.7195632457733154, |
| "learning_rate": 3.866666666666667e-06, |
| "loss": 1.316688060760498, |
| "mean_token_accuracy": 0.7588966429233551, |
| "num_tokens": 352556.0, |
| "step": 30 |
| }, |
| { |
| "entropy": 0.587939453125, |
| "epoch": 0.15065913370998116, |
| "grad_norm": 1.6744133234024048, |
| "learning_rate": 5.2e-06, |
| "loss": 1.123267650604248, |
| "mean_token_accuracy": 0.77352384775877, |
| "num_tokens": 463768.0, |
| "step": 40 |
| }, |
| { |
| "entropy": 0.63876953125, |
| "epoch": 0.18832391713747645, |
| "grad_norm": 1.0927727222442627, |
| "learning_rate": 6.533333333333334e-06, |
| "loss": 0.7759298801422119, |
| "mean_token_accuracy": 0.8159186348319054, |
| "num_tokens": 591532.0, |
| "step": 50 |
| }, |
| { |
| "entropy": 0.645166015625, |
| "epoch": 0.22598870056497175, |
| "grad_norm": 0.7504259347915649, |
| "learning_rate": 7.866666666666667e-06, |
| "loss": 0.5978899478912354, |
| "mean_token_accuracy": 0.8416336104273796, |
| "num_tokens": 709718.0, |
| "step": 60 |
| }, |
| { |
| "entropy": 0.528759765625, |
| "epoch": 0.263653483992467, |
| "grad_norm": 0.601105272769928, |
| "learning_rate": 9.200000000000002e-06, |
| "loss": 0.5329740047454834, |
| "mean_token_accuracy": 0.8565513953566551, |
| "num_tokens": 825895.0, |
| "step": 70 |
| }, |
| { |
| "entropy": 0.462109375, |
| "epoch": 0.3013182674199623, |
| "grad_norm": 0.34783050417900085, |
| "learning_rate": 1.0533333333333333e-05, |
| "loss": 0.48036208152771, |
| "mean_token_accuracy": 0.8669139310717583, |
| "num_tokens": 950437.0, |
| "step": 80 |
| }, |
| { |
| "entropy": 0.502734375, |
| "epoch": 0.3389830508474576, |
| "grad_norm": 0.41616687178611755, |
| "learning_rate": 1.186666666666667e-05, |
| "loss": 0.5197448253631591, |
| "mean_token_accuracy": 0.8561126098036766, |
| "num_tokens": 1070848.0, |
| "step": 90 |
| }, |
| { |
| "entropy": 0.444482421875, |
| "epoch": 0.3766478342749529, |
| "grad_norm": 0.3191535174846649, |
| "learning_rate": 1.3200000000000002e-05, |
| "loss": 0.44791593551635744, |
| "mean_token_accuracy": 0.8714623257517815, |
| "num_tokens": 1194591.0, |
| "step": 100 |
| }, |
| { |
| "entropy": 0.457958984375, |
| "epoch": 0.4143126177024482, |
| "grad_norm": 0.32832491397857666, |
| "learning_rate": 1.4533333333333335e-05, |
| "loss": 0.45896482467651367, |
| "mean_token_accuracy": 0.8700259447097778, |
| "num_tokens": 1316384.0, |
| "step": 110 |
| }, |
| { |
| "entropy": 0.444970703125, |
| "epoch": 0.4519774011299435, |
| "grad_norm": 0.3374221920967102, |
| "learning_rate": 1.586666666666667e-05, |
| "loss": 0.452594518661499, |
| "mean_token_accuracy": 0.8728646501898766, |
| "num_tokens": 1441119.0, |
| "step": 120 |
| }, |
| { |
| "entropy": 0.444189453125, |
| "epoch": 0.4896421845574388, |
| "grad_norm": 0.41891390085220337, |
| "learning_rate": 1.72e-05, |
| "loss": 0.446180534362793, |
| "mean_token_accuracy": 0.8741719871759415, |
| "num_tokens": 1565880.0, |
| "step": 130 |
| }, |
| { |
| "entropy": 0.43759765625, |
| "epoch": 0.527306967984934, |
| "grad_norm": 0.4017740488052368, |
| "learning_rate": 1.8533333333333334e-05, |
| "loss": 0.43535647392272947, |
| "mean_token_accuracy": 0.8766337320208549, |
| "num_tokens": 1680914.0, |
| "step": 140 |
| }, |
| { |
| "entropy": 0.410400390625, |
| "epoch": 0.5649717514124294, |
| "grad_norm": 0.3538770377635956, |
| "learning_rate": 1.9866666666666667e-05, |
| "loss": 0.4241950035095215, |
| "mean_token_accuracy": 0.878371699154377, |
| "num_tokens": 1806507.0, |
| "step": 150 |
| }, |
| { |
| "entropy": 0.4160888671875, |
| "epoch": 0.6026365348399246, |
| "grad_norm": 0.30268988013267517, |
| "learning_rate": 1.999983006999844e-05, |
| "loss": 0.41704425811767576, |
| "mean_token_accuracy": 0.8801073461771012, |
| "num_tokens": 1942059.0, |
| "step": 160 |
| }, |
| { |
| "entropy": 0.4154541015625, |
| "epoch": 0.64030131826742, |
| "grad_norm": 0.4021327793598175, |
| "learning_rate": 1.99992426650619e-05, |
| "loss": 0.43154168128967285, |
| "mean_token_accuracy": 0.8793380454182624, |
| "num_tokens": 2064168.0, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.64030131826742, |
| "eval_entropy": 0.406005859375, |
| "eval_loss": 0.41560086607933044, |
| "eval_mean_token_accuracy": 0.8810630203729652, |
| "eval_num_tokens": 2064168.0, |
| "eval_runtime": 434.75, |
| "eval_samples_per_second": 6.102, |
| "eval_steps_per_second": 1.527, |
| "step": 170 |
| }, |
| { |
| "entropy": 0.37158203125, |
| "epoch": 0.6779661016949152, |
| "grad_norm": 0.353048712015152, |
| "learning_rate": 1.9998235711929593e-05, |
| "loss": 0.380794620513916, |
| "mean_token_accuracy": 0.8898573040962219, |
| "num_tokens": 2192732.0, |
| "step": 180 |
| }, |
| { |
| "entropy": 0.42578125, |
| "epoch": 0.7156308851224106, |
| "grad_norm": 0.40385329723358154, |
| "learning_rate": 1.9996809252851254e-05, |
| "loss": 0.43743000030517576, |
| "mean_token_accuracy": 0.8758690312504769, |
| "num_tokens": 2313401.0, |
| "step": 190 |
| }, |
| { |
| "entropy": 0.38935546875, |
| "epoch": 0.7532956685499058, |
| "grad_norm": 0.3307430148124695, |
| "learning_rate": 1.999496334767825e-05, |
| "loss": 0.39965505599975587, |
| "mean_token_accuracy": 0.8848762065172195, |
| "num_tokens": 2438318.0, |
| "step": 200 |
| }, |
| { |
| "entropy": 0.3981201171875, |
| "epoch": 0.7909604519774012, |
| "grad_norm": 0.4147140085697174, |
| "learning_rate": 1.9992698073861067e-05, |
| "loss": 0.40434741973876953, |
| "mean_token_accuracy": 0.8810549288988113, |
| "num_tokens": 2564058.0, |
| "step": 210 |
| }, |
| { |
| "entropy": 0.3850341796875, |
| "epoch": 0.8286252354048964, |
| "grad_norm": 0.37557435035705566, |
| "learning_rate": 1.9990013526446056e-05, |
| "loss": 0.39697415828704835, |
| "mean_token_accuracy": 0.8842006206512452, |
| "num_tokens": 2680718.0, |
| "step": 220 |
| }, |
| { |
| "entropy": 0.389453125, |
| "epoch": 0.8662900188323918, |
| "grad_norm": 0.353782594203949, |
| "learning_rate": 1.998690981807145e-05, |
| "loss": 0.39929752349853515, |
| "mean_token_accuracy": 0.8828964561223984, |
| "num_tokens": 2808902.0, |
| "step": 230 |
| }, |
| { |
| "entropy": 0.3729248046875, |
| "epoch": 0.903954802259887, |
| "grad_norm": 0.43150216341018677, |
| "learning_rate": 1.9983387078962634e-05, |
| "loss": 0.38041508197784424, |
| "mean_token_accuracy": 0.889741088449955, |
| "num_tokens": 2934271.0, |
| "step": 240 |
| }, |
| { |
| "entropy": 0.3947265625, |
| "epoch": 0.9416195856873822, |
| "grad_norm": 0.45530763268470764, |
| "learning_rate": 1.997944545692669e-05, |
| "loss": 0.40334210395812986, |
| "mean_token_accuracy": 0.883297449350357, |
| "num_tokens": 3048899.0, |
| "step": 250 |
| }, |
| { |
| "entropy": 0.3867431640625, |
| "epoch": 0.9792843691148776, |
| "grad_norm": 0.3603038787841797, |
| "learning_rate": 1.997508511734618e-05, |
| "loss": 0.3969164609909058, |
| "mean_token_accuracy": 0.8817866280674934, |
| "num_tokens": 3166251.0, |
| "step": 260 |
| }, |
| { |
| "entropy": 0.3775699013157895, |
| "epoch": 1.015065913370998, |
| "grad_norm": 0.43462836742401123, |
| "learning_rate": 1.9970306243172223e-05, |
| "loss": 0.38600931167602537, |
| "mean_token_accuracy": 0.8880707009842521, |
| "num_tokens": 3279505.0, |
| "step": 270 |
| }, |
| { |
| "entropy": 0.35439453125, |
| "epoch": 1.0527306967984935, |
| "grad_norm": 0.42387086153030396, |
| "learning_rate": 1.9965109034916806e-05, |
| "loss": 0.36482274532318115, |
| "mean_token_accuracy": 0.8907309755682945, |
| "num_tokens": 3390710.0, |
| "step": 280 |
| }, |
| { |
| "entropy": 0.3351318359375, |
| "epoch": 1.0903954802259888, |
| "grad_norm": 0.3719269633293152, |
| "learning_rate": 1.9959493710644385e-05, |
| "loss": 0.3415523529052734, |
| "mean_token_accuracy": 0.8967518746852875, |
| "num_tokens": 3518278.0, |
| "step": 290 |
| }, |
| { |
| "entropy": 0.366552734375, |
| "epoch": 1.128060263653484, |
| "grad_norm": 0.4268398880958557, |
| "learning_rate": 1.995346050596271e-05, |
| "loss": 0.37390072345733644, |
| "mean_token_accuracy": 0.8893876507878303, |
| "num_tokens": 3633007.0, |
| "step": 300 |
| }, |
| { |
| "entropy": 0.3847412109375, |
| "epoch": 1.1657250470809792, |
| "grad_norm": 0.4580579102039337, |
| "learning_rate": 1.9947009674012975e-05, |
| "loss": 0.3997214317321777, |
| "mean_token_accuracy": 0.884394271671772, |
| "num_tokens": 3755208.0, |
| "step": 310 |
| }, |
| { |
| "entropy": 0.3870849609375, |
| "epoch": 1.2033898305084745, |
| "grad_norm": 0.4420165419578552, |
| "learning_rate": 1.994014148545916e-05, |
| "loss": 0.41088366508483887, |
| "mean_token_accuracy": 0.8840712696313858, |
| "num_tokens": 3888151.0, |
| "step": 320 |
| }, |
| { |
| "entropy": 0.3621337890625, |
| "epoch": 1.24105461393597, |
| "grad_norm": 0.47655192017555237, |
| "learning_rate": 1.9932856228476705e-05, |
| "loss": 0.3697712182998657, |
| "mean_token_accuracy": 0.890459081530571, |
| "num_tokens": 4007631.0, |
| "step": 330 |
| }, |
| { |
| "entropy": 0.361279296875, |
| "epoch": 1.2787193973634652, |
| "grad_norm": 0.38902050256729126, |
| "learning_rate": 1.9925154208740412e-05, |
| "loss": 0.36210358142852783, |
| "mean_token_accuracy": 0.8924632355570793, |
| "num_tokens": 4131339.0, |
| "step": 340 |
| }, |
| { |
| "epoch": 1.2787193973634652, |
| "eval_entropy": 0.35297145613704817, |
| "eval_loss": 0.3814155161380768, |
| "eval_mean_token_accuracy": 0.8880114263619285, |
| "eval_num_tokens": 4131339.0, |
| "eval_runtime": 434.3155, |
| "eval_samples_per_second": 6.108, |
| "eval_steps_per_second": 1.529, |
| "step": 340 |
| }, |
| { |
| "entropy": 0.3543701171875, |
| "epoch": 1.3163841807909604, |
| "grad_norm": 0.39349332451820374, |
| "learning_rate": 1.9917035749411585e-05, |
| "loss": 0.36451337337493894, |
| "mean_token_accuracy": 0.8919237062335015, |
| "num_tokens": 4255599.0, |
| "step": 350 |
| }, |
| { |
| "entropy": 0.3242919921875, |
| "epoch": 1.3540489642184557, |
| "grad_norm": 0.5312153697013855, |
| "learning_rate": 1.9908501191124535e-05, |
| "loss": 0.33121490478515625, |
| "mean_token_accuracy": 0.8983337059617043, |
| "num_tokens": 4374352.0, |
| "step": 360 |
| }, |
| { |
| "entropy": 0.359375, |
| "epoch": 1.3917137476459511, |
| "grad_norm": 0.44787994027137756, |
| "learning_rate": 1.9899550891972224e-05, |
| "loss": 0.36217460632324217, |
| "mean_token_accuracy": 0.8934197455644608, |
| "num_tokens": 4491075.0, |
| "step": 370 |
| }, |
| { |
| "entropy": 0.33974609375, |
| "epoch": 1.4293785310734464, |
| "grad_norm": 0.45037373900413513, |
| "learning_rate": 1.9890185227491285e-05, |
| "loss": 0.3445204973220825, |
| "mean_token_accuracy": 0.8977437034249306, |
| "num_tokens": 4622376.0, |
| "step": 380 |
| }, |
| { |
| "entropy": 0.3542724609375, |
| "epoch": 1.4670433145009416, |
| "grad_norm": 0.4353658854961395, |
| "learning_rate": 1.9880404590646233e-05, |
| "loss": 0.37237210273742677, |
| "mean_token_accuracy": 0.8895931392908096, |
| "num_tokens": 4740521.0, |
| "step": 390 |
| }, |
| { |
| "entropy": 0.353369140625, |
| "epoch": 1.5047080979284368, |
| "grad_norm": 0.4182438254356384, |
| "learning_rate": 1.9870209391813013e-05, |
| "loss": 0.359093451499939, |
| "mean_token_accuracy": 0.8922405913472176, |
| "num_tokens": 4861869.0, |
| "step": 400 |
| }, |
| { |
| "entropy": 0.36416015625, |
| "epoch": 1.542372881355932, |
| "grad_norm": 0.4568746089935303, |
| "learning_rate": 1.985960005876174e-05, |
| "loss": 0.3687510251998901, |
| "mean_token_accuracy": 0.8903782144188881, |
| "num_tokens": 4988253.0, |
| "step": 410 |
| }, |
| { |
| "entropy": 0.352783203125, |
| "epoch": 1.5800376647834273, |
| "grad_norm": 0.3993203639984131, |
| "learning_rate": 1.984857703663879e-05, |
| "loss": 0.36400485038757324, |
| "mean_token_accuracy": 0.8927491247653961, |
| "num_tokens": 5109023.0, |
| "step": 420 |
| }, |
| { |
| "entropy": 0.354736328125, |
| "epoch": 1.6177024482109228, |
| "grad_norm": 0.3715269863605499, |
| "learning_rate": 1.9837140787948082e-05, |
| "loss": 0.3601724624633789, |
| "mean_token_accuracy": 0.8927290469408036, |
| "num_tokens": 5220888.0, |
| "step": 430 |
| }, |
| { |
| "entropy": 0.3735107421875, |
| "epoch": 1.655367231638418, |
| "grad_norm": 0.4928207993507385, |
| "learning_rate": 1.9825291792531717e-05, |
| "loss": 0.3838499546051025, |
| "mean_token_accuracy": 0.8860176280140877, |
| "num_tokens": 5349625.0, |
| "step": 440 |
| }, |
| { |
| "entropy": 0.358154296875, |
| "epoch": 1.6930320150659135, |
| "grad_norm": 0.4274348020553589, |
| "learning_rate": 1.9813030547549806e-05, |
| "loss": 0.3699859619140625, |
| "mean_token_accuracy": 0.8911255195736885, |
| "num_tokens": 5469725.0, |
| "step": 450 |
| }, |
| { |
| "entropy": 0.334619140625, |
| "epoch": 1.7306967984934087, |
| "grad_norm": 0.3487861156463623, |
| "learning_rate": 1.9800357567459633e-05, |
| "loss": 0.3373765468597412, |
| "mean_token_accuracy": 0.8974497631192208, |
| "num_tokens": 5604349.0, |
| "step": 460 |
| }, |
| { |
| "entropy": 0.3565185546875, |
| "epoch": 1.768361581920904, |
| "grad_norm": 0.41273072361946106, |
| "learning_rate": 1.978727338399406e-05, |
| "loss": 0.36286952495574953, |
| "mean_token_accuracy": 0.8902141079306602, |
| "num_tokens": 5715690.0, |
| "step": 470 |
| }, |
| { |
| "entropy": 0.3404541015625, |
| "epoch": 1.8060263653483992, |
| "grad_norm": 0.48571380972862244, |
| "learning_rate": 1.9773778546139228e-05, |
| "loss": 0.354712700843811, |
| "mean_token_accuracy": 0.8953486695885658, |
| "num_tokens": 5830295.0, |
| "step": 480 |
| }, |
| { |
| "entropy": 0.353759765625, |
| "epoch": 1.8436911487758945, |
| "grad_norm": 0.4093576967716217, |
| "learning_rate": 1.9759873620111492e-05, |
| "loss": 0.3602489709854126, |
| "mean_token_accuracy": 0.8912778198719025, |
| "num_tokens": 5954416.0, |
| "step": 490 |
| }, |
| { |
| "entropy": 0.3271240234375, |
| "epoch": 1.8813559322033897, |
| "grad_norm": 0.458849161863327, |
| "learning_rate": 1.974555918933371e-05, |
| "loss": 0.3355377435684204, |
| "mean_token_accuracy": 0.8979063704609871, |
| "num_tokens": 6071230.0, |
| "step": 500 |
| }, |
| { |
| "entropy": 0.3464111328125, |
| "epoch": 1.9190207156308852, |
| "grad_norm": 0.49934810400009155, |
| "learning_rate": 1.9730835854410726e-05, |
| "loss": 0.35040299892425536, |
| "mean_token_accuracy": 0.8943001076579093, |
| "num_tokens": 6191076.0, |
| "step": 510 |
| }, |
| { |
| "epoch": 1.9190207156308852, |
| "eval_entropy": 0.34145419568900603, |
| "eval_loss": 0.3649611175060272, |
| "eval_mean_token_accuracy": 0.8922233424452414, |
| "eval_num_tokens": 6191076.0, |
| "eval_runtime": 435.0399, |
| "eval_samples_per_second": 6.098, |
| "eval_steps_per_second": 1.526, |
| "step": 510 |
| }, |
| { |
| "entropy": 0.3384765625, |
| "epoch": 1.9566854990583804, |
| "grad_norm": 0.3491309881210327, |
| "learning_rate": 1.9715704233104188e-05, |
| "loss": 0.3490856170654297, |
| "mean_token_accuracy": 0.8961027547717094, |
| "num_tokens": 6329338.0, |
| "step": 520 |
| }, |
| { |
| "entropy": 0.3297119140625, |
| "epoch": 1.9943502824858759, |
| "grad_norm": 0.41385412216186523, |
| "learning_rate": 1.9700164960306612e-05, |
| "loss": 0.33870773315429686, |
| "mean_token_accuracy": 0.8972823545336723, |
| "num_tokens": 6458727.0, |
| "step": 530 |
| }, |
| { |
| "entropy": 0.3355263157894737, |
| "epoch": 2.030131826741996, |
| "grad_norm": 0.4988757371902466, |
| "learning_rate": 1.9684218688014773e-05, |
| "loss": 0.3331795930862427, |
| "mean_token_accuracy": 0.8956447431915685, |
| "num_tokens": 6560386.0, |
| "step": 540 |
| }, |
| { |
| "entropy": 0.2864013671875, |
| "epoch": 2.0677966101694913, |
| "grad_norm": 0.45556309819221497, |
| "learning_rate": 1.9667866085302312e-05, |
| "loss": 0.2940270185470581, |
| "mean_token_accuracy": 0.9079476356506347, |
| "num_tokens": 6675383.0, |
| "step": 550 |
| }, |
| { |
| "entropy": 0.326416015625, |
| "epoch": 2.105461393596987, |
| "grad_norm": 0.44398558139801025, |
| "learning_rate": 1.965110783829169e-05, |
| "loss": 0.32726523876190183, |
| "mean_token_accuracy": 0.8985984146595001, |
| "num_tokens": 6802970.0, |
| "step": 560 |
| }, |
| { |
| "entropy": 0.33291015625, |
| "epoch": 2.1431261770244823, |
| "grad_norm": 0.45157620310783386, |
| "learning_rate": 1.963394465012539e-05, |
| "loss": 0.3427454710006714, |
| "mean_token_accuracy": 0.8983364552259445, |
| "num_tokens": 6935993.0, |
| "step": 570 |
| }, |
| { |
| "entropy": 0.3192626953125, |
| "epoch": 2.1807909604519775, |
| "grad_norm": 0.46245673298835754, |
| "learning_rate": 1.961637724093641e-05, |
| "loss": 0.32583916187286377, |
| "mean_token_accuracy": 0.9005291208624839, |
| "num_tokens": 7049577.0, |
| "step": 580 |
| }, |
| { |
| "entropy": 0.3029052734375, |
| "epoch": 2.2184557438794728, |
| "grad_norm": 0.5186663269996643, |
| "learning_rate": 1.9598406347818056e-05, |
| "loss": 0.3109822988510132, |
| "mean_token_accuracy": 0.9043365225195885, |
| "num_tokens": 7152910.0, |
| "step": 590 |
| }, |
| { |
| "entropy": 0.3109375, |
| "epoch": 2.256120527306968, |
| "grad_norm": 0.4700295627117157, |
| "learning_rate": 1.958003272479301e-05, |
| "loss": 0.31394057273864745, |
| "mean_token_accuracy": 0.9028953716158867, |
| "num_tokens": 7272029.0, |
| "step": 600 |
| }, |
| { |
| "entropy": 0.3043212890625, |
| "epoch": 2.2937853107344632, |
| "grad_norm": 0.4783408045768738, |
| "learning_rate": 1.9561257142781706e-05, |
| "loss": 0.3106163740158081, |
| "mean_token_accuracy": 0.9049968093633651, |
| "num_tokens": 7400846.0, |
| "step": 610 |
| }, |
| { |
| "entropy": 0.3078369140625, |
| "epoch": 2.3314500941619585, |
| "grad_norm": 0.6188530325889587, |
| "learning_rate": 1.9542080389569947e-05, |
| "loss": 0.3121727228164673, |
| "mean_token_accuracy": 0.9031846761703491, |
| "num_tokens": 7514935.0, |
| "step": 620 |
| }, |
| { |
| "entropy": 0.3011474609375, |
| "epoch": 2.3691148775894537, |
| "grad_norm": 0.47817936539649963, |
| "learning_rate": 1.9522503269775897e-05, |
| "loss": 0.3063828945159912, |
| "mean_token_accuracy": 0.9046044871211052, |
| "num_tokens": 7642606.0, |
| "step": 630 |
| }, |
| { |
| "entropy": 0.3004150390625, |
| "epoch": 2.406779661016949, |
| "grad_norm": 0.5287877321243286, |
| "learning_rate": 1.9502526604816293e-05, |
| "loss": 0.30952765941619875, |
| "mean_token_accuracy": 0.9043435141444206, |
| "num_tokens": 7762332.0, |
| "step": 640 |
| }, |
| { |
| "entropy": 0.3209716796875, |
| "epoch": 2.4444444444444446, |
| "grad_norm": 0.4484723210334778, |
| "learning_rate": 1.948215123287199e-05, |
| "loss": 0.331793999671936, |
| "mean_token_accuracy": 0.8998603627085686, |
| "num_tokens": 7887841.0, |
| "step": 650 |
| }, |
| { |
| "entropy": 0.31649169921875, |
| "epoch": 2.48210922787194, |
| "grad_norm": 0.4972360134124756, |
| "learning_rate": 1.9461378008852785e-05, |
| "loss": 0.32228312492370603, |
| "mean_token_accuracy": 0.9026615381240845, |
| "num_tokens": 8011819.0, |
| "step": 660 |
| }, |
| { |
| "entropy": 0.3254150390625, |
| "epoch": 2.519774011299435, |
| "grad_norm": 0.449107825756073, |
| "learning_rate": 1.944020780436155e-05, |
| "loss": 0.33617188930511477, |
| "mean_token_accuracy": 0.8992837205529213, |
| "num_tokens": 8140649.0, |
| "step": 670 |
| }, |
| { |
| "entropy": 0.311376953125, |
| "epoch": 2.5574387947269304, |
| "grad_norm": 0.5488213896751404, |
| "learning_rate": 1.9418641507657673e-05, |
| "loss": 0.30985331535339355, |
| "mean_token_accuracy": 0.9042975798249244, |
| "num_tokens": 8261103.0, |
| "step": 680 |
| }, |
| { |
| "epoch": 2.5574387947269304, |
| "eval_entropy": 0.3314003082643072, |
| "eval_loss": 0.3603939414024353, |
| "eval_mean_token_accuracy": 0.8933142435299345, |
| "eval_num_tokens": 8261103.0, |
| "eval_runtime": 434.7902, |
| "eval_samples_per_second": 6.102, |
| "eval_steps_per_second": 1.527, |
| "step": 680 |
| }, |
| { |
| "entropy": 0.2923828125, |
| "epoch": 2.5951035781544256, |
| "grad_norm": 0.4833008050918579, |
| "learning_rate": 1.9396680023619767e-05, |
| "loss": 0.29590415954589844, |
| "mean_token_accuracy": 0.9064638644456864, |
| "num_tokens": 8376817.0, |
| "step": 690 |
| }, |
| { |
| "entropy": 0.3073486328125, |
| "epoch": 2.632768361581921, |
| "grad_norm": 0.5057386159896851, |
| "learning_rate": 1.9374324273707717e-05, |
| "loss": 0.31825664043426516, |
| "mean_token_accuracy": 0.9031327351927757, |
| "num_tokens": 8494354.0, |
| "step": 700 |
| }, |
| { |
| "entropy": 0.335205078125, |
| "epoch": 2.670433145009416, |
| "grad_norm": 0.4985320568084717, |
| "learning_rate": 1.9351575195924014e-05, |
| "loss": 0.35352158546447754, |
| "mean_token_accuracy": 0.8966242641210556, |
| "num_tokens": 8617261.0, |
| "step": 710 |
| }, |
| { |
| "entropy": 0.3291748046875, |
| "epoch": 2.7080979284369113, |
| "grad_norm": 0.48186948895454407, |
| "learning_rate": 1.9328433744774403e-05, |
| "loss": 0.33725249767303467, |
| "mean_token_accuracy": 0.8990528821945191, |
| "num_tokens": 8745040.0, |
| "step": 720 |
| }, |
| { |
| "entropy": 0.330029296875, |
| "epoch": 2.7457627118644066, |
| "grad_norm": 0.555007815361023, |
| "learning_rate": 1.9304900891227825e-05, |
| "loss": 0.33881282806396484, |
| "mean_token_accuracy": 0.8977430954575538, |
| "num_tokens": 8867445.0, |
| "step": 730 |
| }, |
| { |
| "entropy": 0.3059814453125, |
| "epoch": 2.7834274952919023, |
| "grad_norm": 0.5257039666175842, |
| "learning_rate": 1.928097762267568e-05, |
| "loss": 0.31029477119445803, |
| "mean_token_accuracy": 0.903009583055973, |
| "num_tokens": 8992180.0, |
| "step": 740 |
| }, |
| { |
| "entropy": 0.309912109375, |
| "epoch": 2.8210922787193975, |
| "grad_norm": 0.5107100009918213, |
| "learning_rate": 1.9256664942890412e-05, |
| "loss": 0.31228840351104736, |
| "mean_token_accuracy": 0.9031670719385148, |
| "num_tokens": 9112534.0, |
| "step": 750 |
| }, |
| { |
| "entropy": 0.3145751953125, |
| "epoch": 2.8587570621468927, |
| "grad_norm": 0.507240355014801, |
| "learning_rate": 1.9231963871983367e-05, |
| "loss": 0.32340617179870607, |
| "mean_token_accuracy": 0.9009902447462081, |
| "num_tokens": 9234002.0, |
| "step": 760 |
| }, |
| { |
| "entropy": 0.334130859375, |
| "epoch": 2.896421845574388, |
| "grad_norm": 0.4524434208869934, |
| "learning_rate": 1.9206875446362005e-05, |
| "loss": 0.34357576370239257, |
| "mean_token_accuracy": 0.8974097043275833, |
| "num_tokens": 9365959.0, |
| "step": 770 |
| }, |
| { |
| "entropy": 0.298486328125, |
| "epoch": 2.934086629001883, |
| "grad_norm": 0.4092046320438385, |
| "learning_rate": 1.918140071868642e-05, |
| "loss": 0.3051126480102539, |
| "mean_token_accuracy": 0.9063295319676399, |
| "num_tokens": 9496561.0, |
| "step": 780 |
| }, |
| { |
| "entropy": 0.3322021484375, |
| "epoch": 2.9717514124293785, |
| "grad_norm": 0.5656010508537292, |
| "learning_rate": 1.9155540757825168e-05, |
| "loss": 0.34139955043792725, |
| "mean_token_accuracy": 0.8985045611858368, |
| "num_tokens": 9620378.0, |
| "step": 790 |
| }, |
| { |
| "entropy": 0.3062808388157895, |
| "epoch": 3.007532956685499, |
| "grad_norm": 0.43500763177871704, |
| "learning_rate": 1.912929664881041e-05, |
| "loss": 0.31280133724212644, |
| "mean_token_accuracy": 0.9056721659083116, |
| "num_tokens": 9736438.0, |
| "step": 800 |
| }, |
| { |
| "entropy": 0.2744384765625, |
| "epoch": 3.0451977401129944, |
| "grad_norm": 0.5794429183006287, |
| "learning_rate": 1.9102669492792406e-05, |
| "loss": 0.27615647315979003, |
| "mean_token_accuracy": 0.9126084685325623, |
| "num_tokens": 9860464.0, |
| "step": 810 |
| }, |
| { |
| "entropy": 0.27353515625, |
| "epoch": 3.0828625235404896, |
| "grad_norm": 0.5663220286369324, |
| "learning_rate": 1.9075660406993285e-05, |
| "loss": 0.278867244720459, |
| "mean_token_accuracy": 0.9130979672074317, |
| "num_tokens": 9975626.0, |
| "step": 820 |
| }, |
| { |
| "entropy": 0.28662109375, |
| "epoch": 3.120527306967985, |
| "grad_norm": 0.6250424981117249, |
| "learning_rate": 1.9048270524660197e-05, |
| "loss": 0.2926678419113159, |
| "mean_token_accuracy": 0.9088445946574211, |
| "num_tokens": 10093533.0, |
| "step": 830 |
| }, |
| { |
| "entropy": 0.2937255859375, |
| "epoch": 3.15819209039548, |
| "grad_norm": 0.5113135576248169, |
| "learning_rate": 1.9020500995017747e-05, |
| "loss": 0.2945073366165161, |
| "mean_token_accuracy": 0.9113668486475944, |
| "num_tokens": 10229485.0, |
| "step": 840 |
| }, |
| { |
| "entropy": 0.281298828125, |
| "epoch": 3.1958568738229753, |
| "grad_norm": 0.5600440502166748, |
| "learning_rate": 1.8992352983219785e-05, |
| "loss": 0.2873659372329712, |
| "mean_token_accuracy": 0.9089308872818946, |
| "num_tokens": 10341414.0, |
| "step": 850 |
| }, |
| { |
| "epoch": 3.1958568738229753, |
| "eval_entropy": 0.3167504000376506, |
| "eval_loss": 0.3594783842563629, |
| "eval_mean_token_accuracy": 0.8942027627883187, |
| "eval_num_tokens": 10341414.0, |
| "eval_runtime": 434.8714, |
| "eval_samples_per_second": 6.101, |
| "eval_steps_per_second": 1.527, |
| "step": 850 |
| }, |
| { |
| "entropy": 0.2764892578125, |
| "epoch": 3.2335216572504706, |
| "grad_norm": 0.5793660879135132, |
| "learning_rate": 1.8963827670300512e-05, |
| "loss": 0.27801764011383057, |
| "mean_token_accuracy": 0.9128028020262718, |
| "num_tokens": 10458746.0, |
| "step": 860 |
| }, |
| { |
| "entropy": 0.2720458984375, |
| "epoch": 3.2711864406779663, |
| "grad_norm": 0.633267879486084, |
| "learning_rate": 1.8934926253124922e-05, |
| "loss": 0.2757373332977295, |
| "mean_token_accuracy": 0.9125229880213738, |
| "num_tokens": 10574140.0, |
| "step": 870 |
| }, |
| { |
| "entropy": 0.2927490234375, |
| "epoch": 3.3088512241054615, |
| "grad_norm": 0.5631522536277771, |
| "learning_rate": 1.8905649944338596e-05, |
| "loss": 0.3019412994384766, |
| "mean_token_accuracy": 0.9066396117210388, |
| "num_tokens": 10683968.0, |
| "step": 880 |
| }, |
| { |
| "entropy": 0.2663818359375, |
| "epoch": 3.3465160075329567, |
| "grad_norm": 0.648776650428772, |
| "learning_rate": 1.8875999972316826e-05, |
| "loss": 0.27347893714904786, |
| "mean_token_accuracy": 0.9152637153863907, |
| "num_tokens": 10806730.0, |
| "step": 890 |
| }, |
| { |
| "entropy": 0.2777099609375, |
| "epoch": 3.384180790960452, |
| "grad_norm": 0.6328104734420776, |
| "learning_rate": 1.8845977581113048e-05, |
| "loss": 0.28642497062683103, |
| "mean_token_accuracy": 0.9128389939665794, |
| "num_tokens": 10939186.0, |
| "step": 900 |
| }, |
| { |
| "entropy": 0.29140625, |
| "epoch": 3.4218455743879472, |
| "grad_norm": 0.5813119411468506, |
| "learning_rate": 1.8815584030406663e-05, |
| "loss": 0.2928615570068359, |
| "mean_token_accuracy": 0.907462652027607, |
| "num_tokens": 11067083.0, |
| "step": 910 |
| }, |
| { |
| "entropy": 0.281591796875, |
| "epoch": 3.4595103578154425, |
| "grad_norm": 0.5428271889686584, |
| "learning_rate": 1.8784820595450198e-05, |
| "loss": 0.28984737396240234, |
| "mean_token_accuracy": 0.910303819179535, |
| "num_tokens": 11196283.0, |
| "step": 920 |
| }, |
| { |
| "entropy": 0.28740234375, |
| "epoch": 3.4971751412429377, |
| "grad_norm": 0.5831995606422424, |
| "learning_rate": 1.875368856701576e-05, |
| "loss": 0.2924405813217163, |
| "mean_token_accuracy": 0.9076780572533607, |
| "num_tokens": 11322943.0, |
| "step": 930 |
| }, |
| { |
| "entropy": 0.2769775390625, |
| "epoch": 3.5348399246704334, |
| "grad_norm": 0.54417484998703, |
| "learning_rate": 1.872218925134092e-05, |
| "loss": 0.27564899921417235, |
| "mean_token_accuracy": 0.9126177057623863, |
| "num_tokens": 11447033.0, |
| "step": 940 |
| }, |
| { |
| "entropy": 0.277490234375, |
| "epoch": 3.572504708097928, |
| "grad_norm": 0.5877102017402649, |
| "learning_rate": 1.8690323970073874e-05, |
| "loss": 0.2835560321807861, |
| "mean_token_accuracy": 0.9109170585870743, |
| "num_tokens": 11561706.0, |
| "step": 950 |
| }, |
| { |
| "entropy": 0.2740478515625, |
| "epoch": 3.610169491525424, |
| "grad_norm": 0.5150530338287354, |
| "learning_rate": 1.8658094060218e-05, |
| "loss": 0.27728586196899413, |
| "mean_token_accuracy": 0.9116252958774567, |
| "num_tokens": 11678800.0, |
| "step": 960 |
| }, |
| { |
| "entropy": 0.2775146484375, |
| "epoch": 3.647834274952919, |
| "grad_norm": 0.6397176384925842, |
| "learning_rate": 1.862550087407577e-05, |
| "loss": 0.2833576679229736, |
| "mean_token_accuracy": 0.9115300670266151, |
| "num_tokens": 11795260.0, |
| "step": 970 |
| }, |
| { |
| "entropy": 0.2694091796875, |
| "epoch": 3.6854990583804144, |
| "grad_norm": 0.5690704584121704, |
| "learning_rate": 1.8592545779191993e-05, |
| "loss": 0.2770532608032227, |
| "mean_token_accuracy": 0.914972423017025, |
| "num_tokens": 11923344.0, |
| "step": 980 |
| }, |
| { |
| "entropy": 0.2844482421875, |
| "epoch": 3.7231638418079096, |
| "grad_norm": 0.5788920521736145, |
| "learning_rate": 1.8559230158296454e-05, |
| "loss": 0.2866381645202637, |
| "mean_token_accuracy": 0.9088692501187324, |
| "num_tokens": 12047774.0, |
| "step": 990 |
| }, |
| { |
| "entropy": 0.2859130859375, |
| "epoch": 3.760828625235405, |
| "grad_norm": 0.5496282577514648, |
| "learning_rate": 1.8525555409245877e-05, |
| "loss": 0.294761061668396, |
| "mean_token_accuracy": 0.9096270382404328, |
| "num_tokens": 12164978.0, |
| "step": 1000 |
| }, |
| { |
| "entropy": 0.2739013671875, |
| "epoch": 3.7984934086629, |
| "grad_norm": 0.6697402596473694, |
| "learning_rate": 1.849152294496529e-05, |
| "loss": 0.2829042911529541, |
| "mean_token_accuracy": 0.9111244812607765, |
| "num_tokens": 12277877.0, |
| "step": 1010 |
| }, |
| { |
| "entropy": 0.311279296875, |
| "epoch": 3.8361581920903953, |
| "grad_norm": 0.5649982690811157, |
| "learning_rate": 1.845713419338873e-05, |
| "loss": 0.3214085578918457, |
| "mean_token_accuracy": 0.9038550212979317, |
| "num_tokens": 12415855.0, |
| "step": 1020 |
| }, |
| { |
| "epoch": 3.8361581920903953, |
| "eval_entropy": 0.305065476750753, |
| "eval_loss": 0.35731908679008484, |
| "eval_mean_token_accuracy": 0.8951125881039953, |
| "eval_num_tokens": 12415855.0, |
| "eval_runtime": 434.3577, |
| "eval_samples_per_second": 6.108, |
| "eval_steps_per_second": 1.529, |
| "step": 1020 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 5000, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 19, |
| "save_steps": 170, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 9.598053240243487e+17, |
| "train_batch_size": 4, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
