{"event": "train", "step": 1, "max_steps": 20000, "progress_percent": 0.005, "loss": 2.256478786468506, "perplexity": 9.549404403942797, "learning_rate": 6e-07, "gradient_norm": 1.3867446184158325, "step_time": 4.138624000002892, "tokens_per_second": 1979.4018495022199, "steps_per_second": 0.24162620233181395, "batch_wait": 0.003074899999774061, "processed_target_tokens": 6265, "elapsed_seconds": 4.138626399999339, "eta_seconds": 82768.34137605784, "gpu": {}}
{"event": "train", "step": 50, "max_steps": 20000, "progress_percent": 0.25, "loss": 2.3223867416381836, "perplexity": 10.199990017754057, "learning_rate": 2.9999999999999997e-05, "gradient_norm": 1.1412876844406128, "step_time": 3.4009705999997095, "tokens_per_second": 2408.7241448075733, "steps_per_second": 0.29403370908295573, "batch_wait": 0.001219500001752749, "processed_target_tokens": 290170, "elapsed_seconds": 171.40560819999882, "eta_seconds": 67849.3634699942, "gpu": {}}
{"event": "train", "step": 100, "max_steps": 20000, "progress_percent": 0.5, "loss": 1.817542552947998, "perplexity": 6.156710055342754, "learning_rate": 5.9999999999999995e-05, "gradient_norm": 1.1113624572753906, "step_time": 3.47592319999967, "tokens_per_second": 2356.7839473555623, "steps_per_second": 0.28769335294867704, "batch_wait": 0.001194999993458623, "processed_target_tokens": 583705, "elapsed_seconds": 343.8856147000006, "eta_seconds": 69170.87167999343, "gpu": {}}
{"event": "train", "step": 150, "max_steps": 20000, "progress_percent": 0.75, "loss": 1.986922025680542, "perplexity": 7.29305135514172, "learning_rate": 8.999999999999999e-05, "gradient_norm": 1.1220747232437134, "step_time": 3.430251699999644, "tokens_per_second": 2388.162944427912, "steps_per_second": 0.29152379692723535, "batch_wait": 0.0003590000014810357, "processed_target_tokens": 873748, "elapsed_seconds": 517.8077993000006, "eta_seconds": 68090.49624499294, "gpu": {}}
{"event": "train", "step": 200, "max_steps": 20000, "progress_percent": 1.0, "loss": 2.1119954586029053, "perplexity": 8.264716741235816, "learning_rate": 0.00011999999999999999, "gradient_norm": 1.2295078039169312, "step_time": 3.4131790999999794, "tokens_per_second": 2400.1084502128965, "steps_per_second": 0.29298198855137897, "batch_wait": 0.0014868000034766737, "processed_target_tokens": 1168441, "elapsed_seconds": 689.7110724999984, "eta_seconds": 67580.94617999959, "gpu": {}}
{"event": "train", "step": 250, "max_steps": 20000, "progress_percent": 1.25, "loss": 1.7892898321151733, "perplexity": 5.985200459591831, "learning_rate": 0.00015, "gradient_norm": 1.2155787944793701, "step_time": 3.3396486999990884, "tokens_per_second": 2452.952611453485, "steps_per_second": 0.299432691827818, "batch_wait": 0.001226100001076702, "processed_target_tokens": 1456346, "elapsed_seconds": 861.1369439000009, "eta_seconds": 65958.061824982, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3798.72, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 300, "max_steps": 20000, "progress_percent": 1.5, "loss": 1.9895539283752441, "perplexity": 7.312271237992117, "learning_rate": 0.00017999999999999998, "gradient_norm": 1.3982632160186768, "step_time": 3.4701088999972853, "tokens_per_second": 2360.732828876468, "steps_per_second": 0.28817539414995946, "batch_wait": 0.0014370999997481704, "processed_target_tokens": 1752550, "elapsed_seconds": 1033.850635499999, "eta_seconds": 68361.14532994652, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3798.72, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 350, "max_steps": 20000, "progress_percent": 1.7500000000000002, "loss": 1.8942921161651611, "perplexity": 6.647840839557938, "learning_rate": 0.00020999999999999998, "gradient_norm": 1.1794360876083374, "step_time": 3.6177910000005795, "tokens_per_second": 2264.365188591239, "steps_per_second": 0.276411766185454, "batch_wait": 0.0007125000011001248, "processed_target_tokens": 2037854, "elapsed_seconds": 1207.3757588000008, "eta_seconds": 71089.59315001138, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3798.72, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 400, "max_steps": 20000, "progress_percent": 2.0, "loss": 1.962095022201538, "perplexity": 7.114215903752084, "learning_rate": 0.00023999999999999998, "gradient_norm": 1.5300354957580566, "step_time": 3.4551344999999856, "tokens_per_second": 2370.964140469795, "steps_per_second": 0.28942433355344177, "batch_wait": 0.0007965000004332978, "processed_target_tokens": 2331347, "elapsed_seconds": 1376.9422468000012, "eta_seconds": 67720.63619999972, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3798.72, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 450, "max_steps": 20000, "progress_percent": 2.25, "loss": 1.8573267459869385, "perplexity": 6.406587423820064, "learning_rate": 0.00027, "gradient_norm": 1.1610974073410034, "step_time": 3.425512300000264, "tokens_per_second": 2391.467109897509, "steps_per_second": 0.2919271374386608, "batch_wait": 0.0004526999982772395, "processed_target_tokens": 2620442, "elapsed_seconds": 1549.9882249000002, "eta_seconds": 66968.76546500516, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3798.72, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 500, "max_steps": 20000, "progress_percent": 2.5, "loss": 2.218559980392456, "perplexity": 9.194081670742152, "learning_rate": 0.0003, "gradient_norm": 1.2077858448028564, "step_time": 3.9009679999981017, "tokens_per_second": 2099.9915918315623, "steps_per_second": 0.25634662986225126, "batch_wait": 0.0006566000010934658, "processed_target_tokens": 2907724, "elapsed_seconds": 1722.862887899999, "eta_seconds": 76068.87599996298, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3706.86, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "validation", "step": 500, "validation_loss": 2.0969789624214172, "validation_perplexity": 8.141536833851754, "validation_time": 2.6048922000009043, "best_validation_loss": null}
{"step": 500, "val_loss": 2.0969789624214172, "best_val_loss": 2.0969789624214172, "processed_target_tokens": 2907724, "timestamp": 1793367596.1601415}
{"event": "checkpoint", "step": 500, "max_steps": 20000, "progress_percent": 2.5, "train_loss": 2.218559980392456, "train_perplexity": 9.194081670742152, "validation_loss": 2.0969789624214172, "validation_perplexity": 8.141536833851754, "best_validation_loss": 2.0969789624214172, "best_validation_perplexity": 8.141536833851754, "processed_target_tokens": 2907724, "tokens_per_step": 8192, "tokens_per_second": 2099.9915918315623, "steps_per_second": 0.25634662986225126, "elapsed_seconds": 1722.862887899999, "eta_seconds": 76068.87599996298, "learning_rate": 0.0003, "gradient_norm": 1.2077858448028564, "batch_wait": 0.0006566000010934658, "evaluation_time": 2.6048922000009043, "parameter_count": 62113280, "gpu": {"gpu_utilization": 6, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3744.32, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 54}, "is_best": true, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"step": 500, "val_loss": 2.0969789624214172, "best_val_loss": 2.0969789624214172, "processed_target_tokens": 2907724, "timestamp": 1793367598.7500305}
{"event": "checkpoint", "step": 500, "max_steps": 20000, "progress_percent": 2.5, "train_loss": 2.218559980392456, "train_perplexity": 9.194081670742152, "validation_loss": 2.0969789624214172, "validation_perplexity": 8.141536833851754, "best_validation_loss": 2.0969789624214172, "best_validation_perplexity": 8.141536833851754, "processed_target_tokens": 2907724, "tokens_per_step": 8192, "tokens_per_second": 2099.9915918315623, "steps_per_second": 0.25634662986225126, "elapsed_seconds": 1722.862887899999, "eta_seconds": 76068.87599996298, "learning_rate": 0.0003, "gradient_norm": 1.2077858448028564, "batch_wait": 0.0006566000010934658, "evaluation_time": 2.6048922000009043, "parameter_count": 62113280, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 6, "gpu_memory_used_mb": 3699.49, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 52}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 550, "max_steps": 20000, "progress_percent": 2.75, "loss": 2.4125022888183594, "perplexity": 11.161856418002024, "learning_rate": 0.0002999956200217312, "gradient_norm": 1.2063146829605103, "step_time": 3.4077837000004365, "tokens_per_second": 2403.9084405500707, "steps_per_second": 0.2934458545593348, "batch_wait": 0.0011171000005560927, "processed_target_tokens": 3198503, "elapsed_seconds": 1903.063326399999, "eta_seconds": 66281.39296500849, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 6, "gpu_memory_used_mb": 3699.49, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 52}}
{"event": "train", "step": 600, "max_steps": 20000, "progress_percent": 3.0, "loss": 2.0101327896118164, "perplexity": 7.46430846413616, "learning_rate": 0.00029998248037113557, "gradient_norm": 1.501205325126648, "step_time": 3.4335648999986006, "tokens_per_second": 2385.8584994282005, "steps_per_second": 0.2912424926059815, "batch_wait": 0.0015027999979793094, "processed_target_tokens": 3487259, "elapsed_seconds": 2074.9086672999983, "eta_seconds": 66611.15905997285, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 6, "gpu_memory_used_mb": 3699.49, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 52}}
{"event": "train", "step": 650, "max_steps": 20000, "progress_percent": 3.25, "loss": 1.980380654335022, "perplexity": 7.245500491470833, "learning_rate": 0.000299960581900826, "gradient_norm": 1.102168321609497, "step_time": 3.4463814000009734, "tokens_per_second": 2376.985901791858, "steps_per_second": 0.2901594118398264, "batch_wait": 0.0016049000005295966, "processed_target_tokens": 3779317, "elapsed_seconds": 2249.8350322000006, "eta_seconds": 66687.48009001883, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 6, "gpu_memory_used_mb": 3699.49, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 52}}
{"event": "train", "step": 700, "max_steps": 20000, "progress_percent": 3.5000000000000004, "loss": 2.1013615131378174, "perplexity": 8.177295832513655, "learning_rate": 0.00029992992603176304, "gradient_norm": 1.1482175588607788, "step_time": 3.3763905000014347, "tokens_per_second": 2426.2596402864297, "steps_per_second": 0.29617427249590206, "batch_wait": 0.0009156000014627352, "processed_target_tokens": 4065843, "elapsed_seconds": 2424.1243622, "eta_seconds": 65164.33665002769, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 6, "gpu_memory_used_mb": 3699.49, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 52}}
{"event": "train", "step": 750, "max_steps": 20000, "progress_percent": 3.75, "loss": 2.1941146850585938, "perplexity": 8.972054445855669, "learning_rate": 0.0002998905147531619, "gradient_norm": 1.1301285028457642, "step_time": 3.730917100001534, "tokens_per_second": 2195.7067874803843, "steps_per_second": 0.2680306137061016, "batch_wait": 0.0012123999986215495, "processed_target_tokens": 4350406, "elapsed_seconds": 2597.7479872000004, "eta_seconds": 71820.15417502953, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 41, "gpu_memory_used_mb": 3790.84, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 800, "max_steps": 20000, "progress_percent": 4.0, "loss": 1.9402058124542236, "perplexity": 6.960183315645034, "learning_rate": 0.0002998423506223641, "gradient_norm": 1.1127398014068604, "step_time": 3.374460299997736, "tokens_per_second": 2427.647467064732, "steps_per_second": 0.2963436849444253, "batch_wait": 0.0006845999996585306, "processed_target_tokens": 4638888, "elapsed_seconds": 2778.5393726, "eta_seconds": 64789.63775995653, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 41, "gpu_memory_used_mb": 3790.84, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 850, "max_steps": 20000, "progress_percent": 4.25, "loss": 2.0916154384613037, "perplexity": 8.097986402265521, "learning_rate": 0.000299785436764671, "gradient_norm": 1.2281521558761597, "step_time": 3.6682781999988947, "tokens_per_second": 2233.2003063460315, "steps_per_second": 0.2726074592707558, "batch_wait": 0.0006179000010888558, "processed_target_tokens": 4918990, "elapsed_seconds": 2958.5563287000004, "eta_seconds": 70247.52752997883, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 41, "gpu_memory_used_mb": 3790.84, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 900, "max_steps": 20000, "progress_percent": 4.5, "loss": 1.936161756515503, "perplexity": 6.932092783098911, "learning_rate": 0.00029971977687314133, "gradient_norm": 1.1172562837600708, "step_time": 3.5304225999971095, "tokens_per_second": 2320.402095773664, "steps_per_second": 0.2832522089567461, "batch_wait": 0.0008838999965519179, "processed_target_tokens": 5209597, "elapsed_seconds": 3133.5337118999996, "eta_seconds": 67431.07165994479, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 41, "gpu_memory_used_mb": 3790.84, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 950, "max_steps": 20000, "progress_percent": 4.75, "loss": 1.7169514894485474, "perplexity": 5.567529893653337, "learning_rate": 0.0002996453752083513, "gradient_norm": 1.0557023286819458, "step_time": 3.5477874999996857, "tokens_per_second": 2309.0447215344, "steps_per_second": 0.2818658107341797, "batch_wait": 0.0004489999955694657, "processed_target_tokens": 5501364, "elapsed_seconds": 3310.412559299999, "eta_seconds": 67585.35187499401, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 41, "gpu_memory_used_mb": 3790.84, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 1000, "max_steps": 20000, "progress_percent": 5.0, "loss": 2.1309332847595215, "perplexity": 8.42272394428951, "learning_rate": 0.00029956223659811834, "gradient_norm": 1.106326699256897, "step_time": 3.4523608999988937, "tokens_per_second": 2372.868954692027, "steps_per_second": 0.28965685482080405, "batch_wait": 0.00046379999184864573, "processed_target_tokens": 5784428, "elapsed_seconds": 3490.5691678000003, "eta_seconds": 65594.85709997898, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3593.68, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "validation", "step": 1000, "validation_loss": 1.961993670463562, "validation_perplexity": 7.113494902143902, "validation_time": 2.321532299996761, "best_validation_loss": 2.0969789624214172}
{"step": 1000, "val_loss": 1.961993670463562, "best_val_loss": 1.961993670463562, "processed_target_tokens": 5784428, "timestamp": 1793369362.7392247}
{"event": "checkpoint", "step": 1000, "max_steps": 20000, "progress_percent": 5.0, "train_loss": 2.1309332847595215, "train_perplexity": 8.42272394428951, "validation_loss": 1.961993670463562, "validation_perplexity": 7.113494902143902, "best_validation_loss": 1.961993670463562, "best_validation_perplexity": 7.113494902143902, "processed_target_tokens": 5784428, "tokens_per_step": 8192, "tokens_per_second": 2372.868954692027, "steps_per_second": 0.28965685482080405, "elapsed_seconds": 3490.5691678000003, "eta_seconds": 65594.85709997898, "learning_rate": 0.00029956223659811834, "gradient_norm": 1.106326699256897, "batch_wait": 0.00046379999184864573, "evaluation_time": 2.321532299996761, "parameter_count": 62113280, "gpu": {"gpu_utilization": 6, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3591.62, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 54}, "is_best": true, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"step": 1000, "val_loss": 1.961993670463562, "best_val_loss": 1.961993670463562, "processed_target_tokens": 5784428, "timestamp": 1793369365.5567255}
{"event": "checkpoint", "step": 1000, "max_steps": 20000, "progress_percent": 5.0, "train_loss": 2.1309332847595215, "train_perplexity": 8.42272394428951, "validation_loss": 1.961993670463562, "validation_perplexity": 7.113494902143902, "best_validation_loss": 1.961993670463562, "best_validation_perplexity": 7.113494902143902, "processed_target_tokens": 5784428, "tokens_per_step": 8192, "tokens_per_second": 2372.868954692027, "steps_per_second": 0.28965685482080405, "elapsed_seconds": 3490.5691678000003, "eta_seconds": 65594.85709997898, "learning_rate": 0.00029956223659811834, "gradient_norm": 1.106326699256897, "batch_wait": 0.00046379999184864573, "evaluation_time": 2.321532299996761, "parameter_count": 62113280, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 6, "gpu_memory_used_mb": 3769.97, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 52}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 1050, "max_steps": 20000, "progress_percent": 5.25, "loss": 1.9751930236816406, "perplexity": 7.208010836339498, "learning_rate": 0.0002994703664371877, "gradient_norm": 1.009350061416626, "step_time": 3.478802199999336, "tokens_per_second": 2354.8335113739904, "steps_per_second": 0.2874552626188953, "batch_wait": 0.0013755999971181154, "processed_target_tokens": 6070414, "elapsed_seconds": 3670.3132110000006, "eta_seconds": 65923.30168998742, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 6, "gpu_memory_used_mb": 3744.34, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 52}}
{"event": "train", "step": 1100, "max_steps": 20000, "progress_percent": 5.5, "loss": 2.0887699127197266, "perplexity": 8.074976127194164, "learning_rate": 0.00029936977068688254, "gradient_norm": 1.1101609468460083, "step_time": 3.3370672999990347, "tokens_per_second": 2454.8501014655503, "steps_per_second": 0.29966431902655644, "batch_wait": 0.0010323999958927743, "processed_target_tokens": 6355572, "elapsed_seconds": 3849.3149120999988, "eta_seconds": 63070.57196998176, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 6, "gpu_memory_used_mb": 3744.34, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 52}}
{"event": "train", "step": 1150, "max_steps": 20000, "progress_percent": 5.75, "loss": 1.891514539718628, "perplexity": 6.629401573512014, "learning_rate": 0.00029926045587471686, "gradient_norm": 1.22021484375, "step_time": 3.750708499999746, "tokens_per_second": 2184.1206801329813, "steps_per_second": 0.26661629396154557, "batch_wait": 0.0005253000017546583, "processed_target_tokens": 6651027, "elapsed_seconds": 4034.5181483999986, "eta_seconds": 70700.8552249952, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 6, "gpu_memory_used_mb": 3744.34, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 52}}
{"event": "train", "step": 1200, "max_steps": 20000, "progress_percent": 6.0, "loss": 2.0095670223236084, "perplexity": 7.4600865969878365, "learning_rate": 0.00029914242909397223, "gradient_norm": 1.0855426788330078, "step_time": 3.2988798000005772, "tokens_per_second": 2483.267198762006, "steps_per_second": 0.3031332029738777, "batch_wait": 0.0006436999974539503, "processed_target_tokens": 6943367, "elapsed_seconds": 4209.667029, "eta_seconds": 62018.94024001085, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 6, "gpu_memory_used_mb": 3744.34, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 52}}
{"event": "train", "step": 1250, "max_steps": 20000, "progress_percent": 6.25, "loss": 2.042921781539917, "perplexity": 7.7131123374857395, "learning_rate": 0.00029901569800323726, "gradient_norm": 1.1023701429367065, "step_time": 3.2998579000013706, "tokens_per_second": 2482.531141718738, "steps_per_second": 0.30304335226058815, "batch_wait": 0.0011424000003898982, "processed_target_tokens": 7233113, "elapsed_seconds": 4375.290495599998, "eta_seconds": 61872.3356250257, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3477.2, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 1300, "max_steps": 20000, "progress_percent": 6.5, "loss": 2.271566390991211, "perplexity": 9.694574421106532, "learning_rate": 0.00029888027082591086, "gradient_norm": 1.1824874877929688, "step_time": 3.302067000000534, "tokens_per_second": 2480.8703154716954, "steps_per_second": 0.30284061468160345, "batch_wait": 0.0009034999966388568, "processed_target_tokens": 7519374, "elapsed_seconds": 4540.6922103, "eta_seconds": 61748.65290000998, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3477.2, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 1350, "max_steps": 20000, "progress_percent": 6.75, "loss": 2.2774529457092285, "perplexity": 9.751810359870998, "learning_rate": 0.0002987361563496684, "gradient_norm": 1.1554471254348755, "step_time": 3.295453599999746, "tokens_per_second": 2485.8489890437636, "steps_per_second": 0.3034483629203813, "batch_wait": 0.0010783999969135039, "processed_target_tokens": 7806748, "elapsed_seconds": 4706.019347999998, "eta_seconds": 61460.209639995264, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3477.2, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 1400, "max_steps": 20000, "progress_percent": 7.000000000000001, "loss": 2.151118516921997, "perplexity": 8.594466078743627, "learning_rate": 0.00029858336392589186, "gradient_norm": 1.1299587488174438, "step_time": 3.2955038999971293, "tokens_per_second": 2485.811046986513, "steps_per_second": 0.3034437313215958, "batch_wait": 0.0011230999989493284, "processed_target_tokens": 8096588, "elapsed_seconds": 4871.284934799998, "eta_seconds": 61296.372539946606, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3477.2, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 1450, "max_steps": 20000, "progress_percent": 7.249999999999999, "loss": 2.073550224304199, "perplexity": 7.9530080192461785, "learning_rate": 0.0002984219034690626, "gradient_norm": 1.0355428457260132, "step_time": 3.291129599998385, "tokens_per_second": 2489.1149835011115, "steps_per_second": 0.303847043884413, "batch_wait": 0.0008391999981540721, "processed_target_tokens": 8388681, "elapsed_seconds": 5036.093673399999, "eta_seconds": 61050.45407997004, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3477.2, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 1500, "max_steps": 20000, "progress_percent": 7.5, "loss": 2.042604684829712, "perplexity": 7.710666922675068, "learning_rate": 0.0002982517854561182, "gradient_norm": 1.0862542390823364, "step_time": 3.3103254000016022, "tokens_per_second": 2474.6811899507024, "steps_per_second": 0.3020851061951541, "batch_wait": 0.0012073999969288707, "processed_target_tokens": 8671988, "elapsed_seconds": 5200.9160778000005, "eta_seconds": 61241.01990002964, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3472.46, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "validation", "step": 1500, "validation_loss": 2.1048702120780947, "validation_perplexity": 8.206037895872274, "validation_time": 2.2229706999969494, "best_validation_loss": 1.961993670463562}
{"step": 1500, "val_loss": 2.1048702120780947, "best_val_loss": 1.961993670463562, "processed_target_tokens": 8671988, "timestamp": 1793371072.7402291}
{"event": "checkpoint", "step": 1500, "max_steps": 20000, "progress_percent": 7.5, "train_loss": 2.042604684829712, "train_perplexity": 7.710666922675068, "validation_loss": 2.1048702120780947, "validation_perplexity": 8.206037895872274, "best_validation_loss": 1.961993670463562, "best_validation_perplexity": 7.113494902143902, "processed_target_tokens": 8671988, "tokens_per_step": 8192, "tokens_per_second": 2474.6811899507024, "steps_per_second": 0.3020851061951541, "elapsed_seconds": 5200.9160778000005, "eta_seconds": 61241.01990002964, "learning_rate": 0.0002982517854561182, "gradient_norm": 1.0862542390823364, "batch_wait": 0.0012073999969288707, "evaluation_time": 2.2229706999969494, "parameter_count": 62113280, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3472.46, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 54}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 1550, "max_steps": 20000, "progress_percent": 7.75, "loss": 1.9205838441848755, "perplexity": 6.824942008998412, "learning_rate": 0.0002980730209257728, "gradient_norm": 1.0799121856689453, "step_time": 3.2849612000027264, "tokens_per_second": 2493.788967733683, "steps_per_second": 0.3044175986003031, "batch_wait": 0.0016560000040044542, "processed_target_tokens": 8960137, "elapsed_seconds": 5368.642286899998, "eta_seconds": 60607.5341400503, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3472.46, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 54}}
{"event": "train", "step": 1600, "max_steps": 20000, "progress_percent": 8.0, "loss": 1.9823142290115356, "perplexity": 7.259523760890044, "learning_rate": 0.00029788562147780066, "gradient_norm": 1.0481690168380737, "step_time": 3.291726199997356, "tokens_per_second": 2488.663850598078, "steps_per_second": 0.3037919739499607, "batch_wait": 0.0007408999954350293, "processed_target_tokens": 9247287, "elapsed_seconds": 5533.508130399998, "eta_seconds": 60567.76207995135, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3472.46, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 54}}
{"event": "train", "step": 1650, "max_steps": 20000, "progress_percent": 8.25, "loss": 2.050887107849121, "perplexity": 7.7747951297684255, "learning_rate": 0.00029768959927228334, "gradient_norm": 1.1161962747573853, "step_time": 3.4592802000006486, "tokens_per_second": 2368.122709458015, "steps_per_second": 0.2890774791818866, "batch_wait": 0.0007641000011062715, "processed_target_tokens": 9534799, "elapsed_seconds": 5701.448251099999, "eta_seconds": 63477.7916700119, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3472.46, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 54}}
{"event": "train", "step": 1700, "max_steps": 20000, "progress_percent": 8.5, "loss": 2.0234782695770264, "perplexity": 7.564590913538509, "learning_rate": 0.0002974849670288208, "gradient_norm": 1.0935364961624146, "step_time": 5.974488600000768, "tokens_per_second": 1371.1633829209995, "steps_per_second": 0.16737834264172358, "batch_wait": 0.0009218000013788696, "processed_target_tokens": 9825447, "elapsed_seconds": 5969.135788200001, "eta_seconds": 109333.14138001406, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3472.46, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 54}}
{"event": "train", "step": 1750, "max_steps": 20000, "progress_percent": 8.75, "loss": 2.1065635681152344, "perplexity": 8.219945411548798, "learning_rate": 0.0002972717380257061, "gradient_norm": 1.1437135934829712, "step_time": 5.4492484999973385, "tokens_per_second": 1503.3265596171657, "steps_per_second": 0.1835115429220173, "batch_wait": 0.0009226999973179772, "processed_target_tokens": 10115700, "elapsed_seconds": 6258.584627799999, "eta_seconds": 99448.78512495142, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 50, "gpu_memory_used_mb": 3777.28, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 1800, "max_steps": 20000, "progress_percent": 9.0, "loss": 2.146172523498535, "perplexity": 8.552062855513551, "learning_rate": 0.0002970499260990637, "gradient_norm": 1.1216371059417725, "step_time": 6.017737800000759, "tokens_per_second": 1361.3088958443764, "steps_per_second": 0.16617540232475297, "batch_wait": 0.0008853999970597215, "processed_target_tokens": 10414513, "elapsed_seconds": 6548.325427899999, "eta_seconds": 109522.82796001382, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 50, "gpu_memory_used_mb": 3777.28, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 1850, "max_steps": 20000, "progress_percent": 9.25, "loss": 1.9822828769683838, "perplexity": 7.25929616355567, "learning_rate": 0.0002968195456419517, "gradient_norm": 1.0615074634552002, "step_time": 6.4136233999997785, "tokens_per_second": 1277.2811075873715, "steps_per_second": 0.15591810395353656, "batch_wait": 0.001143099998444086, "processed_target_tokens": 10700633, "elapsed_seconds": 6836.314713, "eta_seconds": 116407.26470999597, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 50, "gpu_memory_used_mb": 3777.28, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"step": 1888, "val_loss": 2.1048702120780947, "best_val_loss": 1.961993670463562, "processed_target_tokens": 10923355, "timestamp": 1793372934.112046}
{"event": "checkpoint", "step": 1888, "max_steps": 20000, "progress_percent": 9.44, "train_loss": 2.2355990409851074, "train_perplexity": 9.352082455220557, "validation_loss": 2.1048702120780947, "validation_perplexity": 8.206037895872274, "best_validation_loss": 1.961993670463562, "best_validation_perplexity": 7.113494902143902, "processed_target_tokens": 10923355, "tokens_per_step": 8192, "tokens_per_second": 2370.089128023575, "steps_per_second": 0.2893175205106903, "elapsed_seconds": 7060.9570994999995, "eta_seconds": 62602.4997311933, "learning_rate": 0.0002966339102085154, "gradient_norm": 1.040771722793579, "batch_wait": 0.0009933000037563033, "evaluation_time": 2.2229706999969494, "parameter_count": 62113280, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3393.44, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 55}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 1900, "max_steps": 20000, "progress_percent": 9.5, "loss": 2.420868396759033, "perplexity": 11.255629424215092, "learning_rate": 0.00029658061160342774, "gradient_norm": 1.1650331020355225, "step_time": 3.359739700000091, "tokens_per_second": 2438.284132547464, "steps_per_second": 0.2976421060238604, "batch_wait": 0.0016529000001810346, "processed_target_tokens": 10990246, "elapsed_seconds": 41.25772630000006, "eta_seconds": 60811.28857000165, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3624.82, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 48}}
{"event": "train", "step": 1950, "max_steps": 20000, "progress_percent": 9.75, "loss": 2.0191359519958496, "perplexity": 7.531814272215429, "learning_rate": 0.00029633313948757917, "gradient_norm": 1.0529026985168457, "step_time": 3.3613646999999673, "tokens_per_second": 2437.1053816326685, "steps_per_second": 0.2974982155313316, "batch_wait": 0.0017885999998270563, "processed_target_tokens": 11278948, "elapsed_seconds": 212.17459439999993, "eta_seconds": 60672.63283499941, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3624.82, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 48}}
{"event": "train", "step": 2000, "max_steps": 20000, "progress_percent": 10.0, "loss": 2.05757474899292, "perplexity": 7.826964419871174, "learning_rate": 0.00029607714535251703, "gradient_norm": 1.1183853149414062, "step_time": 3.429228799999919, "tokens_per_second": 2388.875306307994, "steps_per_second": 0.29161075516455004, "batch_wait": 0.00140600000008817, "processed_target_tokens": 11560552, "elapsed_seconds": 383.50239039999997, "eta_seconds": 61726.11839999854, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3624.82, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 48}}
{"event": "validation", "step": 2000, "validation_loss": 1.8589496612548828, "validation_perplexity": 6.416993213939488, "validation_time": 2.3118868999999904, "best_validation_loss": 1.961993670463562}
{"step": 2000, "val_loss": 1.8589496612548828, "best_val_loss": 1.8589496612548828, "processed_target_tokens": 11560552, "timestamp": 1793409431.489282}
{"event": "checkpoint", "step": 2000, "max_steps": 20000, "progress_percent": 10.0, "train_loss": 2.05757474899292, "train_perplexity": 7.826964419871174, "validation_loss": 1.8589496612548828, "validation_perplexity": 6.416993213939488, "best_validation_loss": 1.8589496612548828, "best_validation_perplexity": 6.416993213939488, "processed_target_tokens": 11560552, "tokens_per_step": 8192, "tokens_per_second": 2388.875306307994, "steps_per_second": 0.29161075516455004, "elapsed_seconds": 383.50239039999997, "eta_seconds": 61726.11839999854, "learning_rate": 0.00029607714535251703, "gradient_norm": 1.1183853149414062, "batch_wait": 0.00140600000008817, "evaluation_time": 2.3118868999999904, "parameter_count": 62113280, "gpu": {"gpu_utilization": 12, "gpu_memory_utilization": 14, "gpu_memory_used_mb": 3576.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 55}, "is_best": true, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"step": 2000, "val_loss": 1.8589496612548828, "best_val_loss": 1.8589496612548828, "processed_target_tokens": 11560552, "timestamp": 1793409433.455328}
{"event": "checkpoint", "step": 2000, "max_steps": 20000, "progress_percent": 10.0, "train_loss": 2.05757474899292, "train_perplexity": 7.826964419871174, "validation_loss": 1.8589496612548828, "validation_perplexity": 6.416993213939488, "best_validation_loss": 1.8589496612548828, "best_validation_perplexity": 6.416993213939488, "processed_target_tokens": 11560552, "tokens_per_step": 8192, "tokens_per_second": 2388.875306307994, "steps_per_second": 0.29161075516455004, "elapsed_seconds": 383.50239039999997, "eta_seconds": 61726.11839999854, "learning_rate": 0.00029607714535251703, "gradient_norm": 1.1183853149414062, "batch_wait": 0.00140600000008817, "evaluation_time": 2.3118868999999904, "parameter_count": 62113280, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 13, "gpu_memory_used_mb": 3576.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 54}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 2050, "max_steps": 20000, "progress_percent": 10.25, "loss": 2.1152286529541016, "perplexity": 8.291481421108225, "learning_rate": 0.00029581264580933375, "gradient_norm": 1.082527995109558, "step_time": 3.3894006000000445, "tokens_per_second": 2416.9465244090334, "steps_per_second": 0.2950374175303996, "batch_wait": 0.0010592999997243169, "processed_target_tokens": 11852504, "elapsed_seconds": 563.7218405000001, "eta_seconds": 60839.7407700008, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 13, "gpu_memory_used_mb": 3576.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 54}}
{"event": "train", "step": 2100, "max_steps": 20000, "progress_percent": 10.5, "loss": 2.2173898220062256, "perplexity": 9.183329431110026, "learning_rate": 0.0002955396580210258, "gradient_norm": 1.0359742641448975, "step_time": 3.659216000000015, "tokens_per_second": 2238.7309194100503, "steps_per_second": 0.27328258293579716, "batch_wait": 0.0008465000000796863, "processed_target_tokens": 12145371, "elapsed_seconds": 737.7806737999999, "eta_seconds": 65499.96640000027, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 13, "gpu_memory_used_mb": 3576.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 54}}
{"event": "train", "step": 2150, "max_steps": 20000, "progress_percent": 10.75, "loss": 2.1526594161987305, "perplexity": 8.607719493774715, "learning_rate": 0.00029525819970137947, "gradient_norm": 1.0804308652877808, "step_time": 4.9789058999999725, "tokens_per_second": 1645.3413992017895, "steps_per_second": 0.2008473387697497, "batch_wait": 0.000546999999869513, "processed_target_tokens": 12434075, "elapsed_seconds": 958.7660437, "eta_seconds": 88873.4703149995, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 13, "gpu_memory_used_mb": 3576.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 54}}
{"event": "train", "step": 2200, "max_steps": 20000, "progress_percent": 11.0, "loss": 2.024430274963379, "perplexity": 7.57179587387011, "learning_rate": 0.0002949682891138217, "gradient_norm": 1.106541633605957, "step_time": 5.318772299999637, "tokens_per_second": 1540.205058223786, "steps_per_second": 0.18801331277145825, "batch_wait": 0.0014967999995860737, "processed_target_tokens": 12718891, "elapsed_seconds": 1222.3952923000002, "eta_seconds": 94674.14693999353, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 13, "gpu_memory_used_mb": 3576.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 54}}
{"event": "train", "step": 2250, "max_steps": 20000, "progress_percent": 11.25, "loss": 2.160154342651367, "perplexity": 8.672476088125766, "learning_rate": 0.00029466994507023524, "gradient_norm": 1.028157353401184, "step_time": 5.384667799999988, "tokens_per_second": 1521.3566192514268, "steps_per_second": 0.18571247793596518, "batch_wait": 0.0015281000000868517, "processed_target_tokens": 13014071, "elapsed_seconds": 1486.9323367000002, "eta_seconds": 95577.85344999979, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 38, "gpu_memory_used_mb": 3637.67, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 65}}
{"event": "train", "step": 2300, "max_steps": 20000, "progress_percent": 11.5, "loss": 2.3191184997558594, "perplexity": 10.16670839899345, "learning_rate": 0.0002943631869297374, "gradient_norm": 1.0868799686431885, "step_time": 5.419257200000175, "tokens_per_second": 1511.6462824461876, "steps_per_second": 0.18452713408766938, "batch_wait": 0.0006581000002370274, "processed_target_tokens": 13306604, "elapsed_seconds": 1760.237075, "eta_seconds": 95920.85244000309, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 38, "gpu_memory_used_mb": 3637.67, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 65}}
{"event": "train", "step": 2350, "max_steps": 20000, "progress_percent": 11.75, "loss": 2.0861377716064453, "perplexity": 8.053749598405647, "learning_rate": 0.0002940480345974244, "gradient_norm": 1.0512617826461792, "step_time": 5.754843499999879, "tokens_per_second": 1423.496572930293, "steps_per_second": 0.1737666715002799, "batch_wait": 0.001105499999539461, "processed_target_tokens": 13602527, "elapsed_seconds": 2037.1463683000002, "eta_seconds": 101572.98777499786, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 38, "gpu_memory_used_mb": 3637.67, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 65}}
{"event": "train", "step": 2400, "max_steps": 20000, "progress_percent": 12.0, "loss": 2.0534415245056152, "perplexity": 7.794680583002429, "learning_rate": 0.00029372450852307937, "gradient_norm": 1.053582787513733, "step_time": 4.362044900000001, "tokens_per_second": 1878.018266157691, "steps_per_second": 0.2292502766305775, "batch_wait": 0.001318200000696379, "processed_target_tokens": 13890962, "elapsed_seconds": 2230.0312912, "eta_seconds": 76771.99024000001, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 38, "gpu_memory_used_mb": 3637.67, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 65}}
{"event": "train", "step": 2450, "max_steps": 20000, "progress_percent": 12.25, "loss": 1.9082670211791992, "perplexity": 6.741395973304021, "learning_rate": 0.0002933926296998457, "gradient_norm": 1.122728705406189, "step_time": 3.380979099999877, "tokens_per_second": 2422.9667672303262, "steps_per_second": 0.2957723104529207, "batch_wait": 0.0005468000003929774, "processed_target_tokens": 14179414, "elapsed_seconds": 2420.9648049, "eta_seconds": 59336.18320499784, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 38, "gpu_memory_used_mb": 3637.67, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 65}}
{"event": "train", "step": 2500, "max_steps": 20000, "progress_percent": 12.5, "loss": 2.0443272590637207, "perplexity": 7.7239605651989915, "learning_rate": 0.0002930524196628646, "gradient_norm": 1.0268034934997559, "step_time": 3.344306299999971, "tokens_per_second": 2449.5363956345955, "steps_per_second": 0.2990156732952387, "batch_wait": 0.0011613000001489127, "processed_target_tokens": 14470191, "elapsed_seconds": 2590.8750219, "eta_seconds": 58525.360249999496, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3768.27, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "validation", "step": 2500, "validation_loss": 1.9485508680343628, "validation_perplexity": 7.018509461239991, "validation_time": 2.2351870000002236, "best_validation_loss": 1.8589496612548828}
{"step": 2500, "val_loss": 1.9485508680343628, "best_val_loss": 1.8589496612548828, "processed_target_tokens": 14470191, "timestamp": 1793411636.271583}
{"event": "checkpoint", "step": 2500, "max_steps": 20000, "progress_percent": 12.5, "train_loss": 2.0443272590637207, "train_perplexity": 7.7239605651989915, "validation_loss": 1.9485508680343628, "validation_perplexity": 7.018509461239991, "best_validation_loss": 1.8589496612548828, "best_validation_perplexity": 6.416993213939488, "processed_target_tokens": 14470191, "tokens_per_step": 8192, "tokens_per_second": 2449.5363956345955, "steps_per_second": 0.2990156732952387, "elapsed_seconds": 2590.8750219, "eta_seconds": 58525.360249999496, "learning_rate": 0.0002930524196628646, "gradient_norm": 1.0268034934997559, "batch_wait": 0.0011613000001489127, "evaluation_time": 2.2351870000002236, "parameter_count": 62113280, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3787.52, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 2550, "max_steps": 20000, "progress_percent": 12.75, "loss": 1.9924867153167725, "perplexity": 7.333748049654276, "learning_rate": 0.00029270390048787785, "gradient_norm": 1.0456565618515015, "step_time": 3.315776200000073, "tokens_per_second": 2470.613064898596, "steps_per_second": 0.3015885088987544, "batch_wait": 0.0009607999995751015, "processed_target_tokens": 14761345, "elapsed_seconds": 2772.4251406, "eta_seconds": 57860.294690001276, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3787.52, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 2600, "max_steps": 20000, "progress_percent": 13.0, "loss": 2.126939296722412, "perplexity": 8.389150775662829, "learning_rate": 0.0002923470947897953, "gradient_norm": 1.00432288646698, "step_time": 3.318337600000177, "tokens_per_second": 2468.7060171332664, "steps_per_second": 0.3013557149820882, "batch_wait": 0.000859500000387925, "processed_target_tokens": 15050852, "elapsed_seconds": 2952.1115963, "eta_seconds": 57739.07424000309, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3787.52, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 2650, "max_steps": 20000, "progress_percent": 13.25, "loss": 2.1154420375823975, "perplexity": 8.293250884570726, "learning_rate": 0.00029198202572122727, "gradient_norm": 1.0164934396743774, "step_time": 3.314740100000563, "tokens_per_second": 2471.3853131346887, "steps_per_second": 0.3016827774822618, "batch_wait": 0.0012110999987271498, "processed_target_tokens": 15338894, "elapsed_seconds": 3117.8559895, "eta_seconds": 57510.740735009764, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3787.52, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 2700, "max_steps": 20000, "progress_percent": 13.5, "loss": 1.953789472579956, "perplexity": 7.055373129593202, "learning_rate": 0.0002916087169709826, "gradient_norm": 0.946841835975647, "step_time": 3.4051196999998865, "tokens_per_second": 2405.78914156829, "steps_per_second": 0.2936754323203479, "batch_wait": 0.0007580000010420918, "processed_target_tokens": 15626763, "elapsed_seconds": 3289.519144, "eta_seconds": 58908.57080999804, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3787.52, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 2750, "max_steps": 20000, "progress_percent": 13.750000000000002, "loss": 1.9853944778442383, "perplexity": 7.281919374804976, "learning_rate": 0.000291227192762531, "gradient_norm": 1.0870435237884521, "step_time": 3.4373522000005323, "tokens_per_second": 2383.2297429395603, "steps_per_second": 0.2909215994799268, "batch_wait": 0.0012388999994072947, "processed_target_tokens": 15914983, "elapsed_seconds": 3458.7793885, "eta_seconds": 59294.325450009186, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3802.89, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 2800, "max_steps": 20000, "progress_percent": 14.000000000000002, "loss": 1.8502181768417358, "perplexity": 6.361207239317603, "learning_rate": 0.0002908374778524317, "gradient_norm": 1.0369482040405273, "step_time": 3.300217199999679, "tokens_per_second": 2482.2608645275823, "steps_per_second": 0.30301035943940213, "batch_wait": 0.0008704000001671375, "processed_target_tokens": 16196956, "elapsed_seconds": 3629.3419674999996, "eta_seconds": 56763.73583999448, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3802.89, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 2850, "max_steps": 20000, "progress_percent": 14.249999999999998, "loss": 1.995961308479309, "perplexity": 7.359274161228783, "learning_rate": 0.00029043959752872656, "gradient_norm": 1.04179048538208, "step_time": 3.308244300000297, "tokens_per_second": 2476.2379247503773, "steps_per_second": 0.30227513729863004, "batch_wait": 0.001190000001770386, "processed_target_tokens": 16484717, "elapsed_seconds": 3796.5280952000003, "eta_seconds": 56736.389745005086, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3802.89, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 2900, "max_steps": 20000, "progress_percent": 14.499999999999998, "loss": 1.9861091375350952, "perplexity": 7.28712532907548, "learning_rate": 0.0002900335776092998, "gradient_norm": 0.994956910610199, "step_time": 3.3132261999999173, "tokens_per_second": 2472.514553941474, "steps_per_second": 0.3018206242604338, "batch_wait": 0.0009920000002239249, "processed_target_tokens": 16772507, "elapsed_seconds": 3962.172643, "eta_seconds": 56656.168019998586, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3802.89, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 2950, "max_steps": 20000, "progress_percent": 14.75, "loss": 1.834026575088501, "perplexity": 6.259038475104673, "learning_rate": 0.0002896194444402021, "gradient_norm": 0.980046808719635, "step_time": 3.305633200000557, "tokens_per_second": 2478.193890356202, "steps_per_second": 0.3025139026313723, "batch_wait": 0.001319199998761178, "processed_target_tokens": 17054446, "elapsed_seconds": 4127.6856327, "eta_seconds": 56361.046060009496, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3802.89, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 3000, "max_steps": 20000, "progress_percent": 15.0, "loss": 1.9820325374603271, "perplexity": 7.257479102375839, "learning_rate": 0.00028919722489394124, "gradient_norm": 0.9961909651756287, "step_time": 3.3092145999999047, "tokens_per_second": 2475.5118631472965, "steps_per_second": 0.3021865067318477, "batch_wait": 0.0012679000001298846, "processed_target_tokens": 17347797, "elapsed_seconds": 4293.4357951, "eta_seconds": 56256.64819999838, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3713.36, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "validation", "step": 3000, "validation_loss": 1.8683966398239136, "validation_perplexity": 6.477901658712857, "validation_time": 2.226497699999527, "best_validation_loss": 1.8589496612548828}
{"step": 3000, "val_loss": 1.8683966398239136, "best_val_loss": 1.8589496612548828, "processed_target_tokens": 17347797, "timestamp": 1793413338.6790414}
{"event": "checkpoint", "step": 3000, "max_steps": 20000, "progress_percent": 15.0, "train_loss": 1.9820325374603271, "train_perplexity": 7.257479102375839, "validation_loss": 1.8683966398239136, "validation_perplexity": 6.477901658712857, "best_validation_loss": 1.8589496612548828, "best_validation_perplexity": 6.416993213939488, "processed_target_tokens": 17347797, "tokens_per_step": 8192, "tokens_per_second": 2475.5118631472965, "steps_per_second": 0.3021865067318477, "elapsed_seconds": 4293.4357951, "eta_seconds": 56256.64819999838, "learning_rate": 0.00028919722489394124, "gradient_norm": 0.9961909651756287, "batch_wait": 0.0012679000001298846, "evaluation_time": 2.226497699999527, "parameter_count": 62113280, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3709.05, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 3050, "max_steps": 20000, "progress_percent": 15.25, "loss": 1.9545105695724487, "perplexity": 7.060462572708394, "learning_rate": 0.0002887669463677386, "gradient_norm": 0.9607078433036804, "step_time": 3.297407000000021, "tokens_per_second": 2484.376359970106, "steps_per_second": 0.30326859862916333, "batch_wait": 0.0008730999998078914, "processed_target_tokens": 17634870, "elapsed_seconds": 4461.4359048, "eta_seconds": 55891.04865000036, "gpu": {"gpu_utilization": 8, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3709.05, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 3100, "max_steps": 20000, "progress_percent": 15.5, "loss": 2.153614044189453, "perplexity": 8.615940587158356, "learning_rate": 0.0002883286367817511, "gradient_norm": 1.092179775238037, "step_time": 3.307256099999904, "tokens_per_second": 2476.9778185609025, "steps_per_second": 0.30236545636729767, "batch_wait": 0.0009528999999020016, "processed_target_tokens": 17925282, "elapsed_seconds": 4626.3734397, "eta_seconds": 55892.62808999838, "gpu": {"gpu_utilization": 8, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3709.05, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 3150, "max_steps": 20000, "progress_percent": 15.75, "loss": 2.032228946685791, "perplexity": 7.63107667956002, "learning_rate": 0.00028788232457725965, "gradient_norm": 1.1149026155471802, "step_time": 3.3102638000000297, "tokens_per_second": 2474.727240771544, "steps_per_second": 0.3020907276332451, "batch_wait": 0.0009932000011758646, "processed_target_tokens": 18208571, "elapsed_seconds": 4791.3575947, "eta_seconds": 55777.9450300005, "gpu": {"gpu_utilization": 8, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3709.05, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 3200, "max_steps": 20000, "progress_percent": 16.0, "loss": 1.7964681386947632, "perplexity": 6.0283186360262855, "learning_rate": 0.0002874280387148236, "gradient_norm": 1.0188226699829102, "step_time": 3.295340099999521, "tokens_per_second": 2485.9346080852747, "steps_per_second": 0.3034588144635345, "batch_wait": 0.0010299000005034031, "processed_target_tokens": 18497114, "elapsed_seconds": 4956.3252232, "eta_seconds": 55361.71367999195, "gpu": {"gpu_utilization": 8, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3709.05, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 3250, "max_steps": 20000, "progress_percent": 16.25, "loss": 2.0636303424835205, "perplexity": 7.874505133002547, "learning_rate": 0.0002869658086724016, "gradient_norm": 1.0762771368026733, "step_time": 3.2935920999998416, "tokens_per_second": 2487.253962019278, "steps_per_second": 0.3036198684105564, "batch_wait": 0.0013822000009895419, "processed_target_tokens": 18791142, "elapsed_seconds": 5121.2551085, "eta_seconds": 55167.66767499735, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3705.34, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 3300, "max_steps": 20000, "progress_percent": 16.5, "loss": 2.0118772983551025, "perplexity": 7.477341380227198, "learning_rate": 0.0002864956644434386, "gradient_norm": 1.0609337091445923, "step_time": 3.308606999999938, "tokens_per_second": 2475.966471690398, "steps_per_second": 0.3022420009387693, "batch_wait": 0.0010397999985798378, "processed_target_tokens": 19080013, "elapsed_seconds": 5286.1852697, "eta_seconds": 55253.73689999897, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3705.34, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 3350, "max_steps": 20000, "progress_percent": 16.75, "loss": 1.975789189338684, "perplexity": 7.21230928602245, "learning_rate": 0.00028601763653491994, "gradient_norm": 1.0332965850830078, "step_time": 3.3052176999999574, "tokens_per_second": 2478.505424922572, "steps_per_second": 0.30255193175324363, "batch_wait": 0.0010627000010572374, "processed_target_tokens": 19367252, "elapsed_seconds": 5451.1974016, "eta_seconds": 55031.87470499929, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3705.34, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 3400, "max_steps": 20000, "progress_percent": 17.0, "loss": 1.808990240097046, "perplexity": 6.104280461528298, "learning_rate": 0.00028553175596539163, "gradient_norm": 1.037864089012146, "step_time": 3.309366399999817, "tokens_per_second": 2475.3983118945225, "steps_per_second": 0.30217264549493683, "batch_wait": 0.0010435000012876117, "processed_target_tokens": 19648688, "elapsed_seconds": 5616.1785091, "eta_seconds": 54935.48223999696, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3705.34, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 3450, "max_steps": 20000, "progress_percent": 17.25, "loss": 1.966810703277588, "perplexity": 7.147843503261122, "learning_rate": 0.0002850380542629474, "gradient_norm": 1.0039360523223877, "step_time": 3.2977473000000828, "tokens_per_second": 2484.119993063081, "steps_per_second": 0.30323730384070813, "batch_wait": 0.0014164000003802357, "processed_target_tokens": 19936685, "elapsed_seconds": 5781.1147515, "eta_seconds": 54577.71781500137, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3705.34, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 3500, "max_steps": 20000, "progress_percent": 17.5, "loss": 2.093736171722412, "perplexity": 8.11517829464241, "learning_rate": 0.0002845365634631833, "gradient_norm": 1.0228872299194336, "step_time": 3.289044400000421, "tokens_per_second": 2490.6930414192498, "steps_per_second": 0.30403967790762326, "batch_wait": 0.000646700000288547, "processed_target_tokens": 20226860, "elapsed_seconds": 5946.1049715, "eta_seconds": 54269.232600006944, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3709.62, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "validation", "step": 3500, "validation_loss": 1.8646142482757568, "validation_perplexity": 6.453445977865652, "validation_time": 2.2201922999993258, "best_validation_loss": 1.8589496612548828}
{"step": 3500, "val_loss": 1.8646142482757568, "best_val_loss": 1.8589496612548828, "processed_target_tokens": 20226860, "timestamp": 1793414991.3649306}
{"event": "checkpoint", "step": 3500, "max_steps": 20000, "progress_percent": 17.5, "train_loss": 2.093736171722412, "train_perplexity": 8.11517829464241, "validation_loss": 1.8646142482757568, "validation_perplexity": 6.453445977865652, "best_validation_loss": 1.8589496612548828, "best_validation_perplexity": 6.416993213939488, "processed_target_tokens": 20226860, "tokens_per_step": 8192, "tokens_per_second": 2490.6930414192498, "steps_per_second": 0.30403967790762326, "elapsed_seconds": 5946.1049715, "eta_seconds": 54269.232600006944, "learning_rate": 0.0002845365634631833, "gradient_norm": 1.0228872299194336, "batch_wait": 0.000646700000288547, "evaluation_time": 2.2201922999993258, "parameter_count": 62113280, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3709.62, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 3550, "max_steps": 20000, "progress_percent": 17.75, "loss": 1.9492661952972412, "perplexity": 7.023531788492031, "learning_rate": 0.0002840273161071187, "gradient_norm": 1.109400987625122, "step_time": 3.292768100000103, "tokens_per_second": 2487.8763858286115, "steps_per_second": 0.3036958478794692, "batch_wait": 0.0009468999996897765, "processed_target_tokens": 20520156, "elapsed_seconds": 6113.9913857, "eta_seconds": 54166.035245001694, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3709.62, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 3600, "max_steps": 20000, "progress_percent": 18.0, "loss": 1.973778247833252, "perplexity": 7.197820327036313, "learning_rate": 0.0002835103452390846, "gradient_norm": 1.0913194417953491, "step_time": 3.2941799999998693, "tokens_per_second": 2486.8100710951817, "steps_per_second": 0.30356568250673605, "batch_wait": 0.0004961000004186644, "processed_target_tokens": 20808976, "elapsed_seconds": 6278.9585532, "eta_seconds": 54024.55199999786, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3709.62, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 3650, "max_steps": 20000, "progress_percent": 18.25, "loss": 2.0464160442352295, "perplexity": 7.740111121141676, "learning_rate": 0.0002829856844045799, "gradient_norm": 0.9564743638038635, "step_time": 3.2899575000001278, "tokens_per_second": 2490.0017705394926, "steps_per_second": 0.30395529425530915, "batch_wait": 0.0011016999997082166, "processed_target_tokens": 21103341, "elapsed_seconds": 6443.9548601, "eta_seconds": 53790.80512500209, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3709.62, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 3700, "max_steps": 20000, "progress_percent": 18.5, "loss": 1.8591398000717163, "perplexity": 6.418213449440206, "learning_rate": 0.00028245336764809426, "gradient_norm": 0.9409807324409485, "step_time": 3.285181199999897, "tokens_per_second": 2493.6219652055283, "steps_per_second": 0.30439721254950297, "batch_wait": 0.001027899998916837, "processed_target_tokens": 21397670, "elapsed_seconds": 6608.9488588, "eta_seconds": 53548.45355999832, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3709.62, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 3750, "max_steps": 20000, "progress_percent": 18.75, "loss": 2.0073471069335938, "perplexity": 7.443544204090075, "learning_rate": 0.0002819134295108992, "gradient_norm": 1.0090367794036865, "step_time": 3.7064194000004136, "tokens_per_second": 2210.2193831596837, "steps_per_second": 0.26980217079585983, "batch_wait": 0.0011128000014650752, "processed_target_tokens": 21687754, "elapsed_seconds": 6779.5704535, "eta_seconds": 60229.31525000672, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3726.23, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 3800, "max_steps": 20000, "progress_percent": 19.0, "loss": 2.002244710922241, "perplexity": 7.405661023516457, "learning_rate": 0.0002813659050288066, "gradient_norm": 1.0416814088821411, "step_time": 4.622254699999758, "tokens_per_second": 1772.2952393775336, "steps_per_second": 0.21634463371307783, "batch_wait": 0.0008246999996117665, "processed_target_tokens": 21979971, "elapsed_seconds": 6986.1918532, "eta_seconds": 74880.52613999607, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3726.23, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 3850, "max_steps": 20000, "progress_percent": 19.25, "loss": 1.8112211227416992, "perplexity": 6.117913596175599, "learning_rate": 0.00028081082972989557, "gradient_norm": 1.1646623611450195, "step_time": 4.634551600000123, "tokens_per_second": 1767.592791501077, "steps_per_second": 0.21577060443128382, "batch_wait": 0.00048150000111490954, "processed_target_tokens": 22267034, "elapsed_seconds": 7220.955708900001, "eta_seconds": 74848.00834000198, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3726.23, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 3900, "max_steps": 20000, "progress_percent": 19.5, "loss": 1.8316540718078613, "perplexity": 6.244206487215955, "learning_rate": 0.00028024823963220674, "gradient_norm": 1.0042060613632202, "step_time": 4.42528830000083, "tokens_per_second": 1851.1788260209994, "steps_per_second": 0.22597397778576653, "batch_wait": 0.0007392000006802846, "processed_target_tokens": 22554543, "elapsed_seconds": 7452.087115599999, "eta_seconds": 71247.14163001336, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3726.23, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 3950, "max_steps": 20000, "progress_percent": 19.75, "loss": 1.9454662799835205, "perplexity": 6.996893605964571, "learning_rate": 0.00027967817124140533, "gradient_norm": 0.9839587211608887, "step_time": 4.796845499999108, "tokens_per_second": 1707.7890042532167, "steps_per_second": 0.208470337433254, "batch_wait": 0.0006756000002496876, "processed_target_tokens": 22839232, "elapsed_seconds": 7689.437565599999, "eta_seconds": 76989.37027498569, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3726.23, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 4000, "max_steps": 20000, "progress_percent": 20.0, "loss": 1.6986167430877686, "perplexity": 5.466380750671174, "learning_rate": 0.00027910066154841225, "gradient_norm": 1.0467406511306763, "step_time": 3.604876299999887, "tokens_per_second": 2272.4774217634754, "steps_per_second": 0.27740202902386174, "batch_wait": 0.0012252000014996156, "processed_target_tokens": 23125821, "elapsed_seconds": 7885.704084000001, "eta_seconds": 57678.020799998194, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3715.59, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "validation", "step": 4000, "validation_loss": 2.027543294429779, "validation_perplexity": 7.595403748642831, "validation_time": 2.3817838999984815, "best_validation_loss": 1.8589496612548828}
{"step": 4000, "val_loss": 2.027543294429779, "best_val_loss": 1.8589496612548828, "processed_target_tokens": 23125821, "timestamp": 1793416932.102406}
{"event": "checkpoint", "step": 4000, "max_steps": 20000, "progress_percent": 20.0, "train_loss": 1.6986167430877686, "train_perplexity": 5.466380750671174, "validation_loss": 2.027543294429779, "validation_perplexity": 7.595403748642831, "best_validation_loss": 1.8589496612548828, "best_validation_perplexity": 6.416993213939488, "processed_target_tokens": 23125821, "tokens_per_step": 8192, "tokens_per_second": 2272.4774217634754, "steps_per_second": 0.27740202902386174, "elapsed_seconds": 7885.704084000001, "eta_seconds": 57678.020799998194, "learning_rate": 0.00027910066154841225, "gradient_norm": 1.0467406511306763, "batch_wait": 0.0012252000014996156, "evaluation_time": 2.3817838999984815, "parameter_count": 62113280, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 7, "gpu_memory_used_mb": 3697.41, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 4050, "max_steps": 20000, "progress_percent": 20.25, "loss": 1.7546792030334473, "perplexity": 5.781592727022429, "learning_rate": 0.00027851574802700396, "gradient_norm": 1.0496121644973755, "step_time": 3.580864899999142, "tokens_per_second": 2287.7154622621933, "steps_per_second": 0.2792621413894279, "batch_wait": 0.0013331000009202398, "processed_target_tokens": 23418260, "elapsed_seconds": 8069.238570699999, "eta_seconds": 57114.795154986314, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 7, "gpu_memory_used_mb": 3697.41, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 4100, "max_steps": 20000, "progress_percent": 20.5, "loss": 1.8598601818084717, "perplexity": 6.422838678957195, "learning_rate": 0.00027792346863138057, "gradient_norm": 1.0952386856079102, "step_time": 3.5268572000004497, "tokens_per_second": 2322.7478560796153, "steps_per_second": 0.28353855665034366, "batch_wait": 0.0010528000002523186, "processed_target_tokens": 23704602, "elapsed_seconds": 8247.2081718, "eta_seconds": 56077.02948000715, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 7, "gpu_memory_used_mb": 3697.41, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 4150, "max_steps": 20000, "progress_percent": 20.75, "loss": 1.4820982217788696, "perplexity": 4.402172732034454, "learning_rate": 0.00027732386179370335, "gradient_norm": 0.9145543575286865, "step_time": 3.5827703999984806, "tokens_per_second": 2286.498738519073, "steps_per_second": 0.279113615541879, "batch_wait": 0.0013760999972873833, "processed_target_tokens": 23996151, "elapsed_seconds": 8425.2863679, "eta_seconds": 56786.91083997591, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 7, "gpu_memory_used_mb": 3697.41, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 4200, "max_steps": 20000, "progress_percent": 21.0, "loss": 2.1323113441467285, "perplexity": 8.43433895933893, "learning_rate": 0.0002767169664216007, "gradient_norm": 1.0384563207626343, "step_time": 3.5332651999997324, "tokens_per_second": 2318.5352743973535, "steps_per_second": 0.2830243254879582, "batch_wait": 0.001127900000938098, "processed_target_tokens": 24283189, "elapsed_seconds": 8604.7738099, "eta_seconds": 55825.59015999577, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 7, "gpu_memory_used_mb": 3697.41, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 4250, "max_steps": 20000, "progress_percent": 21.25, "loss": 2.050335645675659, "perplexity": 7.770508806329189, "learning_rate": 0.0002761028218956436, "gradient_norm": 1.0329301357269287, "step_time": 3.5934242000002996, "tokens_per_second": 2279.719716920512, "steps_per_second": 0.27828609825689843, "batch_wait": 0.0008168999993358739, "processed_target_tokens": 24576278, "elapsed_seconds": 8784.080363, "eta_seconds": 56596.43115000472, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3713.02, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 4300, "max_steps": 20000, "progress_percent": 21.5, "loss": 1.8941090106964111, "perplexity": 6.64662369498115, "learning_rate": 0.0002754814680667902, "gradient_norm": 1.073460340499878, "step_time": 3.5001739999988786, "tokens_per_second": 2340.455074519902, "steps_per_second": 0.28570008233885524, "batch_wait": 0.001338699999905657, "processed_target_tokens": 24868301, "elapsed_seconds": 8960.5644715, "eta_seconds": 54952.73179998239, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3713.02, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 4350, "max_steps": 20000, "progress_percent": 21.75, "loss": 1.7505908012390137, "perplexity": 5.75800350690538, "learning_rate": 0.0002748529452538001, "gradient_norm": 1.1209149360656738, "step_time": 3.595135900000969, "tokens_per_second": 2278.6343069806603, "steps_per_second": 0.27815360192635014, "batch_wait": 0.0012530999974842416, "processed_target_tokens": 25151897, "elapsed_seconds": 9137.6857918, "eta_seconds": 56263.876835015166, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3713.02, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 4400, "max_steps": 20000, "progress_percent": 22.0, "loss": 2.034360885620117, "perplexity": 7.647363023620829, "learning_rate": 0.00027421729424061787, "gradient_norm": 0.9905362129211426, "step_time": 3.584039900000789, "tokens_per_second": 2285.688839568498, "steps_per_second": 0.27901475092388894, "batch_wait": 0.001220599997395766, "processed_target_tokens": 25445252, "elapsed_seconds": 9316.8798149, "eta_seconds": 55911.02244001231, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3713.02, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 4450, "max_steps": 20000, "progress_percent": 22.25, "loss": 1.8737174272537231, "perplexity": 6.512461056501162, "learning_rate": 0.00027357455627372685, "gradient_norm": 1.0792056322097778, "step_time": 3.4996956000013597, "tokens_per_second": 2340.7750091170265, "steps_per_second": 0.28573913685510577, "batch_wait": 0.0010873000010178657, "processed_target_tokens": 25737359, "elapsed_seconds": 9496.4600747, "eta_seconds": 54420.26658002114, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3713.02, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 4500, "max_steps": 20000, "progress_percent": 22.5, "loss": 2.068452835083008, "perplexity": 7.91257158955945, "learning_rate": 0.0002729247730594727, "gradient_norm": 1.0213383436203003, "step_time": 3.6340846000002784, "tokens_per_second": 2254.2127940553096, "steps_per_second": 0.2751724602118298, "batch_wait": 0.0006494999997812556, "processed_target_tokens": 26028421, "elapsed_seconds": 9674.6191449, "eta_seconds": 56328.311300004316, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3606.36, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "validation", "step": 4500, "validation_loss": 1.8244182229042054, "validation_perplexity": 6.199187424542538, "validation_time": 2.396307999999408, "best_validation_loss": 1.8589496612548828}
{"step": 4500, "val_loss": 1.8244182229042054, "best_val_loss": 1.8244182229042054, "processed_target_tokens": 26028421, "timestamp": 1793418721.5273426}
{"event": "checkpoint", "step": 4500, "max_steps": 20000, "progress_percent": 22.5, "train_loss": 2.068452835083008, "train_perplexity": 7.91257158955945, "validation_loss": 1.8244182229042054, "validation_perplexity": 6.199187424542538, "best_validation_loss": 1.8244182229042054, "best_validation_perplexity": 6.199187424542538, "processed_target_tokens": 26028421, "tokens_per_step": 8192, "tokens_per_second": 2254.2127940553096, "steps_per_second": 0.2751724602118298, "elapsed_seconds": 9674.6191449, "eta_seconds": 56328.311300004316, "learning_rate": 0.0002729247730594727, "gradient_norm": 1.0213383436203003, "batch_wait": 0.0006494999997812556, "evaluation_time": 2.396307999999408, "parameter_count": 62113280, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3674.11, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}, "is_best": true, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"step": 4500, "val_loss": 1.8244182229042054, "best_val_loss": 1.8244182229042054, "processed_target_tokens": 26028421, "timestamp": 1793418722.3292763}
{"event": "checkpoint", "step": 4500, "max_steps": 20000, "progress_percent": 22.5, "train_loss": 2.068452835083008, "train_perplexity": 7.91257158955945, "validation_loss": 1.8244182229042054, "validation_perplexity": 6.199187424542538, "best_validation_loss": 1.8244182229042054, "best_validation_perplexity": 6.199187424542538, "processed_target_tokens": 26028421, "tokens_per_step": 8192, "tokens_per_second": 2254.2127940553096, "steps_per_second": 0.2751724602118298, "elapsed_seconds": 9674.6191449, "eta_seconds": 56328.311300004316, "learning_rate": 0.0002729247730594727, "gradient_norm": 1.0213383436203003, "batch_wait": 0.0006494999997812556, "evaluation_time": 2.396307999999408, "parameter_count": 62113280, "gpu": {"gpu_utilization": 12, "gpu_memory_utilization": 10, "gpu_memory_used_mb": 3650.55, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 4550, "max_steps": 20000, "progress_percent": 22.75, "loss": 1.8783780336380005, "perplexity": 6.54288391349068, "learning_rate": 0.0002722679867613568, "gradient_norm": 1.0176291465759277, "step_time": 3.5446711999993568, "tokens_per_second": 2311.07471970926, "steps_per_second": 0.2821136132457593, "batch_wait": 0.0009470000004512258, "processed_target_tokens": 26314109, "elapsed_seconds": 9856.2993806, "eta_seconds": 54765.17003999006, "gpu": {"gpu_utilization": 1, "gpu_memory_utilization": 6, "gpu_memory_used_mb": 3650.55, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 4600, "max_steps": 20000, "progress_percent": 23.0, "loss": 1.824805736541748, "perplexity": 6.201590159727489, "learning_rate": 0.00027160423999730106, "gradient_norm": 1.002840280532837, "step_time": 3.5331997999983287, "tokens_per_second": 2318.578190795741, "steps_per_second": 0.28302956430612075, "batch_wait": 0.00043849999929079786, "processed_target_tokens": 26602714, "elapsed_seconds": 10035.9273214, "eta_seconds": 54411.27691997426, "gpu": {"gpu_utilization": 1, "gpu_memory_utilization": 6, "gpu_memory_used_mb": 3650.55, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 4650, "max_steps": 20000, "progress_percent": 23.25, "loss": 1.8991436958312988, "perplexity": 6.680171733536211, "learning_rate": 0.00027093357583688165, "gradient_norm": 1.158972978591919, "step_time": 3.463609099999303, "tokens_per_second": 2365.1629740785843, "steps_per_second": 0.2887161833592022, "batch_wait": 0.0006629999988945201, "processed_target_tokens": 26896684, "elapsed_seconds": 10210.436575, "eta_seconds": 53166.3996849893, "gpu": {"gpu_utilization": 1, "gpu_memory_utilization": 6, "gpu_memory_used_mb": 3650.55, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 4700, "max_steps": 20000, "progress_percent": 23.5, "loss": 1.6891722679138184, "perplexity": 5.414996683101597, "learning_rate": 0.000270256037798535, "gradient_norm": 0.9455683827400208, "step_time": 3.542094899999938, "tokens_per_second": 2312.755652029578, "steps_per_second": 0.28231880517939184, "batch_wait": 0.0008534999997209525, "processed_target_tokens": 27185309, "elapsed_seconds": 10385.5956217, "eta_seconds": 54194.051969999055, "gpu": {"gpu_utilization": 1, "gpu_memory_utilization": 6, "gpu_memory_used_mb": 3650.55, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 4750, "max_steps": 20000, "progress_percent": 23.75, "loss": 1.8972485065460205, "perplexity": 6.667523532796011, "learning_rate": 0.00026957166984673334, "gradient_norm": 0.9144791960716248, "step_time": 3.5379585000009683, "tokens_per_second": 2315.459607566838, "steps_per_second": 0.28264887787681126, "batch_wait": 0.0013625999999931082, "processed_target_tokens": 27477300, "elapsed_seconds": 10563.6893652, "eta_seconds": 53953.86712501477, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3659.98, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 4800, "max_steps": 20000, "progress_percent": 24.0, "loss": 2.0482680797576904, "perplexity": 7.7544593645127815, "learning_rate": 0.0002688805163891324, "gradient_norm": 0.9617116451263428, "step_time": 3.458792400000675, "tokens_per_second": 2368.4566902594097, "steps_per_second": 0.28911824832268185, "batch_wait": 0.0012955000001966255, "processed_target_tokens": 27771177, "elapsed_seconds": 10739.2008762, "eta_seconds": 52573.64448001026, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3659.98, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 4850, "max_steps": 20000, "progress_percent": 24.25, "loss": 2.021247386932373, "perplexity": 7.547734008834839, "learning_rate": 0.00026818262227368945, "gradient_norm": 1.0789895057678223, "step_time": 3.5449941999995644, "tokens_per_second": 2310.8641475354193, "steps_per_second": 0.28208790863469474, "batch_wait": 0.0011211000000912463, "processed_target_tokens": 28060674, "elapsed_seconds": 10913.2130022, "eta_seconds": 53706.662129993405, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3659.98, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 4900, "max_steps": 20000, "progress_percent": 24.5, "loss": 1.939357042312622, "perplexity": 6.954278226252929, "learning_rate": 0.00026747803278575374, "gradient_norm": 1.0795600414276123, "step_time": 3.538090399999419, "tokens_per_second": 2315.3732872402993, "steps_per_second": 0.2826383407275756, "batch_wait": 0.000703499998053303, "processed_target_tokens": 28351081, "elapsed_seconds": 11089.9973688, "eta_seconds": 53425.165039991225, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3659.98, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 4950, "max_steps": 20000, "progress_percent": 24.75, "loss": 2.314774513244629, "perplexity": 10.122640140103822, "learning_rate": 0.00026676679364512725, "gradient_norm": 1.0625371932983398, "step_time": 3.450290799999493, "tokens_per_second": 2374.2926248422896, "steps_per_second": 0.28983064268094355, "batch_wait": 0.0007480000003852183, "processed_target_tokens": 28641193, "elapsed_seconds": 11266.7287839, "eta_seconds": 51926.87653999237, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3659.98, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 5000, "max_steps": 20000, "progress_percent": 25.0, "loss": 2.030819892883301, "perplexity": 7.620331653885028, "learning_rate": 0.0002660489510030986, "gradient_norm": 1.0044598579406738, "step_time": 3.4645189000002574, "tokens_per_second": 2364.541870445386, "steps_per_second": 0.2886403650446028, "batch_wait": 0.0006379000005836133, "processed_target_tokens": 28920143, "elapsed_seconds": 11440.1983851, "eta_seconds": 51967.78350000386, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3526.52, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "validation", "step": 5000, "validation_loss": 2.0035398483276365, "validation_perplexity": 7.415258585859123, "validation_time": 2.3234971999991103, "best_validation_loss": 1.8244182229042054}
{"step": 5000, "val_loss": 2.0035398483276365, "best_val_loss": 1.8244182229042054, "processed_target_tokens": 28920143, "timestamp": 1793420486.5517077}
{"event": "checkpoint", "step": 5000, "max_steps": 20000, "progress_percent": 25.0, "train_loss": 2.030819892883301, "train_perplexity": 7.620331653885028, "validation_loss": 2.0035398483276365, "validation_perplexity": 7.415258585859123, "best_validation_loss": 1.8244182229042054, "best_validation_perplexity": 6.199187424542538, "processed_target_tokens": 28920143, "tokens_per_step": 8192, "tokens_per_second": 2364.541870445386, "steps_per_second": 0.2886403650446028, "elapsed_seconds": 11440.1983851, "eta_seconds": 51967.78350000386, "learning_rate": 0.0002660489510030986, "gradient_norm": 1.0044598579406738, "batch_wait": 0.0006379000005836133, "evaluation_time": 2.3234971999991103, "parameter_count": 62113280, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3521.89, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 5050, "max_steps": 20000, "progress_percent": 25.25, "loss": 1.573365330696106, "perplexity": 4.82285140377468, "learning_rate": 0.0002653245514394482, "gradient_norm": 0.9009289741516113, "step_time": 3.527725300000384, "tokens_per_second": 2322.1762760266815, "steps_per_second": 0.2834687836946633, "batch_wait": 0.0007800000021234155, "processed_target_tokens": 29208424, "elapsed_seconds": 11621.362784, "eta_seconds": 52739.493235005735, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3521.89, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 5100, "max_steps": 20000, "progress_percent": 25.5, "loss": 1.9024325609207153, "perplexity": 6.702178085266704, "learning_rate": 0.00026459364195942554, "gradient_norm": 1.0745196342468262, "step_time": 3.5334286999986944, "tokens_per_second": 2318.427990354815, "steps_per_second": 0.28301122929135925, "batch_wait": 0.0006878000003780471, "processed_target_tokens": 29495267, "elapsed_seconds": 11798.2632565, "eta_seconds": 52648.087629980546, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3521.89, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 5150, "max_steps": 20000, "progress_percent": 25.75, "loss": 1.738459587097168, "perplexity": 5.688573917415442, "learning_rate": 0.0002638562699906994, "gradient_norm": 0.960809588432312, "step_time": 3.5461989000014, "tokens_per_second": 2310.079110338894, "steps_per_second": 0.2819920788987908, "batch_wait": 0.0006712000013067154, "processed_target_tokens": 29779822, "elapsed_seconds": 11975.0746861, "eta_seconds": 52661.053665020794, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3521.89, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 5200, "max_steps": 20000, "progress_percent": 26.0, "loss": 1.6478837728500366, "perplexity": 5.195972326394282, "learning_rate": 0.0002631124833802801, "gradient_norm": 1.0001945495605469, "step_time": 3.465295599999081, "tokens_per_second": 2364.011889779958, "steps_per_second": 0.288575670139155, "batch_wait": 0.0006843999999546213, "processed_target_tokens": 30076361, "elapsed_seconds": 12149.2717993, "eta_seconds": 51286.3748799864, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3521.89, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 5250, "max_steps": 20000, "progress_percent": 26.25, "loss": 1.8066110610961914, "perplexity": 6.089774548563212, "learning_rate": 0.00026236233039141494, "gradient_norm": 0.961996853351593, "step_time": 3.5311567000007926, "tokens_per_second": 2319.919702231895, "steps_per_second": 0.28319332302635436, "batch_wait": 0.0012623999991774326, "processed_target_tokens": 30363982, "elapsed_seconds": 12326.6754833, "eta_seconds": 52084.56132501169, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 43, "gpu_memory_used_mb": 3691.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 5300, "max_steps": 20000, "progress_percent": 26.5, "loss": 1.7983989715576172, "perplexity": 6.039969556126522, "learning_rate": 0.0002616058597004563, "gradient_norm": 0.9413132071495056, "step_time": 3.5311854999999923, "tokens_per_second": 2319.900781196575, "steps_per_second": 0.28319101332966, "batch_wait": 0.0011283000021649059, "processed_target_tokens": 30650453, "elapsed_seconds": 12505.5211677, "eta_seconds": 51908.42684999989, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 43, "gpu_memory_used_mb": 3691.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 5350, "max_steps": 20000, "progress_percent": 26.75, "loss": 1.8828868865966797, "perplexity": 6.572451422666263, "learning_rate": 0.00026084312039370333, "gradient_norm": 0.9721601605415344, "step_time": 3.458734599998934, "tokens_per_second": 2368.4962702840876, "steps_per_second": 0.28912307986866304, "batch_wait": 0.0006059999996068655, "processed_target_tokens": 30939694, "elapsed_seconds": 12679.2904623, "eta_seconds": 50670.46188998438, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 43, "gpu_memory_used_mb": 3691.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 5400, "max_steps": 20000, "progress_percent": 27.0, "loss": 1.8941490650177002, "perplexity": 6.646889926313938, "learning_rate": 0.00026007416196421656, "gradient_norm": 1.0522515773773193, "step_time": 3.5288600999992923, "tokens_per_second": 2321.4295177078975, "steps_per_second": 0.28337762667332733, "batch_wait": 0.0006843000028311508, "processed_target_tokens": 31234182, "elapsed_seconds": 12853.1775531, "eta_seconds": 51521.35745998967, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 43, "gpu_memory_used_mb": 3691.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 5450, "max_steps": 20000, "progress_percent": 27.250000000000004, "loss": 1.9846075773239136, "perplexity": 7.276191482596223, "learning_rate": 0.0002592990343086065, "gradient_norm": 1.033388614654541, "step_time": 3.535883400001694, "tokens_per_second": 2316.818478798276, "steps_per_second": 0.28281475571268017, "batch_wait": 0.0007029000007605646, "processed_target_tokens": 31519965, "elapsed_seconds": 13031.7455176, "eta_seconds": 51447.10347002465, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 43, "gpu_memory_used_mb": 3691.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 5500, "max_steps": 20000, "progress_percent": 27.500000000000004, "loss": 2.1096980571746826, "perplexity": 8.245751163305334, "learning_rate": 0.0002585177877237959, "gradient_norm": 1.0561102628707886, "step_time": 3.4602763000002597, "tokens_per_second": 2367.4410046386715, "steps_per_second": 0.2889942632615566, "batch_wait": 0.0008271999995486112, "processed_target_tokens": 31813012, "elapsed_seconds": 13209.1566932, "eta_seconds": 50174.006350003765, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3556.23, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "validation", "step": 5500, "validation_loss": 1.7975844025611878, "validation_perplexity": 6.0350515874710515, "validation_time": 2.324402199999895, "best_validation_loss": 1.8244182229042054}
{"step": 5500, "val_loss": 1.7975844025611878, "best_val_loss": 1.7975844025611878, "processed_target_tokens": 31813012, "timestamp": 1793422254.6787527}
{"event": "checkpoint", "step": 5500, "max_steps": 20000, "progress_percent": 27.500000000000004, "train_loss": 2.1096980571746826, "train_perplexity": 8.245751163305334, "validation_loss": 1.7975844025611878, "validation_perplexity": 6.0350515874710515, "best_validation_loss": 1.7975844025611878, "best_validation_perplexity": 6.0350515874710515, "processed_target_tokens": 31813012, "tokens_per_step": 8192, "tokens_per_second": 2367.4410046386715, "steps_per_second": 0.2889942632615566, "elapsed_seconds": 13209.1566932, "eta_seconds": 50174.006350003765, "learning_rate": 0.0002585177877237959, "gradient_norm": 1.0561102628707886, "batch_wait": 0.0008271999995486112, "evaluation_time": 2.324402199999895, "parameter_count": 62113280, "gpu": {"gpu_utilization": 11, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3543.42, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}, "is_best": true, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"step": 5500, "val_loss": 1.7975844025611878, "best_val_loss": 1.7975844025611878, "processed_target_tokens": 31813012, "timestamp": 1793422255.6917496}
{"event": "checkpoint", "step": 5500, "max_steps": 20000, "progress_percent": 27.500000000000004, "train_loss": 2.1096980571746826, "train_perplexity": 8.245751163305334, "validation_loss": 1.7975844025611878, "validation_perplexity": 6.0350515874710515, "best_validation_loss": 1.7975844025611878, "best_validation_perplexity": 6.0350515874710515, "processed_target_tokens": 31813012, "tokens_per_step": 8192, "tokens_per_second": 2367.4410046386715, "steps_per_second": 0.2889942632615566, "elapsed_seconds": 13209.1566932, "eta_seconds": 50174.006350003765, "learning_rate": 0.0002585177877237959, "gradient_norm": 1.0561102628707886, "batch_wait": 0.0008271999995486112, "evaluation_time": 2.324402199999895, "parameter_count": 62113280, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3551.86, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 5550, "max_steps": 20000, "progress_percent": 27.750000000000004, "loss": 1.7931571006774902, "perplexity": 6.008391651548667, "learning_rate": 0.0002577304729037561, "gradient_norm": 0.9856989979743958, "step_time": 3.5348470999997517, "tokens_per_second": 2317.4976931818564, "steps_per_second": 0.28289766762473834, "batch_wait": 0.0007862000002205605, "processed_target_tokens": 32105432, "elapsed_seconds": 13388.0953612, "eta_seconds": 51078.54059499641, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3551.86, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 5600, "max_steps": 20000, "progress_percent": 28.000000000000004, "loss": 1.767991304397583, "perplexity": 5.859072440156681, "learning_rate": 0.0002569371409362176, "gradient_norm": 1.0156276226043701, "step_time": 3.5343432999998186, "tokens_per_second": 2317.8280389458546, "steps_per_second": 0.28293799303538264, "batch_wait": 0.0012530000021797605, "processed_target_tokens": 32393535, "elapsed_seconds": 13564.7575065, "eta_seconds": 50894.54351999739, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3551.86, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 5650, "max_steps": 20000, "progress_percent": 28.249999999999996, "loss": 1.5180463790893555, "perplexity": 4.563301519970162, "learning_rate": 0.00025613784329935475, "gradient_norm": 0.8507628440856934, "step_time": 3.542253999999957, "tokens_per_second": 2312.6517748304045, "steps_per_second": 0.2823061248572271, "batch_wait": 0.0009855000007519266, "processed_target_tokens": 32684773, "elapsed_seconds": 13742.4408575, "eta_seconds": 50831.344899999385, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3551.86, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 5700, "max_steps": 20000, "progress_percent": 28.499999999999996, "loss": 1.8484454154968262, "perplexity": 6.349940326741022, "learning_rate": 0.00025533263185844587, "gradient_norm": 0.998304545879364, "step_time": 3.4533182000013767, "tokens_per_second": 2372.2111677970292, "steps_per_second": 0.2895765585689733, "batch_wait": 0.0004957000001013512, "processed_target_tokens": 32973131, "elapsed_seconds": 13921.451554, "eta_seconds": 49382.45026001969, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3551.86, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 5750, "max_steps": 20000, "progress_percent": 28.749999999999996, "loss": 1.9341535568237305, "perplexity": 6.918185725256964, "learning_rate": 0.00025452155886250734, "gradient_norm": 1.0091739892959595, "step_time": 3.5423059000004287, "tokens_per_second": 2312.6178910745707, "steps_per_second": 0.2823019886565638, "batch_wait": 0.001045499999236199, "processed_target_tokens": 33258005, "elapsed_seconds": 14096.915354, "eta_seconds": 50477.85907500611, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3727.72, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 5800, "max_steps": 20000, "progress_percent": 28.999999999999996, "loss": 1.7990792989730835, "perplexity": 6.044080111107214, "learning_rate": 0.00025370467694090366, "gradient_norm": 1.072853922843933, "step_time": 3.5176418000010017, "tokens_per_second": 2328.832913003725, "steps_per_second": 0.28428136145065, "batch_wait": 0.0005382999988796655, "processed_target_tokens": 33547231, "elapsed_seconds": 14277.2279834, "eta_seconds": 49950.513560014224, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3727.72, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 5850, "max_steps": 20000, "progress_percent": 29.25, "loss": 2.061413288116455, "perplexity": 7.857066265610647, "learning_rate": 0.000252882039099932, "gradient_norm": 1.0136151313781738, "step_time": 3.5472573999995802, "tokens_per_second": 2309.389783780836, "steps_per_second": 0.28190793259043406, "batch_wait": 0.0009196999981213594, "processed_target_tokens": 33838910, "elapsed_seconds": 14454.0854404, "eta_seconds": 50193.69220999406, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3727.72, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 5900, "max_steps": 20000, "progress_percent": 29.5, "loss": 1.611393928527832, "perplexity": 5.009789651705536, "learning_rate": 0.000252053698719383, "gradient_norm": 0.9144011735916138, "step_time": 3.530939300000682, "tokens_per_second": 2320.0625397322515, "steps_per_second": 0.2832107592446596, "batch_wait": 0.0006583000013051787, "processed_target_tokens": 34126922, "elapsed_seconds": 14630.6326918, "eta_seconds": 49786.24413000962, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3727.72, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 5950, "max_steps": 20000, "progress_percent": 29.75, "loss": 1.961545467376709, "perplexity": 7.110307326164506, "learning_rate": 0.00025121970954907695, "gradient_norm": 1.0106053352355957, "step_time": 3.455852400000367, "tokens_per_second": 2370.471609261764, "steps_per_second": 0.28936421011496144, "batch_wait": 0.0008022000001801644, "processed_target_tokens": 34425687, "elapsed_seconds": 14804.34762, "eta_seconds": 48554.72622000516, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3727.72, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 6000, "max_steps": 20000, "progress_percent": 30.0, "loss": 1.8175866603851318, "perplexity": 6.156981618033394, "learning_rate": 0.0002503801257053759, "gradient_norm": 0.9714615345001221, "step_time": 3.5271235999989585, "tokens_per_second": 2322.572421335736, "steps_per_second": 0.283517141276335, "batch_wait": 0.0006768000002921326, "processed_target_tokens": 34715559, "elapsed_seconds": 14981.7674517, "eta_seconds": 49379.73039998542, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3596.68, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "validation", "step": 6000, "validation_loss": 1.9161961078643799, "validation_perplexity": 6.795061564754245, "validation_time": 2.361910900001021, "best_validation_loss": 1.7975844025611878}
{"step": 6000, "val_loss": 1.9161961078643799, "best_val_loss": 1.7975844025611878, "processed_target_tokens": 34715559, "timestamp": 1793424028.1110706}
{"event": "checkpoint", "step": 6000, "max_steps": 20000, "progress_percent": 30.0, "train_loss": 1.8175866603851318, "train_perplexity": 6.156981618033394, "validation_loss": 1.9161961078643799, "validation_perplexity": 6.795061564754245, "best_validation_loss": 1.7975844025611878, "best_validation_perplexity": 6.0350515874710515, "processed_target_tokens": 34715559, "tokens_per_step": 8192, "tokens_per_second": 2322.572421335736, "steps_per_second": 0.283517141276335, "elapsed_seconds": 14981.7674517, "eta_seconds": 49379.73039998542, "learning_rate": 0.0002503801257053759, "gradient_norm": 0.9714615345001221, "batch_wait": 0.0006768000002921326, "evaluation_time": 2.361910900001021, "parameter_count": 62113280, "gpu": {"gpu_utilization": 5, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3601.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 6050, "max_steps": 20000, "progress_percent": 30.25, "loss": 1.6880125999450684, "perplexity": 5.408720724614673, "learning_rate": 0.00024953500166767235, "gradient_norm": 0.8636212348937988, "step_time": 3.529211799999757, "tokens_per_second": 2321.1981780182655, "steps_per_second": 0.2833493869651203, "batch_wait": 0.0006187999970279634, "processed_target_tokens": 35001986, "elapsed_seconds": 15162.860473, "eta_seconds": 49232.50460999661, "gpu": {"gpu_utilization": 5, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3601.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 6100, "max_steps": 20000, "progress_percent": 30.5, "loss": 1.737966537475586, "perplexity": 5.685769859524765, "learning_rate": 0.0002486843922748539, "gradient_norm": 0.994300127029419, "step_time": 3.533215100000234, "tokens_per_second": 2318.5681505774887, "steps_per_second": 0.2830283386935411, "batch_wait": 0.0005650999992212746, "processed_target_tokens": 35293548, "elapsed_seconds": 15339.7624606, "eta_seconds": 49111.689890003254, "gpu": {"gpu_utilization": 5, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3601.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 6150, "max_steps": 20000, "progress_percent": 30.75, "loss": 1.9747560024261475, "perplexity": 7.2048614706141985, "learning_rate": 0.00024782835272174516, "gradient_norm": 1.0714508295059204, "step_time": 3.463257800001884, "tokens_per_second": 2365.40288741876, "steps_per_second": 0.2887454696556104, "batch_wait": 0.0009052000023075379, "processed_target_tokens": 35580226, "elapsed_seconds": 15516.260590499998, "eta_seconds": 47966.12053002609, "gpu": {"gpu_utilization": 5, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3601.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 6200, "max_steps": 20000, "progress_percent": 31.0, "loss": 1.9090845584869385, "perplexity": 6.746909569496642, "learning_rate": 0.000246966938555526, "gradient_norm": 0.9984718561172485, "step_time": 3.585176099997625, "tokens_per_second": 2284.9644679951502, "steps_per_second": 0.27892632665956424, "batch_wait": 0.0007932999978947919, "processed_target_tokens": 35869495, "elapsed_seconds": 15692.904774800001, "eta_seconds": 49475.430179967225, "gpu": {"gpu_utilization": 5, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3601.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 6250, "max_steps": 20000, "progress_percent": 31.25, "loss": 1.8477189540863037, "perplexity": 6.345329015307229, "learning_rate": 0.00024610020567212713, "gradient_norm": 0.9784072637557983, "step_time": 3.5480085000017425, "tokens_per_second": 2308.9008946838703, "steps_per_second": 0.28184825374558964, "batch_wait": 0.0008877999971446116, "processed_target_tokens": 36155698, "elapsed_seconds": 15870.9241277, "eta_seconds": 48785.11687502396, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3640.06, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 6300, "max_steps": 20000, "progress_percent": 31.5, "loss": 1.8143718242645264, "perplexity": 6.137219713795833, "learning_rate": 0.00024522821031260333, "gradient_norm": 0.9772037863731384, "step_time": 3.4660055999993347, "tokens_per_second": 2363.5276296153625, "steps_per_second": 0.28851655634953155, "batch_wait": 0.0009298000004491769, "processed_target_tokens": 36439336, "elapsed_seconds": 16045.67858, "eta_seconds": 47484.276719990885, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3640.06, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 6350, "max_steps": 20000, "progress_percent": 31.75, "loss": 1.7754642963409424, "perplexity": 5.903021251324602, "learning_rate": 0.00024435100905948387, "gradient_norm": 0.9776423573493958, "step_time": 3.5333799, "tokens_per_second": 2318.460010484579, "steps_per_second": 0.28301513799860584, "batch_wait": 0.0005180999978620093, "processed_target_tokens": 36728526, "elapsed_seconds": 16219.4571831, "eta_seconds": 48230.635635, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3640.06, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 6400, "max_steps": 20000, "progress_percent": 32.0, "loss": 1.6077542304992676, "perplexity": 4.991588673312257, "learning_rate": 0.00024346865883310094, "gradient_norm": 0.9769265651702881, "step_time": 3.542154100003245, "tokens_per_second": 2312.716998956227, "steps_per_second": 0.2823140867866488, "batch_wait": 0.000794700001279125, "processed_target_tokens": 37013014, "elapsed_seconds": 16397.1251231, "eta_seconds": 48173.29576004413, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3640.06, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 6450, "max_steps": 20000, "progress_percent": 32.25, "loss": 1.922579050064087, "perplexity": 6.838572766982447, "learning_rate": 0.0002425812168878963, "gradient_norm": 0.9413654208183289, "step_time": 3.5477059000004374, "tokens_per_second": 2309.0978313616665, "steps_per_second": 0.28187229386739093, "batch_wait": 0.0008116000026348047, "processed_target_tokens": 37304249, "elapsed_seconds": 16573.9132068, "eta_seconds": 48071.41494500593, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3640.06, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 6500, "max_steps": 20000, "progress_percent": 32.5, "loss": 2.004326581954956, "perplexity": 7.4210947145822255, "learning_rate": 0.000241688740808706, "gradient_norm": 1.0049556493759155, "step_time": 3.460172100003547, "tokens_per_second": 2367.512298013039, "steps_per_second": 0.2890029660660448, "batch_wait": 0.0006741000033798628, "processed_target_tokens": 37597672, "elapsed_seconds": 16750.4050546, "eta_seconds": 46712.32335004788, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3513.0, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "validation", "step": 6500, "validation_loss": 1.8005013465881348, "validation_perplexity": 6.0526811949403285, "validation_time": 2.315755799998442, "best_validation_loss": 1.7975844025611878}
{"step": 6500, "val_loss": 1.8005013465881348, "best_val_loss": 1.7975844025611878, "processed_target_tokens": 37597672, "timestamp": 1793425796.7199705}
{"event": "checkpoint", "step": 6500, "max_steps": 20000, "progress_percent": 32.5, "train_loss": 2.004326581954956, "train_perplexity": 7.4210947145822255, "validation_loss": 1.8005013465881348, "validation_perplexity": 6.0526811949403285, "best_validation_loss": 1.7975844025611878, "best_validation_perplexity": 6.0350515874710515, "processed_target_tokens": 37597672, "tokens_per_step": 8192, "tokens_per_second": 2367.512298013039, "steps_per_second": 0.2890029660660448, "elapsed_seconds": 16750.4050546, "eta_seconds": 46712.32335004788, "learning_rate": 0.000241688740808706, "gradient_norm": 1.0049556493759155, "batch_wait": 0.0006741000033798628, "evaluation_time": 2.315755799998442, "parameter_count": 62113280, "gpu": {"gpu_utilization": 4, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3487.03, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 6550, "max_steps": 20000, "progress_percent": 32.75, "loss": 1.9755154848098755, "perplexity": 7.210335514435131, "learning_rate": 0.00024079128850702393, "gradient_norm": 1.0482518672943115, "step_time": 3.4599310999983572, "tokens_per_second": 2367.6772060587823, "steps_per_second": 0.28902309644272245, "batch_wait": 0.0012093999976059422, "processed_target_tokens": 37893756, "elapsed_seconds": 16927.8365886, "eta_seconds": 46536.073294977905, "gpu": {"gpu_utilization": 4, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3487.03, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 6600, "max_steps": 20000, "progress_percent": 33.0, "loss": 1.9100279808044434, "perplexity": 6.753277758031954, "learning_rate": 0.00023988891821724373, "gradient_norm": 0.9715063571929932, "step_time": 3.31660009999905, "tokens_per_second": 2469.9993224996724, "steps_per_second": 0.3015135891723233, "batch_wait": 0.0012367999988782685, "processed_target_tokens": 38186000, "elapsed_seconds": 17103.0721908, "eta_seconds": 44442.44133998727, "gpu": {"gpu_utilization": 4, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3487.03, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 6650, "max_steps": 20000, "progress_percent": 33.25, "loss": 1.7628531455993652, "perplexity": 5.829044804990264, "learning_rate": 0.00023898168849288045, "gradient_norm": 1.0584192276000977, "step_time": 3.5392683999998553, "tokens_per_second": 2314.6026449987053, "steps_per_second": 0.2825442681883185, "batch_wait": 0.0006840999994892627, "processed_target_tokens": 38478068, "elapsed_seconds": 17279.9515149, "eta_seconds": 47249.23313999807, "gpu": {"gpu_utilization": 4, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3487.03, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 6700, "max_steps": 20000, "progress_percent": 33.5, "loss": 1.7282590866088867, "perplexity": 5.630842561335275, "learning_rate": 0.00023806965820277073, "gradient_norm": 0.9100927114486694, "step_time": 3.45103829999789, "tokens_per_second": 2373.7783495491803, "steps_per_second": 0.2897678649352027, "batch_wait": 0.0007076000001688953, "processed_target_tokens": 38771487, "elapsed_seconds": 17453.9926983, "eta_seconds": 45898.80938997194, "gpu": {"gpu_utilization": 4, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3487.03, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 6750, "max_steps": 20000, "progress_percent": 33.75, "loss": 1.8195008039474487, "perplexity": 6.1687782513838965, "learning_rate": 0.00023715288652725313, "gradient_norm": 1.1504729986190796, "step_time": 3.525378300000739, "tokens_per_second": 2323.722251310812, "steps_per_second": 0.2836575013807144, "batch_wait": 0.0005954999978712294, "processed_target_tokens": 39057519, "elapsed_seconds": 17628.3356971, "eta_seconds": 46711.26247500979, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3629.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 6800, "max_steps": 20000, "progress_percent": 34.0, "loss": 2.034060001373291, "perplexity": 7.645062398685518, "learning_rate": 0.0002362314329543279, "gradient_norm": 0.9792953133583069, "step_time": 3.5442763999999443, "tokens_per_second": 2311.3321523118593, "steps_per_second": 0.28214503812400626, "batch_wait": 0.0007999000008567236, "processed_target_tokens": 39353382, "elapsed_seconds": 17805.364548, "eta_seconds": 46784.448479999264, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3629.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 6850, "max_steps": 20000, "progress_percent": 34.25, "loss": 2.0238122940063477, "perplexity": 7.567118093747919, "learning_rate": 0.0002353053572757969, "gradient_norm": 1.067991018295288, "step_time": 3.4687144000017724, "tokens_per_second": 2361.6818957466817, "steps_per_second": 0.28829124703938985, "batch_wait": 0.001006399998004781, "processed_target_tokens": 39644906, "elapsed_seconds": 17980.1855853, "eta_seconds": 45613.59436002331, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3629.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 6900, "max_steps": 20000, "progress_percent": 34.5, "loss": 1.823479175567627, "perplexity": 6.193368826499643, "learning_rate": 0.00023437471958338374, "gradient_norm": 0.9522172808647156, "step_time": 3.5412407999974675, "tokens_per_second": 2313.31345781565, "steps_per_second": 0.28238689670601197, "batch_wait": 0.001352999999653548, "processed_target_tokens": 39933717, "elapsed_seconds": 18154.0715384, "eta_seconds": 46390.254479966825, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3629.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 6950, "max_steps": 20000, "progress_percent": 34.75, "loss": 2.005265235900879, "perplexity": 7.4280638246988, "learning_rate": 0.0002334395802648347, "gradient_norm": 0.9704801440238953, "step_time": 3.5324261999994633, "tokens_per_second": 2319.0859585406893, "steps_per_second": 0.283091547673424, "batch_wait": 0.0004627999987860676, "processed_target_tokens": 40222257, "elapsed_seconds": 18330.7755822, "eta_seconds": 46098.161909992996, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3629.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 7000, "max_steps": 20000, "progress_percent": 35.0, "loss": 2.056915044784546, "perplexity": 7.821802641315314, "learning_rate": 0.00023249999999999999, "gradient_norm": 1.0139217376708984, "step_time": 3.46514560000287, "tokens_per_second": 2364.114223654329, "steps_per_second": 0.28858816206717885, "batch_wait": 0.0004886000060650986, "processed_target_tokens": 40512209, "elapsed_seconds": 18506.3904064, "eta_seconds": 45046.89280003731, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 37, "gpu_memory_used_mb": 3540.38, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "validation", "step": 7000, "validation_loss": 1.8915138363838195, "validation_perplexity": 6.629396910824768, "validation_time": 2.326350499999535, "best_validation_loss": 1.7975844025611878}
{"step": 7000, "val_loss": 1.8915138363838195, "best_val_loss": 1.7975844025611878, "processed_target_tokens": 40512209, "timestamp": 1793427552.5742016}
{"event": "checkpoint", "step": 7000, "max_steps": 20000, "progress_percent": 35.0, "train_loss": 2.056915044784546, "train_perplexity": 7.821802641315314, "validation_loss": 1.8915138363838195, "validation_perplexity": 6.629396910824768, "best_validation_loss": 1.7975844025611878, "best_validation_perplexity": 6.0350515874710515, "processed_target_tokens": 40512209, "tokens_per_step": 8192, "tokens_per_second": 2364.114223654329, "steps_per_second": 0.28858816206717885, "elapsed_seconds": 18506.3904064, "eta_seconds": 45046.89280003731, "learning_rate": 0.00023249999999999999, "gradient_norm": 1.0139217376708984, "batch_wait": 0.0004886000060650986, "evaluation_time": 2.326350499999535, "parameter_count": 62113280, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3526.7, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 7050, "max_steps": 20000, "progress_percent": 35.25, "loss": 1.7338122129440308, "perplexity": 5.662198322125197, "learning_rate": 0.00023155603975689654, "gradient_norm": 0.9105226397514343, "step_time": 3.541817000001174, "tokens_per_second": 2312.937116739031, "steps_per_second": 0.2823409566331825, "batch_wait": 0.0008445999992545694, "processed_target_tokens": 40801512, "elapsed_seconds": 18683.881448, "eta_seconds": 45866.530150015205, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3526.7, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 7100, "max_steps": 20000, "progress_percent": 35.5, "loss": 1.7679145336151123, "perplexity": 5.858622651846419, "learning_rate": 0.00023060776078775178, "gradient_norm": 0.9756355881690979, "step_time": 3.316032200000336, "tokens_per_second": 2470.4223318456225, "steps_per_second": 0.30156522605537384, "batch_wait": 0.00087340000027325, "processed_target_tokens": 41085878, "elapsed_seconds": 18858.4393316, "eta_seconds": 42776.81538000434, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3526.7, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 7150, "max_steps": 20000, "progress_percent": 35.75, "loss": 1.9090545177459717, "perplexity": 6.746706890378269, "learning_rate": 0.0002296552246250291, "gradient_norm": 1.047318696975708, "step_time": 3.3039742000000842, "tokens_per_second": 2479.4382474293507, "steps_per_second": 0.30266580168815316, "batch_wait": 0.00126290000116569, "processed_target_tokens": 41372980, "elapsed_seconds": 19023.9816225, "eta_seconds": 42456.06847000108, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3526.7, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 7200, "max_steps": 20000, "progress_percent": 36.0, "loss": 1.7524526119232178, "perplexity": 5.7687338051466055, "learning_rate": 0.00022869849307743505, "gradient_norm": 0.9602329730987549, "step_time": 3.3133636000020488, "tokens_per_second": 2472.4120226331133, "steps_per_second": 0.3018081082315812, "batch_wait": 0.001093299997592112, "processed_target_tokens": 41662805, "elapsed_seconds": 19189.3627949, "eta_seconds": 42411.054080026224, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3526.7, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 7250, "max_steps": 20000, "progress_percent": 36.25, "loss": 1.7721608877182007, "perplexity": 5.883553332971076, "learning_rate": 0.00022773762822590873, "gradient_norm": 0.8757309913635254, "step_time": 3.3009467000010773, "tokens_per_second": 2481.712291809294, "steps_per_second": 0.3029433949962517, "batch_wait": 0.0010264000011375174, "processed_target_tokens": 41945442, "elapsed_seconds": 19354.5063138, "eta_seconds": 42087.070425013735, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3574.43, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 7300, "max_steps": 20000, "progress_percent": 36.5, "loss": 1.92526376247406, "perplexity": 6.856957035350861, "learning_rate": 0.00022677269241959337, "gradient_norm": 0.9995074272155762, "step_time": 3.3040811999999278, "tokens_per_second": 2479.3579528251844, "steps_per_second": 0.3026560001007305, "batch_wait": 0.0012202000034449156, "processed_target_tokens": 42231661, "elapsed_seconds": 19519.4779131, "eta_seconds": 41961.83123999908, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3574.43, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 7350, "max_steps": 20000, "progress_percent": 36.75, "loss": 1.5830950736999512, "perplexity": 4.870005535271701, "learning_rate": 0.00022580374827179056, "gradient_norm": 0.8958294987678528, "step_time": 3.299014200001693, "tokens_per_second": 2483.166031839389, "steps_per_second": 0.30312085349601914, "batch_wait": 0.0010222999953839462, "processed_target_tokens": 42523439, "elapsed_seconds": 19684.4377353, "eta_seconds": 41732.52963002142, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3574.43, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 7400, "max_steps": 20000, "progress_percent": 37.0, "loss": 1.8438061475753784, "perplexity": 6.320549480998423, "learning_rate": 0.00022483085865589758, "gradient_norm": 1.0013799667358398, "step_time": 3.3056346000012127, "tokens_per_second": 2478.192840792807, "steps_per_second": 0.3025137745108407, "batch_wait": 0.0006784999932278879, "processed_target_tokens": 42807947, "elapsed_seconds": 19849.4176611, "eta_seconds": 41650.99596001528, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3574.43, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 7450, "max_steps": 20000, "progress_percent": 37.25, "loss": 1.9519076347351074, "perplexity": 7.042108546240385, "learning_rate": 0.00022385408670132735, "gradient_norm": 1.0052236318588257, "step_time": 3.3073068000012427, "tokens_per_second": 2476.9398472488015, "steps_per_second": 0.30236082119736346, "batch_wait": 0.0010025999981735367, "processed_target_tokens": 43096181, "elapsed_seconds": 20014.3781862, "eta_seconds": 41506.700340015595, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3574.43, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 7500, "max_steps": 20000, "progress_percent": 37.5, "loss": 2.0140128135681152, "perplexity": 7.493326418568961, "learning_rate": 0.0002228734957894123, "gradient_norm": 0.9846975207328796, "step_time": 3.309041199998319, "tokens_per_second": 2475.641584639128, "steps_per_second": 0.30220234187489353, "batch_wait": 0.0014229999942472205, "processed_target_tokens": 43378918, "elapsed_seconds": 20179.3360094, "eta_seconds": 41363.01499997899, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3546.77, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "validation", "step": 7500, "validation_loss": 1.700496459007263, "validation_perplexity": 5.4766656569128935, "validation_time": 2.222875899999053, "best_validation_loss": 1.7975844025611878}
{"step": 7500, "val_loss": 1.700496459007263, "best_val_loss": 1.700496459007263, "processed_target_tokens": 43378918, "timestamp": 1793429224.7504876}
{"event": "checkpoint", "step": 7500, "max_steps": 20000, "progress_percent": 37.5, "train_loss": 2.0140128135681152, "train_perplexity": 7.493326418568961, "validation_loss": 1.700496459007263, "validation_perplexity": 5.4766656569128935, "best_validation_loss": 1.700496459007263, "best_validation_perplexity": 5.4766656569128935, "processed_target_tokens": 43378918, "tokens_per_step": 8192, "tokens_per_second": 2475.641584639128, "steps_per_second": 0.30220234187489353, "elapsed_seconds": 20179.3360094, "eta_seconds": 41363.01499997899, "learning_rate": 0.0002228734957894123, "gradient_norm": 0.9846975207328796, "batch_wait": 0.0014229999942472205, "evaluation_time": 2.222875899999053, "parameter_count": 62113280, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3546.77, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}, "is_best": true, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"step": 7500, "val_loss": 1.700496459007263, "best_val_loss": 1.700496459007263, "processed_target_tokens": 43378918, "timestamp": 1793429225.8943112}
{"event": "checkpoint", "step": 7500, "max_steps": 20000, "progress_percent": 37.5, "train_loss": 2.0140128135681152, "train_perplexity": 7.493326418568961, "validation_loss": 1.700496459007263, "validation_perplexity": 5.4766656569128935, "best_validation_loss": 1.700496459007263, "best_validation_perplexity": 5.4766656569128935, "processed_target_tokens": 43378918, "tokens_per_step": 8192, "tokens_per_second": 2475.641584639128, "steps_per_second": 0.30220234187489353, "elapsed_seconds": 20179.3360094, "eta_seconds": 41363.01499997899, "learning_rate": 0.0002228734957894123, "gradient_norm": 0.9846975207328796, "batch_wait": 0.0014229999942472205, "evaluation_time": 2.222875899999053, "parameter_count": 62113280, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 0, "gpu_memory_used_mb": 3546.77, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 7550, "max_steps": 20000, "progress_percent": 37.75, "loss": 1.697974681854248, "perplexity": 5.462872125999981, "learning_rate": 0.00022188914954929142, "gradient_norm": 1.0506789684295654, "step_time": 3.2988232000025164, "tokens_per_second": 2483.3098057494417, "steps_per_second": 0.30313840402214864, "batch_wait": 0.0010546999983489513, "processed_target_tokens": 43668370, "elapsed_seconds": 20348.5483205, "eta_seconds": 41070.34884003133, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 0, "gpu_memory_used_mb": 3546.77, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 7600, "max_steps": 20000, "progress_percent": 38.0, "loss": 1.7926182746887207, "perplexity": 6.005155046037998, "learning_rate": 0.0002209011118537816, "gradient_norm": 0.9949632287025452, "step_time": 3.2975439000001643, "tokens_per_second": 2484.2732192282842, "steps_per_second": 0.30325600820657767, "batch_wait": 0.0009090999992622528, "processed_target_tokens": 43956590, "elapsed_seconds": 20513.5498323, "eta_seconds": 40889.54436000204, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 0, "gpu_memory_used_mb": 3546.77, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 7650, "max_steps": 20000, "progress_percent": 38.25, "loss": 1.7215083837509155, "perplexity": 5.592958432483289, "learning_rate": 0.00021990944681523302, "gradient_norm": 0.924950361251831, "step_time": 3.3033041000016965, "tokens_per_second": 2479.941220063812, "steps_per_second": 0.3027271997148208, "batch_wait": 0.001208499994390877, "processed_target_tokens": 44249224, "elapsed_seconds": 20678.5055373, "eta_seconds": 40795.80563502095, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 0, "gpu_memory_used_mb": 3546.77, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 7700, "max_steps": 20000, "progress_percent": 38.5, "loss": 1.7293903827667236, "perplexity": 5.637216316512892, "learning_rate": 0.00021891421878136887, "gradient_norm": 0.9133812189102173, "step_time": 3.296334900001966, "tokens_per_second": 2485.1843785639358, "steps_per_second": 0.30336723371141794, "batch_wait": 0.0007252000032167416, "processed_target_tokens": 44542511, "elapsed_seconds": 20843.5458794, "eta_seconds": 40544.919270024184, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 0, "gpu_memory_used_mb": 3546.77, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 7750, "max_steps": 20000, "progress_percent": 38.75, "loss": 1.6040091514587402, "perplexity": 4.972929740590411, "learning_rate": 0.0002179154923311101, "gradient_norm": 0.893119752407074, "step_time": 3.2827399000016158, "tokens_per_second": 2495.4764159036686, "steps_per_second": 0.3046235859257408, "batch_wait": 0.0009539000020595267, "processed_target_tokens": 44834038, "elapsed_seconds": 21008.5461631, "eta_seconds": 40213.56377501979, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3561.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 7800, "max_steps": 20000, "progress_percent": 39.0, "loss": 1.891566514968872, "perplexity": 6.629746147272343, "learning_rate": 0.00021691333227038487, "gradient_norm": 0.964909017086029, "step_time": 3.294335599999613, "tokens_per_second": 2486.69261261693, "steps_per_second": 0.3035513443135901, "batch_wait": 0.001472300002205884, "processed_target_tokens": 45116903, "elapsed_seconds": 21173.5499816, "eta_seconds": 40190.89431999528, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3561.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 7850, "max_steps": 20000, "progress_percent": 39.25, "loss": 1.8479934930801392, "perplexity": 6.347071294701519, "learning_rate": 0.00021590780362792323, "gradient_norm": 0.9139741659164429, "step_time": 3.2963756999997713, "tokens_per_second": 2485.1536188670993, "steps_per_second": 0.3033634788656127, "batch_wait": 0.0013107000013405923, "processed_target_tokens": 45406404, "elapsed_seconds": 21338.5709479, "eta_seconds": 40050.96475499722, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3561.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 7900, "max_steps": 20000, "progress_percent": 39.5, "loss": 1.807881236076355, "perplexity": 6.097514542362262, "learning_rate": 0.00021489897165103796, "gradient_norm": 0.9670513868331909, "step_time": 3.2998915999996825, "tokens_per_second": 2482.505788978277, "steps_per_second": 0.30304025744363733, "batch_wait": 0.0013703999975405168, "processed_target_tokens": 45699571, "elapsed_seconds": 21503.5868774, "eta_seconds": 39928.68835999616, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3561.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 7950, "max_steps": 20000, "progress_percent": 39.75, "loss": 1.8772464990615845, "perplexity": 6.535484601191432, "learning_rate": 0.00021388690180139025, "gradient_norm": 1.0346354246139526, "step_time": 3.296095300000161, "tokens_per_second": 2485.3650317694396, "steps_per_second": 0.3033892861046679, "batch_wait": 0.001150300002336735, "processed_target_tokens": 45991667, "elapsed_seconds": 21668.5723233, "eta_seconds": 39717.94836500194, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3561.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 8000, "max_steps": 20000, "progress_percent": 40.0, "loss": 1.832067608833313, "perplexity": 6.246789231786431, "learning_rate": 0.0002128716597507423, "gradient_norm": 1.0408824682235718, "step_time": 3.2955914000012854, "tokens_per_second": 2485.7450471550583, "steps_per_second": 0.3034356747015452, "batch_wait": 0.0009620999990147538, "processed_target_tokens": 46277369, "elapsed_seconds": 21833.5427186, "eta_seconds": 39547.096800015424, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3562.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "validation", "step": 8000, "validation_loss": 1.7224121928215026, "validation_perplexity": 5.598015684096663, "validation_time": 2.2243811000007554, "best_validation_loss": 1.700496459007263}
{"step": 8000, "val_loss": 1.7224121928215026, "best_val_loss": 1.700496459007263, "processed_target_tokens": 46277369, "timestamp": 1793430878.8652344}
{"event": "checkpoint", "step": 8000, "max_steps": 20000, "progress_percent": 40.0, "train_loss": 1.832067608833313, "train_perplexity": 6.246789231786431, "validation_loss": 1.7224121928215026, "validation_perplexity": 5.598015684096663, "best_validation_loss": 1.700496459007263, "best_validation_perplexity": 5.4766656569128935, "processed_target_tokens": 46277369, "tokens_per_step": 8192, "tokens_per_second": 2485.7450471550583, "steps_per_second": 0.3034356747015452, "elapsed_seconds": 21833.5427186, "eta_seconds": 39547.096800015424, "learning_rate": 0.0002128716597507423, "gradient_norm": 1.0408824682235718, "batch_wait": 0.0009620999990147538, "evaluation_time": 2.2243811000007554, "parameter_count": 62113280, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3562.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 8050, "max_steps": 20000, "progress_percent": 40.25, "loss": 1.9075901508331299, "perplexity": 6.736834466227173, "learning_rate": 0.00021185331137669588, "gradient_norm": 0.9378977417945862, "step_time": 3.3042685000000347, "tokens_per_second": 2479.2174122653514, "steps_per_second": 0.3026388442706728, "batch_wait": 0.0009408000005350914, "processed_target_tokens": 46564452, "elapsed_seconds": 22001.4789185, "eta_seconds": 39486.008575000415, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3562.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 8100, "max_steps": 20000, "progress_percent": 40.5, "loss": 1.8683565855026245, "perplexity": 6.477642195954877, "learning_rate": 0.00021083192275841766, "gradient_norm": 1.0183489322662354, "step_time": 3.303741399999126, "tokens_per_second": 2479.6129624437817, "steps_per_second": 0.3026871292045632, "batch_wait": 0.0007825999964552466, "processed_target_tokens": 46848773, "elapsed_seconds": 22166.4840525, "eta_seconds": 39314.5226599896, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3562.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 8150, "max_steps": 20000, "progress_percent": 40.75, "loss": 1.6030646562576294, "perplexity": 4.968235049720461, "learning_rate": 0.0002098075601723513, "gradient_norm": 0.8877339959144592, "step_time": 3.3104408999970474, "tokens_per_second": 2474.5948492864823, "steps_per_second": 0.3020745665632913, "batch_wait": 0.0012473000060708728, "processed_target_tokens": 47141854, "elapsed_seconds": 22331.4848497, "eta_seconds": 39228.72466496501, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3562.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 8200, "max_steps": 20000, "progress_percent": 41.0, "loss": 1.8531709909439087, "perplexity": 6.38001846111311, "learning_rate": 0.00020878029008791716, "gradient_norm": 0.9757744073867798, "step_time": 3.3115064000012353, "tokens_per_second": 2473.7986313409947, "steps_per_second": 0.3019773719898675, "batch_wait": 0.0012199999982840382, "processed_target_tokens": 47434519, "elapsed_seconds": 22496.4768311, "eta_seconds": 39075.77552001458, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3562.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 8250, "max_steps": 20000, "progress_percent": 41.25, "loss": 1.9756566286087036, "perplexity": 7.211353280404451, "learning_rate": 0.0002077501791631988, "gradient_norm": 0.9607507586479187, "step_time": 3.3010281000024406, "tokens_per_second": 2481.6510953038974, "steps_per_second": 0.30293592471971403, "batch_wait": 0.001308499999140622, "processed_target_tokens": 47728597, "elapsed_seconds": 22661.4473973, "eta_seconds": 38787.08017502868, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3561.45, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 8300, "max_steps": 20000, "progress_percent": 41.5, "loss": 1.8382784128189087, "perplexity": 6.285707547155366, "learning_rate": 0.00020671729424061788, "gradient_norm": 0.9125776886940002, "step_time": 3.294620799999393, "tokens_per_second": 2486.4773512027573, "steps_per_second": 0.30352506728549283, "batch_wait": 0.0010274999985995237, "processed_target_tokens": 48017770, "elapsed_seconds": 22826.4234347, "eta_seconds": 38547.0633599929, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3561.45, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 8350, "max_steps": 20000, "progress_percent": 41.75, "loss": 1.67926025390625, "perplexity": 5.361588289770865, "learning_rate": 0.00020568170234259693, "gradient_norm": 1.0413007736206055, "step_time": 3.3013447999983327, "tokens_per_second": 2481.4130290190037, "steps_per_second": 0.30290686389392135, "batch_wait": 0.0006896000049891882, "processed_target_tokens": 48303399, "elapsed_seconds": 22991.391622, "eta_seconds": 38460.66691998058, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3561.45, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 8400, "max_steps": 20000, "progress_percent": 42.0, "loss": 1.8650599718093872, "perplexity": 6.456323071757595, "learning_rate": 0.00020464347066721016, "gradient_norm": 0.8654283881187439, "step_time": 3.293028399999457, "tokens_per_second": 2487.6797296984596, "steps_per_second": 0.3036718420042065, "batch_wait": 0.0019844000016746577, "processed_target_tokens": 48590492, "elapsed_seconds": 23156.3967833, "eta_seconds": 38199.1294399937, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3561.45, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 8450, "max_steps": 20000, "progress_percent": 42.25, "loss": 1.5302343368530273, "perplexity": 4.619259158133676, "learning_rate": 0.00020360266658382302, "gradient_norm": 1.015264868736267, "step_time": 3.3018897000001743, "tokens_per_second": 2481.003529584761, "steps_per_second": 0.3028568761700148, "batch_wait": 0.0016138999963004608, "processed_target_tokens": 48878524, "elapsed_seconds": 23321.3477829, "eta_seconds": 38136.826035002014, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3561.45, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 8500, "max_steps": 20000, "progress_percent": 42.5, "loss": 1.7719169855117798, "perplexity": 5.882118496318594, "learning_rate": 0.0002025593576287211, "gradient_norm": 0.9102436900138855, "step_time": 3.2959114000004774, "tokens_per_second": 2485.5037061975677, "steps_per_second": 0.3034062141354453, "batch_wait": 0.0008623999965493567, "processed_target_tokens": 49169618, "elapsed_seconds": 23486.2441009, "eta_seconds": 37902.98110000549, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 38, "gpu_memory_used_mb": 3559.46, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "validation", "step": 8500, "validation_loss": 1.8683588266372682, "validation_perplexity": 6.477656713239479, "validation_time": 2.2230313000000024, "best_validation_loss": 1.700496459007263}
{"step": 8500, "val_loss": 1.8683588266372682, "best_val_loss": 1.700496459007263, "processed_target_tokens": 49169618, "timestamp": 1793432531.5610726}
{"event": "checkpoint", "step": 8500, "max_steps": 20000, "progress_percent": 42.5, "train_loss": 1.7719169855117798, "train_perplexity": 5.882118496318594, "validation_loss": 1.8683588266372682, "validation_perplexity": 6.477656713239479, "best_validation_loss": 1.700496459007263, "best_validation_perplexity": 5.4766656569128935, "processed_target_tokens": 49169618, "tokens_per_step": 8192, "tokens_per_second": 2485.5037061975677, "steps_per_second": 0.3034062141354453, "elapsed_seconds": 23486.2441009, "eta_seconds": 37902.98110000549, "learning_rate": 0.0002025593576287211, "gradient_norm": 0.9102436900138855, "batch_wait": 0.0008623999965493567, "evaluation_time": 2.2230313000000024, "parameter_count": 62113280, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3559.46, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 8550, "max_steps": 20000, "progress_percent": 42.75, "loss": 2.030169725418091, "perplexity": 7.61537877244522, "learning_rate": 0.0002015136115007273, "gradient_norm": 0.957556426525116, "step_time": 3.2996253999990586, "tokens_per_second": 2482.7060671803342, "steps_per_second": 0.3030647054663494, "batch_wait": 0.0012323000009928364, "processed_target_tokens": 49455561, "elapsed_seconds": 23654.385214, "eta_seconds": 37780.71082998922, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3559.46, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 8600, "max_steps": 20000, "progress_percent": 43.0, "loss": 1.797947883605957, "perplexity": 6.0372456130465, "learning_rate": 0.00020046549605680922, "gradient_norm": 1.0172338485717773, "step_time": 3.2999997999977495, "tokens_per_second": 2482.424392875899, "steps_per_second": 0.30303032139598374, "batch_wait": 0.0008588000018789899, "processed_target_tokens": 49738224, "elapsed_seconds": 23819.3788224, "eta_seconds": 37619.997719974344, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3559.46, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 8650, "max_steps": 20000, "progress_percent": 43.25, "loss": 1.8722972869873047, "perplexity": 6.503219012374125, "learning_rate": 0.00019941507930767605, "gradient_norm": 0.992915689945221, "step_time": 3.3000064000007114, "tokens_per_second": 2482.4194280345137, "steps_per_second": 0.30302971533624434, "batch_wait": 0.0011587000044528395, "processed_target_tokens": 50028867, "elapsed_seconds": 23984.407215, "eta_seconds": 37455.072640008075, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3559.46, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 8700, "max_steps": 20000, "progress_percent": 43.5, "loss": 1.690563678741455, "perplexity": 5.422536412332101, "learning_rate": 0.00019836242941336538, "gradient_norm": 0.8788794875144958, "step_time": 3.300765100000717, "tokens_per_second": 2481.848829532953, "steps_per_second": 0.3029600621988468, "batch_wait": 0.0010557999994489364, "processed_target_tokens": 50321418, "elapsed_seconds": 24149.3730911, "eta_seconds": 37298.645630008104, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3559.46, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 8750, "max_steps": 20000, "progress_percent": 43.75, "loss": 1.7021687030792236, "perplexity": 5.485831640336518, "learning_rate": 0.0001973076146788203, "gradient_norm": 0.9336037039756775, "step_time": 3.310251700000663, "tokens_per_second": 2474.736286669186, "steps_per_second": 0.3020918318687971, "batch_wait": 0.0012607999960891902, "processed_target_tokens": 50612789, "elapsed_seconds": 24314.4264544, "eta_seconds": 37240.33162500746, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3560.13, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 8800, "max_steps": 20000, "progress_percent": 44.0, "loss": 1.8830561637878418, "perplexity": 6.573564082953489, "learning_rate": 0.0001962507035494573, "gradient_norm": 1.1459013223648071, "step_time": 3.309361399999034, "tokens_per_second": 2475.4020518890416, "steps_per_second": 0.3021731020372365, "batch_wait": 0.000937000000703847, "processed_target_tokens": 50899714, "elapsed_seconds": 24479.4886804, "eta_seconds": 37064.84767998918, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3560.13, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 8850, "max_steps": 20000, "progress_percent": 44.25, "loss": 1.8567609786987305, "perplexity": 6.40296381138397, "learning_rate": 0.000195191764606725, "gradient_norm": 1.0082567930221558, "step_time": 3.2983614999975543, "tokens_per_second": 2483.6574159642823, "steps_per_second": 0.30318083690970243, "batch_wait": 0.0011870000053022522, "processed_target_tokens": 51187275, "elapsed_seconds": 24644.514191, "eta_seconds": 36776.73072497273, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3560.13, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 8900, "max_steps": 20000, "progress_percent": 44.5, "loss": 1.897796392440796, "perplexity": 6.671177575800618, "learning_rate": 0.00019413086656365383, "gradient_norm": 0.9313779473304749, "step_time": 3.3011805999994976, "tokens_per_second": 2481.536453958698, "steps_per_second": 0.30292193041488014, "batch_wait": 0.001530300000013085, "processed_target_tokens": 51475735, "elapsed_seconds": 24809.5367071, "eta_seconds": 36643.104659994424, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3560.13, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 8950, "max_steps": 20000, "progress_percent": 44.75, "loss": 1.7709689140319824, "perplexity": 5.876544470236537, "learning_rate": 0.00019306807826039747, "gradient_norm": 0.9997456669807434, "step_time": 3.2991791000022204, "tokens_per_second": 2483.0419179105756, "steps_per_second": 0.3031057028699433, "batch_wait": 0.0011174000028404407, "processed_target_tokens": 51769119, "elapsed_seconds": 24974.5465259, "eta_seconds": 36455.929055024535, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3560.13, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 9000, "max_steps": 20000, "progress_percent": 45.0, "loss": 1.6438963413238525, "perplexity": 5.17529499464986, "learning_rate": 0.000192003468659766, "gradient_norm": 0.9550595879554749, "step_time": 3.3002372000009927, "tokens_per_second": 2482.2458216026216, "steps_per_second": 0.30300852314485127, "batch_wait": 0.0014656999992439523, "processed_target_tokens": 52060672, "elapsed_seconds": 25139.5895776, "eta_seconds": 36302.60920001092, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3559.87, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "validation", "step": 9000, "validation_loss": 1.8791465044021607, "validation_perplexity": 6.547913860927732, "validation_time": 2.223678700000164, "best_validation_loss": 1.700496459007263}
{"step": 9000, "val_loss": 1.8791465044021607, "best_val_loss": 1.700496459007263, "processed_target_tokens": 52060672, "timestamp": 1793434184.8840504}
{"event": "checkpoint", "step": 9000, "max_steps": 20000, "progress_percent": 45.0, "train_loss": 1.6438963413238525, "train_perplexity": 5.17529499464986, "validation_loss": 1.8791465044021607, "validation_perplexity": 6.547913860927732, "best_validation_loss": 1.700496459007263, "best_validation_perplexity": 5.4766656569128935, "processed_target_tokens": 52060672, "tokens_per_step": 8192, "tokens_per_second": 2482.2458216026216, "steps_per_second": 0.30300852314485127, "elapsed_seconds": 25139.5895776, "eta_seconds": 36302.60920001092, "learning_rate": 0.000192003468659766, "gradient_norm": 0.9550595879554749, "batch_wait": 0.0014656999992439523, "evaluation_time": 2.223678700000164, "parameter_count": 62113280, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3559.87, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 9050, "max_steps": 20000, "progress_percent": 45.25, "loss": 1.7112950086593628, "perplexity": 5.536126168782062, "learning_rate": 0.00019093710684275062, "gradient_norm": 0.983163058757782, "step_time": 3.307488000002195, "tokens_per_second": 2476.8041486453053, "steps_per_second": 0.30234425642642887, "batch_wait": 0.001200400001835078, "processed_target_tokens": 52345074, "elapsed_seconds": 25307.6721935, "eta_seconds": 36216.993600024034, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3559.87, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 9100, "max_steps": 20000, "progress_percent": 45.5, "loss": 1.9113433361053467, "perplexity": 6.762166562415849, "learning_rate": 0.00018986906200404156, "gradient_norm": 1.2481108903884888, "step_time": 3.298350799999753, "tokens_per_second": 2483.6654730602377, "steps_per_second": 0.30318182044192354, "batch_wait": 0.0009432000042579602, "processed_target_tokens": 52637264, "elapsed_seconds": 25472.8879132, "eta_seconds": 35952.023719997305, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3559.87, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 9150, "max_steps": 20000, "progress_percent": 45.75, "loss": 1.458593726158142, "perplexity": 4.299908425847199, "learning_rate": 0.00018879940344753782, "gradient_norm": 0.8554109930992126, "step_time": 3.3024175000027753, "tokens_per_second": 2480.6070098626583, "steps_per_second": 0.3028084728836253, "batch_wait": 0.0010933000012300909, "processed_target_tokens": 52926934, "elapsed_seconds": 25638.0744268, "eta_seconds": 35831.22987503011, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3559.87, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 9200, "max_steps": 20000, "progress_percent": 46.0, "loss": 1.887974739074707, "perplexity": 6.605976298415476, "learning_rate": 0.00018772820058185014, "gradient_norm": 0.9741687178611755, "step_time": 3.298350199998822, "tokens_per_second": 2483.6659248623528, "steps_per_second": 0.3031818755935489, "batch_wait": 0.001153699999122182, "processed_target_tokens": 53219447, "elapsed_seconds": 25803.2332933, "eta_seconds": 35622.18215998728, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3559.87, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 9250, "max_steps": 20000, "progress_percent": 46.25, "loss": 1.8447725772857666, "perplexity": 6.326660800407504, "learning_rate": 0.0001866555229157976, "gradient_norm": 0.9213125109672546, "step_time": 3.3047732999984873, "tokens_per_second": 2478.8387148987645, "steps_per_second": 0.3025926165647906, "batch_wait": 0.000805300001957221, "processed_target_tokens": 53508104, "elapsed_seconds": 25968.3877318, "eta_seconds": 35526.31297498374, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3560.1, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 9300, "max_steps": 20000, "progress_percent": 46.5, "loss": 1.8803482055664062, "perplexity": 6.555787226406535, "learning_rate": 0.00018558144005389663, "gradient_norm": 0.908323347568512, "step_time": 3.3125774999971327, "tokens_per_second": 2472.9987449371647, "steps_per_second": 0.3018797296065875, "batch_wait": 0.0008845999982440844, "processed_target_tokens": 53797824, "elapsed_seconds": 26133.5376271, "eta_seconds": 35444.57924996932, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3560.1, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 9350, "max_steps": 20000, "progress_percent": 46.75, "loss": 1.8264490365982056, "perplexity": 6.211789611271499, "learning_rate": 0.00018450602169184485, "gradient_norm": 0.9950300455093384, "step_time": 3.3100943000026746, "tokens_per_second": 2474.853964128267, "steps_per_second": 0.3021061967930013, "batch_wait": 0.0013823000008414965, "processed_target_tokens": 54087489, "elapsed_seconds": 26298.6695767, "eta_seconds": 35252.504295028484, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3560.1, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 9400, "max_steps": 20000, "progress_percent": 47.0, "loss": 2.0357649326324463, "perplexity": 7.65810782216234, "learning_rate": 0.00018342933761199878, "gradient_norm": 1.0301790237426758, "step_time": 3.307518700003129, "tokens_per_second": 2476.7811592394773, "steps_per_second": 0.30234145010247526, "batch_wait": 0.0007826999935787171, "processed_target_tokens": 54382014, "elapsed_seconds": 26463.8765166, "eta_seconds": 35059.69822003317, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3560.1, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 9450, "max_steps": 20000, "progress_percent": 47.25, "loss": 1.6766724586486816, "perplexity": 5.347731533977884, "learning_rate": 0.00018235145767884524, "gradient_norm": 0.9382013082504272, "step_time": 3.3118845999997575, "tokens_per_second": 2473.5161364017936, "steps_per_second": 0.30194288774435957, "batch_wait": 0.001431500004400732, "processed_target_tokens": 54672862, "elapsed_seconds": 26628.9795564, "eta_seconds": 34940.38252999744, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3560.1, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 9500, "max_steps": 20000, "progress_percent": 47.5, "loss": 2.076660394668579, "perplexity": 7.977781734362594, "learning_rate": 0.00018127245183446858, "gradient_norm": 0.9697385430335999, "step_time": 3.30665990000125, "tokens_per_second": 2477.4244245671903, "steps_per_second": 0.3024199737020496, "batch_wait": 0.0019073000003118068, "processed_target_tokens": 54963959, "elapsed_seconds": 26794.1832097, "eta_seconds": 34719.92895001313, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3565.65, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "validation", "step": 9500, "validation_loss": 1.9284355878829955, "validation_perplexity": 6.87874063452747, "validation_time": 2.2291323999997985, "best_validation_loss": 1.700496459007263}
{"step": 9500, "val_loss": 1.9284355878829955, "best_val_loss": 1.700496459007263, "processed_target_tokens": 54963959, "timestamp": 1793435839.495559}
{"event": "checkpoint", "step": 9500, "max_steps": 20000, "progress_percent": 47.5, "train_loss": 2.076660394668579, "train_perplexity": 7.977781734362594, "validation_loss": 1.9284355878829955, "validation_perplexity": 6.87874063452747, "best_validation_loss": 1.700496459007263, "best_validation_perplexity": 5.4766656569128935, "processed_target_tokens": 54963959, "tokens_per_step": 8192, "tokens_per_second": 2477.4244245671903, "steps_per_second": 0.3024199737020496, "elapsed_seconds": 26794.1832097, "eta_seconds": 34719.92895001313, "learning_rate": 0.00018127245183446858, "gradient_norm": 0.9697385430335999, "batch_wait": 0.0019073000003118068, "evaluation_time": 2.2291323999997985, "parameter_count": 62113280, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3565.65, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 9550, "max_steps": 20000, "progress_percent": 47.75, "loss": 2.044933557510376, "perplexity": 7.728645010434134, "learning_rate": 0.00018019239009401172, "gradient_norm": 1.0502690076828003, "step_time": 3.3029736999997112, "tokens_per_second": 2480.1892912440435, "steps_per_second": 0.3027574818413139, "batch_wait": 0.0008903999987524003, "processed_target_tokens": 55255106, "elapsed_seconds": 26962.3398782, "eta_seconds": 34516.07516499698, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3565.65, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 9600, "max_steps": 20000, "progress_percent": 48.0, "loss": 1.4853074550628662, "perplexity": 4.416323024938787, "learning_rate": 0.0001791113425411332, "gradient_norm": 0.9755738377571106, "step_time": 3.303298900002119, "tokens_per_second": 2479.9451239470777, "steps_per_second": 0.302727676263071, "batch_wait": 0.0007836999975552317, "processed_target_tokens": 55540617, "elapsed_seconds": 27127.5069236, "eta_seconds": 34354.30856002204, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3565.65, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 9650, "max_steps": 20000, "progress_percent": 48.25, "loss": 1.8926262855529785, "perplexity": 6.636775881514164, "learning_rate": 0.00017802937932345954, "gradient_norm": 0.9952204823493958, "step_time": 3.292207200000121, "tokens_per_second": 2488.300250360822, "steps_per_second": 0.30374758915537375, "batch_wait": 0.0008612999954493716, "processed_target_tokens": 55831197, "elapsed_seconds": 27292.6683234, "eta_seconds": 34074.34452000125, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3565.65, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 9700, "max_steps": 20000, "progress_percent": 48.5, "loss": 1.748244047164917, "perplexity": 5.744506731717148, "learning_rate": 0.0001769465706480334, "gradient_norm": 0.9689809679985046, "step_time": 3.3025046000002476, "tokens_per_second": 2480.5415865278087, "steps_per_second": 0.3028004866366954, "batch_wait": 0.0009982999981730245, "processed_target_tokens": 56124491, "elapsed_seconds": 27457.8305896, "eta_seconds": 34015.79738000255, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3565.65, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 9750, "max_steps": 20000, "progress_percent": 48.75, "loss": 1.8021551370620728, "perplexity": 6.062699343117994, "learning_rate": 0.000175862986776758, "gradient_norm": 0.9786694645881653, "step_time": 3.299660100001347, "tokens_per_second": 2482.6799584589503, "steps_per_second": 0.3030615183665711, "batch_wait": 0.0012673000019276515, "processed_target_tokens": 56414247, "elapsed_seconds": 27623.0328635, "eta_seconds": 33821.51602501381, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3565.57, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 9800, "max_steps": 20000, "progress_percent": 49.0, "loss": 1.8443918228149414, "perplexity": 6.324252354564712, "learning_rate": 0.00017477869802183795, "gradient_norm": 0.9233778715133667, "step_time": 3.3075929000006, "tokens_per_second": 2476.72559703418, "steps_per_second": 0.3023346676067114, "batch_wait": 0.0013690999985556118, "processed_target_tokens": 56705851, "elapsed_seconds": 27788.2309222, "eta_seconds": 33737.44758000612, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3565.57, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 9850, "max_steps": 20000, "progress_percent": 49.25, "loss": 1.8676669597625732, "perplexity": 6.473176587138077, "learning_rate": 0.00017369377474121662, "gradient_norm": 0.9693276286125183, "step_time": 3.301389399999607, "tokens_per_second": 2481.3795064590004, "steps_per_second": 0.30290277178454594, "batch_wait": 0.0010983000029227696, "processed_target_tokens": 56997851, "elapsed_seconds": 27953.4386156, "eta_seconds": 33509.10240999601, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3565.57, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 9900, "max_steps": 20000, "progress_percent": 49.5, "loss": 1.7232160568237305, "perplexity": 5.602517536585172, "learning_rate": 0.00017260828733401103, "gradient_norm": 0.9601380228996277, "step_time": 3.3059911000018474, "tokens_per_second": 2477.925606029436, "steps_per_second": 0.30248115307976514, "batch_wait": 0.0016954999991867226, "processed_target_tokens": 57285185, "elapsed_seconds": 28118.6253797, "eta_seconds": 33390.51011001866, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3565.57, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 9950, "max_steps": 20000, "progress_percent": 49.75, "loss": 2.041121482849121, "perplexity": 7.699238923334335, "learning_rate": 0.00017152230623594344, "gradient_norm": 1.0396466255187988, "step_time": 3.3050375999991957, "tokens_per_second": 2478.6404850589274, "steps_per_second": 0.30256841858629485, "batch_wait": 0.0009537999976600986, "processed_target_tokens": 57574136, "elapsed_seconds": 28283.7968729, "eta_seconds": 33215.62787999192, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3565.57, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 10000, "max_steps": 20000, "progress_percent": 50.0, "loss": 1.7015886306762695, "perplexity": 5.4826503835635405, "learning_rate": 0.00017043590191477103, "gradient_norm": 0.919183075428009, "step_time": 3.3086996999991243, "tokens_per_second": 2475.8971024182606, "steps_per_second": 0.3022335330100416, "batch_wait": 0.0009127000012085773, "processed_target_tokens": 57863582, "elapsed_seconds": 28448.94987, "eta_seconds": 33086.99699999124, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3578.51, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "validation", "step": 10000, "validation_loss": 1.7763586282730102, "validation_perplexity": 5.908302873134822, "validation_time": 2.2298305999975128, "best_validation_loss": 1.700496459007263}
{"step": 10000, "val_loss": 1.7763586282730102, "best_val_loss": 1.700496459007263, "processed_target_tokens": 57863582, "timestamp": 1793437494.2916775}
{"event": "checkpoint", "step": 10000, "max_steps": 20000, "progress_percent": 50.0, "train_loss": 1.7015886306762695, "train_perplexity": 5.4826503835635405, "validation_loss": 1.7763586282730102, "validation_perplexity": 5.908302873134822, "best_validation_loss": 1.700496459007263, "best_validation_perplexity": 5.4766656569128935, "processed_target_tokens": 57863582, "tokens_per_step": 8192, "tokens_per_second": 2475.8971024182606, "steps_per_second": 0.3022335330100416, "elapsed_seconds": 28448.94987, "eta_seconds": 33086.99699999124, "learning_rate": 0.00017043590191477103, "gradient_norm": 0.919183075428009, "batch_wait": 0.0009127000012085773, "evaluation_time": 2.2298305999975128, "parameter_count": 62113280, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3578.51, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 10050, "max_steps": 20000, "progress_percent": 50.24999999999999, "loss": 1.8038830757141113, "perplexity": 6.07318437178298, "learning_rate": 0.00016934914486571337, "gradient_norm": 0.9684186577796936, "step_time": 3.296715899999981, "tokens_per_second": 2484.897166904812, "steps_per_second": 0.30333217369443505, "batch_wait": 0.001695099999778904, "processed_target_tokens": 58145658, "elapsed_seconds": 28617.2971309, "eta_seconds": 32802.32320499981, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3578.51, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 10100, "max_steps": 20000, "progress_percent": 50.5, "loss": 1.8736242055892944, "perplexity": 6.511853982338619, "learning_rate": 0.00016826210560687787, "gradient_norm": 0.9619124531745911, "step_time": 3.3021450999985973, "tokens_per_second": 2480.8116396833925, "steps_per_second": 0.3028334521097891, "batch_wait": 0.0006952000039746054, "processed_target_tokens": 58439294, "elapsed_seconds": 28782.4780301, "eta_seconds": 32691.236489986113, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3578.51, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 10150, "max_steps": 20000, "progress_percent": 50.74999999999999, "loss": 1.890068531036377, "perplexity": 6.619822328783248, "learning_rate": 0.00016717485467468432, "gradient_norm": 0.9697220921516418, "step_time": 3.308870199998637, "tokens_per_second": 2475.7695239914137, "steps_per_second": 0.3022179594716081, "batch_wait": 0.0012265000004845206, "processed_target_tokens": 58732478, "elapsed_seconds": 28947.6650956, "eta_seconds": 32592.371469986574, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3578.51, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 10200, "max_steps": 20000, "progress_percent": 51.0, "loss": 1.7291481494903564, "perplexity": 5.635850960508957, "learning_rate": 0.00016608746261928764, "gradient_norm": 0.9476467370986938, "step_time": 3.311121800001274, "tokens_per_second": 2474.085972916142, "steps_per_second": 0.30201244786574, "batch_wait": 0.0011597000047913752, "processed_target_tokens": 59025322, "elapsed_seconds": 29112.8067355, "eta_seconds": 32448.993640012486, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3578.51, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 10250, "max_steps": 20000, "progress_percent": 51.24999999999999, "loss": 2.1498169898986816, "perplexity": 8.583287425129084, "learning_rate": 0.000165, "gradient_norm": 1.112826943397522, "step_time": 3.3078823999967426, "tokens_per_second": 2476.5088384061255, "steps_per_second": 0.30230820781324774, "batch_wait": 0.0010505999998713378, "processed_target_tokens": 59314758, "elapsed_seconds": 29278.0039784, "eta_seconds": 32251.85339996824, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3578.75, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 10300, "max_steps": 20000, "progress_percent": 51.5, "loss": 1.5877567529678345, "perplexity": 4.892760937093804, "learning_rate": 0.00016391253738071236, "gradient_norm": 0.9699058532714844, "step_time": 3.296662299999298, "tokens_per_second": 2484.9375685224854, "steps_per_second": 0.30333710553252996, "batch_wait": 0.0012180000012449455, "processed_target_tokens": 59606598, "elapsed_seconds": 29443.1676097, "eta_seconds": 31977.62430999319, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3578.75, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 10350, "max_steps": 20000, "progress_percent": 51.74999999999999, "loss": 1.590346336364746, "perplexity": 4.905447569037632, "learning_rate": 0.00016282514532531568, "gradient_norm": 1.0474411249160767, "step_time": 3.3000621999999566, "tokens_per_second": 2482.377453370457, "steps_per_second": 0.30302459147588584, "batch_wait": 0.001113199999963399, "processed_target_tokens": 59896777, "elapsed_seconds": 29608.3353727, "eta_seconds": 31845.60022999958, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3578.75, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 10400, "max_steps": 20000, "progress_percent": 52.0, "loss": 1.7405917644500732, "perplexity": 5.700715905728761, "learning_rate": 0.00016173789439312216, "gradient_norm": 0.9682046175003052, "step_time": 3.308459399999265, "tokens_per_second": 2476.0769317591808, "steps_per_second": 0.30225548483388437, "batch_wait": 0.0010916000028373674, "processed_target_tokens": 60184140, "elapsed_seconds": 29773.5099919, "eta_seconds": 31761.210239992943, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3578.75, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 10450, "max_steps": 20000, "progress_percent": 52.25, "loss": 2.0265114307403564, "perplexity": 7.587570369492903, "learning_rate": 0.00016065085513428665, "gradient_norm": 0.9414975047111511, "step_time": 3.3026836999997613, "tokens_per_second": 2480.4070701655723, "steps_per_second": 0.30278406618232084, "batch_wait": 0.0017279999992751982, "processed_target_tokens": 60471820, "elapsed_seconds": 29938.652551, "eta_seconds": 31540.62933499772, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3578.75, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 10500, "max_steps": 20000, "progress_percent": 52.5, "loss": 2.121957302093506, "perplexity": 8.347460009375556, "learning_rate": 0.000159564098085229, "gradient_norm": 1.0169590711593628, "step_time": 3.298546300000453, "tokens_per_second": 2483.5182698508356, "steps_per_second": 0.30316385130015083, "batch_wait": 0.0009502999964752235, "processed_target_tokens": 60758142, "elapsed_seconds": 30103.8567312, "eta_seconds": 31336.189850004303, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3576.29, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "validation", "step": 10500, "validation_loss": 1.5567119061946868, "validation_perplexity": 4.743199491451511, "validation_time": 2.229953800000658, "best_validation_loss": 1.700496459007263}
{"step": 10500, "val_loss": 1.5567119061946868, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 60758142, "timestamp": 1793439149.4497862}
{"event": "checkpoint", "step": 10500, "max_steps": 20000, "progress_percent": 52.5, "train_loss": 2.121957302093506, "train_perplexity": 8.347460009375556, "validation_loss": 1.5567119061946868, "validation_perplexity": 4.743199491451511, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 60758142, "tokens_per_step": 8192, "tokens_per_second": 2483.5182698508356, "steps_per_second": 0.30316385130015083, "elapsed_seconds": 30103.8567312, "eta_seconds": 31336.189850004303, "learning_rate": 0.000159564098085229, "gradient_norm": 1.0169590711593628, "batch_wait": 0.0009502999964752235, "evaluation_time": 2.229953800000658, "parameter_count": 62113280, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 0, "gpu_memory_used_mb": 3576.29, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}, "is_best": true, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"step": 10500, "val_loss": 1.5567119061946868, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 60758142, "timestamp": 1793439150.250506}
{"event": "checkpoint", "step": 10500, "max_steps": 20000, "progress_percent": 52.5, "train_loss": 2.121957302093506, "train_perplexity": 8.347460009375556, "validation_loss": 1.5567119061946868, "validation_perplexity": 4.743199491451511, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 60758142, "tokens_per_step": 8192, "tokens_per_second": 2483.5182698508356, "steps_per_second": 0.30316385130015083, "elapsed_seconds": 30103.8567312, "eta_seconds": 31336.189850004303, "learning_rate": 0.000159564098085229, "gradient_norm": 1.0169590711593628, "batch_wait": 0.0009502999964752235, "evaluation_time": 2.229953800000658, "parameter_count": 62113280, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 0, "gpu_memory_used_mb": 3576.29, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 10550, "max_steps": 20000, "progress_percent": 52.75, "loss": 1.6809478998184204, "perplexity": 5.370644391928304, "learning_rate": 0.00015847769376405653, "gradient_norm": 0.9779538512229919, "step_time": 3.3015841000014916, "tokens_per_second": 2481.233175310088, "steps_per_second": 0.3028849090954697, "batch_wait": 0.0013534000026993454, "processed_target_tokens": 61050669, "elapsed_seconds": 30273.1957641, "eta_seconds": 31199.969745014096, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 0, "gpu_memory_used_mb": 3576.29, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 10600, "max_steps": 20000, "progress_percent": 53.0, "loss": 1.7145709991455078, "perplexity": 5.5542922050754635, "learning_rate": 0.00015739171266598894, "gradient_norm": 1.037683367729187, "step_time": 3.3060314000031212, "tokens_per_second": 2477.895400507166, "steps_per_second": 0.3024774658822224, "batch_wait": 0.0014897000000928529, "processed_target_tokens": 61340973, "elapsed_seconds": 30438.4511621, "eta_seconds": 31076.69516002934, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 0, "gpu_memory_used_mb": 3576.29, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 10650, "max_steps": 20000, "progress_percent": 53.25, "loss": 1.542007565498352, "perplexity": 4.673964147667505, "learning_rate": 0.00015630622525878335, "gradient_norm": 0.9065819382667542, "step_time": 3.299158099998749, "tokens_per_second": 2483.0577231212733, "steps_per_second": 0.3031076322169523, "batch_wait": 0.0013816999999107793, "processed_target_tokens": 61632725, "elapsed_seconds": 30603.6785106, "eta_seconds": 30847.128234988304, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 0, "gpu_memory_used_mb": 3576.29, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 10700, "max_steps": 20000, "progress_percent": 53.5, "loss": 1.5800936222076416, "perplexity": 4.855410364196278, "learning_rate": 0.0001552213019781621, "gradient_norm": 0.9516969323158264, "step_time": 3.3092861999975867, "tokens_per_second": 2475.45830276208, "steps_per_second": 0.30217996859888674, "batch_wait": 0.0014155000026221387, "processed_target_tokens": 61920242, "elapsed_seconds": 30768.955657, "eta_seconds": 30776.361659977556, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 0, "gpu_memory_used_mb": 3576.29, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}}
{"event": "train", "step": 10750, "max_steps": 20000, "progress_percent": 53.75, "loss": 1.5202546119689941, "perplexity": 4.573389486616762, "learning_rate": 0.00015413701322324202, "gradient_norm": 0.9685459733009338, "step_time": 3.306726200000412, "tokens_per_second": 2477.374752103449, "steps_per_second": 0.30241391016887803, "batch_wait": 0.0007607999941683374, "processed_target_tokens": 62213911, "elapsed_seconds": 30934.2414937, "eta_seconds": 30587.21735000381, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3576.53, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 10800, "max_steps": 20000, "progress_percent": 54.0, "loss": 1.5680594444274902, "perplexity": 4.797329670318239, "learning_rate": 0.0001530534293519666, "gradient_norm": 0.9866065382957458, "step_time": 3.3073733000019274, "tokens_per_second": 2476.8900444335163, "steps_per_second": 0.3023547417521382, "batch_wait": 0.0017687999970803503, "processed_target_tokens": 62505663, "elapsed_seconds": 31099.4660009, "eta_seconds": 30427.834360017732, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3576.53, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 10850, "max_steps": 20000, "progress_percent": 54.25, "loss": 1.5406848192214966, "perplexity": 4.667785766109461, "learning_rate": 0.00015197062067654043, "gradient_norm": 1.0900384187698364, "step_time": 3.300335100000666, "tokens_per_second": 2482.172189120537, "steps_per_second": 0.30299953480475306, "batch_wait": 0.0012314999985392205, "processed_target_tokens": 62786908, "elapsed_seconds": 31264.7310566, "eta_seconds": 30198.066165006094, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3576.53, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 10900, "max_steps": 20000, "progress_percent": 54.50000000000001, "loss": 1.7287365198135376, "perplexity": 5.633531554400427, "learning_rate": 0.0001508886574588668, "gradient_norm": 0.9734295010566711, "step_time": 3.3145657999994, "tokens_per_second": 2471.5152735846978, "steps_per_second": 0.30169864179500705, "batch_wait": 0.0009137000051850919, "processed_target_tokens": 63081288, "elapsed_seconds": 31430.0203724, "eta_seconds": 30162.54877999454, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3576.53, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 10950, "max_steps": 20000, "progress_percent": 54.75, "loss": 1.9034589529037476, "perplexity": 6.709060678637382, "learning_rate": 0.00014980760990598827, "gradient_norm": 1.1521050930023193, "step_time": 3.30692579999959, "tokens_per_second": 2477.225222289843, "steps_per_second": 0.3023956570178031, "batch_wait": 0.0008338999978150241, "processed_target_tokens": 63374138, "elapsed_seconds": 31595.3061232, "eta_seconds": 29927.678489996288, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3576.53, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 11000, "max_steps": 20000, "progress_percent": 55.00000000000001, "loss": 1.8205773830413818, "perplexity": 6.175423005244206, "learning_rate": 0.00014872754816553141, "gradient_norm": 0.9529239535331726, "step_time": 3.2998254999984056, "tokens_per_second": 2482.5555169520203, "steps_per_second": 0.30304632775293217, "batch_wait": 0.0006660999861196615, "processed_target_tokens": 63669831, "elapsed_seconds": 31760.5493691, "eta_seconds": 29698.42949998565, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3591.29, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "validation", "step": 11000, "validation_loss": 1.6069275736808777, "validation_perplexity": 4.98746404756075, "validation_time": 2.2065408999987994, "best_validation_loss": 1.5567119061946868}
{"step": 11000, "val_loss": 1.6069275736808777, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 63669831, "timestamp": 1793440805.8621724}
{"event": "checkpoint", "step": 11000, "max_steps": 20000, "progress_percent": 55.00000000000001, "train_loss": 1.8205773830413818, "train_perplexity": 6.175423005244206, "validation_loss": 1.6069275736808777, "validation_perplexity": 4.98746404756075, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 63669831, "tokens_per_step": 8192, "tokens_per_second": 2482.5555169520203, "steps_per_second": 0.30304632775293217, "elapsed_seconds": 31760.5493691, "eta_seconds": 29698.42949998565, "learning_rate": 0.00014872754816553141, "gradient_norm": 0.9529239535331726, "batch_wait": 0.0006660999861196615, "evaluation_time": 2.2065408999987994, "parameter_count": 62113280, "gpu": {"gpu_utilization": 8, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3589.22, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 11050, "max_steps": 20000, "progress_percent": 55.25, "loss": 1.8267226219177246, "perplexity": 6.213489298211179, "learning_rate": 0.0001476485423211547, "gradient_norm": 0.9614201784133911, "step_time": 3.3163289000003715, "tokens_per_second": 2470.2013120589704, "steps_per_second": 0.30153824610094854, "batch_wait": 0.0012992000047233887, "processed_target_tokens": 63956742, "elapsed_seconds": 31928.7890194, "eta_seconds": 29681.143655003325, "gpu": {"gpu_utilization": 8, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3589.22, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 11100, "max_steps": 20000, "progress_percent": 55.50000000000001, "loss": 1.7575397491455078, "perplexity": 5.798154916779337, "learning_rate": 0.00014657066238800125, "gradient_norm": 0.9621917605400085, "step_time": 3.29617639999924, "tokens_per_second": 2485.3038811884853, "steps_per_second": 0.3033818214341413, "batch_wait": 0.0009164000075543299, "processed_target_tokens": 64246412, "elapsed_seconds": 32094.0340977, "eta_seconds": 29335.969959993236, "gpu": {"gpu_utilization": 8, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3589.22, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 11150, "max_steps": 20000, "progress_percent": 55.75, "loss": 1.7967971563339233, "perplexity": 6.030302385518408, "learning_rate": 0.00014549397830815515, "gradient_norm": 0.9541011452674866, "step_time": 3.3129913000011584, "tokens_per_second": 2472.6898618771306, "steps_per_second": 0.30184202415492317, "batch_wait": 0.0014778000040678307, "processed_target_tokens": 64539721, "elapsed_seconds": 32259.2945217, "eta_seconds": 29319.97300501025, "gpu": {"gpu_utilization": 8, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3589.22, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 11200, "max_steps": 20000, "progress_percent": 56.00000000000001, "loss": 1.96532142162323, "perplexity": 7.137206273944809, "learning_rate": 0.00014441855994610337, "gradient_norm": 0.9436123371124268, "step_time": 3.3161147000064375, "tokens_per_second": 2470.3608714089705, "steps_per_second": 0.30155772356066535, "batch_wait": 0.0007206999944173731, "processed_target_tokens": 64828443, "elapsed_seconds": 32424.5617776, "eta_seconds": 29181.80936005665, "gpu": {"gpu_utilization": 8, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3589.22, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 11250, "max_steps": 20000, "progress_percent": 56.25, "loss": 1.3848798274993896, "perplexity": 3.9943458654424915, "learning_rate": 0.00014334447708420236, "gradient_norm": 0.9396848082542419, "step_time": 3.294581800000742, "tokens_per_second": 2486.506785170171, "steps_per_second": 0.30352866029909314, "batch_wait": 0.0008600000001024455, "processed_target_tokens": 65118087, "elapsed_seconds": 32589.7934839, "eta_seconds": 28827.590750006493, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3579.8, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 11300, "max_steps": 20000, "progress_percent": 56.49999999999999, "loss": 1.6798983812332153, "perplexity": 5.365010757643505, "learning_rate": 0.00014227179941814978, "gradient_norm": 0.9111768007278442, "step_time": 3.303671599998779, "tokens_per_second": 2479.665351726554, "steps_per_second": 0.30269352438068287, "batch_wait": 0.0013039999976172112, "processed_target_tokens": 65409792, "elapsed_seconds": 32755.0400574, "eta_seconds": 28741.94291998938, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3579.8, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 11350, "max_steps": 20000, "progress_percent": 56.75, "loss": 1.9512808322906494, "perplexity": 7.0376959184568735, "learning_rate": 0.00014120059655246218, "gradient_norm": 0.9922928214073181, "step_time": 3.307756400005019, "tokens_per_second": 2476.6031742807813, "steps_per_second": 0.30231972342294694, "batch_wait": 0.0007924000092316419, "processed_target_tokens": 65705272, "elapsed_seconds": 32920.2993075, "eta_seconds": 28612.092860043413, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3579.8, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 11400, "max_steps": 20000, "progress_percent": 56.99999999999999, "loss": 1.477349042892456, "perplexity": 4.381315592586475, "learning_rate": 0.00014013093799595844, "gradient_norm": 1.027435302734375, "step_time": 3.3016697999992175, "tokens_per_second": 2481.168771026691, "steps_per_second": 0.3028770472444691, "batch_wait": 0.0005813999960082583, "processed_target_tokens": 65988975, "elapsed_seconds": 33085.5654627, "eta_seconds": 28394.36027999327, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3579.8, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 11450, "max_steps": 20000, "progress_percent": 57.25, "loss": 1.7633968591690063, "perplexity": 5.8322149975090305, "learning_rate": 0.00013906289315724938, "gradient_norm": 1.0035903453826904, "step_time": 3.304265499995381, "tokens_per_second": 2479.2196631933634, "steps_per_second": 0.3026391190421586, "batch_wait": 0.0011800999927800149, "processed_target_tokens": 66276233, "elapsed_seconds": 33250.7873143, "eta_seconds": 28251.47002496051, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3579.8, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 11500, "max_steps": 20000, "progress_percent": 57.49999999999999, "loss": 1.8830525875091553, "perplexity": 6.573540574098401, "learning_rate": 0.00013799653134023403, "gradient_norm": 0.9635425209999084, "step_time": 3.299303300002066, "tokens_per_second": 2482.9484455081383, "steps_per_second": 0.30309429266456767, "batch_wait": 0.0013235000078566372, "processed_target_tokens": 66568594, "elapsed_seconds": 33416.0728892, "eta_seconds": 28044.07805001756, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 42, "gpu_memory_used_mb": 3579.91, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "validation", "step": 11500, "validation_loss": 1.7137077093124389, "validation_perplexity": 5.549499310211298, "validation_time": 2.209284900003695, "best_validation_loss": 1.5567119061946868}
{"step": 11500, "val_loss": 1.7137077093124389, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 66568594, "timestamp": 1793442461.3688614}
{"event": "checkpoint", "step": 11500, "max_steps": 20000, "progress_percent": 57.49999999999999, "train_loss": 1.8830525875091553, "train_perplexity": 6.573540574098401, "validation_loss": 1.7137077093124389, "validation_perplexity": 5.549499310211298, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 66568594, "tokens_per_step": 8192, "tokens_per_second": 2482.9484455081383, "steps_per_second": 0.30309429266456767, "elapsed_seconds": 33416.0728892, "eta_seconds": 28044.07805001756, "learning_rate": 0.00013799653134023403, "gradient_norm": 0.9635425209999084, "batch_wait": 0.0013235000078566372, "evaluation_time": 2.209284900003695, "parameter_count": 62113280, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3579.91, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 11550, "max_steps": 20000, "progress_percent": 57.75, "loss": 1.7933099269866943, "perplexity": 6.00930996203824, "learning_rate": 0.00013693192173960253, "gradient_norm": 0.986849308013916, "step_time": 3.293244499996945, "tokens_per_second": 2487.5164901991334, "steps_per_second": 0.3036519153075114, "batch_wait": 0.001382399997964967, "processed_target_tokens": 66854271, "elapsed_seconds": 33584.4046762, "eta_seconds": 27827.916024974184, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3579.91, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 11600, "max_steps": 20000, "progress_percent": 57.99999999999999, "loss": 1.9374134540557861, "perplexity": 6.940775099267845, "learning_rate": 0.00013586913343634617, "gradient_norm": 1.0401208400726318, "step_time": 3.301819100001012, "tokens_per_second": 2481.056578780312, "steps_per_second": 0.30286335190189356, "batch_wait": 0.0008453999980702065, "processed_target_tokens": 67142295, "elapsed_seconds": 33749.6194473, "eta_seconds": 27735.2804400085, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3579.91, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 11650, "max_steps": 20000, "progress_percent": 58.25, "loss": 1.754054069519043, "perplexity": 5.7779795891065975, "learning_rate": 0.000134808235393275, "gradient_norm": 0.938066840171814, "step_time": 4.279627600000822, "tokens_per_second": 1914.185243594192, "steps_per_second": 0.23366519086843163, "batch_wait": 0.0011924999926122837, "processed_target_tokens": 67441674, "elapsed_seconds": 33921.8725517, "eta_seconds": 35734.89046000686, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3579.91, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 11700, "max_steps": 20000, "progress_percent": 58.5, "loss": 1.993127465248108, "perplexity": 7.338448654008658, "learning_rate": 0.00013374929645054273, "gradient_norm": 1.001403570175171, "step_time": 3.5019035000004806, "tokens_per_second": 2339.299184000609, "steps_per_second": 0.2855589824219493, "batch_wait": 0.0004920999999740161, "processed_target_tokens": 67726140, "elapsed_seconds": 34106.1096273, "eta_seconds": 29065.79905000399, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3579.91, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 11750, "max_steps": 20000, "progress_percent": 58.75, "loss": 1.732667326927185, "perplexity": 5.655719459928643, "learning_rate": 0.00013269238532117973, "gradient_norm": 0.9251386523246765, "step_time": 3.5683923000033246, "tokens_per_second": 2295.7117130850124, "steps_per_second": 0.2802382462261978, "batch_wait": 0.0015980000025592744, "processed_target_tokens": 68019225, "elapsed_seconds": 34285.5984655, "eta_seconds": 29439.23647502743, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3564.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 11800, "max_steps": 20000, "progress_percent": 59.0, "loss": 1.6093217134475708, "perplexity": 4.999419038821556, "learning_rate": 0.0001316375705866346, "gradient_norm": 1.0164414644241333, "step_time": 3.5392666999978246, "tokens_per_second": 2314.603756762675, "steps_per_second": 0.2825444039016937, "batch_wait": 0.0008918999956222251, "processed_target_tokens": 68312939, "elapsed_seconds": 34461.6428688, "eta_seconds": 29021.986939982162, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3564.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 11850, "max_steps": 20000, "progress_percent": 59.25, "loss": 1.5367580652236938, "perplexity": 4.649492459854212, "learning_rate": 0.00013058492069232386, "gradient_norm": 0.9589428305625916, "step_time": 3.6534931999995024, "tokens_per_second": 2242.2376480681874, "steps_per_second": 0.27371065039894865, "batch_wait": 0.0017148999977507629, "processed_target_tokens": 68604249, "elapsed_seconds": 34640.4502677, "eta_seconds": 29775.969579995945, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3564.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 11900, "max_steps": 20000, "progress_percent": 59.5, "loss": 1.6614784002304077, "perplexity": 5.267091961196268, "learning_rate": 0.00012953450394319075, "gradient_norm": 0.9341172575950623, "step_time": 3.6172386999969603, "tokens_per_second": 2264.710924387402, "steps_per_second": 0.27645397026213403, "batch_wait": 0.0010245000084978528, "processed_target_tokens": 68891559, "elapsed_seconds": 34820.6210066, "eta_seconds": 29299.63346997538, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3564.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 11950, "max_steps": 20000, "progress_percent": 59.75, "loss": 1.7739582061767578, "perplexity": 5.894137460653149, "learning_rate": 0.00012848638849927269, "gradient_norm": 0.9795879125595093, "step_time": 3.5125476000030176, "tokens_per_second": 2332.210387694949, "steps_per_second": 0.2846936508416686, "batch_wait": 0.0012404999943100847, "processed_target_tokens": 69182747, "elapsed_seconds": 34998.8306017, "eta_seconds": 28276.00818002429, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3564.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 12000, "max_steps": 20000, "progress_percent": 60.0, "loss": 1.721937894821167, "perplexity": 5.595361186013178, "learning_rate": 0.00012744064237127888, "gradient_norm": 0.9295622706413269, "step_time": 3.6048782000070787, "tokens_per_second": 2272.4762240188625, "steps_per_second": 0.27740188281480255, "batch_wait": 0.0010106999980052933, "processed_target_tokens": 69476727, "elapsed_seconds": 35174.9896242, "eta_seconds": 28839.02560005663, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3638.97, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "validation", "step": 12000, "validation_loss": 1.5667545080184937, "validation_perplexity": 4.791073542977429, "validation_time": 2.4065612999984296, "best_validation_loss": 1.5567119061946868}
{"step": 12000, "val_loss": 1.5667545080184937, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 69476727, "timestamp": 1793444221.223234}
{"event": "checkpoint", "step": 12000, "max_steps": 20000, "progress_percent": 60.0, "train_loss": 1.721937894821167, "train_perplexity": 5.595361186013178, "validation_loss": 1.5667545080184937, "validation_perplexity": 4.791073542977429, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 69476727, "tokens_per_step": 8192, "tokens_per_second": 2272.4762240188625, "steps_per_second": 0.27740188281480255, "elapsed_seconds": 35174.9896242, "eta_seconds": 28839.02560005663, "learning_rate": 0.00012744064237127888, "gradient_norm": 0.9295622706413269, "batch_wait": 0.0010106999980052933, "evaluation_time": 2.4065612999984296, "parameter_count": 62113280, "gpu": {"gpu_utilization": 6, "gpu_memory_utilization": 7, "gpu_memory_used_mb": 3630.09, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 12050, "max_steps": 20000, "progress_percent": 60.25, "loss": 1.7464314699172974, "perplexity": 5.734103800422907, "learning_rate": 0.00012639733341617698, "gradient_norm": 0.988540768623352, "step_time": 5.010109700000612, "tokens_per_second": 1635.0939381624717, "steps_per_second": 0.1995964279983486, "batch_wait": 0.0019035000004805624, "processed_target_tokens": 69770542, "elapsed_seconds": 35360.7587903, "eta_seconds": 39830.372115004866, "gpu": {"gpu_utilization": 6, "gpu_memory_utilization": 7, "gpu_memory_used_mb": 3630.09, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 12100, "max_steps": 20000, "progress_percent": 60.5, "loss": 1.801316738128662, "perplexity": 6.057618512634178, "learning_rate": 0.00012535652933278984, "gradient_norm": 1.0179634094238281, "step_time": 3.612820999995165, "tokens_per_second": 2267.480176850988, "steps_per_second": 0.27679201377575535, "batch_wait": 0.001396500003465917, "processed_target_tokens": 70065470, "elapsed_seconds": 35541.9829293, "eta_seconds": 28541.285899961804, "gpu": {"gpu_utilization": 6, "gpu_memory_utilization": 7, "gpu_memory_used_mb": 3630.09, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 12150, "max_steps": 20000, "progress_percent": 60.75000000000001, "loss": 1.7292002439498901, "perplexity": 5.636144564766269, "learning_rate": 0.00012431829765740306, "gradient_norm": 0.9730962514877319, "step_time": 3.5051880000028177, "tokens_per_second": 2337.1071680016635, "steps_per_second": 0.28529140234395306, "batch_wait": 0.0006093000120017678, "processed_target_tokens": 70354677, "elapsed_seconds": 35718.0208835, "eta_seconds": 27515.72580002212, "gpu": {"gpu_utilization": 6, "gpu_memory_utilization": 7, "gpu_memory_used_mb": 3630.09, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 12200, "max_steps": 20000, "progress_percent": 61.0, "loss": 1.6286367177963257, "perplexity": 5.096921437682096, "learning_rate": 0.00012328270575938212, "gradient_norm": 0.9574907422065735, "step_time": 3.6260796999995364, "tokens_per_second": 2259.189173365673, "steps_per_second": 0.2757799283893644, "batch_wait": 0.0006133999995654449, "processed_target_tokens": 70649667, "elapsed_seconds": 35896.2230114, "eta_seconds": 28283.421659996384, "gpu": {"gpu_utilization": 6, "gpu_memory_utilization": 7, "gpu_memory_used_mb": 3630.09, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 12250, "max_steps": 20000, "progress_percent": 61.25000000000001, "loss": 1.5243632793426514, "perplexity": 4.592218677740884, "learning_rate": 0.00012224982083680123, "gradient_norm": 0.889268159866333, "step_time": 3.6048512999986997, "tokens_per_second": 2272.4931816197122, "steps_per_second": 0.27740395283443753, "batch_wait": 0.0009515000056126155, "processed_target_tokens": 70942442, "elapsed_seconds": 36076.1611601, "eta_seconds": 27937.597574989923, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 38, "gpu_memory_used_mb": 3582.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 12300, "max_steps": 20000, "progress_percent": 61.5, "loss": 1.70774507522583, "perplexity": 5.516508131368953, "learning_rate": 0.00012121970991208285, "gradient_norm": 1.031707763671875, "step_time": 3.5133491000015056, "tokens_per_second": 2331.678340759388, "steps_per_second": 0.28462870370598, "batch_wait": 0.001285799990000669, "processed_target_tokens": 71234014, "elapsed_seconds": 36253.1200278, "eta_seconds": 27052.788070011593, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 38, "gpu_memory_used_mb": 3582.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 12350, "max_steps": 20000, "progress_percent": 61.75000000000001, "loss": 1.7312628030776978, "perplexity": 5.647781442933193, "learning_rate": 0.00012019243982764868, "gradient_norm": 0.9919840693473816, "step_time": 3.6007726000025286, "tokens_per_second": 2275.0673008326735, "steps_per_second": 0.27771817637117596, "batch_wait": 0.0014024999909452163, "processed_target_tokens": 71523996, "elapsed_seconds": 36431.4727697, "eta_seconds": 27545.910390019344, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 38, "gpu_memory_used_mb": 3582.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 12400, "max_steps": 20000, "progress_percent": 62.0, "loss": 1.7729718685150146, "perplexity": 5.888326717040937, "learning_rate": 0.00011916807724158233, "gradient_norm": 1.0326594114303589, "step_time": 3.617848800000502, "tokens_per_second": 2264.329012312196, "steps_per_second": 0.2764073501357661, "batch_wait": 0.0010134000040125102, "processed_target_tokens": 71805422, "elapsed_seconds": 36611.538388, "eta_seconds": 27495.650880003814, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 38, "gpu_memory_used_mb": 3582.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 12450, "max_steps": 20000, "progress_percent": 62.25000000000001, "loss": 1.7328145503997803, "perplexity": 5.656552175883721, "learning_rate": 0.00011814668862330408, "gradient_norm": 0.9250268340110779, "step_time": 3.5868663000001106, "tokens_per_second": 2283.8877490359055, "steps_per_second": 0.27879489123973455, "batch_wait": 0.0009401999996043742, "processed_target_tokens": 72088346, "elapsed_seconds": 36791.7389377, "eta_seconds": 27080.840565000835, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 38, "gpu_memory_used_mb": 3582.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 12500, "max_steps": 20000, "progress_percent": 62.5, "loss": 1.6619060039520264, "perplexity": 5.269344670920189, "learning_rate": 0.00011712834024925766, "gradient_norm": 0.958214282989502, "step_time": 3.510708200003137, "tokens_per_second": 2333.4323257035944, "steps_per_second": 0.28484281319623955, "batch_wait": 0.0009408999903826043, "processed_target_tokens": 72379026, "elapsed_seconds": 36968.6565643, "eta_seconds": 26330.311500023527, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 47, "gpu_memory_used_mb": 3527.05, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "validation", "step": 12500, "validation_loss": 1.777506709098816, "validation_perplexity": 5.915089977703329, "validation_time": 2.3405226999966544, "best_validation_loss": 1.5567119061946868}
{"step": 12500, "val_loss": 1.777506709098816, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 72379026, "timestamp": 1793446016.1198223}
{"event": "checkpoint", "step": 12500, "max_steps": 20000, "progress_percent": 62.5, "train_loss": 1.6619060039520264, "train_perplexity": 5.269344670920189, "validation_loss": 1.777506709098816, "validation_perplexity": 5.915089977703329, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 72379026, "tokens_per_step": 8192, "tokens_per_second": 2333.4323257035944, "steps_per_second": 0.28484281319623955, "elapsed_seconds": 36968.6565643, "eta_seconds": 26330.311500023527, "learning_rate": 0.00011712834024925766, "gradient_norm": 0.958214282989502, "batch_wait": 0.0009408999903826043, "evaluation_time": 2.3405226999966544, "parameter_count": 62113280, "gpu": {"gpu_utilization": 5, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3517.49, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 12550, "max_steps": 20000, "progress_percent": 62.74999999999999, "loss": 1.700620174407959, "perplexity": 5.477343246712398, "learning_rate": 0.00011611309819860976, "gradient_norm": 1.0095678567886353, "step_time": 3.584823299999698, "tokens_per_second": 2285.1893425265034, "steps_per_second": 0.2789537771638798, "batch_wait": 0.001891299987619277, "processed_target_tokens": 72670149, "elapsed_seconds": 37151.0214414, "eta_seconds": 26706.93358499775, "gpu": {"gpu_utilization": 5, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3517.49, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 12600, "max_steps": 20000, "progress_percent": 63.0, "loss": 1.7856488227844238, "perplexity": 5.963447913514695, "learning_rate": 0.00011510102834896202, "gradient_norm": 1.063223123550415, "step_time": 3.3090620000002673, "tokens_per_second": 2475.6260233260477, "steps_per_second": 0.30220044230054294, "batch_wait": 0.0010561999952187762, "processed_target_tokens": 72955231, "elapsed_seconds": 37325.8008699, "eta_seconds": 24487.058800001978, "gpu": {"gpu_utilization": 5, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3517.49, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 12650, "max_steps": 20000, "progress_percent": 63.24999999999999, "loss": 1.225523591041565, "perplexity": 3.4059489403591625, "learning_rate": 0.00011409219637207673, "gradient_norm": 0.8560803532600403, "step_time": 3.3195166000004974, "tokens_per_second": 2467.829201396002, "steps_per_second": 0.3012486818110354, "batch_wait": 0.0015652999936719425, "processed_target_tokens": 73241493, "elapsed_seconds": 37491.5411481, "eta_seconds": 24398.447010003656, "gpu": {"gpu_utilization": 5, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3517.49, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 12700, "max_steps": 20000, "progress_percent": 63.5, "loss": 1.7030103206634521, "perplexity": 5.4904505561168735, "learning_rate": 0.00011308666772961514, "gradient_norm": 0.9669539928436279, "step_time": 3.3196420000022044, "tokens_per_second": 2467.7359787575165, "steps_per_second": 0.3012373020944234, "batch_wait": 0.0008986000102595426, "processed_target_tokens": 73529721, "elapsed_seconds": 37657.5134174, "eta_seconds": 24233.386600016092, "gpu": {"gpu_utilization": 5, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3517.49, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 12750, "max_steps": 20000, "progress_percent": 63.74999999999999, "loss": 1.6287691593170166, "perplexity": 5.097596526412047, "learning_rate": 0.00011208450766888988, "gradient_norm": 1.0633227825164795, "step_time": 3.3146674999952666, "tokens_per_second": 2471.4394430245866, "steps_per_second": 0.30168938513483723, "batch_wait": 0.0011871000024257228, "processed_target_tokens": 73816783, "elapsed_seconds": 37823.189012, "eta_seconds": 24031.339374965683, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3503.19, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 12800, "max_steps": 20000, "progress_percent": 64.0, "loss": 1.9669510126113892, "perplexity": 7.148846482783229, "learning_rate": 0.00011108578121863114, "gradient_norm": 1.0127451419830322, "step_time": 3.303518399996392, "tokens_per_second": 2479.7803457092737, "steps_per_second": 0.30270756173208907, "batch_wait": 0.0007891999994171783, "processed_target_tokens": 74107672, "elapsed_seconds": 37988.6057529, "eta_seconds": 23785.332479974022, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3503.19, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 12850, "max_steps": 20000, "progress_percent": 64.25, "loss": 1.497498869895935, "perplexity": 4.470493789150732, "learning_rate": 0.00011009055318476698, "gradient_norm": 0.8670309782028198, "step_time": 3.308324999998149, "tokens_per_second": 2476.1775218591233, "steps_per_second": 0.30226776389881876, "batch_wait": 0.0007251000060932711, "processed_target_tokens": 74400697, "elapsed_seconds": 38153.9090875, "eta_seconds": 23654.523749986765, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3503.19, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 12900, "max_steps": 20000, "progress_percent": 64.5, "loss": 1.6001472473144531, "perplexity": 4.953761798815901, "learning_rate": 0.00010909888814621843, "gradient_norm": 0.9024977087974548, "step_time": 3.3113487000009627, "tokens_per_second": 2473.916443773384, "steps_per_second": 0.30199175339030565, "batch_wait": 0.0008491999979014508, "processed_target_tokens": 74692816, "elapsed_seconds": 38319.1998718, "eta_seconds": 23510.575770006835, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3503.19, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 12950, "max_steps": 20000, "progress_percent": 64.75, "loss": 1.5164865255355835, "perplexity": 4.556188986574701, "learning_rate": 0.0001081108504507086, "gradient_norm": 0.9139864444732666, "step_time": 3.3008330000011483, "tokens_per_second": 2481.797776499796, "steps_per_second": 0.30295383013913524, "batch_wait": 0.0007651000050827861, "processed_target_tokens": 74976741, "elapsed_seconds": 38484.4741882, "eta_seconds": 23270.872650008096, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3503.19, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 13000, "max_steps": 20000, "progress_percent": 65.0, "loss": 1.4273691177368164, "perplexity": 4.167719975182088, "learning_rate": 0.00010712650421058767, "gradient_norm": 0.9161381721496582, "step_time": 3.2998251999961212, "tokens_per_second": 2482.5557426525593, "steps_per_second": 0.3030463553042675, "batch_wait": 0.0020344000004115514, "processed_target_tokens": 75261600, "elapsed_seconds": 38649.7721166, "eta_seconds": 23098.77639997285, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 40, "gpu_memory_used_mb": 3406.21, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "validation", "step": 13000, "validation_loss": 1.6978457927703858, "validation_perplexity": 5.462168066790152, "validation_time": 2.2077112999977544, "best_validation_loss": 1.5567119061946868}
{"step": 13000, "val_loss": 1.6978457927703858, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 75261600, "timestamp": 1793447696.3531911}
{"event": "checkpoint", "step": 13000, "max_steps": 20000, "progress_percent": 65.0, "train_loss": 1.4273691177368164, "train_perplexity": 4.167719975182088, "validation_loss": 1.6978457927703858, "validation_perplexity": 5.462168066790152, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 75261600, "tokens_per_step": 8192, "tokens_per_second": 2482.5557426525593, "steps_per_second": 0.3030463553042675, "elapsed_seconds": 38649.7721166, "eta_seconds": 23098.77639997285, "learning_rate": 0.00010712650421058767, "gradient_norm": 0.9161381721496582, "batch_wait": 0.0020344000004115514, "evaluation_time": 2.2077112999977544, "parameter_count": 62113280, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3406.21, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 13050, "max_steps": 20000, "progress_percent": 65.25, "loss": 1.6278862953186035, "perplexity": 5.09309802803377, "learning_rate": 0.00010614591329867262, "gradient_norm": 0.9518619179725647, "step_time": 3.3033014000029652, "tokens_per_second": 2479.943247077801, "steps_per_second": 0.3027274471530519, "batch_wait": 0.0015869999915594235, "processed_target_tokens": 75559695, "elapsed_seconds": 38818.0107151, "eta_seconds": 22957.94473002061, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3406.21, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 13100, "max_steps": 20000, "progress_percent": 65.5, "loss": 1.9664435386657715, "perplexity": 7.145219549816824, "learning_rate": 0.00010516914134410242, "gradient_norm": 1.1117446422576904, "step_time": 3.304406100003689, "tokens_per_second": 2479.1141742508144, "steps_per_second": 0.30262624197397636, "batch_wait": 0.0013593000039691105, "processed_target_tokens": 75846647, "elapsed_seconds": 38983.2757824, "eta_seconds": 22800.402090025455, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3406.21, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 13150, "max_steps": 20000, "progress_percent": 65.75, "loss": 1.7026772499084473, "perplexity": 5.488622152115685, "learning_rate": 0.00010419625172820942, "gradient_norm": 0.9572184085845947, "step_time": 3.307076100005361, "tokens_per_second": 2477.112637349567, "steps_per_second": 0.30238191373896084, "batch_wait": 0.0007483000081265345, "processed_target_tokens": 76137661, "elapsed_seconds": 39148.6038368, "eta_seconds": 22653.47128503672, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3406.21, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 13200, "max_steps": 20000, "progress_percent": 66.0, "loss": 1.7460346221923828, "perplexity": 5.731828685842166, "learning_rate": 0.00010322730758040664, "gradient_norm": 0.9418200254440308, "step_time": 3.299384600002668, "tokens_per_second": 2482.8872632773323, "steps_per_second": 0.3030868241305337, "batch_wait": 0.0004454000008990988, "processed_target_tokens": 76425364, "elapsed_seconds": 39313.8377444, "eta_seconds": 22435.81528001814, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3406.21, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 13250, "max_steps": 20000, "progress_percent": 66.25, "loss": 1.6477875709533691, "perplexity": 5.195472488044535, "learning_rate": 0.00010226237177409124, "gradient_norm": 1.0264476537704468, "step_time": 3.308980200003134, "tokens_per_second": 2475.6872223025816, "steps_per_second": 0.3022079128787331, "batch_wait": 0.0009448000055272132, "processed_target_tokens": 76713223, "elapsed_seconds": 39479.1684254, "eta_seconds": 22335.616350021155, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3437.9, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 13300, "max_steps": 20000, "progress_percent": 66.5, "loss": 1.3846733570098877, "perplexity": 3.9935212360301624, "learning_rate": 0.00010130150692256492, "gradient_norm": 0.8734497427940369, "step_time": 3.3065905999974348, "tokens_per_second": 2477.476346786432, "steps_per_second": 0.3024263118635781, "batch_wait": 0.0007289999994100071, "processed_target_tokens": 77005906, "elapsed_seconds": 39644.4205538, "eta_seconds": 22154.157019982813, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3437.9, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 13350, "max_steps": 20000, "progress_percent": 66.75, "loss": 1.7008583545684814, "perplexity": 5.478647996582736, "learning_rate": 0.00010034477537497088, "gradient_norm": 0.9825637340545654, "step_time": 3.3131140999976196, "tokens_per_second": 2472.598212058524, "steps_per_second": 0.3018308364329253, "batch_wait": 0.001177600002847612, "processed_target_tokens": 77296021, "elapsed_seconds": 39809.6992651, "eta_seconds": 22032.20876498417, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3437.9, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 13400, "max_steps": 20000, "progress_percent": 67.0, "loss": 1.9387438297271729, "perplexity": 6.950015082562147, "learning_rate": 9.939223921224818e-05, "gradient_norm": 1.1012617349624634, "step_time": 3.3106517999985954, "tokens_per_second": 2474.4372090122783, "steps_per_second": 0.30205532336575663, "batch_wait": 0.0009105000062845647, "processed_target_tokens": 77586206, "elapsed_seconds": 39974.9301644, "eta_seconds": 21850.30187999073, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3437.9, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 13450, "max_steps": 20000, "progress_percent": 67.25, "loss": 1.7504229545593262, "perplexity": 5.757037126239279, "learning_rate": 9.844396024310345e-05, "gradient_norm": 1.0522958040237427, "step_time": 3.3091596999947797, "tokens_per_second": 2475.5529326713736, "steps_per_second": 0.30219152010148603, "batch_wait": 0.0018196999953943305, "processed_target_tokens": 77877923, "elapsed_seconds": 40140.1940989, "eta_seconds": 21674.996034965807, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3437.9, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 13500, "max_steps": 20000, "progress_percent": 67.5, "loss": 1.750346302986145, "perplexity": 5.7565958571988896, "learning_rate": 9.750000000000003e-05, "gradient_norm": 0.955805778503418, "step_time": 3.301783399998385, "tokens_per_second": 2481.083404806023, "steps_per_second": 0.3028666265632352, "batch_wait": 0.0009409999911440536, "processed_target_tokens": 78166310, "elapsed_seconds": 40305.4779926, "eta_seconds": 21461.592099989502, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 44, "gpu_memory_used_mb": 3433.72, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "validation", "step": 13500, "validation_loss": 1.7104616165161133, "validation_perplexity": 5.531514326732353, "validation_time": 2.2118940000000293, "best_validation_loss": 1.5567119061946868}
{"step": 13500, "val_loss": 1.7104616165161133, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 78166310, "timestamp": 1793449351.9939053}
{"event": "checkpoint", "step": 13500, "max_steps": 20000, "progress_percent": 67.5, "train_loss": 1.750346302986145, "train_perplexity": 5.7565958571988896, "validation_loss": 1.7104616165161133, "validation_perplexity": 5.531514326732353, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 78166310, "tokens_per_step": 8192, "tokens_per_second": 2481.083404806023, "steps_per_second": 0.3028666265632352, "elapsed_seconds": 40305.4779926, "eta_seconds": 21461.592099989502, "learning_rate": 9.750000000000003e-05, "gradient_norm": 0.955805778503418, "batch_wait": 0.0009409999911440536, "evaluation_time": 2.2118940000000293, "parameter_count": 62113280, "gpu": {"gpu_utilization": 8, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3433.72, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 13550, "max_steps": 20000, "progress_percent": 67.75, "loss": 1.7947014570236206, "perplexity": 6.017677918132465, "learning_rate": 9.656041973516534e-05, "gradient_norm": 1.0140289068222046, "step_time": 3.302371400000993, "tokens_per_second": 2480.641638308016, "steps_per_second": 0.30281269998877147, "batch_wait": 0.0011818000057246536, "processed_target_tokens": 78447036, "elapsed_seconds": 40473.6313476, "eta_seconds": 21300.295530006406, "gpu": {"gpu_utilization": 8, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3433.72, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 13600, "max_steps": 20000, "progress_percent": 68.0, "loss": 1.658280849456787, "perplexity": 5.250277064790273, "learning_rate": 9.562528041661624e-05, "gradient_norm": 1.1812540292739868, "step_time": 3.310979900001257, "tokens_per_second": 2474.192005815828, "steps_per_second": 0.30202539133493994, "batch_wait": 0.0014736000084667467, "processed_target_tokens": 78735875, "elapsed_seconds": 40638.8840782, "eta_seconds": 21190.271360008046, "gpu": {"gpu_utilization": 8, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3433.72, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 13650, "max_steps": 20000, "progress_percent": 68.25, "loss": 1.702040672302246, "perplexity": 5.485129330008866, "learning_rate": 9.469464272420307e-05, "gradient_norm": 0.9901965260505676, "step_time": 3.2987933000040357, "tokens_per_second": 2483.3323142707904, "steps_per_second": 0.3031411516443836, "batch_wait": 0.000810199991974514, "processed_target_tokens": 79028504, "elapsed_seconds": 40804.1273065, "eta_seconds": 20947.337455025627, "gpu": {"gpu_utilization": 8, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3433.72, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 13700, "max_steps": 20000, "progress_percent": 68.5, "loss": 1.747760534286499, "perplexity": 5.741729860112894, "learning_rate": 9.376856704567206e-05, "gradient_norm": 0.9160374402999878, "step_time": 3.303767900004459, "tokens_per_second": 2479.5930731056933, "steps_per_second": 0.3026847013068473, "batch_wait": 0.001058400004694704, "processed_target_tokens": 79327974, "elapsed_seconds": 40969.4295705, "eta_seconds": 20813.737770028092, "gpu": {"gpu_utilization": 8, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3433.72, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 13750, "max_steps": 20000, "progress_percent": 68.75, "loss": 1.515639305114746, "perplexity": 4.552330524938621, "learning_rate": 9.284711347274684e-05, "gradient_norm": 0.9622532725334167, "step_time": 3.305252300000575, "tokens_per_second": 2478.4794794632094, "steps_per_second": 0.3025487645829113, "batch_wait": 0.0017118999967351556, "processed_target_tokens": 79609378, "elapsed_seconds": 41134.7065831, "eta_seconds": 20657.826875003593, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3432.89, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 13800, "max_steps": 20000, "progress_percent": 69.0, "loss": 1.8146239519119263, "perplexity": 6.138767271646711, "learning_rate": 9.193034179722928e-05, "gradient_norm": 0.9275847673416138, "step_time": 3.3114399000041885, "tokens_per_second": 2473.8483099118416, "steps_per_second": 0.30198343626853535, "batch_wait": 0.0012181000056443736, "processed_target_tokens": 79900734, "elapsed_seconds": 41299.9796545, "eta_seconds": 20530.92738002597, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3432.89, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 13850, "max_steps": 20000, "progress_percent": 69.25, "loss": 1.6257398128509521, "perplexity": 5.082177506957002, "learning_rate": 9.101831150711958e-05, "gradient_norm": 0.9554548263549805, "step_time": 3.310462599998573, "tokens_per_second": 2474.578628377657, "steps_per_second": 0.30207258647188195, "batch_wait": 0.0009085000056074932, "processed_target_tokens": 80191275, "elapsed_seconds": 41465.1640758, "eta_seconds": 20359.344989991223, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3432.89, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 13900, "max_steps": 20000, "progress_percent": 69.5, "loss": 1.6940838098526, "perplexity": 5.441658087114384, "learning_rate": 9.011108178275623e-05, "gradient_norm": 0.9691120386123657, "step_time": 3.307147100000293, "tokens_per_second": 2477.0594570768485, "steps_per_second": 0.30237542200645123, "batch_wait": 0.0015762999973958358, "processed_target_tokens": 80479521, "elapsed_seconds": 41630.420169, "eta_seconds": 20173.597310001787, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3432.89, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 13950, "max_steps": 20000, "progress_percent": 69.75, "loss": 1.9179375171661377, "perplexity": 6.806904857186022, "learning_rate": 8.920871149297608e-05, "gradient_norm": 1.0286935567855835, "step_time": 3.30973339999764, "tokens_per_second": 2475.1238271958223, "steps_per_second": 0.30213913906199, "batch_wait": 0.0008790000065346248, "processed_target_tokens": 80773293, "elapsed_seconds": 41795.7450223, "eta_seconds": 20023.88706998572, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3432.89, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 14000, "max_steps": 20000, "progress_percent": 70.0, "loss": 1.6794788837432861, "perplexity": 5.362760621093546, "learning_rate": 8.831125919129397e-05, "gradient_norm": 0.9529083371162415, "step_time": 3.308827500004554, "tokens_per_second": 2475.8014734792687, "steps_per_second": 0.3022218595555748, "batch_wait": 0.001695599996310193, "processed_target_tokens": 81063442, "elapsed_seconds": 41961.0396977, "eta_seconds": 19852.965000027325, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3432.84, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "validation", "step": 14000, "validation_loss": 1.7432886004447936, "validation_perplexity": 5.716110550664961, "validation_time": 2.2149618999974336, "best_validation_loss": 1.5567119061946868}
{"step": 14000, "val_loss": 1.7432886004447936, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 81063442, "timestamp": 1793451007.669216}
{"event": "checkpoint", "step": 14000, "max_steps": 20000, "progress_percent": 70.0, "train_loss": 1.6794788837432861, "train_perplexity": 5.362760621093546, "validation_loss": 1.7432886004447936, "validation_perplexity": 5.716110550664961, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 81063442, "tokens_per_step": 8192, "tokens_per_second": 2475.8014734792687, "steps_per_second": 0.3022218595555748, "elapsed_seconds": 41961.0396977, "eta_seconds": 19852.965000027325, "learning_rate": 8.831125919129397e-05, "gradient_norm": 0.9529083371162415, "batch_wait": 0.001695599996310193, "evaluation_time": 2.2149618999974336, "parameter_count": 62113280, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3432.84, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 14050, "max_steps": 20000, "progress_percent": 70.25, "loss": 1.612942099571228, "perplexity": 5.017551669893354, "learning_rate": 8.741878311210371e-05, "gradient_norm": 1.0207369327545166, "step_time": 3.2955360999985714, "tokens_per_second": 2485.7867586410453, "steps_per_second": 0.30344076643567447, "batch_wait": 0.0009449999924981967, "processed_target_tokens": 81357121, "elapsed_seconds": 42129.5367038, "eta_seconds": 19608.4397949915, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3432.84, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 14100, "max_steps": 20000, "progress_percent": 70.5, "loss": 1.523078441619873, "perplexity": 4.586322210765237, "learning_rate": 8.65313411668991e-05, "gradient_norm": 0.9628317356109619, "step_time": 3.306550299996161, "tokens_per_second": 2477.506542092982, "steps_per_second": 0.3024299978140847, "batch_wait": 0.001048300000547897, "processed_target_tokens": 81645225, "elapsed_seconds": 42294.7551783, "eta_seconds": 19508.64676997735, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3432.84, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 14150, "max_steps": 20000, "progress_percent": 70.75, "loss": 1.7032105922698975, "perplexity": 5.4915502475846685, "learning_rate": 8.564899094051614e-05, "gradient_norm": 0.926210880279541, "step_time": 3.309332800003176, "tokens_per_second": 2475.423444868445, "steps_per_second": 0.3021757134849176, "batch_wait": 0.0013022999992244877, "processed_target_tokens": 81933881, "elapsed_seconds": 42460.0162114, "eta_seconds": 19359.59688001858, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3432.84, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 14200, "max_steps": 20000, "progress_percent": 71.0, "loss": 1.533172607421875, "perplexity": 4.63285175094638, "learning_rate": 8.477178968739666e-05, "gradient_norm": 0.9142972230911255, "step_time": 3.3036639999991166, "tokens_per_second": 2479.6710561371224, "steps_per_second": 0.30269422071986357, "batch_wait": 0.0019246000083512627, "processed_target_tokens": 82221502, "elapsed_seconds": 42625.2597307, "eta_seconds": 19161.251199994877, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3432.84, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 14250, "max_steps": 20000, "progress_percent": 71.25, "loss": 1.6092180013656616, "perplexity": 4.998900565551139, "learning_rate": 8.389979432787281e-05, "gradient_norm": 0.9620718955993652, "step_time": 3.5943096999981208, "tokens_per_second": 2279.158081454217, "steps_per_second": 0.27821753924001674, "batch_wait": 0.0014716999939992093, "processed_target_tokens": 82514300, "elapsed_seconds": 42795.9037321, "eta_seconds": 20667.280774989194, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3631.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 14300, "max_steps": 20000, "progress_percent": 71.5, "loss": 1.3805599212646484, "perplexity": 3.977127882646505, "learning_rate": 8.303306144447398e-05, "gradient_norm": 0.9444478750228882, "step_time": 3.60830729999725, "tokens_per_second": 2270.316610784853, "steps_per_second": 0.27713825815244786, "batch_wait": 0.0006652000010944903, "processed_target_tokens": 82806063, "elapsed_seconds": 42977.3400258, "eta_seconds": 20567.351609984325, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3631.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 14350, "max_steps": 20000, "progress_percent": 71.75, "loss": 1.673628330230713, "perplexity": 5.331477105279892, "learning_rate": 8.217164727825481e-05, "gradient_norm": 1.1394014358520508, "step_time": 3.5362049999966985, "tokens_per_second": 2316.607775852262, "steps_per_second": 0.28278903513821557, "batch_wait": 0.0009111999970627949, "processed_target_tokens": 83095984, "elapsed_seconds": 43155.8380646, "eta_seconds": 19979.558249981346, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3631.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 14400, "max_steps": 20000, "progress_percent": 72.0, "loss": 1.4580178260803223, "perplexity": 4.2974328211690205, "learning_rate": 8.131560772514603e-05, "gradient_norm": 0.949525773525238, "step_time": 3.6347692999988794, "tokens_per_second": 2253.788156514507, "steps_per_second": 0.2751206245745248, "batch_wait": 0.0011360999997123145, "processed_target_tokens": 83384183, "elapsed_seconds": 43334.2252099, "eta_seconds": 20354.708079993725, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3631.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 14450, "max_steps": 20000, "progress_percent": 72.25, "loss": 1.6320021152496338, "perplexity": 5.114103500125921, "learning_rate": 8.046499833232766e-05, "gradient_norm": 0.9671048521995544, "step_time": 3.6303903000007267, "tokens_per_second": 2256.506690203078, "steps_per_second": 0.2754524768314304, "batch_wait": 0.0008939999897847883, "processed_target_tokens": 83669341, "elapsed_seconds": 43516.0913387, "eta_seconds": 20148.666165004033, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3631.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 14500, "max_steps": 20000, "progress_percent": 72.5, "loss": 1.77750563621521, "perplexity": 5.915083631503669, "learning_rate": 7.961987429462408e-05, "gradient_norm": 1.0044102668762207, "step_time": 3.58819520000543, "tokens_per_second": 2283.0419036254225, "steps_per_second": 0.2786916386261502, "batch_wait": 0.0009655000030761585, "processed_target_tokens": 83961087, "elapsed_seconds": 43697.0681542, "eta_seconds": 19735.073600029864, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3702.1, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "validation", "step": 14500, "validation_loss": 1.6070539593696593, "validation_perplexity": 4.988094431474589, "validation_time": 2.4158944000009797, "best_validation_loss": 1.5567119061946868}
{"step": 14500, "val_loss": 1.6070539593696593, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 83961087, "timestamp": 1793452744.859018}
{"event": "checkpoint", "step": 14500, "max_steps": 20000, "progress_percent": 72.5, "train_loss": 1.77750563621521, "train_perplexity": 5.915083631503669, "validation_loss": 1.6070539593696593, "validation_perplexity": 4.988094431474589, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 83961087, "tokens_per_step": 8192, "tokens_per_second": 2283.0419036254225, "steps_per_second": 0.2786916386261502, "elapsed_seconds": 43697.0681542, "eta_seconds": 19735.073600029864, "learning_rate": 7.961987429462408e-05, "gradient_norm": 1.0044102668762207, "batch_wait": 0.0009655000030761585, "evaluation_time": 2.4158944000009797, "parameter_count": 62113280, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 7, "gpu_memory_used_mb": 3698.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 14550, "max_steps": 20000, "progress_percent": 72.75, "loss": 1.5259675979614258, "perplexity": 4.599591972642742, "learning_rate": 7.878029045092305e-05, "gradient_norm": 0.9006205797195435, "step_time": 3.6442849000013666, "tokens_per_second": 2247.903285496951, "steps_per_second": 0.2744022565303895, "batch_wait": 0.0011518999963300303, "processed_target_tokens": 84255226, "elapsed_seconds": 43882.2680394, "eta_seconds": 19861.352705007448, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 6, "gpu_memory_used_mb": 3698.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 14600, "max_steps": 20000, "progress_percent": 73.0, "loss": 1.690694808959961, "perplexity": 5.423247517339364, "learning_rate": 7.794630128061699e-05, "gradient_norm": 0.9855700731277466, "step_time": 3.510399699996924, "tokens_per_second": 2333.6373917782576, "steps_per_second": 0.28486784567605683, "batch_wait": 0.0008497000017086975, "processed_target_tokens": 84546689, "elapsed_seconds": 44061.4667381, "eta_seconds": 18956.15837998339, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 6, "gpu_memory_used_mb": 3698.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 14650, "max_steps": 20000, "progress_percent": 73.25, "loss": 1.3337763547897339, "perplexity": 3.7953489433017893, "learning_rate": 7.711796090006801e-05, "gradient_norm": 0.8702449202537537, "step_time": 3.597275499996613, "tokens_per_second": 2277.2790129662612, "steps_per_second": 0.27798816076248306, "batch_wait": 0.0013948999985586852, "processed_target_tokens": 84837481, "elapsed_seconds": 44238.2115159, "eta_seconds": 19245.42392498188, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 6, "gpu_memory_used_mb": 3698.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 14700, "max_steps": 20000, "progress_percent": 73.5, "loss": 1.8155921697616577, "perplexity": 6.144713814004147, "learning_rate": 7.629532305909636e-05, "gradient_norm": 1.0409958362579346, "step_time": 3.599103799999284, "tokens_per_second": 2276.1221835284746, "steps_per_second": 0.27784694623150324, "batch_wait": 0.0009376000089105219, "processed_target_tokens": 85125238, "elapsed_seconds": 44418.3139055, "eta_seconds": 19075.250139996206, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 6, "gpu_memory_used_mb": 3698.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 14750, "max_steps": 20000, "progress_percent": 73.75, "loss": 1.8788504600524902, "perplexity": 6.545975674935737, "learning_rate": 7.547844113749268e-05, "gradient_norm": 0.9654471278190613, "step_time": 3.618297499997425, "tokens_per_second": 2264.04821604797, "steps_per_second": 0.2763730732480432, "batch_wait": 0.0004838999957428314, "processed_target_tokens": 85413197, "elapsed_seconds": 44598.4220225, "eta_seconds": 18996.06187498648, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3607.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 14800, "max_steps": 20000, "progress_percent": 74.0, "loss": 1.8041400909423828, "perplexity": 6.074745473255461, "learning_rate": 7.466736814155418e-05, "gradient_norm": 1.0502129793167114, "step_time": 3.596186499999021, "tokens_per_second": 2277.9686203711153, "steps_per_second": 0.2780723413538959, "batch_wait": 0.0014144000015221536, "processed_target_tokens": 85698202, "elapsed_seconds": 44778.6697494, "eta_seconds": 18700.16979999491, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3607.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 14850, "max_steps": 20000, "progress_percent": 74.25, "loss": 1.7799807786941528, "perplexity": 5.9297424401027055, "learning_rate": 7.386215670064524e-05, "gradient_norm": 0.9568449258804321, "step_time": 3.5117408000005526, "tokens_per_second": 2332.746198124506, "steps_per_second": 0.28475905738824536, "batch_wait": 0.0015343999984906986, "processed_target_tokens": 85992465, "elapsed_seconds": 44957.6865723, "eta_seconds": 18085.465120002846, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3607.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 14900, "max_steps": 20000, "progress_percent": 74.5, "loss": 1.8044490814208984, "perplexity": 6.076622801789488, "learning_rate": 7.306285906378236e-05, "gradient_norm": 1.027624487876892, "step_time": 3.665873099998862, "tokens_per_second": 2234.6654607336363, "steps_per_second": 0.27278631112471147, "batch_wait": 0.0007312999950954691, "processed_target_tokens": 86281888, "elapsed_seconds": 45134.0491817, "eta_seconds": 18695.952809994196, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3607.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 14950, "max_steps": 20000, "progress_percent": 74.75, "loss": 1.7586919069290161, "perplexity": 5.804839156006392, "learning_rate": 7.226952709624383e-05, "gradient_norm": 0.967564582824707, "step_time": 3.740973899999517, "tokens_per_second": 2189.8041042203095, "steps_per_second": 0.26731007131595574, "batch_wait": 0.0009926999991876073, "processed_target_tokens": 86576896, "elapsed_seconds": 45317.1542713, "eta_seconds": 18891.91819499756, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3607.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 15000, "max_steps": 20000, "progress_percent": 75.0, "loss": 1.8235232830047607, "perplexity": 6.193642006150387, "learning_rate": 7.148221227620408e-05, "gradient_norm": 0.9753847122192383, "step_time": 3.7612450000015087, "tokens_per_second": 2178.0022306435008, "steps_per_second": 0.2658694129203492, "batch_wait": 0.0009877999982563779, "processed_target_tokens": 86863364, "elapsed_seconds": 45500.1668887, "eta_seconds": 18806.225000007544, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3536.05, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "validation", "step": 15000, "validation_loss": 1.7659905552864075, "validation_perplexity": 5.8473616252874745, "validation_time": 2.4823861999975634, "best_validation_loss": 1.5567119061946868}
{"step": 15000, "val_loss": 1.7659905552864075, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 86863364, "timestamp": 1793454547.6591573}
{"event": "checkpoint", "step": 15000, "max_steps": 20000, "progress_percent": 75.0, "train_loss": 1.8235232830047607, "train_perplexity": 6.193642006150387, "validation_loss": 1.7659905552864075, "validation_perplexity": 5.8473616252874745, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 86863364, "tokens_per_step": 8192, "tokens_per_second": 2178.0022306435008, "steps_per_second": 0.2658694129203492, "elapsed_seconds": 45500.1668887, "eta_seconds": 18806.225000007544, "learning_rate": 7.148221227620408e-05, "gradient_norm": 0.9753847122192383, "batch_wait": 0.0009877999982563779, "evaluation_time": 2.4823861999975634, "parameter_count": 62113280, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3529.92, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 15050, "max_steps": 20000, "progress_percent": 75.25, "loss": 1.4576705694198608, "perplexity": 4.295940768076664, "learning_rate": 7.070096569139352e-05, "gradient_norm": 1.1225861310958862, "step_time": 3.859711299999617, "tokens_per_second": 2122.4385357528718, "steps_per_second": 0.2590867353213955, "batch_wait": 0.0007799000013619661, "processed_target_tokens": 87153902, "elapsed_seconds": 45692.8502254, "eta_seconds": 19105.570934998104, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3529.92, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 15100, "max_steps": 20000, "progress_percent": 75.5, "loss": 1.6409335136413574, "perplexity": 5.159984180227343, "learning_rate": 6.992583803578346e-05, "gradient_norm": 1.115483283996582, "step_time": 3.606294199998956, "tokens_per_second": 2271.583943429344, "steps_per_second": 0.27729296184440233, "batch_wait": 0.0010055000020656735, "processed_target_tokens": 87444821, "elapsed_seconds": 45873.0333803, "eta_seconds": 17670.841579994885, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3529.92, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 15150, "max_steps": 20000, "progress_percent": 75.75, "loss": 1.7444562911987305, "perplexity": 5.72278909858393, "learning_rate": 6.915687960629664e-05, "gradient_norm": 1.1509491205215454, "step_time": 3.3577548999965074, "tokens_per_second": 2439.725424869016, "steps_per_second": 0.29781804502795606, "batch_wait": 0.0013944000020273961, "processed_target_tokens": 87734380, "elapsed_seconds": 46052.9952044, "eta_seconds": 16285.111264983061, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3529.92, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 15200, "max_steps": 20000, "progress_percent": 76.0, "loss": 1.6890294551849365, "perplexity": 5.414223407866484, "learning_rate": 6.839414029954365e-05, "gradient_norm": 0.9792423248291016, "step_time": 3.638146199999028, "tokens_per_second": 2251.6962072613214, "steps_per_second": 0.2748652596754543, "batch_wait": 0.001066900003934279, "processed_target_tokens": 88025707, "elapsed_seconds": 46230.8031096, "eta_seconds": 17463.101759995334, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3529.92, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 15250, "max_steps": 20000, "progress_percent": 76.25, "loss": 1.9576971530914307, "perplexity": 7.08299721156472, "learning_rate": 6.763766960858505e-05, "gradient_norm": 1.100737452507019, "step_time": 3.5380814999953145, "tokens_per_second": 2315.379111535686, "steps_per_second": 0.28263905170113357, "batch_wait": 0.0015186000018729828, "processed_target_tokens": 88317750, "elapsed_seconds": 46409.0930518, "eta_seconds": 16805.887124977744, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3645.25, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 15300, "max_steps": 20000, "progress_percent": 76.5, "loss": 1.4561610221862793, "perplexity": 4.289460734763288, "learning_rate": 6.688751661971989e-05, "gradient_norm": 0.9505186676979065, "step_time": 3.641552100001718, "tokens_per_second": 2249.5902228053073, "steps_per_second": 0.2746081814947885, "batch_wait": 0.0012530999956652522, "processed_target_tokens": 88602741, "elapsed_seconds": 46584.9562849, "eta_seconds": 17115.294870008074, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3645.25, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 15350, "max_steps": 20000, "progress_percent": 76.75, "loss": 1.8414793014526367, "perplexity": 6.305859632081792, "learning_rate": 6.614373000930062e-05, "gradient_norm": 1.0258831977844238, "step_time": 3.4945976000017254, "tokens_per_second": 2344.1897859701944, "steps_per_second": 0.28615597973268975, "batch_wait": 0.0004960999940522015, "processed_target_tokens": 88897782, "elapsed_seconds": 46765.7323061, "eta_seconds": 16249.878840008023, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3645.25, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 15400, "max_steps": 20000, "progress_percent": 77.0, "loss": 1.7124422788619995, "perplexity": 5.542481246171945, "learning_rate": 6.540635804057443e-05, "gradient_norm": 1.1732317209243774, "step_time": 3.7700072000006912, "tokens_per_second": 2172.9401471696124, "steps_per_second": 0.2652514828087906, "batch_wait": 0.000960600002144929, "processed_target_tokens": 89180778, "elapsed_seconds": 46946.0612007, "eta_seconds": 17342.03312000318, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3645.25, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 15450, "max_steps": 20000, "progress_percent": 77.25, "loss": 1.5648767948150635, "perplexity": 4.782085721843159, "learning_rate": 6.467544856055175e-05, "gradient_norm": 0.9464347958564758, "step_time": 3.5431637999936356, "tokens_per_second": 2312.057940988987, "steps_per_second": 0.2822336353746322, "batch_wait": 0.0018151999975088984, "processed_target_tokens": 89469145, "elapsed_seconds": 47123.0522446, "eta_seconds": 16121.395289971042, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3645.25, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 15500, "max_steps": 20000, "progress_percent": 77.5, "loss": 1.630231261253357, "perplexity": 5.105055183493753, "learning_rate": 6.395104899690134e-05, "gradient_norm": 0.9878023266792297, "step_time": 3.37047440000606, "tokens_per_second": 2430.5183863687766, "steps_per_second": 0.2966941389610323, "batch_wait": 0.0013578000070992857, "processed_target_tokens": 89753036, "elapsed_seconds": 47308.2734787, "eta_seconds": 15167.13480002727, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3544.34, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "validation", "step": 15500, "validation_loss": 1.7128706336021424, "validation_perplexity": 5.544855902847307, "validation_time": 2.2404445000065607, "best_validation_loss": 1.5567119061946868}
{"step": 15500, "val_loss": 1.7128706336021424, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 89753036, "timestamp": 1793456354.809678}
{"event": "checkpoint", "step": 15500, "max_steps": 20000, "progress_percent": 77.5, "train_loss": 1.630231261253357, "train_perplexity": 5.105055183493753, "validation_loss": 1.7128706336021424, "validation_perplexity": 5.544855902847307, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 89753036, "tokens_per_step": 8192, "tokens_per_second": 2430.5183863687766, "steps_per_second": 0.2966941389610323, "elapsed_seconds": 47308.2734787, "eta_seconds": 15167.13480002727, "learning_rate": 6.395104899690134e-05, "gradient_norm": 0.9878023266792297, "batch_wait": 0.0013578000070992857, "evaluation_time": 2.2404445000065607, "parameter_count": 62113280, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3521.15, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 15550, "max_steps": 20000, "progress_percent": 77.75, "loss": 1.6147878170013428, "perplexity": 5.0268212043060725, "learning_rate": 6.323320635487272e-05, "gradient_norm": 0.9832611083984375, "step_time": 3.297987199999625, "tokens_per_second": 2483.9392948526092, "steps_per_second": 0.30321524595368765, "batch_wait": 0.000738499998988118, "processed_target_tokens": 90043708, "elapsed_seconds": 47492.647038, "eta_seconds": 14676.04303999833, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3521.15, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 15600, "max_steps": 20000, "progress_percent": 78.0, "loss": 1.6486353874206543, "perplexity": 5.199879162937039, "learning_rate": 6.252196721424626e-05, "gradient_norm": 0.9156558513641357, "step_time": 3.355644099996425, "tokens_per_second": 2441.2600847654635, "steps_per_second": 0.2980053814410966, "batch_wait": 0.0023610999996890314, "processed_target_tokens": 90329632, "elapsed_seconds": 47659.6296919, "eta_seconds": 14764.834039984271, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3521.15, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 15650, "max_steps": 20000, "progress_percent": 78.25, "loss": 1.8643957376480103, "perplexity": 6.45203598538868, "learning_rate": 6.181737772631046e-05, "gradient_norm": 1.144761562347412, "step_time": 3.89427479999722, "tokens_per_second": 2103.6009066452752, "steps_per_second": 0.2567872200494721, "batch_wait": 0.0006204000019351952, "processed_target_tokens": 90621026, "elapsed_seconds": 47833.6861764, "eta_seconds": 16940.095379987906, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3521.15, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 15700, "max_steps": 20000, "progress_percent": 78.5, "loss": 1.581529140472412, "perplexity": 4.862385399654641, "learning_rate": 6.111948361086759e-05, "gradient_norm": 0.9938846230506897, "step_time": 3.741508099999919, "tokens_per_second": 2189.4914513215076, "steps_per_second": 0.26727190567889497, "batch_wait": 0.000763399999414105, "processed_target_tokens": 90905702, "elapsed_seconds": 48025.4424868, "eta_seconds": 16088.484829999652, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3521.15, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 15750, "max_steps": 20000, "progress_percent": 78.75, "loss": 1.53464937210083, "perplexity": 4.6396984370027905, "learning_rate": 6.042833015326665e-05, "gradient_norm": 0.9468088746070862, "step_time": 3.5570842000015546, "tokens_per_second": 2303.009864089363, "steps_per_second": 0.28112913379997106, "batch_wait": 0.0011120999915874563, "processed_target_tokens": 91206740, "elapsed_seconds": 48208.2179513, "eta_seconds": 15117.607850006607, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3695.86, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 15800, "max_steps": 20000, "progress_percent": 79.0, "loss": 1.9068074226379395, "perplexity": 6.731563419111839, "learning_rate": 5.974396220146498e-05, "gradient_norm": 1.1021043062210083, "step_time": 3.573991700002807, "tokens_per_second": 2292.115004070537, "steps_per_second": 0.2797991948328292, "batch_wait": 0.0018159999890485778, "processed_target_tokens": 91496902, "elapsed_seconds": 48391.0264222, "eta_seconds": 15010.76514001179, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3695.86, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 15850, "max_steps": 20000, "progress_percent": 79.25, "loss": 1.6338502168655396, "perplexity": 5.123563622009112, "learning_rate": 5.906642416311832e-05, "gradient_norm": 1.039717197418213, "step_time": 3.599594600003911, "tokens_per_second": 2275.811837252756, "steps_per_second": 0.2778090621646431, "batch_wait": 0.0017135000016423874, "processed_target_tokens": 91781097, "elapsed_seconds": 48572.9328144, "eta_seconds": 14938.317590016231, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3695.86, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 15900, "max_steps": 20000, "progress_percent": 79.5, "loss": 1.8348093032836914, "perplexity": 6.263939518836252, "learning_rate": 5.839576000269893e-05, "gradient_norm": 0.9926276803016663, "step_time": 3.6181322999982513, "tokens_per_second": 2264.151590035544, "steps_per_second": 0.27638569214301073, "batch_wait": 0.0010760999939520843, "processed_target_tokens": 92072840, "elapsed_seconds": 48755.9787789, "eta_seconds": 14834.34242999283, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3695.86, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 15950, "max_steps": 20000, "progress_percent": 79.75, "loss": 1.6999677419662476, "perplexity": 5.47377081579549, "learning_rate": 5.7732013238643196e-05, "gradient_norm": 0.9777299165725708, "step_time": 3.7164211999988765, "tokens_per_second": 2204.271141280347, "steps_per_second": 0.2690760670508236, "batch_wait": 0.001137600003858097, "processed_target_tokens": 92358786, "elapsed_seconds": 48939.598363, "eta_seconds": 15051.50585999545, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3695.86, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 16000, "max_steps": 20000, "progress_percent": 80.0, "loss": 1.7368075847625732, "perplexity": 5.679184138127489, "learning_rate": 5.7075226940527376e-05, "gradient_norm": 1.1033953428268433, "step_time": 3.5351923999987775, "tokens_per_second": 2317.271331541342, "steps_per_second": 0.28287003558854273, "batch_wait": 0.001338500005658716, "processed_target_tokens": 92643929, "elapsed_seconds": 49118.1931699, "eta_seconds": 14140.76959999511, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3534.36, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "validation", "step": 16000, "validation_loss": 1.7178968071937561, "validation_perplexity": 5.572795466886575, "validation_time": 2.313634299993282, "best_validation_loss": 1.5567119061946868}
{"step": 16000, "val_loss": 1.7178968071937561, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 92643929, "timestamp": 1793458164.7752118}
{"event": "checkpoint", "step": 16000, "max_steps": 20000, "progress_percent": 80.0, "train_loss": 1.7368075847625732, "train_perplexity": 5.679184138127489, "validation_loss": 1.7178968071937561, "validation_perplexity": 5.572795466886575, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 92643929, "tokens_per_step": 8192, "tokens_per_second": 2317.271331541342, "steps_per_second": 0.28287003558854273, "elapsed_seconds": 49118.1931699, "eta_seconds": 14140.76959999511, "learning_rate": 5.7075226940527376e-05, "gradient_norm": 1.1033953428268433, "batch_wait": 0.001338500005658716, "evaluation_time": 2.313634299993282, "parameter_count": 62113280, "gpu": {"gpu_utilization": 12, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3526.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 16050, "max_steps": 20000, "progress_percent": 80.25, "loss": 1.8555090427398682, "perplexity": 6.394952726474182, "learning_rate": 5.6425443726273115e-05, "gradient_norm": 1.0441430807113647, "step_time": 3.532092900000862, "tokens_per_second": 2319.3047951819162, "steps_per_second": 0.283118261130605, "batch_wait": 0.0015082000099937432, "processed_target_tokens": 92939859, "elapsed_seconds": 49299.3583242, "eta_seconds": 13951.766955003404, "gpu": {"gpu_utilization": 12, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3526.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 16100, "max_steps": 20000, "progress_percent": 80.5, "loss": 1.7270077466964722, "perplexity": 5.623800869991321, "learning_rate": 5.578270575938211e-05, "gradient_norm": 1.0082398653030396, "step_time": 3.463078400003724, "tokens_per_second": 2365.5254238515627, "steps_per_second": 0.2887604277162552, "batch_wait": 0.0006935999990673736, "processed_target_tokens": 93228544, "elapsed_seconds": 49475.3369432, "eta_seconds": 13506.005760014523, "gpu": {"gpu_utilization": 12, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3526.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 16150, "max_steps": 20000, "progress_percent": 80.75, "loss": 1.608799934387207, "perplexity": 4.996811127089149, "learning_rate": 5.514705474619985e-05, "gradient_norm": 0.9610632658004761, "step_time": 3.454993100000138, "tokens_per_second": 2371.0611752016734, "steps_per_second": 0.28943617861348553, "batch_wait": 0.0009776000006240793, "processed_target_tokens": 93515552, "elapsed_seconds": 49651.3536188, "eta_seconds": 13301.723435000531, "gpu": {"gpu_utilization": 12, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3526.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 16200, "max_steps": 20000, "progress_percent": 81.0, "loss": 1.6696555614471436, "perplexity": 5.3103383968686595, "learning_rate": 5.451853193320974e-05, "gradient_norm": 1.0234403610229492, "step_time": 3.3132211999982246, "tokens_per_second": 2472.518285227799, "steps_per_second": 0.30182107973972155, "batch_wait": 0.0009655999965616502, "processed_target_tokens": 93806585, "elapsed_seconds": 49824.5930467, "eta_seconds": 12590.240559993254, "gpu": {"gpu_utilization": 12, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3526.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 16250, "max_steps": 20000, "progress_percent": 81.25, "loss": 1.7205219268798828, "perplexity": 5.5874439405595675, "learning_rate": 5.3897178104356386e-05, "gradient_norm": 1.0675734281539917, "step_time": 3.8570284999950672, "tokens_per_second": 2123.9148219958647, "steps_per_second": 0.2592669460444171, "batch_wait": 0.0006830999991507269, "processed_target_tokens": 94096673, "elapsed_seconds": 50008.3901498, "eta_seconds": 14463.856874981502, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3590.88, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 16300, "max_steps": 20000, "progress_percent": 81.5, "loss": 1.4434559345245361, "perplexity": 4.235307499353974, "learning_rate": 5.328303357839929e-05, "gradient_norm": 0.9131332635879517, "step_time": 3.769780800001172, "tokens_per_second": 2173.0706464411546, "steps_per_second": 0.26526741289564876, "batch_wait": 0.001404300011927262, "processed_target_tokens": 94379221, "elapsed_seconds": 50194.6412582, "eta_seconds": 13948.188960004336, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3590.88, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 16350, "max_steps": 20000, "progress_percent": 81.75, "loss": 1.576329231262207, "perplexity": 4.837167060392024, "learning_rate": 5.2676138206296635e-05, "gradient_norm": 0.9096693992614746, "step_time": 3.762052499994752, "tokens_per_second": 2177.5347366926508, "steps_per_second": 0.2658123457876771, "batch_wait": 0.0009432000006199814, "processed_target_tokens": 94669519, "elapsed_seconds": 50382.6209974, "eta_seconds": 13731.491624980845, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3590.88, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"step": 16360, "val_loss": 1.7178968071937561, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 94726900, "timestamp": 1793459465.027338}
{"event": "checkpoint", "step": 16360, "max_steps": 20000, "progress_percent": 81.8, "train_loss": 1.6228225231170654, "train_perplexity": 5.06737292781317, "validation_loss": 1.7178968071937561, "validation_perplexity": 5.572795466886575, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 94726900, "tokens_per_step": 8192, "tokens_per_second": 2275.5923442989515, "steps_per_second": 0.2777822685911806, "elapsed_seconds": 50420.0534869, "eta_seconds": 13103.788151997142, "learning_rate": 5.25435979808281e-05, "gradient_norm": 0.9682856798171997, "batch_wait": 0.0007101000010152347, "evaluation_time": 2.313634299993282, "parameter_count": 62113280, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3414.31, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 16400, "max_steps": 20000, "progress_percent": 82.0, "loss": 1.627275824546814, "perplexity": 5.089989789390748, "learning_rate": 5.207653136861939e-05, "gradient_norm": 1.0512953996658325, "step_time": 3.4074152000000595, "tokens_per_second": 2404.1684148148006, "steps_per_second": 0.29347758969907234, "batch_wait": 0.000980400000003101, "processed_target_tokens": 94954218, "elapsed_seconds": 138.60114529999998, "eta_seconds": 12266.694720000214, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 41, "gpu_memory_used_mb": 3620.67, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 16450, "max_steps": 20000, "progress_percent": 82.25, "loss": 1.5231918096542358, "perplexity": 4.58684218257276, "learning_rate": 5.148425197299601e-05, "gradient_norm": 1.0259199142456055, "step_time": 3.637568599999895, "tokens_per_second": 2252.0537482097893, "steps_per_second": 0.2749089048107653, "batch_wait": 0.000750400000015361, "processed_target_tokens": 95243400, "elapsed_seconds": 311.53025979999995, "eta_seconds": 12913.368529999627, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 41, "gpu_memory_used_mb": 3620.67, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 16500, "max_steps": 20000, "progress_percent": 82.5, "loss": 1.6917228698730469, "perplexity": 5.428825813056513, "learning_rate": 5.089933845158773e-05, "gradient_norm": 0.9621163606643677, "step_time": 3.4815733000000364, "tokens_per_second": 2352.959221050987, "steps_per_second": 0.28722646741345054, "batch_wait": 0.0013503000002401677, "processed_target_tokens": 95531816, "elapsed_seconds": 486.3799170000001, "eta_seconds": 12185.506550000127, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 41, "gpu_memory_used_mb": 3620.67, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "validation", "step": 16500, "validation_loss": 1.6333218097686768, "validation_perplexity": 5.120857009789544, "validation_time": 2.326253399999814, "best_validation_loss": 1.5567119061946868}
{"step": 16500, "val_loss": 1.6333218097686768, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 95531816, "timestamp": 1793497030.290106}
{"event": "checkpoint", "step": 16500, "max_steps": 20000, "progress_percent": 82.5, "train_loss": 1.6917228698730469, "train_perplexity": 5.428825813056513, "validation_loss": 1.6333218097686768, "validation_perplexity": 5.120857009789544, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 95531816, "tokens_per_step": 8192, "tokens_per_second": 2352.959221050987, "steps_per_second": 0.28722646741345054, "elapsed_seconds": 486.3799170000001, "eta_seconds": 12185.506550000127, "learning_rate": 5.089933845158773e-05, "gradient_norm": 0.9621163606643677, "batch_wait": 0.0013503000002401677, "evaluation_time": 2.326253399999814, "parameter_count": 62113280, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3581.82, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 16550, "max_steps": 20000, "progress_percent": 82.75, "loss": 1.5415047407150269, "perplexity": 4.6716145534242965, "learning_rate": 5.0321828758594684e-05, "gradient_norm": 0.9826920628547668, "step_time": 3.463243299999931, "tokens_per_second": 2365.4127909523895, "steps_per_second": 0.28874667858305536, "batch_wait": 0.00119370000015806, "processed_target_tokens": 95820201, "elapsed_seconds": 664.959397, "eta_seconds": 11948.189384999761, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3581.82, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 16600, "max_steps": 20000, "progress_percent": 83.0, "loss": 1.6592828035354614, "perplexity": 5.255540237597927, "learning_rate": 4.9751760367793284e-05, "gradient_norm": 0.9731417298316956, "step_time": 3.461050000000114, "tokens_per_second": 2366.9117753282185, "steps_per_second": 0.2889296600742454, "batch_wait": 0.0014491000001726206, "processed_target_tokens": 96113175, "elapsed_seconds": 840.7352445, "eta_seconds": 11767.570000000387, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3581.82, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 16650, "max_steps": 20000, "progress_percent": 83.25, "loss": 1.7877289056777954, "perplexity": 5.975865289613678, "learning_rate": 4.918917027010442e-05, "gradient_norm": 1.084500789642334, "step_time": 3.51947569999993, "tokens_per_second": 2327.619423540888, "steps_per_second": 0.28413323041270605, "batch_wait": 0.0017010000001391745, "processed_target_tokens": 96407401, "elapsed_seconds": 1017.5892620000001, "eta_seconds": 11790.243594999765, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3581.82, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 16700, "max_steps": 20000, "progress_percent": 83.5, "loss": 1.6204397678375244, "perplexity": 5.055312991859931, "learning_rate": 4.863409497119335e-05, "gradient_norm": 1.1001163721084595, "step_time": 3.529397599999811, "tokens_per_second": 2321.075981918398, "steps_per_second": 0.2833344704490232, "batch_wait": 0.0017713000002004264, "processed_target_tokens": 96697710, "elapsed_seconds": 1193.9319707999998, "eta_seconds": 11647.012079999377, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3581.82, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "train", "step": 16750, "max_steps": 20000, "progress_percent": 83.75, "loss": 1.6627864837646484, "perplexity": 5.273986265644044, "learning_rate": 4.808657048910077e-05, "gradient_norm": 1.0608540773391724, "step_time": 3.332272600000124, "tokens_per_second": 2458.3823064174567, "steps_per_second": 0.3000954963888497, "batch_wait": 0.0011894000008396688, "processed_target_tokens": 96979270, "elapsed_seconds": 1362.9301612, "eta_seconds": 10829.885950000404, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3555.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 16800, "max_steps": 20000, "progress_percent": 84.0, "loss": 1.3353204727172852, "perplexity": 3.8012139366017785, "learning_rate": 4.754663235190571e-05, "gradient_norm": 0.9557269215583801, "step_time": 3.3112716000000546, "tokens_per_second": 2473.9740467075744, "steps_per_second": 0.3019987849984832, "batch_wait": 0.0012736999997287057, "processed_target_tokens": 97272093, "elapsed_seconds": 1528.8063325000003, "eta_seconds": 10596.069120000175, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3555.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 16850, "max_steps": 20000, "progress_percent": 84.25, "loss": 1.95383882522583, "perplexity": 7.0557213395172464, "learning_rate": 4.701431559542008e-05, "gradient_norm": 1.1220892667770386, "step_time": 3.3093237999996745, "tokens_per_second": 2475.4301770049838, "steps_per_second": 0.3021765352789287, "batch_wait": 0.0014196999991327175, "processed_target_tokens": 97566111, "elapsed_seconds": 1694.8334771, "eta_seconds": 10424.369969998974, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3555.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 16900, "max_steps": 20000, "progress_percent": 84.5, "loss": 1.4339467287063599, "perplexity": 4.195223971941005, "learning_rate": 4.6489654760915376e-05, "gradient_norm": 0.9755064845085144, "step_time": 3.3117505000000165, "tokens_per_second": 2473.616294464199, "steps_per_second": 0.3019551140703368, "batch_wait": 0.0012577000006785966, "processed_target_tokens": 97857074, "elapsed_seconds": 1860.848911, "eta_seconds": 10266.426550000051, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3555.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 16950, "max_steps": 20000, "progress_percent": 84.75, "loss": 1.6138293743133545, "perplexity": 5.022005592391635, "learning_rate": 4.597268389288132e-05, "gradient_norm": 0.9488036036491394, "step_time": 3.3042154999998274, "tokens_per_second": 2479.257179200457, "steps_per_second": 0.3026436986328683, "batch_wait": 0.0005711000003429945, "processed_target_tokens": 98135980, "elapsed_seconds": 2026.1816889000002, "eta_seconds": 10077.857274999475, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3555.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 17000, "max_steps": 20000, "progress_percent": 85.0, "loss": 1.827567219734192, "perplexity": 6.2187394145113775, "learning_rate": 4.546343653681667e-05, "gradient_norm": 1.0685176849365234, "step_time": 3.484766899999613, "tokens_per_second": 2350.802861448469, "steps_per_second": 0.2869632399229088, "batch_wait": 0.001483499999721971, "processed_target_tokens": 98419906, "elapsed_seconds": 2201.6956489, "eta_seconds": 10454.30069999884, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3618.73, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "validation", "step": 17000, "validation_loss": 1.794072961807251, "validation_perplexity": 6.013897024608469, "validation_time": 2.333172000000104, "best_validation_loss": 1.5567119061946868}
{"step": 17000, "val_loss": 1.794072961807251, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 98419906, "timestamp": 1793498745.1072633}
{"event": "checkpoint", "step": 17000, "max_steps": 20000, "progress_percent": 85.0, "train_loss": 1.827567219734192, "train_perplexity": 6.2187394145113775, "validation_loss": 1.794072961807251, "validation_perplexity": 6.013897024608469, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 98419906, "tokens_per_step": 8192, "tokens_per_second": 2350.802861448469, "steps_per_second": 0.2869632399229088, "elapsed_seconds": 2201.6956489, "eta_seconds": 10454.30069999884, "learning_rate": 4.546343653681667e-05, "gradient_norm": 1.0685176849365234, "batch_wait": 0.001483499999721971, "evaluation_time": 2.333172000000104, "parameter_count": 62113280, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3591.51, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 17050, "max_steps": 20000, "progress_percent": 85.25, "loss": 1.6608401536941528, "perplexity": 5.2637313305654105, "learning_rate": 4.496194573705259e-05, "gradient_norm": 0.9541724324226379, "step_time": 3.4774826000002577, "tokens_per_second": 2355.7270998277295, "steps_per_second": 0.28756434324068963, "batch_wait": 0.0013947999996162253, "processed_target_tokens": 98707627, "elapsed_seconds": 2378.8925659, "eta_seconds": 10258.57367000076, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3591.51, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 17100, "max_steps": 20000, "progress_percent": 85.5, "loss": 1.619215726852417, "perplexity": 5.049128867148528, "learning_rate": 4.446824403460836e-05, "gradient_norm": 1.0524364709854126, "step_time": 3.479511900000034, "tokens_per_second": 2354.3532068391314, "steps_per_second": 0.2873966316942299, "batch_wait": 0.0014378999994733022, "processed_target_tokens": 98995512, "elapsed_seconds": 2553.1895373, "eta_seconds": 10090.584510000099, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3591.51, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 17150, "max_steps": 20000, "progress_percent": 85.75, "loss": 1.761436939239502, "perplexity": 5.8207955173910495, "learning_rate": 4.398236346507999e-05, "gradient_norm": 1.0582166910171509, "step_time": 3.4027012999999897, "tokens_per_second": 2407.499006745031, "steps_per_second": 0.2938841560968055, "batch_wait": 0.0005183000002944027, "processed_target_tokens": 99283059, "elapsed_seconds": 2724.5061137000002, "eta_seconds": 9697.69870499997, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3591.51, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 17200, "max_steps": 20000, "progress_percent": 86.0, "loss": 1.8144863843917847, "perplexity": 6.137922834741301, "learning_rate": 4.3504335556561374e-05, "gradient_norm": 0.9954750537872314, "step_time": 3.388650200000029, "tokens_per_second": 2417.481745386387, "steps_per_second": 0.2951027521223617, "batch_wait": 0.0009816000001592329, "processed_target_tokens": 99568660, "elapsed_seconds": 2896.9668996, "eta_seconds": 9488.220560000082, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3591.51, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 17250, "max_steps": 20000, "progress_percent": 86.25, "loss": 1.460899829864502, "perplexity": 4.309835903101478, "learning_rate": 4.303419132759841e-05, "gradient_norm": 0.9317931532859802, "step_time": 3.493399899999986, "tokens_per_second": 2344.9934832825847, "steps_per_second": 0.28625408731476865, "batch_wait": 0.0009958999999071239, "processed_target_tokens": 99862064, "elapsed_seconds": 3072.2272190000003, "eta_seconds": 9606.84972499996, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 49, "gpu_memory_used_mb": 3602.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 17300, "max_steps": 20000, "progress_percent": 86.5, "loss": 1.5165412425994873, "perplexity": 4.556438294679279, "learning_rate": 4.257196128517637e-05, "gradient_norm": 1.0655732154846191, "step_time": 3.484692399999858, "tokens_per_second": 2350.853119776177, "steps_per_second": 0.2869693749726779, "batch_wait": 0.0012646999998651154, "processed_target_tokens": 100150230, "elapsed_seconds": 3251.4570261, "eta_seconds": 9408.669479999617, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 49, "gpu_memory_used_mb": 3602.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 17350, "max_steps": 20000, "progress_percent": 86.75, "loss": 1.8041574954986572, "perplexity": 6.074851202424985, "learning_rate": 4.211767542274033e-05, "gradient_norm": 1.0590547323226929, "step_time": 3.4757659999995667, "tokens_per_second": 2356.890538661412, "steps_per_second": 0.2877063645826919, "batch_wait": 0.0014887000015733065, "processed_target_tokens": 100432007, "elapsed_seconds": 3427.3787551000005, "eta_seconds": 9210.779899998852, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 49, "gpu_memory_used_mb": 3602.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 17400, "max_steps": 20000, "progress_percent": 87.0, "loss": 1.515929937362671, "perplexity": 4.553653771272095, "learning_rate": 4.167136321824887e-05, "gradient_norm": 1.0424697399139404, "step_time": 3.487002400000165, "tokens_per_second": 2349.29577335525, "steps_per_second": 0.2867792692084045, "batch_wait": 0.001026099999762664, "processed_target_tokens": 100719152, "elapsed_seconds": 3607.1699392, "eta_seconds": 9066.206240000429, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 49, "gpu_memory_used_mb": 3602.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 17450, "max_steps": 20000, "progress_percent": 87.25, "loss": 2.0005545616149902, "perplexity": 7.393154922234117, "learning_rate": 4.123305363226137e-05, "gradient_norm": 1.1003262996673584, "step_time": 3.477521899999374, "tokens_per_second": 2355.7004774007246, "steps_per_second": 0.28756109343270564, "batch_wait": 0.0012413000013111741, "processed_target_tokens": 101007731, "elapsed_seconds": 3782.0657969000004, "eta_seconds": 8867.680844998404, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 49, "gpu_memory_used_mb": 3602.54, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 17500, "max_steps": 20000, "progress_percent": 87.5, "loss": 1.6498970985412598, "perplexity": 5.206444048926656, "learning_rate": 4.080277510605873e-05, "gradient_norm": 1.0248335599899292, "step_time": 3.3189766000004965, "tokens_per_second": 2468.230719071287, "steps_per_second": 0.3012976951991317, "batch_wait": 0.0008844999993016245, "processed_target_tokens": 101296887, "elapsed_seconds": 3952.4611323999998, "eta_seconds": 8297.441500001241, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3286.19, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "validation", "step": 17500, "validation_loss": 1.633911406993866, "validation_perplexity": 5.123877143116732, "validation_time": 2.2302614000000176, "best_validation_loss": 1.5567119061946868}
{"step": 17500, "val_loss": 1.633911406993866, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 101296887, "timestamp": 1793500494.9791384}
{"event": "checkpoint", "step": 17500, "max_steps": 20000, "progress_percent": 87.5, "train_loss": 1.6498970985412598, "train_perplexity": 5.206444048926656, "validation_loss": 1.633911406993866, "validation_perplexity": 5.123877143116732, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 101296887, "tokens_per_step": 8192, "tokens_per_second": 2468.230719071287, "steps_per_second": 0.3012976951991317, "elapsed_seconds": 3952.4611323999998, "eta_seconds": 8297.441500001241, "learning_rate": 4.080277510605873e-05, "gradient_norm": 1.0248335599899292, "batch_wait": 0.0008844999993016245, "evaluation_time": 2.2302614000000176, "parameter_count": 62113280, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3272.69, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 17550, "max_steps": 20000, "progress_percent": 87.75, "loss": 1.780056118965149, "perplexity": 5.9301892053345755, "learning_rate": 4.038055555979791e-05, "gradient_norm": 1.1050429344177246, "step_time": 3.4870004000003973, "tokens_per_second": 2349.2971208145163, "steps_per_second": 0.28677943369317827, "batch_wait": 0.002048500000455533, "processed_target_tokens": 101581526, "elapsed_seconds": 4125.9656199, "eta_seconds": 8543.150980000974, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3272.69, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 17600, "max_steps": 20000, "progress_percent": 88.0, "loss": 1.720057487487793, "perplexity": 5.584849514016057, "learning_rate": 3.996642239070019e-05, "gradient_norm": 1.0493113994598389, "step_time": 3.478434300000117, "tokens_per_second": 2355.0825726389958, "steps_per_second": 0.28748566560534616, "batch_wait": 0.0010857999996005674, "processed_target_tokens": 101868805, "elapsed_seconds": 4300.2036794000005, "eta_seconds": 8348.242320000281, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3272.69, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 17650, "max_steps": 20000, "progress_percent": 88.25, "loss": 1.475678563117981, "perplexity": 4.374002803137431, "learning_rate": 3.956040247127342e-05, "gradient_norm": 0.9710196256637573, "step_time": 3.398989000000256, "tokens_per_second": 2410.128423481036, "steps_per_second": 0.2942051298194624, "batch_wait": 0.001443999999537482, "processed_target_tokens": 102155746, "elapsed_seconds": 4473.2116951, "eta_seconds": 7987.624150000602, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3272.69, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 17700, "max_steps": 20000, "progress_percent": 88.5, "loss": 1.6628659963607788, "perplexity": 5.274405630656161, "learning_rate": 3.916252214756836e-05, "gradient_norm": 0.9844022393226624, "step_time": 3.484116599999652, "tokens_per_second": 2351.2416318101464, "steps_per_second": 0.2870168007580745, "batch_wait": 0.0010791000004246598, "processed_target_tokens": 102441322, "elapsed_seconds": 4645.3291263, "eta_seconds": 8013.4681799991995, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3272.69, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 17750, "max_steps": 20000, "progress_percent": 88.75, "loss": 1.4780378341674805, "perplexity": 4.384334444099714, "learning_rate": 3.8772807237469016e-05, "gradient_norm": 0.9644654393196106, "step_time": 3.6782099000001836, "tokens_per_second": 2227.1703417468348, "steps_per_second": 0.2718713796077679, "batch_wait": 0.0011422999987189542, "processed_target_tokens": 102728587, "elapsed_seconds": 4822.1786043, "eta_seconds": 8275.972275000413, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3451.08, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 17800, "max_steps": 20000, "progress_percent": 89.0, "loss": 1.4706048965454102, "perplexity": 4.351866774227409, "learning_rate": 3.839128302901742e-05, "gradient_norm": 0.9399583339691162, "step_time": 3.4075284999998985, "tokens_per_second": 2404.0884764427483, "steps_per_second": 0.29346783159701517, "batch_wait": 0.0010898000009547104, "processed_target_tokens": 103016040, "elapsed_seconds": 4998.5020418, "eta_seconds": 7496.562699999777, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3451.08, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 17850, "max_steps": 20000, "progress_percent": 89.25, "loss": 1.513875961303711, "perplexity": 4.544310274389436, "learning_rate": 3.801797427877269e-05, "gradient_norm": 0.9043550491333008, "step_time": 3.403866499999822, "tokens_per_second": 2406.6748798757026, "steps_per_second": 0.293783554672327, "batch_wait": 0.000974599999608472, "processed_target_tokens": 103305635, "elapsed_seconds": 5169.2157559, "eta_seconds": 7318.312974999617, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3451.08, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 17900, "max_steps": 20000, "progress_percent": 89.5, "loss": 1.521390438079834, "perplexity": 4.578587012995009, "learning_rate": 3.765290521020467e-05, "gradient_norm": 0.9839680194854736, "step_time": 3.4756446000001233, "tokens_per_second": 2356.9728619547896, "steps_per_second": 0.2877164138128405, "batch_wait": 0.0009844000014709309, "processed_target_tokens": 103595798, "elapsed_seconds": 5343.1630806, "eta_seconds": 7298.853660000259, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3451.08, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 17950, "max_steps": 20000, "progress_percent": 89.75, "loss": 1.6327548027038574, "perplexity": 5.117954270701617, "learning_rate": 3.72960995121221e-05, "gradient_norm": 0.9932989478111267, "step_time": 3.4213368999999147, "tokens_per_second": 2394.3856566712866, "steps_per_second": 0.29228340535538166, "batch_wait": 0.0009329000004072441, "processed_target_tokens": 103882627, "elapsed_seconds": 5517.347515500001, "eta_seconds": 7013.740644999825, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3451.08, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 18000, "max_steps": 20000, "progress_percent": 90.0, "loss": 1.8715269565582275, "perplexity": 6.498211313920824, "learning_rate": 3.694758033713536e-05, "gradient_norm": 1.114031195640564, "step_time": 3.410624799999823, "tokens_per_second": 2401.9059499011514, "steps_per_second": 0.2932014099000429, "batch_wait": 0.001302300001043477, "processed_target_tokens": 104173924, "elapsed_seconds": 5688.0687359, "eta_seconds": 6821.249599999646, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3351.82, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "validation", "step": 18000, "validation_loss": 1.696633791923523, "validation_perplexity": 5.455551924662323, "validation_time": 2.2925717000007353, "best_validation_loss": 1.5567119061946868}
{"step": 18000, "val_loss": 1.696633791923523, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 104173924, "timestamp": 1793502231.602326}
{"event": "checkpoint", "step": 18000, "max_steps": 20000, "progress_percent": 90.0, "train_loss": 1.8715269565582275, "train_perplexity": 6.498211313920824, "validation_loss": 1.696633791923523, "validation_perplexity": 5.455551924662323, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 104173924, "tokens_per_step": 8192, "tokens_per_second": 2401.9059499011514, "steps_per_second": 0.2932014099000429, "elapsed_seconds": 5688.0687359, "eta_seconds": 6821.249599999646, "learning_rate": 3.694758033713536e-05, "gradient_norm": 1.114031195640564, "batch_wait": 0.001302300001043477, "evaluation_time": 2.2925717000007353, "parameter_count": 62113280, "gpu": {"gpu_utilization": 2, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3350.45, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 18050, "max_steps": 20000, "progress_percent": 90.25, "loss": 1.8832954168319702, "perplexity": 6.575137016328155, "learning_rate": 3.660737030015427e-05, "gradient_norm": 1.1511048078536987, "step_time": 3.4985077000001183, "tokens_per_second": 2341.5698070350745, "steps_per_second": 0.28583615808533624, "batch_wait": 0.0011270999993939768, "processed_target_tokens": 104467570, "elapsed_seconds": 5865.4197527, "eta_seconds": 6822.090015000231, "gpu": {"gpu_utilization": 2, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3350.45, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 18100, "max_steps": 20000, "progress_percent": 90.5, "loss": 1.7314634323120117, "perplexity": 5.6489146666747665, "learning_rate": 3.6275491476920604e-05, "gradient_norm": 1.0488048791885376, "step_time": 3.663329099999828, "tokens_per_second": 2236.217324837232, "steps_per_second": 0.2729757476607949, "batch_wait": 0.001006599998618185, "processed_target_tokens": 104758320, "elapsed_seconds": 6043.0174514, "eta_seconds": 6960.325289999673, "gpu": {"gpu_utilization": 2, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3350.45, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 18150, "max_steps": 20000, "progress_percent": 90.75, "loss": 1.5804957151412964, "perplexity": 4.857363082954612, "learning_rate": 3.595196540257558e-05, "gradient_norm": 0.9621977210044861, "step_time": 3.3394337999998243, "tokens_per_second": 2453.1104644147854, "steps_per_second": 0.299451960988133, "batch_wait": 0.0012655999998969492, "processed_target_tokens": 105052689, "elapsed_seconds": 6218.536612, "eta_seconds": 6177.952529999675, "gpu": {"gpu_utilization": 2, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3350.45, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"step": 18196, "val_loss": 1.696633791923523, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 105324717, "timestamp": 1793502925.8895109}
{"event": "checkpoint", "step": 18196, "max_steps": 20000, "progress_percent": 90.98, "train_loss": 1.657578945159912, "train_perplexity": 5.24659316578223, "validation_loss": 1.696633791923523, "validation_perplexity": 5.455551924662323, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 105324717, "tokens_per_step": 8192, "tokens_per_second": 2166.7789202034987, "steps_per_second": 0.26449937990765365, "elapsed_seconds": 6382.331753500001, "eta_seconds": 6820.431868800002, "learning_rate": 3.565548569512419e-05, "gradient_norm": 1.0455026626586914, "batch_wait": 0.0010181999996348168, "evaluation_time": 2.2925717000007353, "parameter_count": 62113280, "gpu": {"gpu_utilization": 12, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3730.99, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 18200, "max_steps": 100000, "progress_percent": 18.2, "loss": 1.614551067352295, "perplexity": 5.025631247016671, "learning_rate": 0.0002794613885927092, "gradient_norm": 0.9345472455024719, "step_time": 3.5517565999998624, "tokens_per_second": 2306.464356256934, "steps_per_second": 0.28155082473839527, "batch_wait": 0.0008614000007582945, "processed_target_tokens": 105349762, "elapsed_seconds": 15.31250589999945, "eta_seconds": 290533.6898799887, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3385.28, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 50}}
{"event": "train", "step": 18250, "max_steps": 100000, "progress_percent": 18.25, "loss": 1.885363221168518, "perplexity": 6.588747179891989, "learning_rate": 0.00027934824455926033, "gradient_norm": 1.1120362281799316, "step_time": 3.4003787999999986, "tokens_per_second": 2409.143357792962, "steps_per_second": 0.2940848825430862, "batch_wait": 0.0012370999993436271, "processed_target_tokens": 105635140, "elapsed_seconds": 186.9664045999998, "eta_seconds": 277980.9668999999, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3385.28, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 50}}
{"event": "train", "step": 18300, "max_steps": 100000, "progress_percent": 18.3, "loss": 1.4115726947784424, "perplexity": 4.102402159808668, "learning_rate": 0.00027923481554015364, "gradient_norm": 1.0766099691390991, "step_time": 3.4824422000001505, "tokens_per_second": 2352.372137001914, "steps_per_second": 0.28715480188011644, "batch_wait": 0.0010927000012088683, "processed_target_tokens": 105926974, "elapsed_seconds": 359.86834550000003, "eta_seconds": 284515.5277400123, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3385.28, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 50}}
{"event": "train", "step": 18350, "max_steps": 100000, "progress_percent": 18.35, "loss": 1.9392033815383911, "perplexity": 6.953209708573195, "learning_rate": 0.00027912110181808387, "gradient_norm": 1.1437602043151855, "step_time": 3.592003299998396, "tokens_per_second": 2280.6215127930586, "steps_per_second": 0.2783961807608714, "batch_wait": 0.0010933000030490803, "processed_target_tokens": 106215102, "elapsed_seconds": 536.6268897999998, "eta_seconds": 293287.069444869, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3385.28, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 50}}
{"event": "train", "step": 18400, "max_steps": 100000, "progress_percent": 18.4, "loss": 1.5721213817596436, "perplexity": 4.8168557528152, "learning_rate": 0.0002790071036764553, "gradient_norm": 1.078163981437683, "step_time": 3.4258116999990307, "tokens_per_second": 2391.2581068020518, "steps_per_second": 0.29190162436548484, "batch_wait": 0.0017537999992782716, "processed_target_tokens": 106506033, "elapsed_seconds": 711.2243871999999, "eta_seconds": 279546.2347199209, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3385.28, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 50}}
{"event": "train", "step": 18450, "max_steps": 100000, "progress_percent": 18.45, "loss": 1.7726805210113525, "perplexity": 5.886611417637413, "learning_rate": 0.000278892821399381, "gradient_norm": 1.195027470588684, "step_time": 3.49020019999989, "tokens_per_second": 2347.1432956769236, "steps_per_second": 0.28651651558556196, "batch_wait": 0.0016626999986328883, "processed_target_tokens": 106788608, "elapsed_seconds": 883.7492249000006, "eta_seconds": 284625.82630999107, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3552.6, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 64}}
{"event": "train", "step": 18500, "max_steps": 100000, "progress_percent": 18.5, "loss": 1.6839126348495483, "perplexity": 5.3865905558921385, "learning_rate": 0.0002787782552716823, "gradient_norm": 1.110693097114563, "step_time": 3.497764699999607, "tokens_per_second": 2342.0672065221884, "steps_per_second": 0.2858968757961656, "batch_wait": 0.0015634999999747379, "processed_target_tokens": 107076632, "elapsed_seconds": 1057.9659367999993, "eta_seconds": 285067.82304996793, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3552.6, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 64}}
{"event": "validation", "step": 18500, "validation_loss": 1.7735712766647338, "validation_perplexity": 5.89185728608356, "validation_time": 2.3271022999997513, "best_validation_loss": 1.5567119061946868}
{"step": 18500, "val_loss": 1.7735712766647338, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 107076632, "timestamp": 1793504609.7600532}
{"event": "checkpoint", "step": 18500, "max_steps": 100000, "progress_percent": 18.5, "train_loss": 1.6839126348495483, "train_perplexity": 5.3865905558921385, "validation_loss": 1.7735712766647338, "validation_perplexity": 5.89185728608356, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 107076632, "tokens_per_step": 8192, "tokens_per_second": 2342.0672065221884, "steps_per_second": 0.2858968757961656, "elapsed_seconds": 1057.9659367999993, "eta_seconds": 285067.82304996793, "learning_rate": 0.0002787782552716823, "gradient_norm": 1.110693097114563, "batch_wait": 0.0015634999999747379, "evaluation_time": 2.3271022999997513, "parameter_count": 62113280, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3515.73, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 18550, "max_steps": 100000, "progress_percent": 18.55, "loss": 1.987581491470337, "perplexity": 7.2978624592185675, "learning_rate": 0.0002786634055788878, "gradient_norm": 1.1553272008895874, "step_time": 3.416603900001064, "tokens_per_second": 2397.702584135506, "steps_per_second": 0.2926883037274788, "batch_wait": 0.0007143000002542976, "processed_target_tokens": 107369047, "elapsed_seconds": 1234.3089646999997, "eta_seconds": 278282.3876550867, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3515.73, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 18600, "max_steps": 100000, "progress_percent": 18.6, "loss": 1.8203213214874268, "perplexity": 6.173841919269458, "learning_rate": 0.00027854827260723295, "gradient_norm": 1.064555048942566, "step_time": 3.5012209999986226, "tokens_per_second": 2339.755188262387, "steps_per_second": 0.2856146470046859, "batch_wait": 0.0015863000026001828, "processed_target_tokens": 107661693, "elapsed_seconds": 1406.3564043999995, "eta_seconds": 284999.3893998879, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3515.73, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 18650, "max_steps": 100000, "progress_percent": 18.65, "loss": 1.9157483577728271, "perplexity": 6.7920197563523335, "learning_rate": 0.0002784328566436591, "gradient_norm": 1.1555334329605103, "step_time": 3.4919365999994625, "tokens_per_second": 2345.976155466643, "steps_per_second": 0.2863740424153617, "batch_wait": 0.0014551000003848458, "processed_target_tokens": 107948648, "elapsed_seconds": 1584.4310475999991, "eta_seconds": 284069.0424099563, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3515.73, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 18700, "max_steps": 100000, "progress_percent": 18.7, "loss": 1.6313356161117554, "perplexity": 5.110696090196094, "learning_rate": 0.000278317157975813, "gradient_norm": 1.0657371282577515, "step_time": 3.4838486000007833, "tokens_per_second": 2351.422504410254, "steps_per_second": 0.28703887993289234, "batch_wait": 0.0012794000012945617, "processed_target_tokens": 108240846, "elapsed_seconds": 1761.3813021999995, "eta_seconds": 283236.8911800637, "gpu": {"gpu_utilization": 3, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3515.73, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 18750, "max_steps": 100000, "progress_percent": 18.75, "loss": 1.8367860317230225, "perplexity": 6.276333872325035, "learning_rate": 0.00027820117689204593, "gradient_norm": 1.0846246480941772, "step_time": 3.4118206999992253, "tokens_per_second": 2401.064041847762, "steps_per_second": 0.2930986379208694, "batch_wait": 0.0018433000004733913, "processed_target_tokens": 108528579, "elapsed_seconds": 1935.973910499999, "eta_seconds": 277210.431874937, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3405.18, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 64}}
{"event": "train", "step": 18800, "max_steps": 100000, "progress_percent": 18.8, "loss": 1.6709113121032715, "perplexity": 5.317011046510024, "learning_rate": 0.0002780849136814129, "gradient_norm": 1.1224443912506104, "step_time": 3.9321941999987757, "tokens_per_second": 2083.315213679566, "steps_per_second": 0.2543109391698689, "batch_wait": 0.00232270000014978, "processed_target_tokens": 108817455, "elapsed_seconds": 2109.8314692999993, "eta_seconds": 319294.1690399006, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3405.18, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 64}}
{"event": "train", "step": 18850, "max_steps": 100000, "progress_percent": 18.85, "loss": 2.0412988662719727, "perplexity": 7.700604761822879, "learning_rate": 0.00027796836863367227, "gradient_norm": 1.2555965185165405, "step_time": 3.8045212000015454, "tokens_per_second": 2153.2275861668672, "steps_per_second": 0.26284516432701016, "batch_wait": 0.00044579999848792795, "processed_target_tokens": 109103482, "elapsed_seconds": 2283.7797193999995, "eta_seconds": 308736.8953801254, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3405.18, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 64}}
{"event": "train", "step": 18900, "max_steps": 100000, "progress_percent": 18.9, "loss": 1.5034157037734985, "perplexity": 4.497023366404928, "learning_rate": 0.0002778515420392846, "gradient_norm": 1.0483903884887695, "step_time": 3.3124992000011844, "tokens_per_second": 2473.0572010393453, "steps_per_second": 0.3018868653612482, "batch_wait": 0.000340599997798563, "processed_target_tokens": 109390454, "elapsed_seconds": 2451.3424195999996, "eta_seconds": 268643.6851200961, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3405.18, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 64}}
{"event": "train", "step": 18950, "max_steps": 100000, "progress_percent": 18.95, "loss": 2.0411062240600586, "perplexity": 7.699121443167966, "learning_rate": 0.00027773443418941236, "gradient_norm": 1.250932216644287, "step_time": 3.3069462000003114, "tokens_per_second": 2477.2099406997395, "steps_per_second": 0.30239379158932367, "batch_wait": 0.0003972000031353673, "processed_target_tokens": 109672011, "elapsed_seconds": 2616.908344199999, "eta_seconds": 268027.98951002525, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3405.18, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 64}}
{"event": "train", "step": 19000, "max_steps": 100000, "progress_percent": 19.0, "loss": 1.8811941146850586, "perplexity": 6.561335172799694, "learning_rate": 0.00027761704537591886, "gradient_norm": 1.0376901626586914, "step_time": 3.3069639000004827, "tokens_per_second": 2477.196681826132, "steps_per_second": 0.302392173074479, "batch_wait": 0.000301799998851493, "processed_target_tokens": 109955799, "elapsed_seconds": 2782.5805488000005, "eta_seconds": 267864.0759000391, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 42, "gpu_memory_used_mb": 3379.08, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "validation", "step": 19000, "validation_loss": 1.9233800649642945, "validation_perplexity": 6.844052760150036, "validation_time": 2.2049437000005128, "best_validation_loss": 1.5567119061946868}
{"step": 19000, "val_loss": 1.9233800649642945, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 109955799, "timestamp": 1793506333.0641482}
{"event": "checkpoint", "step": 19000, "max_steps": 100000, "progress_percent": 19.0, "train_loss": 1.8811941146850586, "train_perplexity": 6.561335172799694, "validation_loss": 1.9233800649642945, "validation_perplexity": 6.844052760150036, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 109955799, "tokens_per_step": 8192, "tokens_per_second": 2477.196681826132, "steps_per_second": 0.302392173074479, "elapsed_seconds": 2782.5805488000005, "eta_seconds": 267864.0759000391, "learning_rate": 0.00027761704537591886, "gradient_norm": 1.0376901626586914, "batch_wait": 0.000301799998851493, "evaluation_time": 2.2049437000005128, "parameter_count": 62113280, "gpu": {"gpu_utilization": 13, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3376.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 19050, "max_steps": 100000, "progress_percent": 19.05, "loss": 1.757030963897705, "perplexity": 5.795205651428177, "learning_rate": 0.0002774993758913676, "gradient_norm": 0.9844722747802734, "step_time": 3.3394599000002927, "tokens_per_second": 2453.091291798198, "steps_per_second": 0.29944962058083474, "batch_wait": 0.0003307000024506124, "processed_target_tokens": 110247785, "elapsed_seconds": 2950.8443714000005, "eta_seconds": 270329.2789050237, "gpu": {"gpu_utilization": 13, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3376.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 19100, "max_steps": 100000, "progress_percent": 19.1, "loss": 1.6318445205688477, "perplexity": 5.113297608121126, "learning_rate": 0.0002773814260290215, "gradient_norm": 1.044673204421997, "step_time": 3.3071424000008847, "tokens_per_second": 2477.062977390332, "steps_per_second": 0.30237585173221826, "batch_wait": 0.00035829999796987977, "processed_target_tokens": 110534631, "elapsed_seconds": 3115.9422577000005, "eta_seconds": 267547.8201600716, "gpu": {"gpu_utilization": 13, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3376.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 19150, "max_steps": 100000, "progress_percent": 19.15, "loss": 1.7878963947296143, "perplexity": 5.976866265448739, "learning_rate": 0.0002772631960828426, "gradient_norm": 1.057029366493225, "step_time": 3.3109819000001153, "tokens_per_second": 2474.1905112799664, "steps_per_second": 0.30202520889648027, "batch_wait": 0.0003063000021938933, "processed_target_tokens": 110823765, "elapsed_seconds": 3281.1216194999997, "eta_seconds": 267692.88661500934, "gpu": {"gpu_utilization": 13, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3376.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 19200, "max_steps": 100000, "progress_percent": 19.2, "loss": 1.6411950588226318, "perplexity": 5.161333925727154, "learning_rate": 0.00027714468634749067, "gradient_norm": 1.0951752662658691, "step_time": 3.3080618000003597, "tokens_per_second": 2476.3745344778954, "steps_per_second": 0.3022918132907587, "batch_wait": 0.0004054000000905944, "processed_target_tokens": 111108438, "elapsed_seconds": 3446.2922419999995, "eta_seconds": 267291.39344002906, "gpu": {"gpu_utilization": 13, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3376.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 19250, "max_steps": 100000, "progress_percent": 19.25, "loss": 1.8553534746170044, "perplexity": 6.393957953062245, "learning_rate": 0.00027702589711832296, "gradient_norm": 1.0424091815948486, "step_time": 3.2985220000009576, "tokens_per_second": 2483.5365657702514, "steps_per_second": 0.3031660846887514, "batch_wait": 0.000353599998561549, "processed_target_tokens": 111396907, "elapsed_seconds": 3611.4330353000005, "eta_seconds": 266355.65150007734, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3321.79, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 19300, "max_steps": 100000, "progress_percent": 19.3, "loss": 2.157254695892334, "perplexity": 8.647365394608698, "learning_rate": 0.0002769068286913931, "gradient_norm": 1.096024513244629, "step_time": 3.2939796000009665, "tokens_per_second": 2486.9613643015873, "steps_per_second": 0.3035841509157211, "batch_wait": 0.0003728000010596588, "processed_target_tokens": 111681424, "elapsed_seconds": 3776.5801422000004, "eta_seconds": 265824.153720078, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3321.79, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 19350, "max_steps": 100000, "progress_percent": 19.35, "loss": 1.7593715190887451, "perplexity": 5.8087855361341525, "learning_rate": 0.00027678748136345085, "gradient_norm": 1.1272526979446411, "step_time": 3.31022639999901, "tokens_per_second": 2474.755201034724, "steps_per_second": 0.3020941407513091, "batch_wait": 0.00040720000106375664, "processed_target_tokens": 111964934, "elapsed_seconds": 3941.6712860999996, "eta_seconds": 266969.75915992016, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3321.79, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 19400, "max_steps": 100000, "progress_percent": 19.400000000000002, "loss": 2.2699241638183594, "perplexity": 9.678666793107558, "learning_rate": 0.00027666785543194075, "gradient_norm": 1.1586909294128418, "step_time": 3.3084349000000657, "tokens_per_second": 2476.095267886286, "steps_per_second": 0.30225772313065014, "batch_wait": 0.0003164000027027214, "processed_target_tokens": 112249644, "elapsed_seconds": 4106.8252474, "eta_seconds": 266659.8529400053, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3321.79, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 19450, "max_steps": 100000, "progress_percent": 19.45, "loss": 1.4287292957305908, "perplexity": 4.173392673241496, "learning_rate": 0.0002765479511950018, "gradient_norm": 0.9481104612350464, "step_time": 3.2926450000013574, "tokens_per_second": 2487.969398461305, "steps_per_second": 0.3037072019606085, "batch_wait": 0.0003834000017377548, "processed_target_tokens": 112536174, "elapsed_seconds": 4271.9526964, "eta_seconds": 265222.5547501093, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3321.79, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "train", "step": 19500, "max_steps": 100000, "progress_percent": 19.5, "loss": 1.6837124824523926, "perplexity": 5.385512524768736, "learning_rate": 0.0002764277689514668, "gradient_norm": 1.0259963274002075, "step_time": 3.2960221999983332, "tokens_per_second": 2485.420152814548, "steps_per_second": 0.30339601474786965, "batch_wait": 0.0003569000000425149, "processed_target_tokens": 112830946, "elapsed_seconds": 4437.0941246, "eta_seconds": 265329.78709986585, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3312.06, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "validation", "step": 19500, "validation_loss": 1.8028398275375366, "validation_perplexity": 6.066851837038121, "validation_time": 2.203730999999607, "best_validation_loss": 1.5567119061946868}
{"step": 19500, "val_loss": 1.8028398275375366, "best_val_loss": 1.5567119061946868, "processed_target_tokens": 112830946, "timestamp": 1793507987.6006255}
{"event": "checkpoint", "step": 19500, "max_steps": 100000, "progress_percent": 19.5, "train_loss": 1.6837124824523926, "train_perplexity": 5.385512524768736, "validation_loss": 1.8028398275375366, "validation_perplexity": 6.066851837038121, "best_validation_loss": 1.5567119061946868, "best_validation_perplexity": 4.743199491451511, "processed_target_tokens": 112830946, "tokens_per_step": 8192, "tokens_per_second": 2485.420152814548, "steps_per_second": 0.30339601474786965, "elapsed_seconds": 4437.0941246, "eta_seconds": 265329.78709986585, "learning_rate": 0.0002764277689514668, "gradient_norm": 1.0259963274002075, "batch_wait": 0.0003569000000425149, "evaluation_time": 2.203730999999607, "parameter_count": 62113280, "gpu": {"gpu_utilization": 12, "gpu_memory_utilization": 2, "gpu_memory_used_mb": 3312.06, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 19550, "max_steps": 100000, "progress_percent": 19.55, "loss": 1.7537932395935059, "perplexity": 5.776472715648005, "learning_rate": 0.0002763073090008611, "gradient_norm": 1.1701122522354126, "step_time": 3.3037389000000985, "tokens_per_second": 2479.6148388117945, "steps_per_second": 0.3026873582533929, "batch_wait": 0.00042300000131945126, "processed_target_tokens": 113116439, "elapsed_seconds": 4604.6527893, "eta_seconds": 265785.7945050079, "gpu": {"gpu_utilization": 12, "gpu_memory_utilization": 2, "gpu_memory_used_mb": 3312.06, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 19600, "max_steps": 100000, "progress_percent": 19.6, "loss": 1.7392408847808838, "perplexity": 5.693020123719935, "learning_rate": 0.00027618657164340236, "gradient_norm": 1.0480995178222656, "step_time": 3.2945280999992974, "tokens_per_second": 2486.5473146220083, "steps_per_second": 0.3035336077419444, "batch_wait": 0.0004079999998793937, "processed_target_tokens": 113411606, "elapsed_seconds": 4769.6068399, "eta_seconds": 264880.0592399435, "gpu": {"gpu_utilization": 12, "gpu_memory_utilization": 2, "gpu_memory_used_mb": 3312.06, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 19650, "max_steps": 100000, "progress_percent": 19.650000000000002, "loss": 1.740471363067627, "perplexity": 5.700029572971314, "learning_rate": 0.00027606555717999966, "gradient_norm": 1.019464373588562, "step_time": 3.291571500001737, "tokens_per_second": 2488.7808148769295, "steps_per_second": 0.30380625181603144, "batch_wait": 0.00033570000050531235, "processed_target_tokens": 113703568, "elapsed_seconds": 4934.5615277, "eta_seconds": 264477.77002513956, "gpu": {"gpu_utilization": 12, "gpu_memory_utilization": 2, "gpu_memory_used_mb": 3312.06, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 19700, "max_steps": 100000, "progress_percent": 19.7, "loss": 1.9635428190231323, "perplexity": 7.124523302634883, "learning_rate": 0.00027594426591225253, "gradient_norm": 1.0955053567886353, "step_time": 3.306563300000562, "tokens_per_second": 2477.4968015881045, "steps_per_second": 0.3024288087876104, "batch_wait": 0.0004102000002603745, "processed_target_tokens": 113989432, "elapsed_seconds": 5099.5340631, "eta_seconds": 265517.03299004515, "gpu": {"gpu_utilization": 12, "gpu_memory_utilization": 2, "gpu_memory_used_mb": 3312.06, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 19750, "max_steps": 100000, "progress_percent": 19.75, "loss": 1.7781916856765747, "perplexity": 5.91914306376943, "learning_rate": 0.00027582269814245043, "gradient_norm": 1.0294997692108154, "step_time": 3.302936199999749, "tokens_per_second": 2480.2174501586264, "steps_per_second": 0.3027609192088167, "batch_wait": 0.0004062999996676808, "processed_target_tokens": 114285910, "elapsed_seconds": 5264.5287608, "eta_seconds": 265060.63004997984, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3341.07, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 19800, "max_steps": 100000, "progress_percent": 19.8, "loss": 1.8024076223373413, "perplexity": 6.064230278691723, "learning_rate": 0.000275700854173572, "gradient_norm": 1.0199744701385498, "step_time": 3.3078989000005095, "tokens_per_second": 2476.4964854272716, "steps_per_second": 0.30230669988125874, "batch_wait": 0.0004102000002603745, "processed_target_tokens": 114576844, "elapsed_seconds": 5429.490939699999, "eta_seconds": 265293.49178004084, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3341.07, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 19850, "max_steps": 100000, "progress_percent": 19.85, "loss": 1.8700147867202759, "perplexity": 6.488392340619962, "learning_rate": 0.0002755787343092843, "gradient_norm": 1.127205491065979, "step_time": 3.3008465000002616, "tokens_per_second": 2481.787626295058, "steps_per_second": 0.30295259110047096, "batch_wait": 0.0003571999986888841, "processed_target_tokens": 114864926, "elapsed_seconds": 5594.4757379, "eta_seconds": 264562.84697502095, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3341.07, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 19900, "max_steps": 100000, "progress_percent": 19.900000000000002, "loss": 1.5531096458435059, "perplexity": 4.726143989607821, "learning_rate": 0.0002754563388539417, "gradient_norm": 0.9800540208816528, "step_time": 3.301508200000171, "tokens_per_second": 2481.290217604056, "steps_per_second": 0.30289187226612013, "batch_wait": 0.0003673999999591615, "processed_target_tokens": 115159946, "elapsed_seconds": 5759.4433924, "eta_seconds": 264450.80682001373, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3341.07, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 19950, "max_steps": 100000, "progress_percent": 19.950000000000003, "loss": 1.8185999393463135, "perplexity": 6.163223519831149, "learning_rate": 0.0002753336681125857, "gradient_norm": 1.0049034357070923, "step_time": 3.3019494000000122, "tokens_per_second": 2480.958672473894, "steps_per_second": 0.30285140044847336, "batch_wait": 0.00045620000128110405, "processed_target_tokens": 115457333, "elapsed_seconds": 5924.4308002, "eta_seconds": 264321.04947000096, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3341.07, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 62}}
{"event": "train", "step": 20000, "max_steps": 100000, "progress_percent": 20.0, "loss": 1.4993047714233398, "perplexity": 4.478574354869718, "learning_rate": 0.00027521072239094385, "gradient_norm": 0.9624646902084351, "step_time": 3.302194799998688, "tokens_per_second": 2480.774301989469, "steps_per_second": 0.3028288942858239, "batch_wait": 0.0003394999985175673, "processed_target_tokens": 115751842, "elapsed_seconds": 6089.424654999999, "eta_seconds": 264175.583999895, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3341.19, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 63}}
{"event": "validation", "step": 20000, "validation_loss": 1.551450753211975, "validation_perplexity": 4.7183103235698125, "validation_time": 2.200659899999664, "best_validation_loss": 1.5567119061946868}
{"step": 20000, "val_loss": 1.551450753211975, "best_val_loss": 1.551450753211975, "processed_target_tokens": 115751842, "timestamp": 1793509639.9815931}
{"event": "checkpoint", "step": 20000, "max_steps": 100000, "progress_percent": 20.0, "train_loss": 1.4993047714233398, "train_perplexity": 4.478574354869718, "validation_loss": 1.551450753211975, "validation_perplexity": 4.7183103235698125, "best_validation_loss": 1.551450753211975, "best_validation_perplexity": 4.7183103235698125, "processed_target_tokens": 115751842, "tokens_per_step": 8192, "tokens_per_second": 2480.774301989469, "steps_per_second": 0.3028288942858239, "elapsed_seconds": 6089.424654999999, "eta_seconds": 264175.583999895, "learning_rate": 0.00027521072239094385, "gradient_norm": 0.9624646902084351, "batch_wait": 0.0003394999985175673, "evaluation_time": 2.200659899999664, "parameter_count": 62113280, "gpu": {"gpu_utilization": 13, "gpu_memory_utilization": 2, "gpu_memory_used_mb": 3341.19, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}, "is_best": true, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"step": 20000, "val_loss": 1.551450753211975, "best_val_loss": 1.551450753211975, "processed_target_tokens": 115751842, "timestamp": 1793509642.9871445}
{"event": "checkpoint", "step": 20000, "max_steps": 100000, "progress_percent": 20.0, "train_loss": 1.4993047714233398, "train_perplexity": 4.478574354869718, "validation_loss": 1.551450753211975, "validation_perplexity": 4.7183103235698125, "best_validation_loss": 1.551450753211975, "best_validation_perplexity": 4.7183103235698125, "processed_target_tokens": 115751842, "tokens_per_step": 8192, "tokens_per_second": 2480.774301989469, "steps_per_second": 0.3028288942858239, "elapsed_seconds": 6089.424654999999, "eta_seconds": 264175.583999895, "learning_rate": 0.00027521072239094385, "gradient_norm": 0.9624646902084351, "batch_wait": 0.0003394999985175673, "evaluation_time": 2.200659899999664, "parameter_count": 62113280, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 0, "gpu_memory_used_mb": 3341.19, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 20050, "max_steps": 100000, "progress_percent": 20.05, "loss": 1.8604726791381836, "perplexity": 6.426773855516769, "learning_rate": 0.00027508750199542887, "gradient_norm": 0.995585024356842, "step_time": 3.292718700000478, "tokens_per_second": 2487.9137109400845, "steps_per_second": 0.3037004041674908, "batch_wait": 0.00035490000118443277, "processed_target_tokens": 116040623, "elapsed_seconds": 6260.326334199999, "eta_seconds": 263252.8600650382, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 0, "gpu_memory_used_mb": 3341.19, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 20100, "max_steps": 100000, "progress_percent": 20.1, "loss": 1.6888957023620605, "perplexity": 5.413499288629577, "learning_rate": 0.0002749640072331382, "gradient_norm": 1.0096209049224854, "step_time": 3.5330446000007214, "tokens_per_second": 2318.6800415704706, "steps_per_second": 0.28304199726202034, "batch_wait": 0.0006417999975383282, "processed_target_tokens": 116334926, "elapsed_seconds": 6434.905714, "eta_seconds": 282290.26354005764, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 0, "gpu_memory_used_mb": 3341.19, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 20150, "max_steps": 100000, "progress_percent": 20.150000000000002, "loss": 2.169127941131592, "perplexity": 8.750649630782345, "learning_rate": 0.0002748402384118529, "gradient_norm": 1.1114423274993896, "step_time": 3.595243699999628, "tokens_per_second": 2278.565984275516, "steps_per_second": 0.2781452617523823, "batch_wait": 0.0006898000010551186, "processed_target_tokens": 116630573, "elapsed_seconds": 6611.957840699999, "eta_seconds": 287080.2094449703, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 0, "gpu_memory_used_mb": 3341.19, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 20200, "max_steps": 100000, "progress_percent": 20.200000000000003, "loss": 1.5554357767105103, "perplexity": 4.737150415254296, "learning_rate": 0.00027471619584003727, "gradient_norm": 1.2008309364318848, "step_time": 3.559129600000233, "tokens_per_second": 2301.68634488597, "steps_per_second": 0.2809675713972131, "batch_wait": 0.0008400999995501479, "processed_target_tokens": 116914291, "elapsed_seconds": 6783.6230326, "eta_seconds": 284018.5420800186, "gpu": {"gpu_utilization": 0, "gpu_memory_utilization": 0, "gpu_memory_used_mb": 3341.19, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 20250, "max_steps": 100000, "progress_percent": 20.25, "loss": 1.863242506980896, "perplexity": 6.444599588389706, "learning_rate": 0.0002745918798268378, "gradient_norm": 1.0396417379379272, "step_time": 3.534872700000051, "tokens_per_second": 2317.480909567092, "steps_per_second": 0.2828956188436392, "batch_wait": 0.0011010999987774994, "processed_target_tokens": 117196631, "elapsed_seconds": 6960.172862199999, "eta_seconds": 281906.0978250041, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3459.16, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 20300, "max_steps": 100000, "progress_percent": 20.3, "loss": 2.0829954147338867, "perplexity": 8.028481564387642, "learning_rate": 0.0002744672906820823, "gradient_norm": 1.1195566654205322, "step_time": 3.5459294999982376, "tokens_per_second": 2310.2546173024793, "steps_per_second": 0.28201350308868156, "batch_wait": 0.0010354999994888203, "processed_target_tokens": 117488277, "elapsed_seconds": 7137.389299500001, "eta_seconds": 282610.5811498595, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3459.16, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 20350, "max_steps": 100000, "progress_percent": 20.349999999999998, "loss": 2.2478551864624023, "perplexity": 9.467408219362248, "learning_rate": 0.00027434242871627945, "gradient_norm": 1.1106518507003784, "step_time": 3.541152100000545, "tokens_per_second": 2313.3714024875517, "steps_per_second": 0.2823939700302187, "batch_wait": 0.0005275999992591096, "processed_target_tokens": 117781084, "elapsed_seconds": 7314.3267166000005, "eta_seconds": 282052.7647650434, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3459.16, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 20400, "max_steps": 100000, "progress_percent": 20.4, "loss": 2.1200015544891357, "perplexity": 8.33115043836047, "learning_rate": 0.00027421729424061787, "gradient_norm": 1.1207135915756226, "step_time": 3.4661292000000685, "tokens_per_second": 2363.443347697437, "steps_per_second": 0.28850626802947227, "batch_wait": 0.0006131000009190757, "processed_target_tokens": 118066470, "elapsed_seconds": 7490.6740393, "eta_seconds": 275903.8843200054, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3459.16, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 20450, "max_steps": 100000, "progress_percent": 20.45, "loss": 1.955393671989441, "perplexity": 7.066700438203089, "learning_rate": 0.00027409188756696536, "gradient_norm": 1.1468254327774048, "step_time": 3.4604583000000275, "tokens_per_second": 2367.31649099772, "steps_per_second": 0.2889790638424951, "batch_wait": 0.0009565999989717966, "processed_target_tokens": 118348973, "elapsed_seconds": 7663.913967500001, "eta_seconds": 275279.4577650022, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3459.16, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 20500, "max_steps": 100000, "progress_percent": 20.5, "loss": 2.115022659301758, "perplexity": 8.289773604472682, "learning_rate": 0.000273966209007868, "gradient_norm": 1.0512101650238037, "step_time": 3.386784500000431, "tokens_per_second": 2418.8134792747983, "steps_per_second": 0.2952653172942869, "batch_wait": 0.00046520000068994705, "processed_target_tokens": 118637831, "elapsed_seconds": 7844.3023416, "eta_seconds": 269249.36775003426, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3480.5, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "validation", "step": 20500, "validation_loss": 1.883409810066223, "validation_perplexity": 6.575889210539252, "validation_time": 2.608511799999178, "best_validation_loss": 1.551450753211975}
{"step": 20500, "val_loss": 1.883409810066223, "best_val_loss": 1.551450753211975, "processed_target_tokens": 118637831, "timestamp": 1793511396.597306}
{"event": "checkpoint", "step": 20500, "max_steps": 100000, "progress_percent": 20.5, "train_loss": 2.115022659301758, "train_perplexity": 8.289773604472682, "validation_loss": 1.883409810066223, "validation_perplexity": 6.575889210539252, "best_validation_loss": 1.551450753211975, "best_validation_perplexity": 4.7183103235698125, "processed_target_tokens": 118637831, "tokens_per_step": 8192, "tokens_per_second": 2418.8134792747983, "steps_per_second": 0.2952653172942869, "elapsed_seconds": 7844.3023416, "eta_seconds": 269249.36775003426, "learning_rate": 0.000273966209007868, "gradient_norm": 1.0512101650238037, "batch_wait": 0.00046520000068994705, "evaluation_time": 2.608511799999178, "parameter_count": 62113280, "gpu": {"gpu_utilization": 6, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3610.59, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 20550, "max_steps": 100000, "progress_percent": 20.549999999999997, "loss": 1.952082872390747, "perplexity": 7.043342696964469, "learning_rate": 0.0002738402588765496, "gradient_norm": 1.1865026950836182, "step_time": 3.4738692000009905, "tokens_per_second": 2358.1774466343363, "steps_per_second": 0.2878634578411055, "batch_wait": 0.0009549000005790731, "processed_target_tokens": 118924682, "elapsed_seconds": 8028.333023199999, "eta_seconds": 275998.9079400787, "gpu": {"gpu_utilization": 6, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3610.59, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 20600, "max_steps": 100000, "progress_percent": 20.599999999999998, "loss": 2.0977001190185547, "perplexity": 8.147410274430987, "learning_rate": 0.0002737140374869107, "gradient_norm": 1.0472140312194824, "step_time": 3.348804500001279, "tokens_per_second": 2446.2461155904657, "steps_per_second": 0.29861402778203927, "batch_wait": 0.0007595000024593901, "processed_target_tokens": 119213911, "elapsed_seconds": 8201.5188715, "eta_seconds": 265895.0773001015, "gpu": {"gpu_utilization": 6, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3610.59, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 20650, "max_steps": 100000, "progress_percent": 20.65, "loss": 1.8841369152069092, "perplexity": 6.580672312086951, "learning_rate": 0.00027358754515352795, "gradient_norm": 1.0012766122817993, "step_time": 3.476263000000472, "tokens_per_second": 2356.5535749161922, "steps_per_second": 0.28766523131301175, "batch_wait": 0.0005827000004501315, "processed_target_tokens": 119501742, "elapsed_seconds": 8373.6126627, "eta_seconds": 275841.46905003744, "gpu": {"gpu_utilization": 6, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3610.59, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 20700, "max_steps": 100000, "progress_percent": 20.7, "loss": 1.6907130479812622, "perplexity": 5.4233464329684145, "learning_rate": 0.00027346078219165325, "gradient_norm": 1.1330313682556152, "step_time": 3.414570900000399, "tokens_per_second": 2399.1301513168296, "steps_per_second": 0.2928625672994177, "batch_wait": 0.0011003999989043223, "processed_target_tokens": 119795049, "elapsed_seconds": 8545.531481099999, "eta_seconds": 270775.47237003164, "gpu": {"gpu_utilization": 6, "gpu_memory_utilization": 4, "gpu_memory_used_mb": 3610.59, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 20750, "max_steps": 100000, "progress_percent": 20.75, "loss": 1.7220306396484375, "perplexity": 5.595880150885173, "learning_rate": 0.00027333374891721303, "gradient_norm": 1.1591933965682983, "step_time": 3.3577981000016734, "tokens_per_second": 2439.6940363972203, "steps_per_second": 0.29781421342739506, "batch_wait": 0.0006840999958512839, "processed_target_tokens": 120083988, "elapsed_seconds": 8715.683051200002, "eta_seconds": 266105.49942513264, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3417.11, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 20800, "max_steps": 100000, "progress_percent": 20.8, "loss": 2.0109405517578125, "perplexity": 7.470340285770151, "learning_rate": 0.00027320644564680723, "gradient_norm": 1.1567721366882324, "step_time": 3.4298285999975633, "tokens_per_second": 2388.4575456644743, "steps_per_second": 0.2915597589922454, "batch_wait": 0.0008305000010295771, "processed_target_tokens": 120376503, "elapsed_seconds": 8883.904791199999, "eta_seconds": 271642.425119807, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3417.11, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 20850, "max_steps": 100000, "progress_percent": 20.849999999999998, "loss": 1.637650728225708, "perplexity": 5.14307283276526, "learning_rate": 0.0002730788726977089, "gradient_norm": 0.9626670479774475, "step_time": 3.8526580999969156, "tokens_per_second": 2126.324160456013, "steps_per_second": 0.25956105474316565, "batch_wait": 0.0007271000031323638, "processed_target_tokens": 120662317, "elapsed_seconds": 9067.8587314, "eta_seconds": 304937.8886147559, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3417.11, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 20900, "max_steps": 100000, "progress_percent": 20.9, "loss": 1.606448769569397, "perplexity": 4.985076600874172, "learning_rate": 0.0002729510303878631, "gradient_norm": 1.1470566987991333, "step_time": 3.9362322999986645, "tokens_per_second": 2081.1779833224728, "steps_per_second": 0.25405004679229404, "batch_wait": 0.000540900004125433, "processed_target_tokens": 120952841, "elapsed_seconds": 9254.466450299999, "eta_seconds": 311355.9749298944, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3417.11, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 20950, "max_steps": 100000, "progress_percent": 20.95, "loss": 1.7898248434066772, "perplexity": 5.988403466165567, "learning_rate": 0.00027282291903588617, "gradient_norm": 1.0431500673294067, "step_time": 3.30986810000104, "tokens_per_second": 2475.0230983516913, "steps_per_second": 0.3021268430605092, "batch_wait": 0.0010771999986900482, "processed_target_tokens": 121241417, "elapsed_seconds": 9435.0508439, "eta_seconds": 261645.07330508222, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3417.11, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 21000, "max_steps": 100000, "progress_percent": 21.0, "loss": 1.7872028350830078, "perplexity": 5.972722389372582, "learning_rate": 0.0002726945389610651, "gradient_norm": 0.9830436706542969, "step_time": 3.3558770999989065, "tokens_per_second": 2441.090587019015, "steps_per_second": 0.2979846907982196, "batch_wait": 0.0006494999979622662, "processed_target_tokens": 121528022, "elapsed_seconds": 9604.2864767, "eta_seconds": 265114.29089991364, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 48, "gpu_memory_used_mb": 3430.1, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "validation", "step": 21000, "validation_loss": 1.8286024570465087, "validation_perplexity": 6.225180619101256, "validation_time": 2.260174499999266, "best_validation_loss": 1.551450753211975}
{"step": 21000, "val_loss": 1.8286024570465087, "best_val_loss": 1.551450753211975, "processed_target_tokens": 121528022, "timestamp": 1793513156.0543754}
{"event": "checkpoint", "step": 21000, "max_steps": 100000, "progress_percent": 21.0, "train_loss": 1.7872028350830078, "train_perplexity": 5.972722389372582, "validation_loss": 1.8286024570465087, "validation_perplexity": 6.225180619101256, "best_validation_loss": 1.551450753211975, "best_validation_perplexity": 4.7183103235698125, "processed_target_tokens": 121528022, "tokens_per_step": 8192, "tokens_per_second": 2441.090587019015, "steps_per_second": 0.2979846907982196, "elapsed_seconds": 9604.2864767, "eta_seconds": 265114.29089991364, "learning_rate": 0.0002726945389610651, "gradient_norm": 0.9830436706542969, "batch_wait": 0.0006494999979622662, "evaluation_time": 2.260174499999266, "parameter_count": 62113280, "gpu": {"gpu_utilization": 1, "gpu_memory_utilization": 2, "gpu_memory_used_mb": 3405.47, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 21050, "max_steps": 100000, "progress_percent": 21.05, "loss": 1.9445687532424927, "perplexity": 6.990616524194841, "learning_rate": 0.00027256589048335644, "gradient_norm": 1.1415735483169556, "step_time": 3.429478100002598, "tokens_per_second": 2388.7016511328047, "steps_per_second": 0.29158955702304745, "batch_wait": 0.0010786999991978519, "processed_target_tokens": 121817295, "elapsed_seconds": 9778.925043300002, "eta_seconds": 270757.2959952051, "gpu": {"gpu_utilization": 1, "gpu_memory_utilization": 2, "gpu_memory_used_mb": 3405.47, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 21100, "max_steps": 100000, "progress_percent": 21.099999999999998, "loss": 2.0347156524658203, "perplexity": 7.65007653578228, "learning_rate": 0.00027243697392338587, "gradient_norm": 1.1164826154708862, "step_time": 3.4277627000010398, "tokens_per_second": 2389.897060259602, "steps_per_second": 0.29173548098872093, "batch_wait": 0.0008834000000206288, "processed_target_tokens": 122102817, "elapsed_seconds": 9952.626878399999, "eta_seconds": 270450.477030082, "gpu": {"gpu_utilization": 1, "gpu_memory_utilization": 2, "gpu_memory_used_mb": 3405.47, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 21150, "max_steps": 100000, "progress_percent": 21.15, "loss": 1.6846510171890259, "perplexity": 5.390569387997511, "learning_rate": 0.000272307789602447, "gradient_norm": 0.9806025624275208, "step_time": 3.807398999997531, "tokens_per_second": 2151.600081842043, "steps_per_second": 0.26264649436548376, "batch_wait": 0.0009823000036703888, "processed_target_tokens": 122395087, "elapsed_seconds": 10129.043216900001, "eta_seconds": 300213.4111498053, "gpu": {"gpu_utilization": 1, "gpu_memory_utilization": 2, "gpu_memory_used_mb": 3405.47, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 21200, "max_steps": 100000, "progress_percent": 21.2, "loss": 1.7399473190307617, "perplexity": 5.697043289004137, "learning_rate": 0.00027217833784250086, "gradient_norm": 1.019201397895813, "step_time": 3.835075200000574, "tokens_per_second": 2136.0728467589825, "steps_per_second": 0.2607510799266336, "batch_wait": 0.0016224000028159935, "processed_target_tokens": 122686583, "elapsed_seconds": 10317.322171299998, "eta_seconds": 302203.92576004524, "gpu": {"gpu_utilization": 1, "gpu_memory_utilization": 2, "gpu_memory_used_mb": 3405.47, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 21250, "max_steps": 100000, "progress_percent": 21.25, "loss": 1.877698302268982, "perplexity": 6.538438021228231, "learning_rate": 0.0002720486189661751, "gradient_norm": 0.9800621271133423, "step_time": 3.7843346999979985, "tokens_per_second": 2164.7133907062534, "steps_per_second": 0.26424724007644695, "batch_wait": 0.0008950999908847734, "processed_target_tokens": 122977740, "elapsed_seconds": 10505.4910871, "eta_seconds": 298016.3576248424, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3656.59, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 21300, "max_steps": 100000, "progress_percent": 21.3, "loss": 1.5128140449523926, "perplexity": 4.539487158329741, "learning_rate": 0.0002719186332967629, "gradient_norm": 1.106856346130371, "step_time": 3.32503579999684, "tokens_per_second": 2463.73287169052, "steps_per_second": 0.30074864156378417, "batch_wait": 0.0012922999958391301, "processed_target_tokens": 123273792, "elapsed_seconds": 10684.900211799999, "eta_seconds": 261680.31745975133, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3656.59, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 21350, "max_steps": 100000, "progress_percent": 21.349999999999998, "loss": 1.7509968280792236, "perplexity": 5.760341885565331, "learning_rate": 0.00027178838115822246, "gradient_norm": 0.978859007358551, "step_time": 3.5117035000002943, "tokens_per_second": 2332.7709756815498, "steps_per_second": 0.2847620819923767, "batch_wait": 0.000680899993312778, "processed_target_tokens": 123558632, "elapsed_seconds": 10854.625463600001, "eta_seconds": 276195.48027502315, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3656.59, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 21400, "max_steps": 100000, "progress_percent": 21.4, "loss": 1.8192778825759888, "perplexity": 6.167403252139919, "learning_rate": 0.0002716578628751762, "gradient_norm": 1.069981575012207, "step_time": 3.740048999999999, "tokens_per_second": 2190.345634509067, "steps_per_second": 0.2673761760875326, "batch_wait": 0.0013964000027044676, "processed_target_tokens": 123853482, "elapsed_seconds": 11032.9195139, "eta_seconds": 293967.8513999999, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3656.59, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 21450, "max_steps": 100000, "progress_percent": 21.45, "loss": 2.032118320465088, "perplexity": 7.630232529080514, "learning_rate": 0.00027152707877290967, "gradient_norm": 0.9749024510383606, "step_time": 3.787967500000377, "tokens_per_second": 2162.6373510330236, "steps_per_second": 0.2639938172647734, "batch_wait": 0.000598100003116997, "processed_target_tokens": 124146653, "elapsed_seconds": 11208.3561227, "eta_seconds": 297544.84712502966, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3656.59, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 21500, "max_steps": 100000, "progress_percent": 21.5, "loss": 1.825066089630127, "perplexity": 6.203204973080135, "learning_rate": 0.00027139602917737094, "gradient_norm": 1.0430586338043213, "step_time": 3.646989000000758, "tokens_per_second": 2246.2365529477324, "steps_per_second": 0.2741987979672525, "batch_wait": 0.0010168999979214277, "processed_target_tokens": 124433651, "elapsed_seconds": 11389.071635699998, "eta_seconds": 286288.6365000595, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3607.18, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "validation", "step": 21500, "validation_loss": 1.784425300359726, "validation_perplexity": 5.95615596310609, "validation_time": 2.4197684999999183, "best_validation_loss": 1.551450753211975}
{"step": 21500, "val_loss": 1.784425300359726, "best_val_loss": 1.551450753211975, "processed_target_tokens": 124433651, "timestamp": 1793514941.0953066}
{"event": "checkpoint", "step": 21500, "max_steps": 100000, "progress_percent": 21.5, "train_loss": 1.825066089630127, "train_perplexity": 6.203204973080135, "validation_loss": 1.784425300359726, "validation_perplexity": 5.95615596310609, "best_validation_loss": 1.551450753211975, "best_validation_perplexity": 4.7183103235698125, "processed_target_tokens": 124433651, "tokens_per_step": 8192, "tokens_per_second": 2246.2365529477324, "steps_per_second": 0.2741987979672525, "elapsed_seconds": 11389.071635699998, "eta_seconds": 286288.6365000595, "learning_rate": 0.00027139602917737094, "gradient_norm": 1.0430586338043213, "batch_wait": 0.0010168999979214277, "evaluation_time": 2.4197684999999183, "parameter_count": 62113280, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3530.5, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 21550, "max_steps": 100000, "progress_percent": 21.55, "loss": 1.858748197555542, "perplexity": 6.4157005529644655, "learning_rate": 0.00027126471441516983, "gradient_norm": 1.0190991163253784, "step_time": 3.7898688999994192, "tokens_per_second": 2161.552342879527, "steps_per_second": 0.263861369980411, "batch_wait": 0.0005572999980358873, "processed_target_tokens": 124724141, "elapsed_seconds": 11577.9379181, "eta_seconds": 297315.21520495444, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3530.5, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 21600, "max_steps": 100000, "progress_percent": 21.6, "loss": 2.070702075958252, "perplexity": 7.9303888992059495, "learning_rate": 0.0002711331348135769, "gradient_norm": 1.1331461668014526, "step_time": 3.4455434000010428, "tokens_per_second": 2377.5640150106715, "steps_per_second": 0.29022998230110736, "batch_wait": 0.0008949000002758112, "processed_target_tokens": 125013686, "elapsed_seconds": 11759.3179978, "eta_seconds": 270130.60256008175, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3530.5, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 21650, "max_steps": 100000, "progress_percent": 21.65, "loss": 1.8239812850952148, "perplexity": 6.196479356843405, "learning_rate": 0.00027100129070052305, "gradient_norm": 1.0182278156280518, "step_time": 3.4802658000007796, "tokens_per_second": 2353.8432035846704, "steps_per_second": 0.28733437543758183, "batch_wait": 0.000957899999775691, "processed_target_tokens": 125308915, "elapsed_seconds": 11931.834424699999, "eta_seconds": 272678.8254300611, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3530.5, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 21700, "max_steps": 100000, "progress_percent": 21.7, "loss": 1.9461771249771118, "perplexity": 7.001869080941291, "learning_rate": 0.000270869182404598, "gradient_norm": 1.005271315574646, "step_time": 3.451764599998569, "tokens_per_second": 2373.2788730736147, "steps_per_second": 0.289706893685744, "batch_wait": 0.0009556000004522502, "processed_target_tokens": 125599236, "elapsed_seconds": 12109.8866706, "eta_seconds": 270273.1681798879, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 5, "gpu_memory_used_mb": 3530.5, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 21750, "max_steps": 100000, "progress_percent": 21.75, "loss": 1.578803300857544, "perplexity": 4.849149364759169, "learning_rate": 0.0002707368102550502, "gradient_norm": 0.9702857136726379, "step_time": 3.5003179000013915, "tokens_per_second": 2340.3588571188757, "steps_per_second": 0.285688337050644, "batch_wait": 0.0009509999945294112, "processed_target_tokens": 125893865, "elapsed_seconds": 12280.6694293, "eta_seconds": 273899.87567510887, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3765.64, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 21800, "max_steps": 100000, "progress_percent": 21.8, "loss": 2.1142773628234863, "perplexity": 8.283597567174906, "learning_rate": 0.0002706041745817855, "gradient_norm": 1.073946237564087, "step_time": 3.4421048000003793, "tokens_per_second": 2379.939158156689, "steps_per_second": 0.2905199167671739, "batch_wait": 0.0007812000003468711, "processed_target_tokens": 126186549, "elapsed_seconds": 12457.3869959, "eta_seconds": 269172.59536002966, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3765.64, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 21850, "max_steps": 100000, "progress_percent": 21.85, "loss": 1.522364854812622, "perplexity": 4.583050639155828, "learning_rate": 0.00027047127571536665, "gradient_norm": 1.0873587131500244, "step_time": 3.4466910000010103, "tokens_per_second": 2376.7723883567164, "steps_per_second": 0.29013334818807573, "batch_wait": 0.0009694000000308733, "processed_target_tokens": 126477383, "elapsed_seconds": 12632.4160138, "eta_seconds": 269358.90165007894, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3765.64, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 21900, "max_steps": 100000, "progress_percent": 21.9, "loss": 2.0250473022460938, "perplexity": 7.576469320177099, "learning_rate": 0.00027033811398701233, "gradient_norm": 1.0425676107406616, "step_time": 3.966734900001029, "tokens_per_second": 2065.1745595597718, "steps_per_second": 0.2520965038525112, "batch_wait": 0.0006596999992325436, "processed_target_tokens": 126769784, "elapsed_seconds": 12810.474377800001, "eta_seconds": 309801.99569008034, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3765.64, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 21950, "max_steps": 100000, "progress_percent": 21.95, "loss": 1.8529483079910278, "perplexity": 6.378597897936227, "learning_rate": 0.00027020468972859625, "gradient_norm": 0.9699684977531433, "step_time": 3.3252831999998307, "tokens_per_second": 2463.5495707554824, "steps_per_second": 0.3007262659613626, "batch_wait": 0.0014364000016939826, "processed_target_tokens": 127061981, "elapsed_seconds": 12985.8374047, "eta_seconds": 259538.35375998679, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3765.64, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 22000, "max_steps": 100000, "progress_percent": 22.0, "loss": 2.129105806350708, "perplexity": 8.407345654164386, "learning_rate": 0.0002700710032726465, "gradient_norm": 0.995527446269989, "step_time": 3.655767199998081, "tokens_per_second": 2240.8429070659367, "steps_per_second": 0.27354039392894736, "batch_wait": 0.0008126999964588322, "processed_target_tokens": 127345104, "elapsed_seconds": 13163.060939899999, "eta_seconds": 285149.84159985033, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3406.61, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "validation", "step": 22000, "validation_loss": 1.792828118801117, "validation_perplexity": 6.006415324694839, "validation_time": 2.4184814000000188, "best_validation_loss": 1.551450753211975}
{"step": 22000, "val_loss": 1.792828118801117, "best_val_loss": 1.551450753211975, "processed_target_tokens": 127345104, "timestamp": 1793516715.3336053}
{"event": "checkpoint", "step": 22000, "max_steps": 100000, "progress_percent": 22.0, "train_loss": 2.129105806350708, "train_perplexity": 8.407345654164386, "validation_loss": 1.792828118801117, "validation_perplexity": 6.006415324694839, "best_validation_loss": 1.551450753211975, "best_validation_perplexity": 4.7183103235698125, "processed_target_tokens": 127345104, "tokens_per_step": 8192, "tokens_per_second": 2240.8429070659367, "steps_per_second": 0.27354039392894736, "elapsed_seconds": 13163.060939899999, "eta_seconds": 285149.84159985033, "learning_rate": 0.0002700710032726465, "gradient_norm": 0.995527446269989, "batch_wait": 0.0008126999964588322, "evaluation_time": 2.4184814000000188, "parameter_count": 62113280, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3388.11, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 22050, "max_steps": 100000, "progress_percent": 22.05, "loss": 1.5369040966033936, "perplexity": 4.650171481231036, "learning_rate": 0.00026993705495234464, "gradient_norm": 0.9633966684341431, "step_time": 3.543532600000617, "tokens_per_second": 2311.8173090882733, "steps_per_second": 0.2822042613633146, "batch_wait": 0.0010037999963969924, "processed_target_tokens": 127636271, "elapsed_seconds": 13344.170469100001, "eta_seconds": 276218.36617004813, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3388.11, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 22100, "max_steps": 100000, "progress_percent": 22.1, "loss": 1.89976167678833, "perplexity": 6.684301228300376, "learning_rate": 0.00026980284510152476, "gradient_norm": 1.0479828119277954, "step_time": 3.457298000001174, "tokens_per_second": 2369.480443975966, "steps_per_second": 0.2892432182587849, "batch_wait": 0.0015704000034020282, "processed_target_tokens": 127930002, "elapsed_seconds": 13523.837923399999, "eta_seconds": 269323.5142000915, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3388.11, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 22150, "max_steps": 100000, "progress_percent": 22.15, "loss": 1.995553731918335, "perplexity": 7.356275304748203, "learning_rate": 0.0002696683740546728, "gradient_norm": 0.9423718452453613, "step_time": 3.8844462000015483, "tokens_per_second": 2108.923532007403, "steps_per_second": 0.25743695459074745, "batch_wait": 0.0016088999982457608, "processed_target_tokens": 128222240, "elapsed_seconds": 13706.536397200001, "eta_seconds": 302404.1366701205, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3388.11, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 22200, "max_steps": 100000, "progress_percent": 22.2, "loss": 1.8074257373809814, "perplexity": 6.094737764900461, "learning_rate": 0.0002695336421469258, "gradient_norm": 1.3289620876312256, "step_time": 3.3332641999986663, "tokens_per_second": 2457.650971682136, "steps_per_second": 0.300006222129167, "batch_wait": 0.0012204000013298355, "processed_target_tokens": 128508542, "elapsed_seconds": 13894.5182025, "eta_seconds": 259327.95475989624, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3388.11, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 22250, "max_steps": 100000, "progress_percent": 22.25, "loss": 1.7864503860473633, "perplexity": 5.968229910563163, "learning_rate": 0.00026939864971407075, "gradient_norm": 0.9813380241394043, "step_time": 3.53059790000043, "tokens_per_second": 2320.286883986138, "steps_per_second": 0.2832381450178391, "batch_wait": 0.0012383999965095427, "processed_target_tokens": 128804264, "elapsed_seconds": 14068.449476900001, "eta_seconds": 274503.98672503344, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3446.88, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 22300, "max_steps": 100000, "progress_percent": 22.3, "loss": 1.8607584238052368, "perplexity": 6.4286105342704225, "learning_rate": 0.00026926339709254407, "gradient_norm": 0.9512794613838196, "step_time": 3.6958851000017603, "tokens_per_second": 2216.5191228472167, "steps_per_second": 0.27057118198818564, "batch_wait": 0.0006978000019444153, "processed_target_tokens": 129094152, "elapsed_seconds": 14251.223062000001, "eta_seconds": 287170.27227013675, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3446.88, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 22350, "max_steps": 100000, "progress_percent": 22.35, "loss": 1.5469112396240234, "perplexity": 4.69694003180716, "learning_rate": 0.0002691278846194304, "gradient_norm": 0.9114102721214294, "step_time": 3.5982678000000305, "tokens_per_second": 2276.651004130357, "steps_per_second": 0.27791149952763144, "batch_wait": 0.0011120000053779222, "processed_target_tokens": 129388835, "elapsed_seconds": 14434.606424900001, "eta_seconds": 279405.4946700024, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3446.88, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 22400, "max_steps": 100000, "progress_percent": 22.400000000000002, "loss": 1.8450870513916016, "perplexity": 6.328650684272709, "learning_rate": 0.00026899211263246245, "gradient_norm": 1.0586990118026733, "step_time": 3.712308899997879, "tokens_per_second": 2206.712916590718, "steps_per_second": 0.2693741353260154, "batch_wait": 0.0008328000003530178, "processed_target_tokens": 129678098, "elapsed_seconds": 14618.4663171, "eta_seconds": 288075.1706398354, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3446.88, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 22450, "max_steps": 100000, "progress_percent": 22.45, "loss": 1.6661336421966553, "perplexity": 5.291668709705529, "learning_rate": 0.0002688560814700193, "gradient_norm": 1.130319356918335, "step_time": 3.4049665999991703, "tokens_per_second": 2405.897314822999, "steps_per_second": 0.29368863706335435, "batch_wait": 0.0010546999947109725, "processed_target_tokens": 129968857, "elapsed_seconds": 14802.288054199998, "eta_seconds": 264055.15982993564, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 33, "gpu_memory_used_mb": 3446.88, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 22500, "max_steps": 100000, "progress_percent": 22.5, "loss": 1.8586952686309814, "perplexity": 6.415360985820436, "learning_rate": 0.000268719791471126, "gradient_norm": 1.0793734788894653, "step_time": 3.384818099999393, "tokens_per_second": 2420.218681766524, "steps_per_second": 0.2954368508015776, "batch_wait": 0.0005425000017567072, "processed_target_tokens": 130260434, "elapsed_seconds": 14979.739440099998, "eta_seconds": 262323.402749953, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3449.0, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "validation", "step": 22500, "validation_loss": 1.7839295744895936, "validation_perplexity": 5.953204074232926, "validation_time": 2.296581900001911, "best_validation_loss": 1.551450753211975}
{"step": 22500, "val_loss": 1.7839295744895936, "best_val_loss": 1.551450753211975, "processed_target_tokens": 130260434, "timestamp": 1793518531.5707438}
{"event": "checkpoint", "step": 22500, "max_steps": 100000, "progress_percent": 22.5, "train_loss": 1.8586952686309814, "train_perplexity": 6.415360985820436, "validation_loss": 1.7839295744895936, "validation_perplexity": 5.953204074232926, "best_validation_loss": 1.551450753211975, "best_validation_perplexity": 4.7183103235698125, "processed_target_tokens": 130260434, "tokens_per_step": 8192, "tokens_per_second": 2420.218681766524, "steps_per_second": 0.2954368508015776, "elapsed_seconds": 14979.739440099998, "eta_seconds": 262323.402749953, "learning_rate": 0.000268719791471126, "gradient_norm": 1.0793734788894653, "batch_wait": 0.0005425000017567072, "evaluation_time": 2.296581900001911, "parameter_count": 62113280, "gpu": {"gpu_utilization": 2, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3453.59, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 22550, "max_steps": 100000, "progress_percent": 22.55, "loss": 1.6046850681304932, "perplexity": 4.97629216294018, "learning_rate": 0.00026858324297545294, "gradient_norm": 1.0728298425674438, "step_time": 3.332309500001429, "tokens_per_second": 2458.3550837629236, "steps_per_second": 0.30009217331090376, "batch_wait": 0.0005544000050576869, "processed_target_tokens": 130548271, "elapsed_seconds": 15153.6719953, "eta_seconds": 258087.37077511067, "gpu": {"gpu_utilization": 2, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3453.59, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 22600, "max_steps": 100000, "progress_percent": 22.6, "loss": 1.8137935400009155, "perplexity": 6.133671682197318, "learning_rate": 0.0002684464363233145, "gradient_norm": 0.9907260537147522, "step_time": 3.317378899999312, "tokens_per_second": 2469.4194564273917, "steps_per_second": 0.30144280473967183, "batch_wait": 0.0012649999953282531, "processed_target_tokens": 130840728, "elapsed_seconds": 15322.6246991, "eta_seconds": 256765.12685994676, "gpu": {"gpu_utilization": 2, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3453.59, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 22650, "max_steps": 100000, "progress_percent": 22.650000000000002, "loss": 1.7824792861938477, "perplexity": 5.944576469810965, "learning_rate": 0.0002683093718556686, "gradient_norm": 1.025700330734253, "step_time": 3.631195000001753, "tokens_per_second": 2256.0066314246537, "steps_per_second": 0.2753914345000798, "batch_wait": 0.0007306000043172389, "processed_target_tokens": 131137855, "elapsed_seconds": 15496.495120799998, "eta_seconds": 280872.9332501356, "gpu": {"gpu_utilization": 2, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3453.59, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 22700, "max_steps": 100000, "progress_percent": 22.7, "loss": 1.6993739604949951, "perplexity": 5.470521556877526, "learning_rate": 0.00026817204991411565, "gradient_norm": 1.100460171699524, "step_time": 3.500531300000148, "tokens_per_second": 2340.216183754636, "steps_per_second": 0.28567092086848583, "batch_wait": 0.0010785999984364025, "processed_target_tokens": 131428233, "elapsed_seconds": 15679.1319806, "eta_seconds": 270591.06949001143, "gpu": {"gpu_utilization": 2, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3453.59, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 22750, "max_steps": 100000, "progress_percent": 22.75, "loss": 1.755394458770752, "perplexity": 5.785729523648588, "learning_rate": 0.00026803447084089774, "gradient_norm": 1.0434494018554688, "step_time": 3.5579264999978477, "tokens_per_second": 2302.464651814745, "steps_per_second": 0.28106257956722963, "batch_wait": 0.0011540999985300004, "processed_target_tokens": 131717041, "elapsed_seconds": 15852.021278100001, "eta_seconds": 274849.8221248337, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3396.84, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 22800, "max_steps": 100000, "progress_percent": 22.8, "loss": 1.7596402168273926, "perplexity": 5.81034655338351, "learning_rate": 0.0002678966349788978, "gradient_norm": 1.0946518182754517, "step_time": 3.5696337999979733, "tokens_per_second": 2294.9132765396416, "steps_per_second": 0.280140780827593, "batch_wait": 0.0010864000032597687, "processed_target_tokens": 132005152, "elapsed_seconds": 16026.4809914, "eta_seconds": 275575.72935984354, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3396.84, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 22850, "max_steps": 100000, "progress_percent": 22.85, "loss": 1.810799241065979, "perplexity": 6.115333104904997, "learning_rate": 0.00026775854267163874, "gradient_norm": 0.9875612258911133, "step_time": 3.3150040000000445, "tokens_per_second": 2471.188571718131, "steps_per_second": 0.3016587611960609, "batch_wait": 0.0007315000002563465, "processed_target_tokens": 132296146, "elapsed_seconds": 16192.4039238, "eta_seconds": 255752.55860000342, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3396.84, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 22900, "max_steps": 100000, "progress_percent": 22.900000000000002, "loss": 1.634669542312622, "perplexity": 5.127763208243375, "learning_rate": 0.00026762019426328264, "gradient_norm": 1.017867922782898, "step_time": 3.313323500002298, "tokens_per_second": 2472.4419453742803, "steps_per_second": 0.3018117609099463, "batch_wait": 0.0013419999995676335, "processed_target_tokens": 132588684, "elapsed_seconds": 16358.198042600001, "eta_seconds": 255457.24185017718, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3396.84, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 22950, "max_steps": 100000, "progress_percent": 22.95, "loss": 1.8730368614196777, "perplexity": 6.508030405856293, "learning_rate": 0.00026748159009862987, "gradient_norm": 1.0799477100372314, "step_time": 3.3106120999982522, "tokens_per_second": 2474.4668818205323, "steps_per_second": 0.30205894553473295, "batch_wait": 0.0013838999984727707, "processed_target_tokens": 132880226, "elapsed_seconds": 16524.1047486, "eta_seconds": 255082.66230486534, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 36, "gpu_memory_used_mb": 3396.84, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 61}}
{"event": "train", "step": 23000, "max_steps": 100000, "progress_percent": 23.0, "loss": 1.7174243927001953, "perplexity": 5.5701634192960325, "learning_rate": 0.00026734273052311813, "gradient_norm": 0.9604368805885315, "step_time": 3.293307800002367, "tokens_per_second": 2487.468678146061, "steps_per_second": 0.3036460788752516, "batch_wait": 0.0018444999986968469, "processed_target_tokens": 133171415, "elapsed_seconds": 16689.6378023, "eta_seconds": 253584.70060018226, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3292.38, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "validation", "step": 23000, "validation_loss": 1.78531357049942, "validation_perplexity": 5.9614489890659055, "validation_time": 2.2255339000003005, "best_validation_loss": 1.551450753211975}
{"step": 23000, "val_loss": 1.78531357049942, "best_val_loss": 1.551450753211975, "processed_target_tokens": 133171415, "timestamp": 1793520240.3510954}
{"event": "checkpoint", "step": 23000, "max_steps": 100000, "progress_percent": 23.0, "train_loss": 1.7174243927001953, "train_perplexity": 5.5701634192960325, "validation_loss": 1.78531357049942, "validation_perplexity": 5.9614489890659055, "best_validation_loss": 1.551450753211975, "best_validation_perplexity": 4.7183103235698125, "processed_target_tokens": 133171415, "tokens_per_step": 8192, "tokens_per_second": 2487.468678146061, "steps_per_second": 0.3036460788752516, "elapsed_seconds": 16689.6378023, "eta_seconds": 253584.70060018226, "learning_rate": 0.00026734273052311813, "gradient_norm": 0.9604368805885315, "batch_wait": 0.0018444999986968469, "evaluation_time": 2.2255339000003005, "parameter_count": 62113280, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3292.38, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 23050, "max_steps": 100000, "progress_percent": 23.05, "loss": 1.9016375541687012, "perplexity": 6.696851925882579, "learning_rate": 0.00026720361588282167, "gradient_norm": 1.052943468093872, "step_time": 3.2905590000009397, "tokens_per_second": 2489.5466089493184, "steps_per_second": 0.3038997325377586, "batch_wait": 0.0007618000017828308, "processed_target_tokens": 133459023, "elapsed_seconds": 16857.6919447, "eta_seconds": 253208.5150500723, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3292.38, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 23100, "max_steps": 100000, "progress_percent": 23.1, "loss": 1.5039010047912598, "perplexity": 4.499206306070118, "learning_rate": 0.00026706424652445056, "gradient_norm": 0.9272951483726501, "step_time": 3.296623099999124, "tokens_per_second": 2484.967116805733, "steps_per_second": 0.3033407125006998, "batch_wait": 0.0015718999966338743, "processed_target_tokens": 133750189, "elapsed_seconds": 17022.7777169, "eta_seconds": 253510.31638993265, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3292.38, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 23150, "max_steps": 100000, "progress_percent": 23.150000000000002, "loss": 1.6301361322402954, "perplexity": 5.104569567730963, "learning_rate": 0.0002669246227953495, "gradient_norm": 0.9807296991348267, "step_time": 3.31527979999737, "tokens_per_second": 2470.982992146394, "steps_per_second": 0.30163366603349534, "batch_wait": 0.0010401999970781617, "processed_target_tokens": 134034753, "elapsed_seconds": 17187.8634974, "eta_seconds": 254779.25262979788, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3292.38, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 23200, "max_steps": 100000, "progress_percent": 23.200000000000003, "loss": 1.8909227848052979, "perplexity": 6.625479753051157, "learning_rate": 0.00026678474504349733, "gradient_norm": 1.0364348888397217, "step_time": 3.4372799999982817, "tokens_per_second": 2383.2798026358328, "steps_per_second": 0.29092771028269443, "batch_wait": 0.0010127000023203436, "processed_target_tokens": 134328222, "elapsed_seconds": 17355.1496714, "eta_seconds": 263983.10399986804, "gpu": {"gpu_utilization": 9, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3292.38, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 23250, "max_steps": 100000, "progress_percent": 23.25, "loss": 1.7507379055023193, "perplexity": 5.758850596073061, "learning_rate": 0.0002666446136175059, "gradient_norm": 0.9953755736351013, "step_time": 3.4437851999973645, "tokens_per_second": 2378.777863383079, "steps_per_second": 0.29037815715125476, "batch_wait": 0.0013559000071836635, "processed_target_tokens": 134615353, "elapsed_seconds": 17526.8885133, "eta_seconds": 264310.5140997977, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3481.8, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 23300, "max_steps": 100000, "progress_percent": 23.3, "loss": 1.849951982498169, "perplexity": 6.359514147287999, "learning_rate": 0.00026650422886661916, "gradient_norm": 1.0514072179794312, "step_time": 3.4371517000035965, "tokens_per_second": 2383.368764314775, "steps_per_second": 0.29093856986264344, "batch_wait": 0.0016354000035789795, "processed_target_tokens": 134898321, "elapsed_seconds": 17698.4207198, "eta_seconds": 263629.5353902759, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3481.8, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 23350, "max_steps": 100000, "progress_percent": 23.35, "loss": 1.8360519409179688, "perplexity": 6.271728164050792, "learning_rate": 0.0002663635911407127, "gradient_norm": 1.0003629922866821, "step_time": 3.3092341999981727, "tokens_per_second": 2475.4972011363, "steps_per_second": 0.3021847169355835, "batch_wait": 0.0008893999984138645, "processed_target_tokens": 135184495, "elapsed_seconds": 17866.4658796, "eta_seconds": 253652.80142985994, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3481.8, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 23400, "max_steps": 100000, "progress_percent": 23.400000000000002, "loss": 1.7440571784973145, "perplexity": 5.72050551650078, "learning_rate": 0.0002662227007902924, "gradient_norm": 0.9935742616653442, "step_time": 3.3127925000007963, "tokens_per_second": 2472.8382474900045, "steps_per_second": 0.3018601376330572, "batch_wait": 0.0013762999951723032, "processed_target_tokens": 135473868, "elapsed_seconds": 18032.113274, "eta_seconds": 253759.905500061, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3481.8, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 23450, "max_steps": 100000, "progress_percent": 23.45, "loss": 1.880387306213379, "perplexity": 6.55604356694001, "learning_rate": 0.0002660815581664936, "gradient_norm": 0.9603741765022278, "step_time": 3.3069692000026407, "tokens_per_second": 2477.192711680973, "steps_per_second": 0.30239168843761877, "batch_wait": 0.0010000999973271973, "processed_target_tokens": 135762560, "elapsed_seconds": 18197.6298671, "eta_seconds": 253148.49226020215, "gpu": {"gpu_utilization": 100, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3481.8, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 23500, "max_steps": 100000, "progress_percent": 23.5, "loss": 1.5950425863265991, "perplexity": 4.928538956019639, "learning_rate": 0.0002659401636210807, "gradient_norm": 0.9890889525413513, "step_time": 3.2962650999979815, "tokens_per_second": 2485.237003542287, "steps_per_second": 0.30337365765897056, "batch_wait": 0.001042200001393212, "processed_target_tokens": 136051687, "elapsed_seconds": 18363.0437103, "eta_seconds": 252164.28014984558, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3384.08, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "validation", "step": 23500, "validation_loss": 1.8107313752174377, "validation_perplexity": 6.114918096717321, "validation_time": 2.2226468000008026, "best_validation_loss": 1.551450753211975}
{"step": 23500, "val_loss": 1.8107313752174377, "best_val_loss": 1.551450753211975, "processed_target_tokens": 136051687, "timestamp": 1793521913.731711}
{"event": "checkpoint", "step": 23500, "max_steps": 100000, "progress_percent": 23.5, "train_loss": 1.5950425863265991, "train_perplexity": 4.928538956019639, "validation_loss": 1.8107313752174377, "validation_perplexity": 6.114918096717321, "best_validation_loss": 1.551450753211975, "best_validation_perplexity": 4.7183103235698125, "processed_target_tokens": 136051687, "tokens_per_step": 8192, "tokens_per_second": 2485.237003542287, "steps_per_second": 0.30337365765897056, "elapsed_seconds": 18363.0437103, "eta_seconds": 252164.28014984558, "learning_rate": 0.0002659401636210807, "gradient_norm": 0.9890889525413513, "batch_wait": 0.001042200001393212, "evaluation_time": 2.2226468000008026, "parameter_count": 62113280, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 2, "gpu_memory_used_mb": 3384.08, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 23550, "max_steps": 100000, "progress_percent": 23.549999999999997, "loss": 1.8710417747497559, "perplexity": 6.495059264724002, "learning_rate": 0.00026579851750644577, "gradient_norm": 1.047950267791748, "step_time": 3.3021012999997765, "tokens_per_second": 2480.8445458655538, "steps_per_second": 0.30283746897772873, "batch_wait": 0.0008398000027227681, "processed_target_tokens": 136342419, "elapsed_seconds": 18530.9246572, "eta_seconds": 252445.6443849829, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 2, "gpu_memory_used_mb": 3384.08, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 23600, "max_steps": 100000, "progress_percent": 23.599999999999998, "loss": 1.6572307348251343, "perplexity": 5.244766565858471, "learning_rate": 0.0002656566201756079, "gradient_norm": 0.9633628129959106, "step_time": 3.302694699999847, "tokens_per_second": 2480.3988088878996, "steps_per_second": 0.3027830577255737, "batch_wait": 0.0014040000023669563, "processed_target_tokens": 136631357, "elapsed_seconds": 18695.8928509, "eta_seconds": 252325.87507998833, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 2, "gpu_memory_used_mb": 3384.08, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 23650, "max_steps": 100000, "progress_percent": 23.65, "loss": 1.6562501192092896, "perplexity": 5.239625986740543, "learning_rate": 0.00026551447198221216, "gradient_norm": 1.0898213386535645, "step_time": 3.309469600000739, "tokens_per_second": 2475.3211209428155, "steps_per_second": 0.3021632227713398, "batch_wait": 0.0011100000047008507, "processed_target_tokens": 136924672, "elapsed_seconds": 18860.8346064, "eta_seconds": 252678.0039600564, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 2, "gpu_memory_used_mb": 3384.08, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 23700, "max_steps": 100000, "progress_percent": 23.7, "loss": 1.7373952865600586, "perplexity": 5.682522785822247, "learning_rate": 0.00026537207328052904, "gradient_norm": 1.0377566814422607, "step_time": 3.2980453000018315, "tokens_per_second": 2483.8955365456777, "steps_per_second": 0.30320990436348605, "batch_wait": 0.0010021000052802265, "processed_target_tokens": 137219248, "elapsed_seconds": 19025.7701992, "eta_seconds": 251640.85639013973, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 2, "gpu_memory_used_mb": 3384.08, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 23750, "max_steps": 100000, "progress_percent": 23.75, "loss": 1.4759771823883057, "perplexity": 4.375309159704817, "learning_rate": 0.0002652294244254532, "gradient_norm": 0.9900556802749634, "step_time": 3.3093386000000464, "tokens_per_second": 2475.4191064038855, "steps_per_second": 0.30217518388719306, "batch_wait": 0.000982600002316758, "processed_target_tokens": 137504665, "elapsed_seconds": 19190.7302982, "eta_seconds": 252337.06825000353, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3307.64, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 23800, "max_steps": 100000, "progress_percent": 23.799999999999997, "loss": 1.5755066871643066, "perplexity": 4.833189913089698, "learning_rate": 0.00026508652577250284, "gradient_norm": 0.9952799677848816, "step_time": 3.28612780000185, "tokens_per_second": 2492.903653958738, "steps_per_second": 0.304309528071135, "batch_wait": 0.00098249999791733, "processed_target_tokens": 137790763, "elapsed_seconds": 19355.6428156, "eta_seconds": 250402.93836014098, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3307.64, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 23850, "max_steps": 100000, "progress_percent": 23.849999999999998, "loss": 1.7074534893035889, "perplexity": 5.514899829748368, "learning_rate": 0.00026494337767781867, "gradient_norm": 1.0034840106964111, "step_time": 3.2926578000005975, "tokens_per_second": 2487.959726637403, "steps_per_second": 0.30370602131804236, "batch_wait": 0.0010244999975839164, "processed_target_tokens": 138083433, "elapsed_seconds": 19520.5980221, "eta_seconds": 250735.8914700455, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3307.64, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 23900, "max_steps": 100000, "progress_percent": 23.9, "loss": 2.1214871406555176, "perplexity": 8.343536278039862, "learning_rate": 0.000264799980498163, "gradient_norm": 1.0706473588943481, "step_time": 3.29088739999861, "tokens_per_second": 2489.298175319964, "steps_per_second": 0.30386940616698777, "batch_wait": 0.0017449000042688567, "processed_target_tokens": 138380178, "elapsed_seconds": 19685.5490038, "eta_seconds": 250436.5311398942, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3307.64, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 23950, "max_steps": 100000, "progress_percent": 23.95, "loss": 1.549365758895874, "perplexity": 4.708482921961957, "learning_rate": 0.00026465633459091915, "gradient_norm": 1.0908793210983276, "step_time": 3.2976855999986583, "tokens_per_second": 2484.166471177038, "steps_per_second": 0.3032429774386033, "batch_wait": 0.001967800002603326, "processed_target_tokens": 138666276, "elapsed_seconds": 19850.5052744, "eta_seconds": 250788.98987989797, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3307.64, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 24000, "max_steps": 100000, "progress_percent": 24.0, "loss": 1.9864119291305542, "perplexity": 7.289332143465745, "learning_rate": 0.0002645124403140901, "gradient_norm": 1.0507270097732544, "step_time": 3.3085756999971636, "tokens_per_second": 2475.9898949892618, "steps_per_second": 0.30224486022818137, "batch_wait": 0.0008844000003591646, "processed_target_tokens": 138959671, "elapsed_seconds": 20015.4592155, "eta_seconds": 251451.75319978443, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 32, "gpu_memory_used_mb": 3307.64, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "validation", "step": 24000, "validation_loss": 1.9131675481796264, "validation_perplexity": 6.774513446550913, "validation_time": 2.226276899997174, "best_validation_loss": 1.551450753211975}
{"step": 24000, "val_loss": 1.9131675481796264, "best_val_loss": 1.551450753211975, "processed_target_tokens": 138959671, "timestamp": 1793523566.183663}
{"event": "checkpoint", "step": 24000, "max_steps": 100000, "progress_percent": 24.0, "train_loss": 1.9864119291305542, "train_perplexity": 7.289332143465745, "validation_loss": 1.9131675481796264, "validation_perplexity": 6.774513446550913, "best_validation_loss": 1.551450753211975, "best_validation_perplexity": 4.7183103235698125, "processed_target_tokens": 138959671, "tokens_per_step": 8192, "tokens_per_second": 2475.9898949892618, "steps_per_second": 0.30224486022818137, "elapsed_seconds": 20015.4592155, "eta_seconds": 251451.75319978443, "learning_rate": 0.0002645124403140901, "gradient_norm": 1.0507270097732544, "batch_wait": 0.0008844000003591646, "evaluation_time": 2.226276899997174, "parameter_count": 62113280, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3307.64, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 55}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 24050, "max_steps": 100000, "progress_percent": 24.05, "loss": 2.0179431438446045, "perplexity": 7.522835618727962, "learning_rate": 0.00026436829802629804, "gradient_norm": 1.0262682437896729, "step_time": 3.2926446000019496, "tokens_per_second": 2487.969700706584, "steps_per_second": 0.30370723885578416, "batch_wait": 0.0012443999985407572, "processed_target_tokens": 139250482, "elapsed_seconds": 20183.3609335, "eta_seconds": 250076.35737014806, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3307.64, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 55}}
{"event": "train", "step": 24100, "max_steps": 100000, "progress_percent": 24.099999999999998, "loss": 1.7184476852416992, "perplexity": 5.575866243308964, "learning_rate": 0.000264223908086783, "gradient_norm": 1.0053590536117554, "step_time": 3.2815934999998717, "tokens_per_second": 2496.348191816055, "steps_per_second": 0.3047300038837958, "batch_wait": 0.0008447000036539976, "processed_target_tokens": 139542558, "elapsed_seconds": 20348.3208352, "eta_seconds": 249072.94664999028, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3307.64, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 55}}
{"event": "train", "step": 24150, "max_steps": 100000, "progress_percent": 24.15, "loss": 1.9017040729522705, "perplexity": 6.697297407142704, "learning_rate": 0.0002640792708554025, "gradient_norm": 0.9573531746864319, "step_time": 3.293841800001246, "tokens_per_second": 2487.0654079369874, "steps_per_second": 0.30359685155480803, "batch_wait": 0.001569199997902615, "processed_target_tokens": 139829838, "elapsed_seconds": 20513.3019511, "eta_seconds": 249837.9005300945, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3307.64, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 55}}
{"event": "train", "step": 24200, "max_steps": 100000, "progress_percent": 24.2, "loss": 1.9696898460388184, "perplexity": 7.1684528194870785, "learning_rate": 0.0002639343866926303, "gradient_norm": 0.9900045394897461, "step_time": 3.2972045999995316, "tokens_per_second": 2484.528864238866, "steps_per_second": 0.30328721487290844, "batch_wait": 0.0015493999999307562, "processed_target_tokens": 140117786, "elapsed_seconds": 20678.270909, "eta_seconds": 249928.1086799645, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3307.64, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 55}}
{"event": "train", "step": 24250, "max_steps": 100000, "progress_percent": 24.25, "loss": 1.6651266813278198, "perplexity": 5.286342888280918, "learning_rate": 0.00026378925595955537, "gradient_norm": 0.976429283618927, "step_time": 3.298682500000723, "tokens_per_second": 2483.415727339083, "steps_per_second": 0.30315133390369664, "batch_wait": 0.0010378000006312504, "processed_target_tokens": 140406192, "elapsed_seconds": 20843.2579873, "eta_seconds": 249875.19937505477, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3314.39, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 24300, "max_steps": 100000, "progress_percent": 24.3, "loss": 2.0539140701293945, "perplexity": 7.7983647956114, "learning_rate": 0.0002636438790178814, "gradient_norm": 1.0193575620651245, "step_time": 3.292000100002042, "tokens_per_second": 2488.456789535006, "steps_per_second": 0.3037666979412849, "batch_wait": 0.0012558999987959396, "processed_target_tokens": 140700183, "elapsed_seconds": 21008.2313909, "eta_seconds": 249204.40757015458, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3314.39, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 24350, "max_steps": 100000, "progress_percent": 24.349999999999998, "loss": 1.790816068649292, "perplexity": 5.994342265701945, "learning_rate": 0.00026349825622992576, "gradient_norm": 1.0364537239074707, "step_time": 3.2953472000008333, "tokens_per_second": 2485.929252006565, "steps_per_second": 0.30345816064533265, "batch_wait": 0.0015682999946875498, "processed_target_tokens": 140988472, "elapsed_seconds": 21173.2286889, "eta_seconds": 249293.01568006305, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3314.39, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 24400, "max_steps": 100000, "progress_percent": 24.4, "loss": 1.8102291822433472, "perplexity": 6.1118479987673435, "learning_rate": 0.0002633523879586183, "gradient_norm": 1.0099999904632568, "step_time": 3.3072793999999703, "tokens_per_second": 2476.960368089879, "steps_per_second": 0.30236332618284656, "batch_wait": 0.0014960000007704366, "processed_target_tokens": 141279328, "elapsed_seconds": 21338.2495503, "eta_seconds": 250030.32263999776, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3314.39, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 24450, "max_steps": 100000, "progress_percent": 24.45, "loss": 1.9617692232131958, "perplexity": 7.111898476935948, "learning_rate": 0.00026320627456750094, "gradient_norm": 1.0217881202697754, "step_time": 3.301723700002185, "tokens_per_second": 2481.1282664247706, "steps_per_second": 0.302872102835055, "batch_wait": 0.0010942000008071773, "processed_target_tokens": 141567916, "elapsed_seconds": 21503.2212742, "eta_seconds": 249445.22553516508, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 31, "gpu_memory_used_mb": 3314.39, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 24500, "max_steps": 100000, "progress_percent": 24.5, "loss": 1.8230013847351074, "perplexity": 6.190410398463107, "learning_rate": 0.0002630599164207263, "gradient_norm": 1.0033857822418213, "step_time": 3.302752399999008, "tokens_per_second": 2480.3554756337353, "steps_per_second": 0.3027777680216962, "batch_wait": 0.0006505000019387808, "processed_target_tokens": 141858803, "elapsed_seconds": 21668.1833979, "eta_seconds": 249357.8061999251, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 30, "gpu_memory_used_mb": 3312.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "validation", "step": 24500, "validation_loss": 1.8160194039344788, "validation_perplexity": 6.147339606601919, "validation_time": 2.222316100000171, "best_validation_loss": 1.551450753211975}
{"step": 24500, "val_loss": 1.8160194039344788, "best_val_loss": 1.551450753211975, "processed_target_tokens": 141858803, "timestamp": 1793525218.959253}
{"event": "checkpoint", "step": 24500, "max_steps": 100000, "progress_percent": 24.5, "train_loss": 1.8230013847351074, "train_perplexity": 6.190410398463107, "validation_loss": 1.8160194039344788, "validation_perplexity": 6.147339606601919, "best_validation_loss": 1.551450753211975, "best_validation_perplexity": 4.7183103235698125, "processed_target_tokens": 141858803, "tokens_per_step": 8192, "tokens_per_second": 2480.3554756337353, "steps_per_second": 0.3027777680216962, "elapsed_seconds": 21668.1833979, "eta_seconds": 249357.8061999251, "learning_rate": 0.0002630599164207263, "gradient_norm": 1.0033857822418213, "batch_wait": 0.0006505000019387808, "evaluation_time": 2.222316100000171, "parameter_count": 62113280, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 2, "gpu_memory_used_mb": 3312.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 55}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 24550, "max_steps": 100000, "progress_percent": 24.55, "loss": 1.7612948417663574, "perplexity": 5.819968455819408, "learning_rate": 0.00026291331388305706, "gradient_norm": 0.8808547854423523, "step_time": 3.2986655999993673, "tokens_per_second": 2483.4284505836454, "steps_per_second": 0.3031528870341364, "batch_wait": 0.0010585999953036662, "processed_target_tokens": 142154351, "elapsed_seconds": 21836.140054299998, "eta_seconds": 248884.31951995226, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3312.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 55}}
{"event": "train", "step": 24600, "max_steps": 100000, "progress_percent": 24.6, "loss": 1.7294909954071045, "perplexity": 5.63778352026435, "learning_rate": 0.000262766467319865, "gradient_norm": 0.9680976867675781, "step_time": 3.3002550000019255, "tokens_per_second": 2482.232433552929, "steps_per_second": 0.3030068888614415, "batch_wait": 0.0010045000017271377, "processed_target_tokens": 142452329, "elapsed_seconds": 22001.1338694, "eta_seconds": 248839.22700014518, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3312.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 55}}
{"event": "train", "step": 24650, "max_steps": 100000, "progress_percent": 24.65, "loss": 1.8173458576202393, "perplexity": 6.155499178331332, "learning_rate": 0.00026261937709713, "gradient_norm": 0.9905794262886047, "step_time": 3.299341199999617, "tokens_per_second": 2482.9199235292645, "steps_per_second": 0.3030908109776934, "batch_wait": 0.000997399998595938, "processed_target_tokens": 142745124, "elapsed_seconds": 22166.1070218, "eta_seconds": 248605.35941997112, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3312.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 55}}
{"event": "train", "step": 24700, "max_steps": 100000, "progress_percent": 24.7, "loss": 1.787587285041809, "perplexity": 5.97501904369515, "learning_rate": 0.00026247204358143943, "gradient_norm": 0.93772953748703, "step_time": 3.2999885999997787, "tokens_per_second": 2482.432818101417, "steps_per_second": 0.30303134986589564, "batch_wait": 0.0012655000027734786, "processed_target_tokens": 143032631, "elapsed_seconds": 22331.0786808, "eta_seconds": 248489.14157998335, "gpu": {"gpu_utilization": 7, "gpu_memory_utilization": 1, "gpu_memory_used_mb": 3312.95, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 55}}
{"event": "train", "step": 24750, "max_steps": 100000, "progress_percent": 24.75, "loss": 1.9634504318237305, "perplexity": 7.123865118284252, "learning_rate": 0.0002623244671399867, "gradient_norm": 1.0023972988128662, "step_time": 3.5615610999993805, "tokens_per_second": 2300.1149692480144, "steps_per_second": 0.280775753082033, "batch_wait": 0.0006791999985580333, "processed_target_tokens": 143319899, "elapsed_seconds": 22508.4070894, "eta_seconds": 268007.4727749534, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 50, "gpu_memory_used_mb": 3859.34, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 24800, "max_steps": 100000, "progress_percent": 24.8, "loss": 1.8853356838226318, "perplexity": 6.588565745780057, "learning_rate": 0.00026217664814057087, "gradient_norm": 0.9351972341537476, "step_time": 3.9637471999994887, "tokens_per_second": 2066.731198195752, "steps_per_second": 0.25228652321725487, "batch_wait": 0.0012047999989590608, "processed_target_tokens": 143607625, "elapsed_seconds": 22724.9557375, "eta_seconds": 298073.78943996155, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 50, "gpu_memory_used_mb": 3859.34, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 24850, "max_steps": 100000, "progress_percent": 24.85, "loss": 1.4840154647827148, "perplexity": 4.410620862878726, "learning_rate": 0.00026202858695159543, "gradient_norm": 0.9118713736534119, "step_time": 5.751139899999544, "tokens_per_second": 1424.4132715325966, "steps_per_second": 0.17387857318513142, "batch_wait": 0.0007306000043172389, "processed_target_tokens": 143894090, "elapsed_seconds": 23038.9959375, "eta_seconds": 432198.1634849657, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 50, "gpu_memory_used_mb": 3859.34, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 24900, "max_steps": 100000, "progress_percent": 24.9, "loss": 1.6329466104507446, "perplexity": 5.118936028130283, "learning_rate": 0.0002618802839420675, "gradient_norm": 0.913074791431427, "step_time": 5.188163700000587, "tokens_per_second": 1578.9787049315876, "steps_per_second": 0.19274642394184419, "batch_wait": 0.0011478999949758872, "processed_target_tokens": 144176830, "elapsed_seconds": 23465.229142199998, "eta_seconds": 389631.09387004405, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 50, "gpu_memory_used_mb": 3859.34, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 24950, "max_steps": 100000, "progress_percent": 24.95, "loss": 1.7960954904556274, "perplexity": 6.026072612216022, "learning_rate": 0.00026173173948159685, "gradient_norm": 1.1403106451034546, "step_time": 4.67754519999653, "tokens_per_second": 1751.3459837878377, "steps_per_second": 0.21378735153660128, "batch_wait": 0.0013407000005827285, "processed_target_tokens": 144460537, "elapsed_seconds": 23822.691805899998, "eta_seconds": 351049.7672597396, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 50, "gpu_memory_used_mb": 3859.34, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 25000, "max_steps": 100000, "progress_percent": 25.0, "loss": 1.872642993927002, "perplexity": 6.5054676089724035, "learning_rate": 0.0002615829539403951, "gradient_norm": 0.9359154105186462, "step_time": 5.97407309999835, "tokens_per_second": 1371.258748072946, "steps_per_second": 0.1673899838956233, "batch_wait": 0.0009921999990183394, "processed_target_tokens": 144751898, "elapsed_seconds": 24088.3130096, "eta_seconds": 448055.48249987623, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 41, "gpu_memory_used_mb": 3608.29, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 58}}
{"event": "validation", "step": 25000, "validation_loss": 1.7801084995269776, "validation_perplexity": 5.930499840112442, "validation_time": 3.2208535999998276, "best_validation_loss": 1.551450753211975}
{"step": 25000, "val_loss": 1.7801084995269776, "best_val_loss": 1.551450753211975, "processed_target_tokens": 144751898, "timestamp": 1793527642.6096122}
{"event": "checkpoint", "step": 25000, "max_steps": 100000, "progress_percent": 25.0, "train_loss": 1.872642993927002, "train_perplexity": 6.5054676089724035, "validation_loss": 1.7801084995269776, "validation_perplexity": 5.930499840112442, "best_validation_loss": 1.551450753211975, "best_validation_perplexity": 4.7183103235698125, "processed_target_tokens": 144751898, "tokens_per_step": 8192, "tokens_per_second": 1371.258748072946, "steps_per_second": 0.1673899838956233, "elapsed_seconds": 24088.3130096, "eta_seconds": 448055.48249987623, "learning_rate": 0.0002615829539403951, "gradient_norm": 0.9359154105186462, "batch_wait": 0.0009921999990183394, "evaluation_time": 3.2208535999998276, "parameter_count": 62113280, "gpu": {"gpu_utilization": 35, "gpu_memory_utilization": 13, "gpu_memory_used_mb": 3707.79, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 54}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 25050, "max_steps": 100000, "progress_percent": 25.05, "loss": 2.091581344604492, "perplexity": 8.097710315383122, "learning_rate": 0.00026143392768927456, "gradient_norm": 1.0844745635986328, "step_time": 4.58766900000046, "tokens_per_second": 1785.656288629188, "steps_per_second": 0.2179756211705552, "batch_wait": 0.000846799994178582, "processed_target_tokens": 145043367, "elapsed_seconds": 24355.4501121, "eta_seconds": 343845.7915500345, "gpu": {"gpu_utilization": 35, "gpu_memory_utilization": 13, "gpu_memory_used_mb": 3707.79, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 54}}
{"event": "train", "step": 25100, "max_steps": 100000, "progress_percent": 25.1, "loss": 1.6098597049713135, "perplexity": 5.002109407520967, "learning_rate": 0.00026128466109964755, "gradient_norm": 1.1103312969207764, "step_time": 6.246685399997659, "tokens_per_second": 1311.4154908462447, "steps_per_second": 0.16008489878494198, "batch_wait": 0.0009974999957194086, "processed_target_tokens": 145332125, "elapsed_seconds": 24604.7340242, "eta_seconds": 467876.73645982466, "gpu": {"gpu_utilization": 35, "gpu_memory_utilization": 13, "gpu_memory_used_mb": 3707.79, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 54}}
{"event": "train", "step": 25150, "max_steps": 100000, "progress_percent": 25.15, "loss": 1.8398723602294922, "perplexity": 6.295734623617164, "learning_rate": 0.0002611351545435254, "gradient_norm": 1.0226786136627197, "step_time": 4.741063199999189, "tokens_per_second": 1727.8824715944306, "steps_per_second": 0.21092315327080452, "batch_wait": 0.0010164999985136092, "processed_target_tokens": 145626593, "elapsed_seconds": 24871.7823561, "eta_seconds": 354868.5805199393, "gpu": {"gpu_utilization": 35, "gpu_memory_utilization": 13, "gpu_memory_used_mb": 3707.79, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 54}}
{"event": "train", "step": 25200, "max_steps": 100000, "progress_percent": 25.2, "loss": 1.7803103923797607, "perplexity": 5.93169728651746, "learning_rate": 0.00026098540839351734, "gradient_norm": 1.0596764087677002, "step_time": 3.4340188999995007, "tokens_per_second": 2385.5430731616507, "steps_per_second": 0.29120398842305306, "batch_wait": 0.0008010000019567087, "processed_target_tokens": 145916149, "elapsed_seconds": 25069.3313002, "eta_seconds": 256864.61371996265, "gpu": {"gpu_utilization": 35, "gpu_memory_utilization": 13, "gpu_memory_used_mb": 3707.79, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 54}}
{"event": "train", "step": 25250, "max_steps": 100000, "progress_percent": 25.25, "loss": 1.5396461486816406, "perplexity": 4.662939991565125, "learning_rate": 0.00026083542302282993, "gradient_norm": 0.9927694201469421, "step_time": 3.47692129999632, "tokens_per_second": 2356.1073988095936, "steps_per_second": 0.2876107664562492, "batch_wait": 0.0007327000057557598, "processed_target_tokens": 146204972, "elapsed_seconds": 25249.2281833, "eta_seconds": 259899.86717472493, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3486.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 25300, "max_steps": 100000, "progress_percent": 25.3, "loss": 1.7249208688735962, "perplexity": 5.612076922153709, "learning_rate": 0.00026068519880526584, "gradient_norm": 1.1390255689620972, "step_time": 3.542039499996463, "tokens_per_second": 2312.7918251640563, "steps_per_second": 0.2823232208452217, "batch_wait": 0.0006744999991497025, "processed_target_tokens": 146495766, "elapsed_seconds": 25432.7148104, "eta_seconds": 264590.3506497358, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3486.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 25350, "max_steps": 100000, "progress_percent": 25.35, "loss": 1.7781416177749634, "perplexity": 5.918846712115804, "learning_rate": 0.00026053473611522296, "gradient_norm": 0.9582082629203796, "step_time": 3.51881599999615, "tokens_per_second": 2328.0558005900175, "steps_per_second": 0.2841864990954611, "batch_wait": 0.0006498999937321059, "processed_target_tokens": 146780194, "elapsed_seconds": 25605.905203000002, "eta_seconds": 262679.6143997126, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3486.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 25400, "max_steps": 100000, "progress_percent": 25.4, "loss": 1.9094877243041992, "perplexity": 6.749630241211278, "learning_rate": 0.00026038403532769363, "gradient_norm": 1.0045360326766968, "step_time": 3.3128305000063847, "tokens_per_second": 2472.8098826620353, "steps_per_second": 0.301856675129643, "batch_wait": 0.0009428999983356334, "processed_target_tokens": 147068055, "elapsed_seconds": 25774.2892133, "eta_seconds": 247137.1553004763, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3486.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 25450, "max_steps": 100000, "progress_percent": 25.45, "loss": 1.9871532917022705, "perplexity": 7.2947381851606, "learning_rate": 0.0002602330968182635, "gradient_norm": 1.0230613946914673, "step_time": 3.316082799996366, "tokens_per_second": 2470.3846357542634, "steps_per_second": 0.3015606244817216, "batch_wait": 0.000874100012879353, "processed_target_tokens": 147355081, "elapsed_seconds": 25939.996647, "eta_seconds": 247213.97273972907, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 34, "gpu_memory_used_mb": 3486.3, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "train", "step": 25500, "max_steps": 100000, "progress_percent": 25.5, "loss": 1.9031994342803955, "perplexity": 6.70731977835387, "learning_rate": 0.00026008192096311074, "gradient_norm": 1.0543465614318848, "step_time": 3.304547599997022, "tokens_per_second": 2479.0080191332036, "steps_per_second": 0.30261328358559614, "batch_wait": 0.0009855999960564077, "processed_target_tokens": 147643708, "elapsed_seconds": 26105.546157700002, "eta_seconds": 246188.79619977815, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 45, "gpu_memory_used_mb": 3420.39, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "validation", "step": 25500, "validation_loss": 1.8119084000587464, "validation_perplexity": 6.122119744647559, "validation_time": 2.2094049000006635, "best_validation_loss": 1.551450753211975}
{"step": 25500, "val_loss": 1.8119084000587464, "best_val_loss": 1.551450753211975, "processed_target_tokens": 147643708, "timestamp": 1793529656.2413342}
{"event": "checkpoint", "step": 25500, "max_steps": 100000, "progress_percent": 25.5, "train_loss": 1.9031994342803955, "train_perplexity": 6.70731977835387, "validation_loss": 1.8119084000587464, "validation_perplexity": 6.122119744647559, "best_validation_loss": 1.551450753211975, "best_validation_perplexity": 4.7183103235698125, "processed_target_tokens": 147643708, "tokens_per_step": 8192, "tokens_per_second": 2479.0080191332036, "steps_per_second": 0.30261328358559614, "elapsed_seconds": 26105.546157700002, "eta_seconds": 246188.79619977815, "learning_rate": 0.00026008192096311074, "gradient_norm": 1.0543465614318848, "batch_wait": 0.0009855999960564077, "evaluation_time": 2.2094049000006635, "parameter_count": 62113280, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3417.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}
{"event": "train", "step": 25550, "max_steps": 100000, "progress_percent": 25.55, "loss": 1.6477322578430176, "perplexity": 5.195185118249207, "learning_rate": 0.00025993050813900495, "gradient_norm": 0.9486275911331177, "step_time": 5.5448113000020385, "tokens_per_second": 1477.4172747766886, "steps_per_second": 0.18034878842488875, "batch_wait": 0.0005645999990520068, "processed_target_tokens": 147928755, "elapsed_seconds": 26302.146964400003, "eta_seconds": 412811.20128515176, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3417.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 25600, "max_steps": 100000, "progress_percent": 25.6, "loss": 1.7380329370498657, "perplexity": 5.686147404757173, "learning_rate": 0.00025977885872330644, "gradient_norm": 0.9507898092269897, "step_time": 4.570024299995566, "tokens_per_second": 1792.5506435508337, "steps_per_second": 0.21881721723032638, "batch_wait": 0.0006882000016048551, "processed_target_tokens": 148218049, "elapsed_seconds": 26544.017192400002, "eta_seconds": 340009.80791967013, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3417.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 25650, "max_steps": 100000, "progress_percent": 25.650000000000002, "loss": 1.4261261224746704, "perplexity": 4.162542737306645, "learning_rate": 0.00025962697309396516, "gradient_norm": 0.9626970291137695, "step_time": 4.480503500002669, "tokens_per_second": 1828.3659414606238, "steps_per_second": 0.22318920183845506, "batch_wait": 0.0008937999955378473, "processed_target_tokens": 148505949, "elapsed_seconds": 26770.8617016, "eta_seconds": 333125.43522519845, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3417.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 25700, "max_steps": 100000, "progress_percent": 25.7, "loss": 1.7036328315734863, "perplexity": 5.493869485539084, "learning_rate": 0.0002594748516295197, "gradient_norm": 0.9562471508979797, "step_time": 4.383667699999933, "tokens_per_second": 1868.7547872299092, "steps_per_second": 0.22811948086302603, "batch_wait": 0.0004155999995418824, "processed_target_tokens": 148793615, "elapsed_seconds": 26994.370469499998, "eta_seconds": 325706.510109995, "gpu": {"gpu_utilization": 10, "gpu_memory_utilization": 3, "gpu_memory_used_mb": 3417.14, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 56}}
{"event": "train", "step": 25750, "max_steps": 100000, "progress_percent": 25.75, "loss": 1.5615767240524292, "perplexity": 4.766330511521029, "learning_rate": 0.00025932249470909637, "gradient_norm": 0.9342523217201233, "step_time": 4.65014009999868, "tokens_per_second": 1761.6673527755272, "steps_per_second": 0.21504728427435635, "batch_wait": 0.0005026999933761545, "processed_target_tokens": 149081799, "elapsed_seconds": 27232.6131732, "eta_seconds": 345272.90242490196, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 45, "gpu_memory_used_mb": 3816.55, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 25800, "max_steps": 100000, "progress_percent": 25.8, "loss": 1.6694905757904053, "perplexity": 5.30946233947119, "learning_rate": 0.0002591699027124084, "gradient_norm": 0.9760781526565552, "step_time": 3.5695889000053285, "tokens_per_second": 2294.942143054, "steps_per_second": 0.2801443045720215, "batch_wait": 0.0009024000028148293, "processed_target_tokens": 149370488, "elapsed_seconds": 27433.7818443, "eta_seconds": 264863.4963803954, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 45, "gpu_memory_used_mb": 3816.55, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 25850, "max_steps": 100000, "progress_percent": 25.85, "loss": 1.9147677421569824, "perplexity": 6.785362660275772, "learning_rate": 0.00025901707601975484, "gradient_norm": 0.9892861843109131, "step_time": 3.30857379999361, "tokens_per_second": 2475.991316867655, "steps_per_second": 0.30224503379732115, "batch_wait": 0.0010066999966511503, "processed_target_tokens": 149651307, "elapsed_seconds": 27606.461426100002, "eta_seconds": 245330.74726952618, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 45, "gpu_memory_used_mb": 3816.55, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 25900, "max_steps": 100000, "progress_percent": 25.900000000000002, "loss": 2.0679397583007812, "perplexity": 7.908512874094514, "learning_rate": 0.0002588640150120198, "gradient_norm": 1.0086671113967896, "step_time": 3.3097561999966274, "tokens_per_second": 2475.1067767494014, "steps_per_second": 0.30213705770866717, "batch_wait": 0.0010806000063894317, "processed_target_tokens": 149940679, "elapsed_seconds": 27778.1797332, "eta_seconds": 245252.9344197501, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 45, "gpu_memory_used_mb": 3816.55, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 25950, "max_steps": 100000, "progress_percent": 25.95, "loss": 1.7286816835403442, "perplexity": 5.633222640994974, "learning_rate": 0.0002587107200706711, "gradient_norm": 0.9482209086418152, "step_time": 3.3071264999962295, "tokens_per_second": 2477.0748866151143, "steps_per_second": 0.3023773054950091, "batch_wait": 0.000836799998069182, "processed_target_tokens": 150225319, "elapsed_seconds": 27950.895242600003, "eta_seconds": 244892.7173247208, "gpu": {"gpu_utilization": 98, "gpu_memory_utilization": 45, "gpu_memory_used_mb": 3816.55, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 59}}
{"event": "train", "step": 26000, "max_steps": 100000, "progress_percent": 26.0, "loss": 2.0003621578216553, "perplexity": 7.3917325880180025, "learning_rate": 0.00025855719157775976, "gradient_norm": 1.043837070465088, "step_time": 3.8313902000008966, "tokens_per_second": 2138.1273042871185, "steps_per_second": 0.26100186819911114, "batch_wait": 0.0005849000081070699, "processed_target_tokens": 150516824, "elapsed_seconds": 28124.9619389, "eta_seconds": 283522.87480006635, "gpu": {"gpu_utilization": 99, "gpu_memory_utilization": 35, "gpu_memory_used_mb": 3657.59, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 60}}
{"event": "validation", "step": 26000, "validation_loss": 1.7366663694381714, "validation_perplexity": 5.678382206920906, "validation_time": 2.639796800001932, "best_validation_loss": 1.551450753211975}
{"step": 26000, "val_loss": 1.7366663694381714, "best_val_loss": 1.551450753211975, "processed_target_tokens": 150516824, "timestamp": 1793531677.794433}
{"event": "checkpoint", "step": 26000, "max_steps": 26000, "progress_percent": 100.0, "train_loss": 2.0003621578216553, "train_perplexity": 7.3917325880180025, "validation_loss": 1.7366663694381714, "validation_perplexity": 5.678382206920906, "best_validation_loss": 1.551450753211975, "best_validation_perplexity": 4.7183103235698125, "processed_target_tokens": 150516824, "tokens_per_step": 8192, "tokens_per_second": 2138.1273042871185, "steps_per_second": 0.26100186819911114, "elapsed_seconds": 28124.9619389, "eta_seconds": 0.0, "learning_rate": 0.00025855719157775976, "gradient_norm": 1.043837070465088, "batch_wait": 0.0005849000081070699, "evaluation_time": 2.639796800001932, "parameter_count": 62113280, "gpu": {"gpu_utilization": 21, "gpu_memory_utilization": 9, "gpu_memory_used_mb": 3661.28, "gpu_memory_total_mb": 4096.0, "gpu_temperature_c": 57}, "is_best": false, "checkpoint_path": "F:\\LIAM\\Documents\\OrangeGlassesStudio\\models\\McConversationBot-V3-62.1M\\instruct\\checkpoint.pt"}