cvis-ablate / c2_uniform_difficulty /round_2 /trainer_state.json
Nhckdvrl's picture
Add c2_uniform_difficulty/round_2: MathVision 54.61%, MathVista 83.20%, MMMU 71.44%
4aac3a4 verified
Raw History Blame Contribute Delete
85.3 kB
{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 80.0, "global_step": 160, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6328125, "completions/max_length": 2048.0, "completions/mean_length": 1654.703125, "completions/min_length": 156.0, "epoch": 0.00625, "frac_reward_zero_std": 0.3125, "grad_norm": 1.3100795864283492, "kl": 0.0, "learning_rate": 2e-07, "loss": -9.313225746154785e-09, "reward": 0.24609375, "reward_std": 0.29196497797966003, "rewards/MajorityAccuracy/mean": 0.24609375, "rewards/MajorityAccuracy/std": 0.43157756328582764, "step": 1, "step_time": 658.7488895617425 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0125, "grad_norm": 1.5411547555701335, "kl": 0.0, "learning_rate": 4e-07, "loss": -9.313225746154785e-10, "step": 2, "step_time": 92.33193903788924 }, { "clip_ratio/high_max": 0.0020130398188484833, "clip_ratio/high_mean": 0.0006289757548074704, "clip_ratio/low_mean": 0.00033599255402805284, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0009649683051975444, "epoch": 0.01875, "grad_norm": 2.6111678582441216, "kl": 0.0003152433109789854, "learning_rate": 6e-07, "loss": 0.0002443257544655353, "step": 3, "step_time": 94.15333711355925 }, { "clip_ratio/high_max": 0.0007583651677123271, "clip_ratio/high_mean": 0.00018959129192808177, "clip_ratio/low_mean": 0.000274658203125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0004642494968720712, "epoch": 0.025, "grad_norm": 0.8986616830589469, "kl": 0.00016798217166069662, "learning_rate": 8e-07, "loss": -2.4342021788470447e-05, "step": 4, "step_time": 94.268355473876 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.03125, "grad_norm": 0.0018725214294947354, "kl": 0.00017453282998758368, "learning_rate": 1e-06, "loss": 1.7468580608692719e-06, "step": 5, "step_time": 94.58316105231643 }, { "clip_ratio/high_max": 0.00047015461314003915, "clip_ratio/high_mean": 0.00017650091831455939, "clip_ratio/low_mean": 0.00047675427776994184, "clip_ratio/low_min": 6.103515625e-05, "clip_ratio/region_mean": 0.00065325519972248, "epoch": 0.0375, "grad_norm": 1.7121795828917972, "kl": 0.0001731301636027638, "learning_rate": 9.998973021172564e-07, "loss": 5.064532160758972e-05, "step": 6, "step_time": 92.9098944440484 }, { "clip_ratio/high_max": 0.00038758504524594173, "clip_ratio/high_mean": 9.689626131148543e-05, "clip_ratio/low_mean": 0.000457763671875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0005546599350054748, "epoch": 0.04375, "grad_norm": 1.8199670759007545, "kl": 0.0002303927940374706, "learning_rate": 9.99589250656446e-07, "loss": 0.00013443428906612098, "step": 7, "step_time": 93.93526208028197 }, { "clip_ratio/high_max": 0.002908887399826199, "clip_ratio/high_mean": 0.0011502621491672471, "clip_ratio/low_mean": 0.0006572946240339661, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018075567786581814, "epoch": 0.05, "grad_norm": 2.4427065860182764, "kl": 0.0003413308750168653, "learning_rate": 9.990759721625005e-07, "loss": 0.0002440623939037323, "step": 8, "step_time": 91.96858477964997 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.546875, "completions/max_length": 2048.0, "completions/mean_length": 1467.01171875, "completions/min_length": 121.0, "epoch": 0.05625, "frac_reward_zero_std": 0.4375, "grad_norm": 4.264321450023842, "kl": 0.0004341038875281811, "learning_rate": 9.983576774858775e-07, "loss": 4.324340352468425e-06, "reward": 0.30078125, "reward_std": 0.26078638434410095, "rewards/MajorityAccuracy/mean": 0.30078125, "rewards/MajorityAccuracy/std": 0.45949608087539673, "step": 9, "step_time": 622.7205950282514 }, { "clip_ratio/high_max": 0.0009290587331634015, "clip_ratio/high_mean": 0.00023226468329085037, "clip_ratio/low_mean": 0.00018310546875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00041537015204085037, "epoch": 0.0625, "grad_norm": 2.9928999585436697, "kl": 0.0011514702491695061, "learning_rate": 9.974346616959475e-07, "loss": 4.9060281526180916e-06, "step": 10, "step_time": 93.75498847290874 }, { "clip_ratio/high_max": 0.0031044440547702834, "clip_ratio/high_mean": 0.0011000082340615336, "clip_ratio/low_mean": 0.0006405206077033654, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001740528841764899, "epoch": 0.06875, "grad_norm": 2.9737724349426595, "kl": 0.0011983619624515995, "learning_rate": 9.963073039597796e-07, "loss": -0.00011477246880531311, "step": 11, "step_time": 91.85147937387228 }, { "clip_ratio/high_max": 0.0029156453092582524, "clip_ratio/high_mean": 0.000764934389735572, "clip_ratio/low_mean": 0.0004477714974200353, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0012127058726036921, "epoch": 0.075, "grad_norm": 1.6000987459477636, "kl": 0.0008952836506068707, "learning_rate": 9.949760673863846e-07, "loss": -0.00019322650041431189, "step": 12, "step_time": 92.96458574011922 }, { "clip_ratio/high_max": 0.0035665895557031035, "clip_ratio/high_mean": 0.0009804258006624877, "clip_ratio/low_mean": 0.0009190084092551842, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018994341953657568, "epoch": 0.08125, "grad_norm": 4.38259464311324, "kl": 0.0010516378679312766, "learning_rate": 9.934414988364722e-07, "loss": 0.0003124253125861287, "step": 13, "step_time": 94.32031967118382 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0875, "grad_norm": 0.005807164405218075, "kl": 0.0006890623990329914, "learning_rate": 9.917042286978063e-07, "loss": 6.893005775054917e-06, "step": 14, "step_time": 94.38057020679116 }, { "clip_ratio/high_max": 0.0025868398952297866, "clip_ratio/high_mean": 0.0008635227604827378, "clip_ratio/low_mean": 0.0009184245573123917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001781947328709066, "epoch": 0.09375, "grad_norm": 1.517705731164097, "kl": 0.0009860388527158648, "learning_rate": 9.897649706262473e-07, "loss": 0.00016279795090667903, "step": 15, "step_time": 94.34362488985062 }, { "clip_ratio/high_max": 0.0034740214759949595, "clip_ratio/high_mean": 0.0010759402721305378, "clip_ratio/low_mean": 0.0011128166370326653, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021887569018872455, "epoch": 0.1, "grad_norm": 1.5028357338282627, "kl": 0.0009736354113556445, "learning_rate": 9.87624521252587e-07, "loss": 0.000274136196821928, "step": 16, "step_time": 93.7784904576838 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 2048.0, "completions/mean_length": 1459.67578125, "completions/min_length": 146.0, "epoch": 0.10625, "frac_reward_zero_std": 0.25, "grad_norm": 2.4569630850117146, "kl": 0.0012972507684025913, "learning_rate": 9.852837598553008e-07, "loss": 1.2962424079887569e-05, "reward": 0.328125, "reward_std": 0.3593018651008606, "rewards/MajorityAccuracy/mean": 0.328125, "rewards/MajorityAccuracy/std": 0.47045034170150757, "step": 17, "step_time": 621.3462286517024 }, { "clip_ratio/high_max": 0.0033847040322143584, "clip_ratio/high_mean": 0.0011478922024252824, "clip_ratio/low_mean": 0.0004856683208345203, "clip_ratio/low_min": 6.103515625e-05, "clip_ratio/region_mean": 0.0016335605250787921, "epoch": 0.1125, "grad_norm": 1.8815849930796809, "kl": 0.0015622450882801786, "learning_rate": 9.827436479993468e-07, "loss": -0.00016696564853191376, "step": 18, "step_time": 92.72470863908529 }, { "clip_ratio/high_max": 0.00338425840891432, "clip_ratio/high_mean": 0.001017358594253892, "clip_ratio/low_mean": 0.001907983809360303, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0029253424145281315, "epoch": 0.11875, "grad_norm": 4.14030277522454, "kl": 0.0015874559903750196, "learning_rate": 9.800052291411629e-07, "loss": 0.0001017330214381218, "step": 19, "step_time": 91.63464274257421 }, { "clip_ratio/high_max": 0.0016688636678736657, "clip_ratio/high_mean": 0.0004172159169684164, "clip_ratio/low_mean": 0.0004843677015742287, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0009015836185426451, "epoch": 0.125, "grad_norm": 1.3889433611050135, "kl": 0.0012330952158663422, "learning_rate": 9.770696282000244e-07, "loss": 0.0004582633846439421, "step": 20, "step_time": 93.65893891453743 }, { "clip_ratio/high_max": 0.0029612480284413323, "clip_ratio/high_mean": 0.0011885990461451001, "clip_ratio/low_mean": 0.0004622797277988866, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016508787593920715, "epoch": 0.13125, "grad_norm": 1.2082473259977442, "kl": 0.0015428411425091326, "learning_rate": 9.739380510959364e-07, "loss": 0.0002680208417586982, "step": 21, "step_time": 93.41697767749429 }, { "clip_ratio/high_max": 0.0029848671692889184, "clip_ratio/high_mean": 0.0007462167923222296, "clip_ratio/low_mean": 0.001083866496628616, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018300832889508456, "epoch": 0.1375, "grad_norm": 1.9017353259024574, "kl": 0.0021649488189723343, "learning_rate": 9.706117842542516e-07, "loss": 0.0006249882280826569, "step": 22, "step_time": 93.9457158409059 }, { "clip_ratio/high_max": 0.005744260328356177, "clip_ratio/high_mean": 0.0018928383506136015, "clip_ratio/low_mean": 0.0012449784553609788, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003137816849630326, "epoch": 0.14375, "grad_norm": 2.054950337412609, "kl": 0.0027847185847349465, "learning_rate": 9.670921940772186e-07, "loss": 0.00011766652460210025, "step": 23, "step_time": 93.04386404901743 }, { "clip_ratio/high_max": 0.010986729059368372, "clip_ratio/high_mean": 0.0040083376807160676, "clip_ratio/low_mean": 0.001972825964912772, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00598116364562884, "epoch": 0.15, "grad_norm": 1.8922782039529462, "kl": 0.0031771485810168087, "learning_rate": 9.633807263826744e-07, "loss": 0.00031996238976716995, "step": 24, "step_time": 93.98081273958087 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.46484375, "completions/max_length": 2048.0, "completions/mean_length": 1453.46484375, "completions/min_length": 109.0, "epoch": 0.15625, "frac_reward_zero_std": 0.34375, "grad_norm": 1.2964562789267087, "kl": 0.002803271432640031, "learning_rate": 9.594789058101153e-07, "loss": 2.8027689040754922e-05, "reward": 0.31640625, "reward_std": 0.2797369360923767, "rewards/MajorityAccuracy/mean": 0.31640625, "rewards/MajorityAccuracy/std": 0.4659844934940338, "step": 25, "step_time": 617.827291239053 }, { "clip_ratio/high_max": 0.0006390817288774997, "clip_ratio/high_mean": 0.00015977043221937492, "clip_ratio/low_mean": 0.00026324604914407246, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.000423016488639405, "epoch": 0.1625, "grad_norm": 1.9912160430787458, "kl": 0.002381405880441889, "learning_rate": 9.55388335194388e-07, "loss": 0.0001286741899093613, "step": 26, "step_time": 93.9811980612576 }, { "clip_ratio/high_max": 0.001661909009271767, "clip_ratio/high_mean": 0.000472711681140936, "clip_ratio/low_mean": 0.0016703100191080011, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.0021430217166198418, "epoch": 0.16875, "grad_norm": 2.0503012191165757, "kl": 0.0032892244053073227, "learning_rate": 9.511106949072587e-07, "loss": 0.0002345023676753044, "step": 27, "step_time": 92.6028545461595 }, { "clip_ratio/high_max": 0.006517984889796935, "clip_ratio/high_mean": 0.001789206111425301, "clip_ratio/low_mean": 0.000878961116541177, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026681672170525417, "epoch": 0.175, "grad_norm": 2.748440326184567, "kl": 0.0036212419799994677, "learning_rate": 9.466477421671295e-07, "loss": 0.00044183951104059815, "step": 28, "step_time": 92.81928509101272 }, { "clip_ratio/high_max": 0.0004529151483438909, "clip_ratio/high_mean": 0.00011322878708597273, "clip_ratio/low_mean": 0.0010233546636300161, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0011365834579919465, "epoch": 0.18125, "grad_norm": 0.860641424192639, "kl": 0.0034606599947437644, "learning_rate": 9.420013103171891e-07, "loss": 3.905543053406291e-05, "step": 29, "step_time": 92.85946570336819 }, { "clip_ratio/high_max": 0.004000585613539442, "clip_ratio/high_mean": 0.0010001464033848606, "clip_ratio/low_mean": 0.0021262645605020225, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0031264109711628407, "epoch": 0.1875, "grad_norm": 3.0078501887416382, "kl": 0.006158369709737599, "learning_rate": 9.37173308072291e-07, "loss": 0.00016213944763876498, "step": 30, "step_time": 92.99975112825632 }, { "clip_ratio/high_max": 0.003398418804863468, "clip_ratio/high_mean": 0.000971318433585111, "clip_ratio/low_mean": 0.0021372123737819493, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0031085308000911027, "epoch": 0.19375, "grad_norm": 1.4133200562481247, "kl": 0.00555389528744854, "learning_rate": 9.321657187348688e-07, "loss": 0.00024008890613913536, "step": 31, "step_time": 93.06574066355824 }, { "clip_ratio/high_max": 0.0015180265763774514, "clip_ratio/high_mean": 0.00037950664409436285, "clip_ratio/low_mean": 0.00152587890625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0019053855357924476, "epoch": 0.2, "grad_norm": 0.9522472528712549, "kl": 0.005124813644215465, "learning_rate": 9.269805993802128e-07, "loss": -1.963978138519451e-05, "step": 32, "step_time": 92.73657894134521 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.35546875, "completions/max_length": 2048.0, "completions/mean_length": 1222.10546875, "completions/min_length": 80.0, "epoch": 0.20625, "frac_reward_zero_std": 0.25, "grad_norm": 3.302370658757549, "kl": 0.0064106733771041036, "learning_rate": 9.216200800114411e-07, "loss": 6.407064938684925e-05, "reward": 0.3828125, "reward_std": 0.35559141635894775, "rewards/MajorityAccuracy/mean": 0.3828125, "rewards/MajorityAccuracy/std": 0.48702529072761536, "step": 33, "step_time": 616.3648856654763 }, { "clip_ratio/high_max": 0.001350025508145336, "clip_ratio/high_mean": 0.000337506377036334, "clip_ratio/low_mean": 1.52587890625e-05, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0003527651697368128, "epoch": 0.2125, "grad_norm": 1.0165361590476292, "kl": 0.0038545840070582926, "learning_rate": 9.160863626845119e-07, "loss": 0.00012713775504380465, "step": 34, "step_time": 93.40912913531065 }, { "clip_ratio/high_max": 0.0020399912900757045, "clip_ratio/high_mean": 0.0005099978225189261, "clip_ratio/low_mean": 0.0003658540081232786, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008758518306422047, "epoch": 0.21875, "grad_norm": 2.992501633643151, "kl": 0.00766081886831671, "learning_rate": 9.103817206036382e-07, "loss": 0.00018003427248913795, "step": 35, "step_time": 92.59871354699135 }, { "clip_ratio/high_max": 0.0041379718750249594, "clip_ratio/high_mean": 0.0012029078716295771, "clip_ratio/low_mean": 0.0013312944429344498, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002534202314564027, "epoch": 0.225, "grad_norm": 8.277260615428975, "kl": 0.007375495799351484, "learning_rate": 9.045084971874737e-07, "loss": -0.0001662311697145924, "step": 36, "step_time": 92.11769533902407 }, { "clip_ratio/high_max": 0.009861077734967694, "clip_ratio/high_mean": 0.0031316077220253646, "clip_ratio/low_mean": 0.0016079964552773163, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004739604220958427, "epoch": 0.23125, "grad_norm": 2.429327262596793, "kl": 0.0071481295744888484, "learning_rate": 8.984691051064574e-07, "loss": 0.00011012516915798187, "step": 37, "step_time": 92.33589129894972 }, { "clip_ratio/high_max": 0.00429630684084259, "clip_ratio/high_mean": 0.0012346687726676464, "clip_ratio/low_mean": 0.0007634796274942346, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0019981484074378386, "epoch": 0.2375, "grad_norm": 3.0964274600608634, "kl": 0.003663207055069506, "learning_rate": 8.922660252917087e-07, "loss": 0.0006947489455342293, "step": 38, "step_time": 92.89865544438362 }, { "clip_ratio/high_max": 0.006928527189302258, "clip_ratio/high_mean": 0.0021163037126825657, "clip_ratio/low_mean": 0.0019737277543754317, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00409003155073151, "epoch": 0.24375, "grad_norm": 4.9401792619886065, "kl": 0.008130333619192243, "learning_rate": 8.859018059158809e-07, "loss": -0.00045569127541966736, "step": 39, "step_time": 92.20122557505965 }, { "clip_ratio/high_max": 0.00597179578471696, "clip_ratio/high_mean": 0.0019424547917878954, "clip_ratio/low_mean": 0.0018403581925667822, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037828130007255822, "epoch": 0.25, "grad_norm": 2.757081628085267, "kl": 0.008970192284323275, "learning_rate": 8.793790613463954e-07, "loss": 0.0009219266357831657, "step": 40, "step_time": 93.54538454860449 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.328125, "completions/max_length": 2048.0, "completions/mean_length": 1150.78125, "completions/min_length": 60.0, "epoch": 0.25625, "frac_reward_zero_std": 0.25, "grad_norm": 1.859427860458678, "kl": 0.006656086829025298, "learning_rate": 8.727004710714798e-07, "loss": 6.660228973487392e-05, "reward": 0.40234375, "reward_std": 0.35836464166641235, "rewards/MajorityAccuracy/mean": 0.40234375, "rewards/MajorityAccuracy/std": 0.4913311004638672, "step": 41, "step_time": 611.8897213041782 }, { "clip_ratio/high_max": 0.0029029125144006684, "clip_ratio/high_mean": 0.0007641154697921593, "clip_ratio/low_mean": 0.0008481685181322973, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001612283980648499, "epoch": 0.2625, "grad_norm": 3.7894734282257816, "kl": 0.006653814809396863, "learning_rate": 8.658687785994578e-07, "loss": -0.0002620406448841095, "step": 42, "step_time": 92.48946865275502 }, { "clip_ratio/high_max": 0.003402838818146847, "clip_ratio/high_mean": 0.0009420839742233511, "clip_ratio/low_mean": 0.0004132877929805545, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013553717581089586, "epoch": 0.26875, "grad_norm": 1.8726335015035525, "kl": 0.012738483143039048, "learning_rate": 8.588867903317394e-07, "loss": 0.00044092678581364453, "step": 43, "step_time": 90.9456696473062 }, { "clip_ratio/high_max": 0.0018718604987952858, "clip_ratio/high_mean": 0.0007007965759839863, "clip_ratio/low_mean": 0.0003671859158203006, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010679824918042868, "epoch": 0.275, "grad_norm": 1.6541159956694966, "kl": 0.006954809359740466, "learning_rate": 8.51757374409974e-07, "loss": 0.0001968394499272108, "step": 44, "step_time": 93.55221934616566 }, { "clip_ratio/high_max": 0.011631626926828176, "clip_ratio/high_mean": 0.004481182622839697, "clip_ratio/low_mean": 0.003848518434097059, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008329701260663569, "epoch": 0.28125, "grad_norm": 6.631061003152457, "kl": 0.0238770074211061, "learning_rate": 8.444834595378433e-07, "loss": 0.0012177564203739166, "step": 45, "step_time": 90.05261616781354 }, { "clip_ratio/high_max": 0.0017544087022542953, "clip_ratio/high_mean": 0.00043860217556357384, "clip_ratio/low_mean": 0.0008167341875378042, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001255336363101378, "epoch": 0.2875, "grad_norm": 2.6330688305169407, "kl": 0.008567059237975627, "learning_rate": 8.370680337779736e-07, "loss": 0.00047486479161307216, "step": 46, "step_time": 94.22641252726316 }, { "clip_ratio/high_max": 0.012057308224029839, "clip_ratio/high_mean": 0.0045319272903725505, "clip_ratio/low_mean": 0.0019763954333029687, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006508322723675519, "epoch": 0.29375, "grad_norm": 3.330546858807778, "kl": 0.008558106143027544, "learning_rate": 8.295141433244659e-07, "loss": 0.00037969648838043213, "step": 47, "step_time": 91.56187829002738 }, { "clip_ratio/high_max": 0.006419014403945766, "clip_ratio/high_mean": 0.0020658404646383133, "clip_ratio/low_mean": 0.0002145807957276702, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022804212785558775, "epoch": 0.3, "grad_norm": 1.251083944189168, "kl": 0.007730341050773859, "learning_rate": 8.218248912515442e-07, "loss": 8.880916720954701e-05, "step": 48, "step_time": 92.9748519435525 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.40234375, "completions/max_length": 2048.0, "completions/mean_length": 1286.18359375, "completions/min_length": 104.0, "epoch": 0.30625, "frac_reward_zero_std": 0.15625, "grad_norm": 2.719234032268336, "kl": 0.011544098146259785, "learning_rate": 8.140034362388397e-07, "loss": 0.00011551938951015472, "reward": 0.328125, "reward_std": 0.3836146593093872, "rewards/MajorityAccuracy/mean": 0.328125, "rewards/MajorityAccuracy/std": 0.47045034170150757, "step": 49, "step_time": 614.6333135180175 }, { "clip_ratio/high_max": 0.0034686223370954394, "clip_ratio/high_mean": 0.0011745667725335807, "clip_ratio/low_mean": 0.0010260311464662664, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002200597918999847, "epoch": 0.3125, "grad_norm": 3.884534896238289, "kl": 0.009100979310460389, "learning_rate": 8.060529912738314e-07, "loss": 0.0004946409608237445, "step": 50, "step_time": 90.42219868302345 }, { "clip_ratio/high_max": 0.0014942463603802025, "clip_ratio/high_mean": 0.0005015951319364831, "clip_ratio/low_mean": 0.0005751347271143459, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001076729859050829, "epoch": 0.31875, "grad_norm": 1.497725721913539, "kl": 0.006711360183544457, "learning_rate": 7.979768223319785e-07, "loss": 0.00017556367674842477, "step": 51, "step_time": 94.44091248512268 }, { "clip_ratio/high_max": 0.0016341325244866312, "clip_ratio/high_mean": 0.0005012631008867174, "clip_ratio/low_mean": 0.0008674138298374601, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013686769380001351, "epoch": 0.325, "grad_norm": 1.3287664789986702, "kl": 0.011083073797635734, "learning_rate": 7.897782470350849e-07, "loss": 4.79040463687852e-05, "step": 52, "step_time": 92.27732546627522 }, { "clip_ratio/high_max": 0.002434810041449964, "clip_ratio/high_mean": 0.0007305688195629045, "clip_ratio/low_mean": 0.0010165427956962958, "clip_ratio/low_min": 0.00018310546875, "clip_ratio/region_mean": 0.0017471116152592003, "epoch": 0.33125, "grad_norm": 1.6503822551742107, "kl": 0.007973313506226987, "learning_rate": 7.814606332884488e-07, "loss": 0.0003444608300924301, "step": 53, "step_time": 94.04076413437724 }, { "clip_ratio/high_max": 0.003116298554232344, "clip_ratio/high_mean": 0.0010158811855944805, "clip_ratio/low_mean": 0.0017048650333890691, "clip_ratio/low_min": 0.00030517578125, "clip_ratio/region_mean": 0.002720746153499931, "epoch": 0.3375, "grad_norm": 2.0744380005816048, "kl": 0.009434459381736815, "learning_rate": 7.730273978973552e-07, "loss": 0.00022509507834911346, "step": 54, "step_time": 93.2253330424428 }, { "clip_ratio/high_max": 0.004632533964468166, "clip_ratio/high_mean": 0.0012234644891577773, "clip_ratio/low_mean": 0.0033401524560758844, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004563616937957704, "epoch": 0.34375, "grad_norm": 4.15764024618867, "kl": 0.015519127366133034, "learning_rate": 7.644820051634812e-07, "loss": 0.0004719048156403005, "step": 55, "step_time": 91.49989666044712 }, { "clip_ratio/high_max": 0.004213306470774114, "clip_ratio/high_mean": 0.0014793382870266214, "clip_ratio/low_mean": 0.001022398930217605, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.00250173726817593, "epoch": 0.35, "grad_norm": 1.8496091265684385, "kl": 0.010392811964266002, "learning_rate": 7.558279654617912e-07, "loss": 0.0007420321926474571, "step": 56, "step_time": 93.6295682489872 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.35546875, "completions/max_length": 2048.0, "completions/mean_length": 1263.3984375, "completions/min_length": 118.0, "epoch": 0.35625, "frac_reward_zero_std": 0.21875, "grad_norm": 2.2982808813307654, "kl": 0.012292009894736111, "learning_rate": 7.470688337985029e-07, "loss": 0.0001230384223163128, "reward": 0.39453125, "reward_std": 0.3608660101890564, "rewards/MajorityAccuracy/mean": 0.39453125, "rewards/MajorityAccuracy/std": 0.48970717191696167, "step": 57, "step_time": 613.7040184140205 }, { "clip_ratio/high_max": 0.0017978516989387572, "clip_ratio/high_mean": 0.0005260931648081169, "clip_ratio/low_mean": 0.00037405104376375675, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0009001442122098524, "epoch": 0.3625, "grad_norm": 1.5403496264297905, "kl": 0.009012477821670473, "learning_rate": 7.382082083507225e-07, "loss": 0.00010491425928194076, "step": 58, "step_time": 94.07058154419065 }, { "clip_ratio/high_max": 0.008262533025117591, "clip_ratio/high_mean": 0.0025283747309003957, "clip_ratio/low_mean": 0.0013638798336614855, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003892254600941669, "epoch": 0.36875, "grad_norm": 4.235867991207229, "kl": 0.010181457735598087, "learning_rate": 7.292497289883432e-07, "loss": -0.0007176409708335996, "step": 59, "step_time": 93.23570550233126 }, { "clip_ratio/high_max": 0.002238327491795644, "clip_ratio/high_mean": 0.0006946974535821937, "clip_ratio/low_mean": 0.0007288153865374625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014235128546715714, "epoch": 0.375, "grad_norm": 1.6921096927643915, "kl": 0.009526177134830505, "learning_rate": 7.201970757788171e-07, "loss": -4.7677407565061e-05, "step": 60, "step_time": 90.65709735825658 }, { "clip_ratio/high_max": 0.002813726823660545, "clip_ratio/high_mean": 0.0007456044404534623, "clip_ratio/low_mean": 0.0020276440191082656, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0027732484159059823, "epoch": 0.38125, "grad_norm": 2.6607120188526276, "kl": 0.010607969015836716, "learning_rate": 7.110539674754159e-07, "loss": 3.1639181543141603e-05, "step": 61, "step_time": 93.4264040030539 }, { "clip_ratio/high_max": 0.0034356716932961717, "clip_ratio/high_mean": 0.0010488269072084222, "clip_ratio/low_mean": 0.0006706072126689833, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017194340980495326, "epoch": 0.3875, "grad_norm": 1.7737197282791841, "kl": 0.010582862712908536, "learning_rate": 7.018241599895973e-07, "loss": 0.0003158450126647949, "step": 62, "step_time": 92.02484929934144 }, { "clip_ratio/high_max": 0.004709801651188172, "clip_ratio/high_mean": 0.0014211888519639615, "clip_ratio/low_mean": 0.0008389163413085043, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002260105175082572, "epoch": 0.39375, "grad_norm": 2.8826619979150743, "kl": 0.009747990523464978, "learning_rate": 6.925114448481088e-07, "loss": 0.0004790116799995303, "step": 63, "step_time": 94.08708464726806 }, { "clip_ratio/high_max": 0.003639666858362034, "clip_ratio/high_mean": 0.0012354190184851177, "clip_ratio/low_mean": 0.0006006814946886152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018361004767939448, "epoch": 0.4, "grad_norm": 1.5002538398142493, "kl": 0.008990575443021953, "learning_rate": 6.831196476354614e-07, "loss": 0.00014480750542134047, "step": 64, "step_time": 92.3987370096147 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 2048.0, "completions/mean_length": 1136.3984375, "completions/min_length": 55.0, "epoch": 0.40625, "frac_reward_zero_std": 0.25, "grad_norm": 1.1921934843549156, "kl": 0.007464712602086365, "learning_rate": 6.7365262642241e-07, "loss": 7.464888039976358e-05, "reward": 0.40625, "reward_std": 0.3356248140335083, "rewards/MajorityAccuracy/mean": 0.40625, "rewards/MajorityAccuracy/std": 0.49209436774253845, "step": 65, "step_time": 610.0534768141806 }, { "clip_ratio/high_max": 0.0003804290317930281, "clip_ratio/high_mean": 0.0001458326769352425, "clip_ratio/low_mean": 0.00043385004028095864, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0005796827172162011, "epoch": 0.4125, "grad_norm": 1.1906495481806871, "kl": 0.01139814278576523, "learning_rate": 6.641142701810931e-07, "loss": 0.00011332621943438426, "step": 66, "step_time": 94.087756190449 }, { "clip_ratio/high_max": 0.0008897843799786642, "clip_ratio/high_mean": 0.00022244609499466605, "clip_ratio/low_mean": 0.0013514130405383185, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0015738591318950057, "epoch": 0.41875, "grad_norm": 2.9153801457083066, "kl": 0.01175980397965759, "learning_rate": 6.545084971874736e-07, "loss": -0.000203379662707448, "step": 67, "step_time": 92.17559164762497 }, { "clip_ratio/high_max": 0.003755491808988154, "clip_ratio/high_mean": 0.000985631049843505, "clip_ratio/low_mean": 0.0005482421038323082, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0015338731609517708, "epoch": 0.425, "grad_norm": 1.844084411407215, "kl": 0.01434664789121598, "learning_rate": 6.448392534117461e-07, "loss": 0.00016619544476270676, "step": 68, "step_time": 92.53664095699787 }, { "clip_ratio/high_max": 0.0032698738941689953, "clip_ratio/high_mean": 0.0009191275566990953, "clip_ratio/low_mean": 0.0005206421701586805, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014397697159438394, "epoch": 0.43125, "grad_norm": 1.1914642082452271, "kl": 0.017741689924150705, "learning_rate": 6.351105108973644e-07, "loss": 0.00023308704840019345, "step": 69, "step_time": 90.66614850237966 }, { "clip_ratio/high_max": 0.0008525633020326495, "clip_ratio/high_mean": 0.00026223744498565793, "clip_ratio/low_mean": 0.000509055178554263, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0007712926235399209, "epoch": 0.4375, "grad_norm": 0.9973491023483292, "kl": 0.008108913607429713, "learning_rate": 6.253262661293602e-07, "loss": 0.0001665100280661136, "step": 70, "step_time": 94.09870118647814 }, { "clip_ratio/high_max": 0.0022358243877533823, "clip_ratio/high_mean": 0.000848653755383566, "clip_ratio/low_mean": 0.0008109116970445029, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016595654742559418, "epoch": 0.44375, "grad_norm": 1.9432986799793814, "kl": 0.01005218317732215, "learning_rate": 6.154905383926216e-07, "loss": 0.0003785146400332451, "step": 71, "step_time": 92.10614557564259 }, { "clip_ratio/high_max": 0.003222787694539875, "clip_ratio/high_mean": 0.0013212211561040021, "clip_ratio/low_mean": 0.0002702496785786934, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001591470827406738, "epoch": 0.45, "grad_norm": 1.4281791970257205, "kl": 0.014184305793605745, "learning_rate": 6.056073681208037e-07, "loss": 0.00016889601829461753, "step": 72, "step_time": 91.3973440490663 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.31640625, "completions/max_length": 2048.0, "completions/mean_length": 1066.34375, "completions/min_length": 112.0, "epoch": 0.45625, "frac_reward_zero_std": 0.3125, "grad_norm": 2.642847513595557, "kl": 0.01342642621602863, "learning_rate": 5.956808152365532e-07, "loss": 0.00013418588787317276, "reward": 0.36328125, "reward_std": 0.3215371370315552, "rewards/MajorityAccuracy/mean": 0.36328125, "rewards/MajorityAccuracy/std": 0.48188698291778564, "step": 73, "step_time": 611.9634358435869 }, { "clip_ratio/high_max": 0.0009874284733086824, "clip_ratio/high_mean": 0.0002468571183271706, "clip_ratio/low_mean": 0.000494759064167738, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0007416161788569298, "epoch": 0.4625, "grad_norm": 1.9597178583945314, "kl": 0.012999030528590083, "learning_rate": 5.857149574837268e-07, "loss": 0.0001453967997804284, "step": 74, "step_time": 92.64086220040917 }, { "clip_ratio/high_max": 0.0025115635653492063, "clip_ratio/high_mean": 0.0007736421393929049, "clip_ratio/low_mean": 0.0012951051903655753, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0020687473079306073, "epoch": 0.46875, "grad_norm": 2.498754490503407, "kl": 0.015487454598769546, "learning_rate": 5.757138887522883e-07, "loss": 0.0003957059234380722, "step": 75, "step_time": 93.89070768654346 }, { "clip_ratio/high_max": 0.004436239047208801, "clip_ratio/high_mean": 0.001356831744487863, "clip_ratio/low_mean": 0.0015089941516635008, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002865825910703279, "epoch": 0.475, "grad_norm": 4.607890716401143, "kl": 0.014953777892515063, "learning_rate": 5.656817173965732e-07, "loss": 0.000576346181333065, "step": 76, "step_time": 91.45148377120495 }, { "clip_ratio/high_max": 0.005128351214807481, "clip_ratio/high_mean": 0.0016222216218011454, "clip_ratio/low_mean": 0.0005544026789721102, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021766243153251708, "epoch": 0.48125, "grad_norm": 2.2983437130829496, "kl": 0.0149973772931844, "learning_rate": 5.556225645476118e-07, "loss": -0.0001673492370173335, "step": 77, "step_time": 92.35000036656857 }, { "clip_ratio/high_max": 0.0022115021420177072, "clip_ratio/high_mean": 0.0005528755355044268, "clip_ratio/low_mean": 3.0517578125e-05, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0005833931136294268, "epoch": 0.4875, "grad_norm": 0.8257651602431088, "kl": 0.009783417102880776, "learning_rate": 5.455405624202032e-07, "loss": 0.00015582122432533652, "step": 78, "step_time": 93.27886363491416 }, { "clip_ratio/high_max": 0.0024499712453689426, "clip_ratio/high_mean": 0.00078512239997508, "clip_ratio/low_mean": 0.0007999921217560768, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001585114536283072, "epoch": 0.49375, "grad_norm": 1.564633805475925, "kl": 0.015155967092141509, "learning_rate": 5.354398526154365e-07, "loss": 8.463409903924912e-05, "step": 79, "step_time": 92.35423078015447 }, { "clip_ratio/high_max": 0.004600720130838454, "clip_ratio/high_mean": 0.0011501800327096134, "clip_ratio/low_mean": 0.0024043756566243246, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0035545557184377685, "epoch": 0.5, "grad_norm": 3.8010728728916154, "kl": 0.02167099015787244, "learning_rate": 5.253245844193564e-07, "loss": 0.00043530072434805334, "step": 80, "step_time": 93.22243381291628 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3828125, "completions/max_length": 2048.0, "completions/mean_length": 1255.43359375, "completions/min_length": 59.0, "epoch": 0.50625, "frac_reward_zero_std": 0.4375, "grad_norm": 1.2523826414557966, "kl": 0.01102160767186433, "learning_rate": 5.151989130984714e-07, "loss": 0.00011026033462258056, "reward": 0.35546875, "reward_std": 0.26302918791770935, "rewards/MajorityAccuracy/mean": 0.35546875, "rewards/MajorityAccuracy/std": 0.4795927405357361, "step": 81, "step_time": 615.0859273932874 }, { "clip_ratio/high_max": 0.0012824924997403286, "clip_ratio/high_mean": 0.00037652651917596813, "clip_ratio/low_mean": 0.0006108994057285599, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.000987425915809581, "epoch": 0.5125, "grad_norm": 1.9000220707913271, "kl": 0.012164968997240067, "learning_rate": 5.050669981928055e-07, "loss": 5.370471626520157e-06, "step": 82, "step_time": 93.24058908224106 }, { "clip_ratio/high_max": 0.00025826445198617876, "clip_ratio/high_mean": 0.00011867866851389408, "clip_ratio/low_mean": 0.0001605257493793033, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0002792044178931974, "epoch": 0.51875, "grad_norm": 1.6919567498106731, "kl": 0.011212784214876592, "learning_rate": 4.949330018071946e-07, "loss": 3.3622258342802525e-05, "step": 83, "step_time": 93.52923339605331 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.525, "grad_norm": 0.09622306154667316, "kl": 0.018376620137132704, "learning_rate": 4.848010869015287e-07, "loss": 0.00018373577040620148, "step": 84, "step_time": 90.6525271460414 }, { "clip_ratio/high_max": 0.0014120639098109677, "clip_ratio/high_mean": 0.0004622789492714219, "clip_ratio/low_mean": 0.00042016530642285943, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008824442484183237, "epoch": 0.53125, "grad_norm": 1.35591064476649, "kl": 0.009382898802869022, "learning_rate": 4.746754155806437e-07, "loss": 0.00013049633707851171, "step": 85, "step_time": 93.15757415443659 }, { "clip_ratio/high_max": 0.0060318153409752995, "clip_ratio/high_mean": 0.0018719161234912463, "clip_ratio/low_mean": 0.0009927319042617455, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002864648005925119, "epoch": 0.5375, "grad_norm": 2.003296204577839, "kl": 0.012280631228350103, "learning_rate": 4.645601473845635e-07, "loss": 4.669843474403024e-05, "step": 86, "step_time": 93.71386248618364 }, { "clip_ratio/high_max": 0.0013314772513695061, "clip_ratio/high_mean": 0.00043034279224229977, "clip_ratio/low_mean": 0.0006957517689443193, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0011260945684625767, "epoch": 0.54375, "grad_norm": 1.2680892318395183, "kl": 0.01831695984583348, "learning_rate": 4.544594375797968e-07, "loss": 0.00015155720757320523, "step": 87, "step_time": 93.65547093003988 }, { "clip_ratio/high_max": 0.002111706657160539, "clip_ratio/high_mean": 0.000677336671287776, "clip_ratio/low_mean": 0.0006334945719572715, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013108312414260581, "epoch": 0.55, "grad_norm": 1.1694401746313074, "kl": 0.012249161140061915, "learning_rate": 4.443774354523882e-07, "loss": 0.00015203689690679312, "step": 88, "step_time": 93.276855122298 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 2048.0, "completions/mean_length": 1179.61328125, "completions/min_length": 119.0, "epoch": 0.55625, "frac_reward_zero_std": 0.25, "grad_norm": 2.6924041216630656, "kl": 0.017416922724805772, "learning_rate": 4.3431828260342675e-07, "loss": 0.0001742010936141014, "reward": 0.453125, "reward_std": 0.35665225982666016, "rewards/MajorityAccuracy/mean": 0.453125, "rewards/MajorityAccuracy/std": 0.4987730085849762, "step": 89, "step_time": 615.0254106856883 }, { "clip_ratio/high_max": 0.0030996048080851324, "clip_ratio/high_mean": 0.0008338758489117026, "clip_ratio/low_mean": 7.62939453125e-05, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0009101697942242026, "epoch": 0.5625, "grad_norm": 3.3216250199339403, "kl": 0.01463937561493367, "learning_rate": 4.242861112477118e-07, "loss": 0.0005419193767011166, "step": 90, "step_time": 93.59632669761777 }, { "clip_ratio/high_max": 0.0011998490881524049, "clip_ratio/high_mean": 0.00048519180018047336, "clip_ratio/low_mean": 0.0004931019357172772, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0009782937413547188, "epoch": 0.56875, "grad_norm": 1.7713593631218327, "kl": 0.012044742470607162, "learning_rate": 4.1428504251627325e-07, "loss": 0.00042311474680900574, "step": 91, "step_time": 93.73527916520834 }, { "clip_ratio/high_max": 0.0017271043034270406, "clip_ratio/high_mean": 0.0005031044493080117, "clip_ratio/low_mean": 0.0005052064370829612, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010083109009428881, "epoch": 0.575, "grad_norm": 1.727346452963389, "kl": 0.013314120471477509, "learning_rate": 4.0431918476344685e-07, "loss": -1.6570818843320012e-05, "step": 92, "step_time": 92.78242744505405 }, { "clip_ratio/high_max": 0.001120240609452594, "clip_ratio/high_mean": 0.00036493607694865204, "clip_ratio/low_mean": 4.57763671875e-05, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00041071245141210966, "epoch": 0.58125, "grad_norm": 1.9925686240285962, "kl": 0.010302539682015777, "learning_rate": 3.943926318791963e-07, "loss": 6.236076296772808e-05, "step": 93, "step_time": 93.4716045409441 }, { "clip_ratio/high_max": 0.002020443571382202, "clip_ratio/high_mean": 0.0005051108928455506, "clip_ratio/low_mean": 0.0002316872705705464, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0007367981597781181, "epoch": 0.5875, "grad_norm": 2.322261074937204, "kl": 0.01709748397115618, "learning_rate": 3.845094616073783e-07, "loss": 2.763605152722448e-05, "step": 94, "step_time": 94.15186908841133 }, { "clip_ratio/high_max": 0.0029420676291920245, "clip_ratio/high_mean": 0.0008981563150882721, "clip_ratio/low_mean": 0.00018712575547397137, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010852820705622435, "epoch": 0.59375, "grad_norm": 2.740360785924372, "kl": 0.025021834764629602, "learning_rate": 3.7467373387063964e-07, "loss": 0.0005356235196813941, "step": 95, "step_time": 89.97958190366626 }, { "clip_ratio/high_max": 0.003168705472489819, "clip_ratio/high_mean": 0.001022216230921913, "clip_ratio/low_mean": 0.000868924063979648, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001891140294901561, "epoch": 0.6, "grad_norm": 3.595080262058795, "kl": 0.017588063841685653, "learning_rate": 3.648894891026358e-07, "loss": -4.210804036119953e-05, "step": 96, "step_time": 91.87338061258197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.32421875, "completions/max_length": 2048.0, "completions/mean_length": 1341.87109375, "completions/min_length": 118.0, "epoch": 0.60625, "frac_reward_zero_std": 0.1875, "grad_norm": 1.9041271667094042, "kl": 0.011970077292062342, "learning_rate": 3.5516074658825397e-07, "loss": 0.00011968174658250064, "reward": 0.40234375, "reward_std": 0.3890015184879303, "rewards/MajorityAccuracy/mean": 0.40234375, "rewards/MajorityAccuracy/std": 0.4913311004638672, "step": 97, "step_time": 614.6720772720873 }, { "clip_ratio/high_max": 0.0024992451071739197, "clip_ratio/high_mean": 0.0007198409512056969, "clip_ratio/low_mean": 0.0003131345074507408, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010329754586564377, "epoch": 0.6125, "grad_norm": 1.64035126615968, "kl": 0.012485804269090295, "learning_rate": 3.454915028125263e-07, "loss": 0.0002270648255944252, "step": 98, "step_time": 93.75987619161606 }, { "clip_ratio/high_max": 0.0030884536827215925, "clip_ratio/high_mean": 0.0010755518051155377, "clip_ratio/low_mean": 0.00043156929314136505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0015071211018948816, "epoch": 0.61875, "grad_norm": 1.7011838236001176, "kl": 0.014667014009319246, "learning_rate": 3.3588572981890684e-07, "loss": 7.991164602572098e-05, "step": 99, "step_time": 92.66376127302647 }, { "clip_ratio/high_max": 0.0019916903984267265, "clip_ratio/high_mean": 0.0005638210459437687, "clip_ratio/low_mean": 0.00039002980338409543, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0009538508456898853, "epoch": 0.625, "grad_norm": 1.7353615344167388, "kl": 0.011498167063109577, "learning_rate": 3.263473735775899e-07, "loss": 0.0004321001470088959, "step": 100, "step_time": 92.68661327660084 }, { "clip_ratio/high_max": 0.002410385321127251, "clip_ratio/high_mean": 0.000639069898170419, "clip_ratio/low_mean": 0.0007436634150508326, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001382733324135188, "epoch": 0.63125, "grad_norm": 1.9721588224157716, "kl": 0.01671644428279251, "learning_rate": 3.1688035236453865e-07, "loss": -4.7711106162751094e-05, "step": 101, "step_time": 93.2302768342197 }, { "clip_ratio/high_max": 0.001496560034865979, "clip_ratio/high_mean": 0.0004228160232742084, "clip_ratio/low_mean": 0.0001983642578125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0006211802829056978, "epoch": 0.6375, "grad_norm": 0.8540452329567962, "kl": 0.008400197606533766, "learning_rate": 3.0748855515189096e-07, "loss": 0.0001011439089779742, "step": 102, "step_time": 93.90861555188894 }, { "clip_ratio/high_max": 0.005467358831083402, "clip_ratio/high_mean": 0.0015238446576404385, "clip_ratio/low_mean": 0.00027849554317072034, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018023401826212648, "epoch": 0.64375, "grad_norm": 3.0897210531091877, "kl": 0.01647684210911393, "learning_rate": 2.981758400104028e-07, "loss": 0.0010852112900465727, "step": 103, "step_time": 91.6669692657888 }, { "clip_ratio/high_max": 0.0036274789526942186, "clip_ratio/high_mean": 0.0015477956494578393, "clip_ratio/low_mean": 0.0007103813368303236, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002258176959003322, "epoch": 0.65, "grad_norm": 2.803343100349624, "kl": 0.015464656404219568, "learning_rate": 2.8894603252458403e-07, "loss": 0.00033554062247276306, "step": 104, "step_time": 92.3558457531035 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.265625, "completions/max_length": 2048.0, "completions/mean_length": 1161.86328125, "completions/min_length": 82.0, "epoch": 0.65625, "frac_reward_zero_std": 0.4375, "grad_norm": 0.9953478035669736, "kl": 0.01897247019223869, "learning_rate": 2.7980292422118277e-07, "loss": 0.0001898403570521623, "reward": 0.4296875, "reward_std": 0.24553045630455017, "rewards/MajorityAccuracy/mean": 0.4296875, "rewards/MajorityAccuracy/std": 0.49600115418434143, "step": 105, "step_time": 612.4460465572774 }, { "clip_ratio/high_max": 0.00185802296618931, "clip_ratio/high_mean": 0.0004645057415473275, "clip_ratio/low_mean": 0.00020461766689550132, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0006691234084428288, "epoch": 0.6625, "grad_norm": 1.9952625589557627, "kl": 0.023894183104857802, "learning_rate": 2.70750271011657e-07, "loss": 0.0002466175938025117, "step": 106, "step_time": 90.8364669047296 }, { "clip_ratio/high_max": 0.0031699044484412298, "clip_ratio/high_mean": 0.0010033947728516068, "clip_ratio/low_mean": 0.00044922902270627674, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014526237955578836, "epoch": 0.66875, "grad_norm": 3.7347908682254376, "kl": 0.0203474466688931, "learning_rate": 2.6179179164927754e-07, "loss": 0.00040223821997642517, "step": 107, "step_time": 92.8140075802803 }, { "clip_ratio/high_max": 0.0012015424290439114, "clip_ratio/high_mean": 0.0004433425929164514, "clip_ratio/low_mean": 0.00020861567281826865, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0006519582675537094, "epoch": 0.675, "grad_norm": 1.7110008178464626, "kl": 0.014502450358122587, "learning_rate": 2.529311662014972e-07, "loss": 0.0001835283328546211, "step": 108, "step_time": 93.18753177300096 }, { "clip_ratio/high_max": 0.0015884720487520099, "clip_ratio/high_mean": 0.0004675802410929464, "clip_ratio/low_mean": 0.0008400637179875048, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013076439572614618, "epoch": 0.68125, "grad_norm": 3.148782974780195, "kl": 0.020499307429417968, "learning_rate": 2.441720345382089e-07, "loss": 0.0003582688805181533, "step": 109, "step_time": 93.13162574544549 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.6875, "grad_norm": 0.7418606441015603, "kl": 0.0229470815975219, "learning_rate": 2.355179948365189e-07, "loss": 0.0002808579010888934, "step": 110, "step_time": 90.89935997873545 }, { "clip_ratio/high_max": 0.0038748688821215183, "clip_ratio/high_mean": 0.0009687172205303796, "clip_ratio/low_mean": 0.0004697469703387469, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014384641835931689, "epoch": 0.69375, "grad_norm": 2.6779814557193364, "kl": 0.020392637350596488, "learning_rate": 2.26972602102645e-07, "loss": 0.0003422262379899621, "step": 111, "step_time": 91.41898300498724 }, { "clip_ratio/high_max": 0.00023551598860649392, "clip_ratio/high_mean": 5.887899715162348e-05, "clip_ratio/low_mean": 0.000244140625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0003030196203326341, "epoch": 0.7, "grad_norm": 1.3256546047039033, "kl": 0.010641569388099015, "learning_rate": 2.1853936671155127e-07, "loss": 0.00010535872570471838, "step": 112, "step_time": 94.13643909990788 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 2048.0, "completions/mean_length": 1342.140625, "completions/min_length": 93.0, "epoch": 0.70625, "frac_reward_zero_std": 0.3125, "grad_norm": 2.245549235754567, "kl": 0.014892687788233161, "learning_rate": 2.1022175296491512e-07, "loss": 0.00014891475439071655, "reward": 0.36328125, "reward_std": 0.33060982823371887, "rewards/MajorityAccuracy/mean": 0.36328125, "rewards/MajorityAccuracy/std": 0.48188698291778564, "step": 113, "step_time": 615.1040812954307 }, { "clip_ratio/high_max": 0.003307644830783829, "clip_ratio/high_mean": 0.0009047443090821616, "clip_ratio/low_mean": 0.0002961102727567777, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0012008545818389393, "epoch": 0.7125, "grad_norm": 2.6867600161358642, "kl": 0.01982644130475819, "learning_rate": 2.0202317766802152e-07, "loss": 0.00014446870773099363, "step": 114, "step_time": 92.57970526814461 }, { "clip_ratio/high_max": 0.00015331678878283128, "clip_ratio/high_mean": 3.832919719570782e-05, "clip_ratio/low_mean": 0.00021035045574535616, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.000248679652941064, "epoch": 0.71875, "grad_norm": 0.9377801345510165, "kl": 0.010885063675232232, "learning_rate": 1.9394700872616853e-07, "loss": 0.00014289292448665947, "step": 115, "step_time": 94.0477049909532 }, { "clip_ratio/high_max": 0.0010816625654115342, "clip_ratio/high_mean": 0.00031712715099274646, "clip_ratio/low_mean": 0.0004424620055942796, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0007595891565870261, "epoch": 0.725, "grad_norm": 2.297693852321618, "kl": 0.014041848364286125, "learning_rate": 1.8599656376116024e-07, "loss": 0.00036753626773133874, "step": 116, "step_time": 93.44860926643014 }, { "clip_ratio/high_max": 0.0008833039755700156, "clip_ratio/high_mean": 0.0002208259938925039, "clip_ratio/low_mean": 0.00016155646881088614, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00038238246270339005, "epoch": 0.73125, "grad_norm": 1.9807708520993408, "kl": 0.016110291937366128, "learning_rate": 1.7817510874845582e-07, "loss": 9.620553464628756e-05, "step": 117, "step_time": 92.60692274570465 }, { "clip_ratio/high_max": 0.0036938314224244095, "clip_ratio/high_mean": 0.0011539608458406292, "clip_ratio/low_mean": 0.0004603685265465174, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016143293687491678, "epoch": 0.7375, "grad_norm": 1.8709827020941996, "kl": 0.012255050824023783, "learning_rate": 1.7048585667553412e-07, "loss": 0.00045996904373168945, "step": 118, "step_time": 93.99895841255784 }, { "clip_ratio/high_max": 0.0042964271997334436, "clip_ratio/high_mean": 0.0012224528036313131, "clip_ratio/low_mean": 0.0004734609683509916, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016959137719823048, "epoch": 0.74375, "grad_norm": 2.8504708352297983, "kl": 0.019536805106326938, "learning_rate": 1.6293196622202632e-07, "loss": 0.0004344708868302405, "step": 119, "step_time": 91.23333411291242 }, { "clip_ratio/high_max": 0.0007603122212458402, "clip_ratio/high_mean": 0.00024734745238674805, "clip_ratio/low_mean": 4.57763671875e-05, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00029312381957424805, "epoch": 0.75, "grad_norm": 1.8694984556624243, "kl": 0.01329256547614932, "learning_rate": 1.555165404621567e-07, "loss": 9.478094580117613e-05, "step": 120, "step_time": 94.47286136075854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.23828125, "completions/max_length": 2048.0, "completions/mean_length": 1010.72265625, "completions/min_length": 114.0, "epoch": 0.75625, "frac_reward_zero_std": 0.21875, "grad_norm": 2.3083044339734324, "kl": 0.01559533434920013, "learning_rate": 1.4824262559002592e-07, "loss": 0.00015597330639138818, "reward": 0.41015625, "reward_std": 0.3540361225605011, "rewards/MajorityAccuracy/mean": 0.41015625, "rewards/MajorityAccuracy/std": 0.49282535910606384, "step": 121, "step_time": 609.0974177420139 }, { "clip_ratio/high_max": 0.007790313538862392, "clip_ratio/high_mean": 0.0021038101040176116, "clip_ratio/low_mean": 0.0005485672954819165, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026523773703956977, "epoch": 0.7625, "grad_norm": 3.0632415102800326, "kl": 0.026748670265078545, "learning_rate": 1.4111320966826057e-07, "loss": 0.0005028623854741454, "step": 122, "step_time": 92.73864832147956 }, { "clip_ratio/high_max": 0.003624320263043046, "clip_ratio/high_mean": 0.00100397938513197, "clip_ratio/low_mean": 0.000825379291200079, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001829358676332049, "epoch": 0.76875, "grad_norm": 2.4601087878230135, "kl": 0.012210196116939187, "learning_rate": 1.3413122140054217e-07, "loss": 6.41755759716034e-05, "step": 123, "step_time": 93.45426305755973 }, { "clip_ratio/high_max": 0.0018462733132764697, "clip_ratio/high_mean": 0.0006783678036299534, "clip_ratio/low_mean": 0.00022967774930293672, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0009080455565708689, "epoch": 0.775, "grad_norm": 1.6026839833387687, "kl": 0.018804894061759114, "learning_rate": 1.272995289285202e-07, "loss": 0.0005071645136922598, "step": 124, "step_time": 93.19347396120429 }, { "clip_ratio/high_max": 0.002319955194252543, "clip_ratio/high_mean": 0.0007834393873054069, "clip_ratio/low_mean": 0.00014439101505558938, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0009278304023609962, "epoch": 0.78125, "grad_norm": 3.1667993200415614, "kl": 0.03483924665488303, "learning_rate": 1.2062093865360457e-07, "loss": 0.00062437733868137, "step": 125, "step_time": 91.44077399373055 }, { "clip_ratio/high_max": 0.002308216266101226, "clip_ratio/high_mean": 0.0006199582276167348, "clip_ratio/low_mean": 0.00047605226063751616, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010960104846162722, "epoch": 0.7875, "grad_norm": 1.883831652345258, "kl": 0.03138963528908789, "learning_rate": 1.1409819408411897e-07, "loss": 0.000468918529804796, "step": 126, "step_time": 92.60021637380123 }, { "clip_ratio/high_max": 0.00342684247152647, "clip_ratio/high_mean": 0.0009535548797430238, "clip_ratio/low_mean": 0.0008377721242140979, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017913270039571216, "epoch": 0.79375, "grad_norm": 2.6611909251920602, "kl": 0.02214129827916622, "learning_rate": 1.0773397470829143e-07, "loss": 6.995504372753203e-05, "step": 127, "step_time": 91.6484418027103 }, { "clip_ratio/high_max": 0.0009090593084692955, "clip_ratio/high_mean": 0.00022726482711732388, "clip_ratio/low_mean": 0.0002288818359375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00045614665577886626, "epoch": 0.8, "grad_norm": 1.5112708759953353, "kl": 0.013814892387017608, "learning_rate": 1.0153089489354256e-07, "loss": 0.00023454276379197836, "step": 128, "step_time": 93.67365654557943 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.23046875, "completions/max_length": 2048.0, "completions/mean_length": 1018.03125, "completions/min_length": 65.0, "epoch": 0.80625, "frac_reward_zero_std": 0.1875, "grad_norm": 3.941082056513956, "kl": 0.027054909383878112, "learning_rate": 9.549150281252632e-08, "loss": 0.0002704653888940811, "reward": 0.43359375, "reward_std": 0.364554226398468, "rewards/MajorityAccuracy/mean": 0.43359375, "rewards/MajorityAccuracy/std": 0.4965413510799408, "step": 129, "step_time": 606.0889938920736 }, { "clip_ratio/high_max": 0.0014833517780061811, "clip_ratio/high_mean": 0.0005835227639181539, "clip_ratio/low_mean": 0.0003513442716212012, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0009348670391773339, "epoch": 0.8125, "grad_norm": 2.2846556700458454, "kl": 0.021552134305238724, "learning_rate": 8.961827939636196e-08, "loss": 0.0005635624984279275, "step": 130, "step_time": 91.86953563988209 }, { "clip_ratio/high_max": 0.0017435181434848346, "clip_ratio/high_mean": 0.00043587953587120865, "clip_ratio/low_mean": 0.00022521338360093068, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0006610929158341605, "epoch": 0.81875, "grad_norm": 2.4886708868384013, "kl": 0.01299207448028028, "learning_rate": 8.391363731548811e-08, "loss": 0.0001300731673836708, "step": 131, "step_time": 93.82235095649958 }, { "clip_ratio/high_max": 0.0031675055506639183, "clip_ratio/high_mean": 0.0009905886254273355, "clip_ratio/low_mean": 0.0003309748353785835, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013215634717198554, "epoch": 0.825, "grad_norm": 2.9601791114067844, "kl": 0.017751977313309908, "learning_rate": 7.837991998855897e-08, "loss": 0.00024236459285020828, "step": 132, "step_time": 91.65208266302943 }, { "clip_ratio/high_max": 0.0051080945413559675, "clip_ratio/high_mean": 0.0015815897641004995, "clip_ratio/low_mean": 0.0008449319284409285, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002426521692541428, "epoch": 0.83125, "grad_norm": 3.753769191814631, "kl": 0.027281400514766574, "learning_rate": 7.301940061978722e-08, "loss": 0.00014307070523500443, "step": 133, "step_time": 91.03292602300644 }, { "clip_ratio/high_max": 0.0011397445778129622, "clip_ratio/high_mean": 0.00028493614445324056, "clip_ratio/low_mean": 0.0005138979286130052, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0007988340730662458, "epoch": 0.8375, "grad_norm": 1.7423897379911264, "kl": 0.02061401621904224, "learning_rate": 6.783428126513125e-08, "loss": 0.0001985627313842997, "step": 134, "step_time": 91.61052031069994 }, { "clip_ratio/high_max": 0.0008280314723378979, "clip_ratio/high_mean": 0.00020700786808447447, "clip_ratio/low_mean": 0.00013304472849995364, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0003400525984034175, "epoch": 0.84375, "grad_norm": 1.1645810050664553, "kl": 0.013928022352047265, "learning_rate": 6.282669192770895e-08, "loss": 0.00018185722001362592, "step": 135, "step_time": 91.6570111438632 }, { "clip_ratio/high_max": 0.0015455730754183605, "clip_ratio/high_mean": 0.00043887017454835586, "clip_ratio/low_mean": 0.0003444630929152481, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.000783333267463604, "epoch": 0.85, "grad_norm": 1.7398503328046129, "kl": 0.013384801102802157, "learning_rate": 5.799868968281074e-08, "loss": 0.00021634310542140156, "step": 136, "step_time": 93.12740936875343 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.328125, "completions/max_length": 2048.0, "completions/mean_length": 1301.8203125, "completions/min_length": 104.0, "epoch": 0.85625, "frac_reward_zero_std": 0.3125, "grad_norm": 1.4320975286079416, "kl": 0.013878306723199785, "learning_rate": 5.335225783287051e-08, "loss": 0.0001387240772601217, "reward": 0.3984375, "reward_std": 0.31075698137283325, "rewards/MajorityAccuracy/mean": 0.3984375, "rewards/MajorityAccuracy/std": 0.4905354082584381, "step": 137, "step_time": 614.1431872360408 }, { "clip_ratio/high_max": 0.0018042457813862711, "clip_ratio/high_mean": 0.0006544681746163405, "clip_ratio/low_mean": 0.0001983642578125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008528324324288405, "epoch": 0.8625, "grad_norm": 2.0370963826004242, "kl": 0.021261168411001563, "learning_rate": 4.888930509274125e-08, "loss": 5.2794814109802246e-05, "step": 138, "step_time": 92.8821756131947 }, { "clip_ratio/high_max": 0.0032449815480504185, "clip_ratio/high_mean": 0.0008112453870126046, "clip_ratio/low_mean": 0.00021038726845290512, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010216326554655097, "epoch": 0.86875, "grad_norm": 2.645106311875411, "kl": 0.023260330082848668, "learning_rate": 4.4611664805611794e-08, "loss": 0.00023684202460572124, "step": 139, "step_time": 90.97562847286463 }, { "clip_ratio/high_max": 0.0026665623590815812, "clip_ratio/high_mean": 0.0006666405897703953, "clip_ratio/low_mean": 0.00033828747473307885, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010049280645034742, "epoch": 0.875, "grad_norm": 1.2603193687182697, "kl": 0.012804505531676114, "learning_rate": 4.0521094189884696e-08, "loss": 2.6322311896365136e-05, "step": 140, "step_time": 93.82410955801606 }, { "clip_ratio/high_max": 0.00023408239940181375, "clip_ratio/high_mean": 5.8520599850453436e-05, "clip_ratio/low_mean": 0.000273304307484068, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0003318249073345214, "epoch": 0.88125, "grad_norm": 0.975151806829916, "kl": 0.009800380561500788, "learning_rate": 3.6619273617325695e-08, "loss": 6.465690967161208e-05, "step": 141, "step_time": 94.4879607707262 }, { "clip_ratio/high_max": 0.0016792996248113923, "clip_ratio/high_mean": 0.00049556459816813, "clip_ratio/low_mean": 3.0622604754171334e-05, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0005261872047412908, "epoch": 0.8875, "grad_norm": 2.2674745767782567, "kl": 0.019610450486652553, "learning_rate": 3.2907805922781476e-08, "loss": 0.00011976382666034624, "step": 142, "step_time": 93.05212846025825 }, { "clip_ratio/high_max": 0.005828458713949658, "clip_ratio/high_mean": 0.0020048600163136143, "clip_ratio/low_mean": 0.0006082619911467191, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026131219638045877, "epoch": 0.89375, "grad_norm": 3.6391422191197957, "kl": 0.02122194366529584, "learning_rate": 2.9388215745748345e-08, "loss": 0.0006820010021328926, "step": 143, "step_time": 93.60796646401286 }, { "clip_ratio/high_max": 0.0011784754096879624, "clip_ratio/high_mean": 0.0002946188524219906, "clip_ratio/low_mean": 3.0517578125e-05, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00032513643236598, "epoch": 0.9, "grad_norm": 0.8647733049543943, "kl": 0.010236152564175427, "learning_rate": 2.6061948904063658e-08, "loss": 0.0001808695524232462, "step": 144, "step_time": 93.17840433865786 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.26953125, "completions/max_length": 2048.0, "completions/mean_length": 1188.2890625, "completions/min_length": 94.0, "epoch": 0.90625, "frac_reward_zero_std": 0.1875, "grad_norm": 2.943047513436166, "kl": 0.02456643688492477, "learning_rate": 2.293037179997559e-08, "loss": 0.00024553845287300646, "reward": 0.421875, "reward_std": 0.38440707325935364, "rewards/MajorityAccuracy/mean": 0.421875, "rewards/MajorityAccuracy/std": 0.49482616782188416, "step": 145, "step_time": 608.3999883718789 }, { "clip_ratio/high_max": 0.004304405949369539, "clip_ratio/high_mean": 0.0013214530790719436, "clip_ratio/low_mean": 8.701669503352605e-05, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014084697741054697, "epoch": 0.9125, "grad_norm": 1.6543977136872499, "kl": 0.013960942393168807, "learning_rate": 1.9994770858837107e-08, "loss": 0.00016222721023950726, "step": 146, "step_time": 92.20001374557614 }, { "clip_ratio/high_max": 0.0027994008851237595, "clip_ratio/high_mean": 0.0007548678404418752, "clip_ratio/low_mean": 0.00030059157870709896, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010554594191489741, "epoch": 0.91875, "grad_norm": 2.5265212239171233, "kl": 0.012740381294861436, "learning_rate": 1.725635200065323e-08, "loss": 0.0008408460998907685, "step": 147, "step_time": 92.4580844938755 }, { "clip_ratio/high_max": 0.006912427925271913, "clip_ratio/high_mean": 0.002186286816140637, "clip_ratio/low_mean": 0.0002703096361074131, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002456596466799965, "epoch": 0.925, "grad_norm": 4.574971544275735, "kl": 0.014113029814325273, "learning_rate": 1.4716240144699187e-08, "loss": 0.0007535265758633614, "step": 148, "step_time": 91.15843005478382 }, { "clip_ratio/high_max": 0.000528227785252966, "clip_ratio/high_mean": 0.00017033166295732372, "clip_ratio/low_mean": 0.00017045388995029498, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0003407855510886293, "epoch": 0.93125, "grad_norm": 1.2478533344454932, "kl": 0.009942804812453687, "learning_rate": 1.2375478747413015e-08, "loss": 5.637717549689114e-06, "step": 149, "step_time": 94.3928376659751 }, { "clip_ratio/high_max": 0.004107552915229462, "clip_ratio/high_mean": 0.0010816824178618845, "clip_ratio/low_mean": 0.0012094910198356956, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.002291173441335559, "epoch": 0.9375, "grad_norm": 9.710265063615376, "kl": 0.020444985362701118, "learning_rate": 1.0235029373752757e-08, "loss": 0.0008776523172855377, "step": 150, "step_time": 93.11101581901312 }, { "clip_ratio/high_max": 0.002559824089985341, "clip_ratio/high_mean": 0.0006811687198933214, "clip_ratio/low_mean": 0.0003419848289922811, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010231535488856025, "epoch": 0.94375, "grad_norm": 1.9244053961372938, "kl": 0.011974780005402863, "learning_rate": 8.295771302193721e-09, "loss": 0.00013202853733673692, "step": 151, "step_time": 92.69347719848156 }, { "clip_ratio/high_max": 0.0035128566523781046, "clip_ratio/high_mean": 0.0010411365510663018, "clip_ratio/low_mean": 0.0005964913289062679, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016376278435927816, "epoch": 0.95, "grad_norm": 4.569389996612099, "kl": 0.03170365002006292, "learning_rate": 6.558501163527963e-09, "loss": -0.00047912728041410446, "step": 152, "step_time": 91.62623711675406 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.296875, "completions/max_length": 2048.0, "completions/mean_length": 1143.296875, "completions/min_length": 83.0, "epoch": 0.95625, "frac_reward_zero_std": 0.15625, "grad_norm": 3.9346865942474873, "kl": 0.01996052695903927, "learning_rate": 5.023932613615445e-09, "loss": 0.00019967369735240936, "reward": 0.42578125, "reward_std": 0.3994034230709076, "rewards/MajorityAccuracy/mean": 0.42578125, "rewards/MajorityAccuracy/std": 0.49542948603630066, "step": 153, "step_time": 610.5072671175003 }, { "clip_ratio/high_max": 0.0021787754230899736, "clip_ratio/high_mean": 0.0008257212903117761, "clip_ratio/low_mean": 0.000264132580923615, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010898538603214547, "epoch": 0.9625, "grad_norm": 2.1484729712795887, "kl": 0.013331101043149829, "learning_rate": 3.6926960402202666e-09, "loss": 0.00021863356232643127, "step": 154, "step_time": 92.12357895076275 }, { "clip_ratio/high_max": 0.0008119291087496094, "clip_ratio/high_mean": 0.00031235500318871345, "clip_ratio/low_mean": 0.0001373291015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00044968410293222405, "epoch": 0.96875, "grad_norm": 1.2887329470782247, "kl": 0.010540736315306276, "learning_rate": 2.5653383040524224e-09, "loss": 0.0001984368427656591, "step": 155, "step_time": 93.6834515221417 }, { "clip_ratio/high_max": 0.0019768475758610293, "clip_ratio/high_mean": 0.0005762601540482137, "clip_ratio/low_mean": 0.0001678466796875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0007441068300977349, "epoch": 0.975, "grad_norm": 2.660644656519148, "kl": 0.013566589914262295, "learning_rate": 1.642322514122385e-09, "loss": 0.00043874606490135193, "step": 156, "step_time": 93.4388095960021 }, { "clip_ratio/high_max": 0.0004832191189052537, "clip_ratio/high_mean": 0.00012080477972631343, "clip_ratio/low_mean": 0.00034263752058905084, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00046344230031536426, "epoch": 0.98125, "grad_norm": 3.7733653730970462, "kl": 0.01926533435471356, "learning_rate": 9.240278374995636e-10, "loss": 0.00035821678466163576, "step": 157, "step_time": 92.32515724003315 }, { "clip_ratio/high_max": 0.00141456624987768, "clip_ratio/high_mean": 0.0004933464751957217, "clip_ratio/low_mean": 0.0005064742144895718, "clip_ratio/low_min": 6.103515625e-05, "clip_ratio/region_mean": 0.0009998206951422617, "epoch": 0.9875, "grad_norm": 3.559171557703873, "kl": 0.0149534068768844, "learning_rate": 4.1074934355384006e-10, "loss": 0.00033688731491565704, "step": 158, "step_time": 93.15880297496915 }, { "clip_ratio/high_max": 0.0017868130162241869, "clip_ratio/high_mean": 0.0005238304911472369, "clip_ratio/low_mean": 0.0002303986402694136, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0007542291314166505, "epoch": 0.99375, "grad_norm": 1.5222360035136502, "kl": 0.021274650702252984, "learning_rate": 1.0269788274358537e-10, "loss": 0.00045027249143458903, "step": 159, "step_time": 92.57221077755094 }, { "clip_ratio/high_max": 0.0036335844051791355, "clip_ratio/high_mean": 0.0010374509765824769, "clip_ratio/low_mean": 0.0005417419888544828, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0015791929654369596, "epoch": 1.0, "grad_norm": 2.6760299525418834, "kl": 0.015428774990141392, "learning_rate": 0.0, "loss": 0.00022567404084838927, "step": 160, "step_time": 91.40290051326156 } ], "logging_steps": 1, "max_steps": 160, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 80, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }
{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 80.0,
"global_step": 160,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.6328125,
"completions/max_length": 2048.0,
"completions/mean_length": 1654.703125,
"completions/min_length": 156.0,
"epoch": 0.00625,
"frac_reward_zero_std": 0.3125,
"grad_norm": 1.3100795864283492,
"kl": 0.0,
"learning_rate": 2e-07,
"loss": -9.313225746154785e-09,
"reward": 0.24609375,
"reward_std": 0.29196497797966003,
"rewards/MajorityAccuracy/mean": 0.24609375,
"rewards/MajorityAccuracy/std": 0.43157756328582764,
"step": 1,
"step_time": 658.7488895617425
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"epoch": 0.0125,
"grad_norm": 1.5411547555701335,
"kl": 0.0,
"learning_rate": 4e-07,
"loss": -9.313225746154785e-10,
"step": 2,
"step_time": 92.33193903788924
},
{
"clip_ratio/high_max": 0.0020130398188484833,
"clip_ratio/high_mean": 0.0006289757548074704,
"clip_ratio/low_mean": 0.00033599255402805284,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0009649683051975444,
"epoch": 0.01875,
"grad_norm": 2.6111678582441216,
"kl": 0.0003152433109789854,
"learning_rate": 6e-07,
"loss": 0.0002443257544655353,
"step": 3,
"step_time": 94.15333711355925
},
{
"clip_ratio/high_max": 0.0007583651677123271,
"clip_ratio/high_mean": 0.00018959129192808177,
"clip_ratio/low_mean": 0.000274658203125,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0004642494968720712,
"epoch": 0.025,
"grad_norm": 0.8986616830589469,
"kl": 0.00016798217166069662,
"learning_rate": 8e-07,
"loss": -2.4342021788470447e-05,
"step": 4,
"step_time": 94.268355473876
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"epoch": 0.03125,
"grad_norm": 0.0018725214294947354,
"kl": 0.00017453282998758368,
"learning_rate": 1e-06,
"loss": 1.7468580608692719e-06,
"step": 5,
"step_time": 94.58316105231643
},
{
"clip_ratio/high_max": 0.00047015461314003915,
"clip_ratio/high_mean": 0.00017650091831455939,
"clip_ratio/low_mean": 0.00047675427776994184,
"clip_ratio/low_min": 6.103515625e-05,
"clip_ratio/region_mean": 0.00065325519972248,
"epoch": 0.0375,
"grad_norm": 1.7121795828917972,
"kl": 0.0001731301636027638,
"learning_rate": 9.998973021172564e-07,